@livekit/agents 1.0.16 → 1.0.18

package/dist/stt/stt.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame, AudioResampler } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { calculateAudioDurationSeconds } from '../audio.js';\nimport { log } from '../log.js';\nimport type { STTMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  /**\n   * Preflight transcript, emitted before final transcript when STT has high confidence\n   * but hasn't fully committed yet. Includes all pre-committed transcripts including\n   * final transcript from the previous STT run.\n   */\n  PREFLIGHT_TRANSCRIPT = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport interface STTError {\n  type: 'stt_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type STTCallbacks = {\n  ['metrics_collected']: (metrics: STTMetrics) => void;\n  ['error']: (error: STTError) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit('metrics_collected', {\n      type: 'stt_metrics',\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      durationMs,\n      label: this.label,\n      audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),\n      streamed: false,\n    });\n    return event;\n  }\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected neededSampleRate?: number;\n  protected resampler?: AudioResampler;\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private _connOptions: APIConnectOptions;\n\n  constructor(\n    stt: STT,\n    sampleRate?: number,\n    connectionOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#stt = stt;\n    this._connOptions = connectionOptions;\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    this.neededSampleRate = sampleRate;\n    this.monitorMetrics();\n    this.pumpInput();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#stt.label, attempt: i + 1, error },\n              `failed to recognize speech, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#stt.emit('error', {\n      type: 'stt_error',\n      timestamp: Date.now(),\n      label: this.#stt.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected async pumpInput() {\n    // TODO(AJS-35): Implement STT with webstreams API\n    const inputStream = this.deferredInputStream.stream;\n    const reader = inputStream.getReader();\n\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done) break;\n        this.pushFrame(value);\n      }\n    } catch (error) {\n      this.logger.error('Error in STTStream mainTask:', error);\n    } finally {\n      reader.releaseLock();\n    }\n  }\n\n  protected async monitorMetrics() {\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const metrics: STTMetrics = {\n        type: 'stt_metrics',\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        durationMs: 0,\n        label: this.#stt.label,\n        audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),\n        streamed: true,\n      };\n      this.#stt.emit('metrics_collected', metrics);\n    }\n    this.output.close();\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {\n      if (!this.resampler) {\n        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);\n      }\n    }\n\n    if (this.resampler) {\n      const frames = this.resampler.push(frame);\n      for (const frame of frames) {\n        this.input.put(frame);\n      }\n    } else {\n      this.input.put(frame);\n    }\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAAgD;AAEhD,yBAA6B;AAE7B,wBAA6C;AAC7C,mBAA8C;AAC9C,iBAAoB;AAEpB,6BAAuC;AACvC,mBAAoE;AAEpE,mBAA8D;AAGvD,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AAMA,EAAAA,kCAAA,0BAAuB,KAAvB;AA5BU,SAAAA;AAAA,GAAA;AAmFL,MAAe,YAAa,gCAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,aAAa,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACjF,SAAK,KAAK,qBAAqB;AAAA,MAC7B,MAAM;AAAA,MACN,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK,UAAM,4CAA8B,KAAK,IAAI,GAAI;AAAA,MACvE,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AAQF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,SAAS,IAAI,gCAAgC;AAAA,EAC7C,QAAQ,IAAI,gCAAgC;AAAA,EAC5C;AAAA,EACA;AAAA,EAEA,SAAS;AAAA,EACnB;AAAA,EACQ;AAAA,EACA,aAAS,gBAAI;AAAA,EACb;AAAA,EAER,YACE,KACA,YACA,oBAAuC,0CACvC;AACA,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,8CAAmC;AAClE,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,UAAU;AAMf,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,oCAAoC,KAAK,aAAa,WAAW,CAAC;AAAA,cAC3E,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,2CAA2C,aAAa;AAAA,YAC1D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,YAAY;AAE1B,UAAM,cAAc,KAAK,oBAAoB;AAC7C,UAAM,SAAS,YAAY,UAAU;AAErC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,aAAK,UAAU,KAAK;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,KAAK;AAAA,IACzD,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,UAAsB;AAAA,QAC1B,MAAM;AAAA,QACN,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,YAAY;AAAA,QACZ,OAAO,KAAK,KAAK;AAAA,QACjB,iBAAiB,KAAK,MAAM,MAAM,iBAAkB,gBAAgB,GAAI;AAAA,QACxE,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,IAC7C;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAIA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,oBAAoB,MAAM,eAAe,KAAK,kBAAkB;AACvE,UAAI,CAAC,KAAK,WAAW;AACnB,aAAK,YAAY,IAAI,+BAAe,MAAM,YAAY,KAAK,gBAAgB;AAAA,MAC7E;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,SAAS,KAAK,UAAU,KAAK,KAAK;AACxC,iBAAWC,UAAS,QAAQ;AAC1B,aAAK,MAAM,IAAIA,MAAK;AAAA,MACtB;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType","frame"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame, AudioResampler } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { calculateAudioDurationSeconds } from '../audio.js';\nimport { log } from '../log.js';\nimport type { STTMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  /**\n   * Preflight transcript, emitted before final transcript when STT has high confidence\n   * but hasn't fully committed yet. Includes all pre-committed transcripts including\n   * final transcript from the previous STT run.\n   */\n  PREFLIGHT_TRANSCRIPT = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport interface STTError {\n  type: 'stt_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type STTCallbacks = {\n  ['metrics_collected']: (metrics: STTMetrics) => void;\n  ['error']: (error: STTError) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit('metrics_collected', {\n      type: 'stt_metrics',\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      durationMs,\n      label: this.label,\n      audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),\n      streamed: false,\n    });\n    return event;\n  }\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected neededSampleRate?: number;\n  protected resampler?: AudioResampler;\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private _connOptions: APIConnectOptions;\n\n  constructor(\n    stt: STT,\n    sampleRate?: number,\n    connectionOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#stt = stt;\n    this._connOptions = connectionOptions;\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    this.neededSampleRate = sampleRate;\n    this.monitorMetrics();\n    this.pumpInput();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#stt.label, attempt: i + 1, error },\n              `failed to recognize speech, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#stt.emit('error', {\n      type: 'stt_error',\n      timestamp: Date.now(),\n      label: this.#stt.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected async pumpInput() {\n    // TODO(AJS-35): Implement STT with webstreams API\n    const inputStream = this.deferredInputStream.stream;\n    const reader = inputStream.getReader();\n\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done) break;\n        this.pushFrame(value);\n      }\n    } catch (error) {\n      this.logger.error('Error in STTStream mainTask:', error);\n    } finally {\n      reader.releaseLock();\n    }\n  }\n\n  protected async monitorMetrics() {\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const metrics: STTMetrics = {\n        type: 'stt_metrics',\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        durationMs: 0,\n        label: this.#stt.label,\n        audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),\n        streamed: true,\n      };\n      this.#stt.emit('metrics_collected', metrics);\n    }\n    this.output.close();\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {\n      if (!this.resampler) {\n        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);\n      }\n    }\n\n    if (this.resampler) {\n      const frames = this.resampler.push(frame);\n      for (const frame of frames) {\n        this.input.put(frame);\n      }\n    } else {\n      this.input.put(frame);\n    }\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAAgD;AAEhD,yBAA6B;AAE7B,wBAA6C;AAC7C,mBAA8C;AAC9C,iBAAoB;AAEpB,6BAAuC;AACvC,mBAAoE;AAEpE,mBAA8D;AAGvD,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AAMA,EAAAA,kCAAA,0BAAuB,KAAvB;AA5BU,SAAAA;AAAA,GAAA;AAmFL,MAAe,YAAa,gCAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,aAAa,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACjF,SAAK,KAAK,qBAAqB;AAAA,MAC7B,MAAM;AAAA,MACN,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK,UAAM,4CAA8B,KAAK,IAAI,GAAI;AAAA,MACvE,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AAQF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,SAAS,IAAI,gCAAgC;AAAA,EAC7C,QAAQ,IAAI,gCAAgC;AAAA,EAC5C;AAAA,EACA;AAAA,EAEA,SAAS;AAAA,EACnB;AAAA,EACQ;AAAA,EACA,aAAS,gBAAI;AAAA,EACb;AAAA,EAER,YACE,KACA,YACA,oBAAuC,0CACvC;AACA,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,8CAAmC;AAClE,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,UAAU;AAMf,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,oCAAoC,KAAK,aAAa,WAAW,CAAC;AAAA,cAC3E,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,2CAA2C,aAAa;AAAA,YAC1D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,YAAY;AAE1B,UAAM,cAAc,KAAK,oBAAoB;AAC7C,UAAM,SAAS,YAAY,UAAU;AAErC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,aAAK,UAAU,KAAK;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,KAAK;AAAA,IACzD,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,UAAsB;AAAA,QAC1B,MAAM;AAAA,QACN,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,YAAY;AAAA,QACZ,OAAO,KAAK,KAAK;AAAA,QACjB,iBAAiB,KAAK,MAAM,MAAM,iBAAkB,gBAAgB,GAAI;AAAA,QACxE,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,IAC7C;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAIA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,oBAAoB,MAAM,eAAe,KAAK,kBAAkB;AACvE,UAAI,CAAC,KAAK,WAAW;AACnB,aAAK,YAAY,IAAI,+BAAe,MAAM,YAAY,KAAK,gBAAgB;AAAA,MAC7E;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,SAAS,KAAK,UAAU,KAAK,KAAK;AACxC,iBAAWC,UAAS,QAAQ;AAC1B,aAAK,MAAM,IAAIA,MAAK;AAAA,MACtB;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType","frame"]}

package/dist/stt/stt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,KAAK,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAItD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAE5E,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;IACjB,mEAAmE;IACnE,iBAAiB,IAAI;IACrB;;;;OAIG;IACH,oBAAoB,IAAI;CACzB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IACtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAGX,YAAY,EAAE,eAAe;IAKzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IACxF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAezD,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAEvE;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,MAAM,kCAAyC;IACzD,SAAS,CAAC,KAAK,kCAAyC;IACxD,SAAS,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,MAAM,UAAS;IAEzB,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAoB;gBAGtC,GAAG,EAAE,GAAG,EACR,UAAU,CAAC,EAAE,MAAM,EACnB,iBAAiB,GAAE,iBAA+C;YAgBtD,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAUD,SAAS;cAkBT,cAAc;IAkB9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAIzD,iBAAiB;IAIjB,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAwB3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}
+{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,KAAK,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAItD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAE5E,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;IACjB,mEAAmE;IACnE,iBAAiB,IAAI;IACrB;;;;OAIG;IACH,oBAAoB,IAAI;CACzB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IACtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAGX,YAAY,EAAE,eAAe;IAKzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IACxF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAezD,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAEvE;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,MAAM,kCAAyC;IACzD,SAAS,CAAC,KAAK,kCAAyC;IACxD,SAAS,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,MAAM,UAAS;IAEzB,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAoB;gBAGtC,GAAG,EAAE,GAAG,EACR,UAAU,CAAC,EAAE,MAAM,EACnB,iBAAiB,GAAE,iBAA+C;YAgBtD,QAAQ;IAqCtB,OAAO,CAAC,SAAS;cAUD,SAAS;cAkBT,cAAc;IAkB9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAIzD,iBAAiB;IAIjB,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAwB3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}

package/dist/stt/stt.js

@@ -80,7 +80,6 @@ class SpeechStream {
               options: { retryable: false }
             });
           } else {
-            this.emitError({ error, recoverable: true });
             this.logger.warn(
               { tts: this.#stt.label, attempt: i + 1, error },
               `failed to recognize speech, retrying in ${retryInterval}s`

package/dist/stt/stt.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame, AudioResampler } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { calculateAudioDurationSeconds } from '../audio.js';\nimport { log } from '../log.js';\nimport type { STTMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  /**\n   * Preflight transcript, emitted before final transcript when STT has high confidence\n   * but hasn't fully committed yet. Includes all pre-committed transcripts including\n   * final transcript from the previous STT run.\n   */\n  PREFLIGHT_TRANSCRIPT = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport interface STTError {\n  type: 'stt_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type STTCallbacks = {\n  ['metrics_collected']: (metrics: STTMetrics) => void;\n  ['error']: (error: STTError) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit('metrics_collected', {\n      type: 'stt_metrics',\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      durationMs,\n      label: this.label,\n      audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),\n      streamed: false,\n    });\n    return event;\n  }\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected neededSampleRate?: number;\n  protected resampler?: AudioResampler;\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private _connOptions: APIConnectOptions;\n\n  constructor(\n    stt: STT,\n    sampleRate?: number,\n    connectionOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#stt = stt;\n    this._connOptions = connectionOptions;\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    this.neededSampleRate = sampleRate;\n    this.monitorMetrics();\n    this.pumpInput();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#stt.label, attempt: i + 1, error },\n              `failed to recognize speech, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#stt.emit('error', {\n      type: 'stt_error',\n      timestamp: Date.now(),\n      label: this.#stt.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected async pumpInput() {\n    // TODO(AJS-35): Implement STT with webstreams API\n    const inputStream = this.deferredInputStream.stream;\n    const reader = inputStream.getReader();\n\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done) break;\n        this.pushFrame(value);\n      }\n    } catch (error) {\n      this.logger.error('Error in STTStream mainTask:', error);\n    } finally {\n      reader.releaseLock();\n    }\n  }\n\n  protected async monitorMetrics() {\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const metrics: STTMetrics = {\n        type: 'stt_metrics',\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        durationMs: 0,\n        label: this.#stt.label,\n        audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),\n        streamed: true,\n      };\n      this.#stt.emit('metrics_collected', metrics);\n    }\n    this.output.close();\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {\n      if (!this.resampler) {\n        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);\n      }\n    }\n\n    if (this.resampler) {\n      const frames = this.resampler.push(frame);\n      for (const frame of frames) {\n        this.input.put(frame);\n      }\n    } else {\n      this.input.put(frame);\n    }\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAGA,SAA0B,sBAAsB;AAEhD,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,qCAAqC;AAC9C,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAiC,mCAAmC;AAEpE,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAGvD,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AAMA,EAAAA,kCAAA,0BAAuB,KAAvB;AA5BU,SAAAA;AAAA,GAAA;AAmFL,MAAe,YAAa,aAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,aAAa,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACjF,SAAK,KAAK,qBAAqB;AAAA,MAC7B,MAAM;AAAA,MACN,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK,MAAM,8BAA8B,KAAK,IAAI,GAAI;AAAA,MACvE,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AAQF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,SAAS,IAAI,mBAAgC;AAAA,EAC7C,QAAQ,IAAI,mBAAgC;AAAA,EAC5C;AAAA,EACA;AAAA,EAEA,SAAS;AAAA,EACnB;AAAA,EACQ;AAAA,EACA,SAAS,IAAI;AAAA,EACb;AAAA,EAER,YACE,KACA,YACA,oBAAuC,6BACvC;AACA,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,UAAU;AAMf,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,oCAAoC,KAAK,aAAa,WAAW,CAAC;AAAA,cAC3E,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,2CAA2C,aAAa;AAAA,YAC1D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,YAAY;AAE1B,UAAM,cAAc,KAAK,oBAAoB;AAC7C,UAAM,SAAS,YAAY,UAAU;AAErC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,aAAK,UAAU,KAAK;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,KAAK;AAAA,IACzD,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,UAAsB;AAAA,QAC1B,MAAM;AAAA,QACN,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,YAAY;AAAA,QACZ,OAAO,KAAK,KAAK;AAAA,QACjB,iBAAiB,KAAK,MAAM,MAAM,iBAAkB,gBAAgB,GAAI;AAAA,QACxE,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,IAC7C;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAIA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,oBAAoB,MAAM,eAAe,KAAK,kBAAkB;AACvE,UAAI,CAAC,KAAK,WAAW;AACnB,aAAK,YAAY,IAAI,eAAe,MAAM,YAAY,KAAK,gBAAgB;AAAA,MAC7E;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,SAAS,KAAK,UAAU,KAAK,KAAK;AACxC,iBAAWC,UAAS,QAAQ;AAC1B,aAAK,MAAM,IAAIA,MAAK;AAAA,MACtB;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType","frame"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame, AudioResampler } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { calculateAudioDurationSeconds } from '../audio.js';\nimport { log } from '../log.js';\nimport type { STTMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  /**\n   * Preflight transcript, emitted before final transcript when STT has high confidence\n   * but hasn't fully committed yet. Includes all pre-committed transcripts including\n   * final transcript from the previous STT run.\n   */\n  PREFLIGHT_TRANSCRIPT = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport interface STTError {\n  type: 'stt_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type STTCallbacks = {\n  ['metrics_collected']: (metrics: STTMetrics) => void;\n  ['error']: (error: STTError) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit('metrics_collected', {\n      type: 'stt_metrics',\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      durationMs,\n      label: this.label,\n      audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),\n      streamed: false,\n    });\n    return event;\n  }\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected neededSampleRate?: number;\n  protected resampler?: AudioResampler;\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private _connOptions: APIConnectOptions;\n\n  constructor(\n    stt: STT,\n    sampleRate?: number,\n    connectionOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#stt = stt;\n    this._connOptions = connectionOptions;\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    this.neededSampleRate = sampleRate;\n    this.monitorMetrics();\n    this.pumpInput();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#stt.label, attempt: i + 1, error },\n              `failed to recognize speech, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#stt.emit('error', {\n      type: 'stt_error',\n      timestamp: Date.now(),\n      label: this.#stt.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected async pumpInput() {\n    // TODO(AJS-35): Implement STT with webstreams API\n    const inputStream = this.deferredInputStream.stream;\n    const reader = inputStream.getReader();\n\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done) break;\n        this.pushFrame(value);\n      }\n    } catch (error) {\n      this.logger.error('Error in STTStream mainTask:', error);\n    } finally {\n      reader.releaseLock();\n    }\n  }\n\n  protected async monitorMetrics() {\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const metrics: STTMetrics = {\n        type: 'stt_metrics',\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        durationMs: 0,\n        label: this.#stt.label,\n        audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),\n        streamed: true,\n      };\n      this.#stt.emit('metrics_collected', metrics);\n    }\n    this.output.close();\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {\n      if (!this.resampler) {\n        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);\n      }\n    }\n\n    if (this.resampler) {\n      const frames = this.resampler.push(frame);\n      for (const frame of frames) {\n        this.input.put(frame);\n      }\n    } else {\n      this.input.put(frame);\n    }\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAGA,SAA0B,sBAAsB;AAEhD,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,qCAAqC;AAC9C,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAiC,mCAAmC;AAEpE,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAGvD,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AAMA,EAAAA,kCAAA,0BAAuB,KAAvB;AA5BU,SAAAA;AAAA,GAAA;AAmFL,MAAe,YAAa,aAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,aAAa,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AACjF,SAAK,KAAK,qBAAqB;AAAA,MAC7B,MAAM;AAAA,MACN,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK,MAAM,8BAA8B,KAAK,IAAI,GAAI;AAAA,MACvE,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AAQF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,SAAS,IAAI,mBAAgC;AAAA,EAC7C,QAAQ,IAAI,mBAAgC;AAAA,EAC5C;AAAA,EACA;AAAA,EAEA,SAAS;AAAA,EACnB;AAAA,EACQ;AAAA,EACA,SAAS,IAAI;AAAA,EACb;AAAA,EAER,YACE,KACA,YACA,oBAAuC,6BACvC;AACA,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,UAAU;AAMf,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,oCAAoC,KAAK,aAAa,WAAW,CAAC;AAAA,cAC3E,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,2CAA2C,aAAa;AAAA,YAC1D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,YAAY;AAE1B,UAAM,cAAc,KAAK,oBAAoB;AAC7C,UAAM,SAAS,YAAY,UAAU;AAErC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,aAAK,UAAU,KAAK;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,KAAK;AAAA,IACzD,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,UAAsB;AAAA,QAC1B,MAAM;AAAA,QACN,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,YAAY;AAAA,QACZ,OAAO,KAAK,KAAK;AAAA,QACjB,iBAAiB,KAAK,MAAM,MAAM,iBAAkB,gBAAgB,GAAI;AAAA,QACxE,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,IAC7C;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAIA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,oBAAoB,MAAM,eAAe,KAAK,kBAAkB;AACvE,UAAI,CAAC,KAAK,WAAW;AACnB,aAAK,YAAY,IAAI,eAAe,MAAM,YAAY,KAAK,gBAAgB;AAAA,MAC7E;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,SAAS,KAAK,UAAU,KAAK,KAAK;AACxC,iBAAWC,UAAS,QAAQ;AAC1B,aAAK,MAAM,IAAIA,MAAK;AAAA,MACtB;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType","frame"]}

package/dist/tts/tts.cjs

@@ -85,7 +85,7 @@ class SynthesizeStream {
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof import_exceptions.APIStatusError) {
+        if (error instanceof import_exceptions.APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -97,7 +97,6 @@ class SynthesizeStream {
               options: { retryable: false }
             });
           } else {
-            this.emitError({ error, recoverable: true });
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to synthesize speech, retrying in  ${retryInterval}s`
@@ -262,7 +261,7 @@ class ChunkedStream {
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof import_exceptions.APIStatusError) {
+        if (error instanceof import_exceptions.APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -274,7 +273,6 @@ class ChunkedStream {
               options: { retryable: false }
             });
           } else {
-            this.emitError({ error, recoverable: true });
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to generate TTS completion, retrying in ${retryInterval}s`

package/dist/tts/tts.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIStatusError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport interface TTSError {\n  type: 'tts_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type TTSCallbacks = {\n  ['metrics_collected']: (metrics: TTSMetrics) => void;\n  ['error']: (error: TTSError) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n  private _connOptions: APIConnectOptions;\n  protected abortController = new AbortController();\n\n  private deferredInputStream: DeferredReadableStream<\n    string | typeof SynthesizeStream.FLUSH_SENTINEL\n  >;\n  private logger = log();\n\n  constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {\n    this.#tts = tts;\n    this._connOptions = connOptions;\n    this.deferredInputStream = new DeferredReadableStream();\n    this.pumpInput();\n    this.abortController.signal.addEventListener('abort', () => {\n      this.deferredInputStream.detachSource();\n      // TODO (AJS-36) clean this up when we refactor with streams\n      this.input.close();\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIStatusError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to synthesize speech, retrying in  ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  // TODO(AJS-37) Remove when refactoring TTS to use streams\n  protected async pumpInput() {\n    const reader = this.deferredInputStream.stream.getReader();\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done || value === SynthesizeStream.FLUSH_SENTINEL) {\n          break;\n        }\n        this.pushText(value);\n      }\n      this.endInput();\n    } catch (error) {\n      this.logger.error(error, 'Error reading deferred input stream');\n    } finally {\n      reader.releaseLock();\n      // Ensure output is closed when the stream ends\n      if (!this.#monitorMetricsTask) {\n        // No text was received, close the output directly\n        this.output.close();\n      }\n    }\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const roundedAudioDurationMs = Math.round(audioDurationMs);\n        const metrics: TTSMetrics = {\n          type: 'tts_metrics',\n          timestamp: Date.now(),\n          requestId,\n          ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n          durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDurationMs: roundedAudioDurationMs,\n          cancelled: this.abortController.signal.aborted,\n          label: this.#tts.label,\n          streamed: false,\n        };\n        this.#tts.emit('metrics_collected', metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      // TODO(AJS-102): use frame.durationMs once available in rtc-node\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(text: ReadableStream<string>) {\n    this.deferredInputStream.setSource(text);\n  }\n\n  /** Push a string of text to the TTS */\n  /** @deprecated Use `updateInputStream` instead */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n      // Close output when metrics task completes\n      this.#monitorMetricsTask.finally(() => this.output.close());\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    this.flush();\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n  private _connOptions: APIConnectOptions;\n  private logger = log();\n\n  constructor(\n    text: string,\n    tts: TTS,\n    connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#text = text;\n    this.#tts = tts;\n    this._connOptions = connOptions;\n\n    this.monitorMetrics();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIStatusError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to generate TTS completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected abstract run(): Promise<void>;\n\n  get inputText(): string {\n    return this.#text;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      type: 'tts_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDurationMs: Math.round(audioDurationMs),\n      cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here\n      label: this.#tts.label,\n      streamed: false,\n    };\n    this.#tts.emit('metrics_collected', metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,wBAAmD;AACnD,iBAAoB;AAEpB,6BAAuC;AACvC,mBAAoE;AACpE,mBAA2E;AA+CpE,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAEpB;AAAA,EACQ,SAAS,IAAI,gCAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EACQ;AAAA,EACE,kBAAkB,IAAI,gBAAgB;AAAA,EAExC;AAAA,EAGA,aAAS,gBAAI;AAAA,EAErB,YAAY,KAAU,cAAiC,0CAA6B;AAClF,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,8CAAuB;AACtD,SAAK,UAAU;AACf,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAC1D,WAAK,oBAAoB,aAAa;AAEtC,WAAK,MAAM,MAAM;AACjB,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,kCAAgB;AACnC,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,6CAA6C,aAAa;AAAA,YAC5D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAgB,YAAY;AAC1B,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,QAAQ,UAAU,iBAAiB,gBAAgB;AACrD;AAAA,QACF;AACA,aAAK,SAAS,KAAK;AAAA,MACrB;AACA,WAAK,SAAS;AAAA,IAChB,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,qCAAqC;AAAA,IAChE,UAAE;AACA,aAAO,YAAY;AAEnB,UAAI,CAAC,KAAK,qBAAqB;AAE7B,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,yBAAyB,KAAK,MAAM,eAAe;AACzD,cAAM,UAAsB;AAAA,UAC1B,MAAM;AAAA,UACN,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,UAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACzD,iBAAiB,KAAK;AAAA,UACtB,iBAAiB;AAAA,UACjB,WAAW,KAAK,gBAAgB,OAAO;AAAA,UACvC,OAAO,KAAK,KAAK;AAAA,UACjB,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AAEA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAC9E,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAIA,kBAAkB,MAA8B;AAC9C,SAAK,oBAAoB,UAAU,IAAI;AAAA,EACzC;AAAA;AAAA;AAAA,EAIA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAE/C,WAAK,oBAAoB,QAAQ,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAC5D;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,SAAK,MAAM;AACX,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,gCAAqC;AAAA,EACjD,SAAS,IAAI,gCAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACQ;AAAA,EACA,aAAS,gBAAI;AAAA,EAErB,YACE,MACA,KACA,cAAiC,0CACjC;AACA,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,eAAe;AAMpB,YAAQ,QAAQ,EAAE,KAAK,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAC7E;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,kCAAgB;AACnC,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAIA,IAAI,YAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAAA,IAChF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACzD,iBAAiB,KAAK,MAAM;AAAA,MAC5B,iBAAiB,KAAK,MAAM,eAAe;AAAA,MAC3C,WAAW;AAAA;AAAA,MACX,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,eAAO,0BAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport interface TTSError {\n  type: 'tts_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type TTSCallbacks = {\n  ['metrics_collected']: (metrics: TTSMetrics) => void;\n  ['error']: (error: TTSError) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n  private _connOptions: APIConnectOptions;\n  protected abortController = new AbortController();\n\n  private deferredInputStream: DeferredReadableStream<\n    string | typeof SynthesizeStream.FLUSH_SENTINEL\n  >;\n  private logger = log();\n\n  constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {\n    this.#tts = tts;\n    this._connOptions = connOptions;\n    this.deferredInputStream = new DeferredReadableStream();\n    this.pumpInput();\n    this.abortController.signal.addEventListener('abort', () => {\n      this.deferredInputStream.detachSource();\n      // TODO (AJS-36) clean this up when we refactor with streams\n      this.input.close();\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to synthesize speech, retrying in  ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  // TODO(AJS-37) Remove when refactoring TTS to use streams\n  protected async pumpInput() {\n    const reader = this.deferredInputStream.stream.getReader();\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done || value === SynthesizeStream.FLUSH_SENTINEL) {\n          break;\n        }\n        this.pushText(value);\n      }\n      this.endInput();\n    } catch (error) {\n      this.logger.error(error, 'Error reading deferred input stream');\n    } finally {\n      reader.releaseLock();\n      // Ensure output is closed when the stream ends\n      if (!this.#monitorMetricsTask) {\n        // No text was received, close the output directly\n        this.output.close();\n      }\n    }\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const roundedAudioDurationMs = Math.round(audioDurationMs);\n        const metrics: TTSMetrics = {\n          type: 'tts_metrics',\n          timestamp: Date.now(),\n          requestId,\n          ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n          durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDurationMs: roundedAudioDurationMs,\n          cancelled: this.abortController.signal.aborted,\n          label: this.#tts.label,\n          streamed: false,\n        };\n        this.#tts.emit('metrics_collected', metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      // TODO(AJS-102): use frame.durationMs once available in rtc-node\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(text: ReadableStream<string>) {\n    this.deferredInputStream.setSource(text);\n  }\n\n  /** Push a string of text to the TTS */\n  /** @deprecated Use `updateInputStream` instead */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n      // Close output when metrics task completes\n      this.#monitorMetricsTask.finally(() => this.output.close());\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    this.flush();\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n  private _connOptions: APIConnectOptions;\n  private logger = log();\n\n  constructor(\n    text: string,\n    tts: TTS,\n    connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#text = text;\n    this.#tts = tts;\n    this._connOptions = connOptions;\n\n    this.monitorMetrics();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to generate TTS completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected abstract run(): Promise<void>;\n\n  get inputText(): string {\n    return this.#text;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      type: 'tts_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDurationMs: Math.round(audioDurationMs),\n      cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here\n      label: this.#tts.label,\n      streamed: false,\n    };\n    this.#tts.emit('metrics_collected', metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,wBAA6C;AAC7C,iBAAoB;AAEpB,6BAAuC;AACvC,mBAAoE;AACpE,mBAA2E;AA+CpE,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAEpB;AAAA,EACQ,SAAS,IAAI,gCAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EACQ;AAAA,EACE,kBAAkB,IAAI,gBAAgB;AAAA,EAExC;AAAA,EAGA,aAAS,gBAAI;AAAA,EAErB,YAAY,KAAU,cAAiC,0CAA6B;AAClF,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,8CAAuB;AACtD,SAAK,UAAU;AACf,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAC1D,WAAK,oBAAoB,aAAa;AAEtC,WAAK,MAAM,MAAM;AACjB,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,6CAA6C,aAAa;AAAA,YAC5D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAgB,YAAY;AAC1B,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,QAAQ,UAAU,iBAAiB,gBAAgB;AACrD;AAAA,QACF;AACA,aAAK,SAAS,KAAK;AAAA,MACrB;AACA,WAAK,SAAS;AAAA,IAChB,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,qCAAqC;AAAA,IAChE,UAAE;AACA,aAAO,YAAY;AAEnB,UAAI,CAAC,KAAK,qBAAqB;AAE7B,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,yBAAyB,KAAK,MAAM,eAAe;AACzD,cAAM,UAAsB;AAAA,UAC1B,MAAM;AAAA,UACN,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,UAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACzD,iBAAiB,KAAK;AAAA,UACtB,iBAAiB;AAAA,UACjB,WAAW,KAAK,gBAAgB,OAAO;AAAA,UACvC,OAAO,KAAK,KAAK;AAAA,UACjB,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AAEA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAC9E,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAIA,kBAAkB,MAA8B;AAC9C,SAAK,oBAAoB,UAAU,IAAI;AAAA,EACzC;AAAA;AAAA;AAAA,EAIA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAE/C,WAAK,oBAAoB,QAAQ,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAC5D;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,SAAK,MAAM;AACX,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,gCAAqC;AAAA,EACjD,SAAS,IAAI,gCAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACQ;AAAA,EACA,aAAS,gBAAI;AAAA,EAErB,YACE,MACA,KACA,cAAiC,0CACjC;AACA,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,eAAe;AAMpB,YAAQ,QAAQ,EAAE,KAAK,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAC7E;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAIA,IAAI,YAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAAA,IAChF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACzD,iBAAiB,KAAK,MAAM;AAAA,MAC5B,iBAAiB,KAAK,MAAM,eAAe;AAAA,MAC3C,WAAW;AAAA;AAAA,MACX,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,eAAO,0BAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/tts/tts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAA0C,MAAM,aAAa,CAAC;AAEzF,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,CAAC;IAClB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IAItF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe;IAOlF,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,mEAAmE;IACnE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,qEAAqE;IACrE,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAEhD;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,gBAAgB;CACpC;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,gBACpB,YAAW,qBAAqB,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC;;IAE1F,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,MAAM,CAAC,QAAQ,CAAC,aAAa,gBAA2B;IACxD,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,KAAK,+EAEX;IACJ,SAAS,CAAC,MAAM,+EAEZ;IACJ,SAAS,CAAC,MAAM,UAAS;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAKvB,OAAO,CAAC,YAAY,CAAoB;IACxC,SAAS,CAAC,eAAe,kBAAyB;IAElD,OAAO,CAAC,mBAAmB,CAEzB;IACF,OAAO,CAAC,MAAM,CAAS;gBAEX,GAAG,EAAE,GAAG,EAAE,WAAW,GAAE,iBAA+C;YAoBpE,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAWD,SAAS;cAuBT,cAAc;IAiD9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,CAAC,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC;IAI9C,uCAAuC;IACvC,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM;IAiBrB,4DAA4D;IAC5D,KAAK;IAcL,2DAA2D;IAC3D,QAAQ;IAWR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAIzF,wDAAwD;IACxD,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,gBAAgB;CAG3C;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,aAAc,YAAW,qBAAqB,CAAC,gBAAgB,CAAC;;IACpF,SAAS,CAAC,KAAK,uCAA8C;IAC7D,SAAS,CAAC,MAAM,uCAA8C;IAC9D,SAAS,CAAC,MAAM,UAAS;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAGvB,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,MAAM,CAAS;gBAGrB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,GAAG,EACR,WAAW,GAAE,iBAA+C;YAehD,QAAQ;IAoCtB,OAAO,CAAC,SAAS;IAUjB,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,IAAI,SAAS,IAAI,MAAM,CAEtB;cAEe,cAAc;IAgC9B,oDAAoD;IAC9C,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC;IAQpC,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAIjD,wDAAwD;IACxD,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa;CAGxC"}
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAA0C,MAAM,aAAa,CAAC;AAEzF,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,CAAC;IAClB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IAItF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe;IAOlF,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,mEAAmE;IACnE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,qEAAqE;IACrE,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAEhD;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,gBAAgB;CACpC;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,gBACpB,YAAW,qBAAqB,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC;;IAE1F,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,MAAM,CAAC,QAAQ,CAAC,aAAa,gBAA2B;IACxD,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,KAAK,+EAEX;IACJ,SAAS,CAAC,MAAM,+EAEZ;IACJ,SAAS,CAAC,MAAM,UAAS;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAKvB,OAAO,CAAC,YAAY,CAAoB;IACxC,SAAS,CAAC,eAAe,kBAAyB;IAElD,OAAO,CAAC,mBAAmB,CAEzB;IACF,OAAO,CAAC,MAAM,CAAS;gBAEX,GAAG,EAAE,GAAG,EAAE,WAAW,GAAE,iBAA+C;YAoBpE,QAAQ;IAqCtB,OAAO,CAAC,SAAS;cAWD,SAAS;cAuBT,cAAc;IAiD9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,CAAC,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC;IAI9C,uCAAuC;IACvC,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM;IAiBrB,4DAA4D;IAC5D,KAAK;IAcL,2DAA2D;IAC3D,QAAQ;IAWR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAIzF,wDAAwD;IACxD,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,gBAAgB;CAG3C;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,aAAc,YAAW,qBAAqB,CAAC,gBAAgB,CAAC;;IACpF,SAAS,CAAC,KAAK,uCAA8C;IAC7D,SAAS,CAAC,MAAM,uCAA8C;IAC9D,SAAS,CAAC,MAAM,UAAS;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAGvB,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,MAAM,CAAS;gBAGrB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,GAAG,EACR,WAAW,GAAE,iBAA+C;YAehD,QAAQ;IAqCtB,OAAO,CAAC,SAAS;IAUjB,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,IAAI,SAAS,IAAI,MAAM,CAEtB;cAEe,cAAc;IAgC9B,oDAAoD;IAC9C,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC;IAQpC,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAIjD,wDAAwD;IACxD,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa;CAGxC"}

package/dist/tts/tts.js

@@ -1,5 +1,5 @@
 import { EventEmitter } from "node:events";
-import { APIConnectionError, APIStatusError } from "../_exceptions.js";
+import { APIConnectionError, APIError } from "../_exceptions.js";
 import { log } from "../log.js";
 import { DeferredReadableStream } from "../stream/deferred_stream.js";
 import { DEFAULT_API_CONNECT_OPTIONS } from "../types.js";
@@ -60,7 +60,7 @@ class SynthesizeStream {
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof APIStatusError) {
+        if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -72,7 +72,6 @@ class SynthesizeStream {
               options: { retryable: false }
             });
           } else {
-            this.emitError({ error, recoverable: true });
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to synthesize speech, retrying in  ${retryInterval}s`
@@ -237,7 +236,7 @@ class ChunkedStream {
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof APIStatusError) {
+        if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -249,7 +248,6 @@ class ChunkedStream {
               options: { retryable: false }
             });
           } else {
-            this.emitError({ error, recoverable: true });
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to generate TTS completion, retrying in ${retryInterval}s`

package/dist/tts/tts.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIStatusError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport interface TTSError {\n  type: 'tts_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type TTSCallbacks = {\n  ['metrics_collected']: (metrics: TTSMetrics) => void;\n  ['error']: (error: TTSError) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n  private _connOptions: APIConnectOptions;\n  protected abortController = new AbortController();\n\n  private deferredInputStream: DeferredReadableStream<\n    string | typeof SynthesizeStream.FLUSH_SENTINEL\n  >;\n  private logger = log();\n\n  constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {\n    this.#tts = tts;\n    this._connOptions = connOptions;\n    this.deferredInputStream = new DeferredReadableStream();\n    this.pumpInput();\n    this.abortController.signal.addEventListener('abort', () => {\n      this.deferredInputStream.detachSource();\n      // TODO (AJS-36) clean this up when we refactor with streams\n      this.input.close();\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIStatusError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to synthesize speech, retrying in  ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  // TODO(AJS-37) Remove when refactoring TTS to use streams\n  protected async pumpInput() {\n    const reader = this.deferredInputStream.stream.getReader();\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done || value === SynthesizeStream.FLUSH_SENTINEL) {\n          break;\n        }\n        this.pushText(value);\n      }\n      this.endInput();\n    } catch (error) {\n      this.logger.error(error, 'Error reading deferred input stream');\n    } finally {\n      reader.releaseLock();\n      // Ensure output is closed when the stream ends\n      if (!this.#monitorMetricsTask) {\n        // No text was received, close the output directly\n        this.output.close();\n      }\n    }\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const roundedAudioDurationMs = Math.round(audioDurationMs);\n        const metrics: TTSMetrics = {\n          type: 'tts_metrics',\n          timestamp: Date.now(),\n          requestId,\n          ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n          durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDurationMs: roundedAudioDurationMs,\n          cancelled: this.abortController.signal.aborted,\n          label: this.#tts.label,\n          streamed: false,\n        };\n        this.#tts.emit('metrics_collected', metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      // TODO(AJS-102): use frame.durationMs once available in rtc-node\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(text: ReadableStream<string>) {\n    this.deferredInputStream.setSource(text);\n  }\n\n  /** Push a string of text to the TTS */\n  /** @deprecated Use `updateInputStream` instead */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n      // Close output when metrics task completes\n      this.#monitorMetricsTask.finally(() => this.output.close());\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    this.flush();\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n  private _connOptions: APIConnectOptions;\n  private logger = log();\n\n  constructor(\n    text: string,\n    tts: TTS,\n    connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#text = text;\n    this.#tts = tts;\n    this._connOptions = connOptions;\n\n    this.monitorMetrics();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIStatusError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to generate TTS completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected abstract run(): Promise<void>;\n\n  get inputText(): string {\n    return this.#text;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      type: 'tts_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDurationMs: Math.round(audioDurationMs),\n      cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here\n      label: this.#tts.label,\n      streamed: false,\n    };\n    this.#tts.emit('metrics_collected', metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,sBAAsB;AACnD,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAiC,mCAAmC;AACpE,SAAS,oBAAoB,OAAO,aAAa,WAAW,eAAe;AA+CpE,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,mBAAoE;AAAA,EAChF,QAAQ,IAAI,mBAEpB;AAAA,EACQ,SAAS,IAAI,mBAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EACQ;AAAA,EACE,kBAAkB,IAAI,gBAAgB;AAAA,EAExC;AAAA,EAGA,SAAS,IAAI;AAAA,EAErB,YAAY,KAAU,cAAiC,6BAA6B;AAClF,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,uBAAuB;AACtD,SAAK,UAAU;AACf,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAC1D,WAAK,oBAAoB,aAAa;AAEtC,WAAK,MAAM,MAAM;AACjB,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,gBAAgB;AACnC,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,6CAA6C,aAAa;AAAA,YAC5D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAgB,YAAY;AAC1B,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,QAAQ,UAAU,iBAAiB,gBAAgB;AACrD;AAAA,QACF;AACA,aAAK,SAAS,KAAK;AAAA,MACrB;AACA,WAAK,SAAS;AAAA,IAChB,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,qCAAqC;AAAA,IAChE,UAAE;AACA,aAAO,YAAY;AAEnB,UAAI,CAAC,KAAK,qBAAqB;AAE7B,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,yBAAyB,KAAK,MAAM,eAAe;AACzD,cAAM,UAAsB;AAAA,UAC1B,MAAM;AAAA,UACN,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,UAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACzD,iBAAiB,KAAK;AAAA,UACtB,iBAAiB;AAAA,UACjB,WAAW,KAAK,gBAAgB,OAAO;AAAA,UACvC,OAAO,KAAK,KAAK;AAAA,UACjB,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AAEA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAC9E,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAIA,kBAAkB,MAA8B;AAC9C,SAAK,oBAAoB,UAAU,IAAI;AAAA,EACzC;AAAA;AAAA;AAAA,EAIA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAE/C,WAAK,oBAAoB,QAAQ,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAC5D;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,SAAK,MAAM;AACX,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,mBAAqC;AAAA,EACjD,SAAS,IAAI,mBAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACQ;AAAA,EACA,SAAS,IAAI;AAAA,EAErB,YACE,MACA,KACA,cAAiC,6BACjC;AACA,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,eAAe;AAMpB,YAAQ,QAAQ,EAAE,KAAK,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAC7E;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,gBAAgB;AACnC,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAIA,IAAI,YAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAAA,IAChF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACzD,iBAAiB,KAAK,MAAM;AAAA,MAC5B,iBAAiB,KAAK,MAAM,eAAe;AAAA,MAC3C,WAAW;AAAA;AAAA,MACX,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,WAAO,YAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport interface TTSError {\n  type: 'tts_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type TTSCallbacks = {\n  ['metrics_collected']: (metrics: TTSMetrics) => void;\n  ['error']: (error: TTSError) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n  private _connOptions: APIConnectOptions;\n  protected abortController = new AbortController();\n\n  private deferredInputStream: DeferredReadableStream<\n    string | typeof SynthesizeStream.FLUSH_SENTINEL\n  >;\n  private logger = log();\n\n  constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {\n    this.#tts = tts;\n    this._connOptions = connOptions;\n    this.deferredInputStream = new DeferredReadableStream();\n    this.pumpInput();\n    this.abortController.signal.addEventListener('abort', () => {\n      this.deferredInputStream.detachSource();\n      // TODO (AJS-36) clean this up when we refactor with streams\n      this.input.close();\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to synthesize speech, retrying in  ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  // TODO(AJS-37) Remove when refactoring TTS to use streams\n  protected async pumpInput() {\n    const reader = this.deferredInputStream.stream.getReader();\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done || value === SynthesizeStream.FLUSH_SENTINEL) {\n          break;\n        }\n        this.pushText(value);\n      }\n      this.endInput();\n    } catch (error) {\n      this.logger.error(error, 'Error reading deferred input stream');\n    } finally {\n      reader.releaseLock();\n      // Ensure output is closed when the stream ends\n      if (!this.#monitorMetricsTask) {\n        // No text was received, close the output directly\n        this.output.close();\n      }\n    }\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const roundedAudioDurationMs = Math.round(audioDurationMs);\n        const metrics: TTSMetrics = {\n          type: 'tts_metrics',\n          timestamp: Date.now(),\n          requestId,\n          ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n          durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDurationMs: roundedAudioDurationMs,\n          cancelled: this.abortController.signal.aborted,\n          label: this.#tts.label,\n          streamed: false,\n        };\n        this.#tts.emit('metrics_collected', metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      // TODO(AJS-102): use frame.durationMs once available in rtc-node\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(text: ReadableStream<string>) {\n    this.deferredInputStream.setSource(text);\n  }\n\n  /** Push a string of text to the TTS */\n  /** @deprecated Use `updateInputStream` instead */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n      // Close output when metrics task completes\n      this.#monitorMetricsTask.finally(() => this.output.close());\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    this.flush();\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n  private _connOptions: APIConnectOptions;\n  private logger = log();\n\n  constructor(\n    text: string,\n    tts: TTS,\n    connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#text = text;\n    this.#tts = tts;\n    this._connOptions = connOptions;\n\n    this.monitorMetrics();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            // Don't emit error event for recoverable errors during retry loop\n            // to avoid ERR_UNHANDLED_ERROR or premature session termination\n            this.logger.warn(\n              { tts: this.#tts.label, attempt: i + 1, error },\n              `failed to generate TTS completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#tts.emit('error', {\n      type: 'tts_error',\n      timestamp: Date.now(),\n      label: this.#tts.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected abstract run(): Promise<void>;\n\n  get inputText(): string {\n    return this.#text;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDurationMs = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      type: 'tts_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      durationMs: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDurationMs: Math.round(audioDurationMs),\n      cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here\n      label: this.#tts.label,\n      streamed: false,\n    };\n    this.#tts.emit('metrics_collected', metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAiC,mCAAmC;AACpE,SAAS,oBAAoB,OAAO,aAAa,WAAW,eAAe;AA+CpE,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,mBAAoE;AAAA,EAChF,QAAQ,IAAI,mBAEpB;AAAA,EACQ,SAAS,IAAI,mBAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EACQ;AAAA,EACE,kBAAkB,IAAI,gBAAgB;AAAA,EAExC;AAAA,EAGA,SAAS,IAAI;AAAA,EAErB,YAAY,KAAU,cAAiC,6BAA6B;AAClF,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,uBAAuB;AACtD,SAAK,UAAU;AACf,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAC1D,WAAK,oBAAoB,aAAa;AAEtC,WAAK,MAAM,MAAM;AACjB,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,6CAA6C,aAAa;AAAA,YAC5D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAgB,YAAY;AAC1B,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,QAAQ,UAAU,iBAAiB,gBAAgB;AACrD;AAAA,QACF;AACA,aAAK,SAAS,KAAK;AAAA,MACrB;AACA,WAAK,SAAS;AAAA,IAChB,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,qCAAqC;AAAA,IAChE,UAAE;AACA,aAAO,YAAY;AAEnB,UAAI,CAAC,KAAK,qBAAqB;AAE7B,aAAK,OAAO,MAAM;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,yBAAyB,KAAK,MAAM,eAAe;AACzD,cAAM,UAAsB;AAAA,UAC1B,MAAM;AAAA,UACN,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,UAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACzD,iBAAiB,KAAK;AAAA,UACtB,iBAAiB;AAAA,UACjB,WAAW,KAAK,gBAAgB,OAAO;AAAA,UACvC,OAAO,KAAK,KAAK;AAAA,UACjB,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AAEA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAC9E,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AAAA,EACF;AAAA,EAIA,kBAAkB,MAA8B;AAC9C,SAAK,oBAAoB,UAAU,IAAI;AAAA,EACzC;AAAA;AAAA;AAAA,EAIA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAE/C,WAAK,oBAAoB,QAAQ,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAC5D;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,SAAK,MAAM;AACX,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,mBAAqC;AAAA,EACjD,SAAS,IAAI,mBAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACQ;AAAA,EACA,SAAS,IAAI;AAAA,EAErB,YACE,MACA,KACA,cAAiC,6BACjC;AACA,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,eAAe;AAMpB,YAAQ,QAAQ,EAAE,KAAK,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAC7E;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AAGL,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAIA,IAAI,YAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,kBAAkB;AACtB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,yBAAoB,MAAM,MAAM,oBAAoB,MAAM,MAAM,aAAc;AAAA,IAChF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E,YAAY,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACzD,iBAAiB,KAAK,MAAM;AAAA,MAC5B,iBAAiB,KAAK,MAAM,eAAe;AAAA,MAC3C,WAAW;AAAA;AAAA,MACX,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,WAAO,YAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":[]}