@livekit/agents 0.7.9 → 1.0.0-next.0

package/src/tts/tts.ts

@@ -3,9 +3,15 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { delay } from '@std/async';
 import { EventEmitter } from 'node:events';
+import type { ReadableStream } from 'node:stream/web';
+import { APIConnectionError, APIStatusError } from '../_exceptions.js';
+import { log } from '../log.js';
 import type { TTSMetrics } from '../metrics/base.js';
-import { AsyncIterableQueue, mergeFrames } from '../utils.js';
+import { DeferredReadableStream } from '../stream/deferred_stream.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { AsyncIterableQueue, mergeFrames, startSoon, toError } from '../utils.js';
 
 /** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
 export interface SynthesizedAudio {
@@ -32,12 +38,17 @@ export interface TTSCapabilities {
   streaming: boolean;
 }
 
-export enum TTSEvent {
-  METRICS_COLLECTED,
+export interface TTSError {
+  type: 'tts_error';
+  timestamp: number;
+  label: string;
+  error: Error;
+  recoverable: boolean;
 }
 
 export type TTSCallbacks = {
-  [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;
+  ['metrics_collected']: (metrics: TTSMetrics) => void;
+  ['error']: (error: TTSError) => void;
 };
 
 /**
@@ -118,15 +129,108 @@ export abstract class SynthesizeStream
   #metricsPendingTexts: string[] = [];
   #metricsText = '';
   #monitorMetricsTask?: Promise<void>;
+  private _connOptions: APIConnectOptions;
+  protected abortController = new AbortController();
 
-  constructor(tts: TTS) {
+  private deferredInputStream: DeferredReadableStream<
+    string | typeof SynthesizeStream.FLUSH_SENTINEL
+  >;
+  private logger = log();
+
+  constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {
     this.#tts = tts;
+    this._connOptions = connOptions;
+    this.deferredInputStream = new DeferredReadableStream();
+    this.pumpInput();
+    this.abortController.signal.addEventListener('abort', () => {
+      this.deferredInputStream.detachSource();
+      // TODO (AJS-36) clean this up when we refactor with streams
+      this.input.close();
+      this.output.close();
+      this.closed = true;
+    });
+
+    // this is a hack to immitate asyncio.create_task so that mainTask
+    // is run **after** the constructor has finished. Otherwise we get
+    // runtime error when trying to access class variables in the
+    // `run` method.
+    startSoon(() => this.mainTask().then(() => this.queue.close()));
+  }
+
+  private async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof APIStatusError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new APIConnectionError({
+              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false },
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { tts: this.#tts.label, attempt: i + 1, error },
+              `failed to synthesize speech, retrying in  ${retryInterval}s`,
+            );
+          }
+
+          if (retryInterval > 0) {
+            await delay(retryInterval);
+          }
+        } else {
+          this.emitError({ error: toError(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+
+  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
+    this.#tts.emit('error', {
+      type: 'tts_error',
+      timestamp: Date.now(),
+      label: this.#tts.label,
+      error,
+      recoverable,
+    });
+  }
+
+  // TODO(AJS-37) Remove when refactoring TTS to use streams
+  protected async pumpInput() {
+    const reader = this.deferredInputStream.stream.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done || value === SynthesizeStream.FLUSH_SENTINEL) {
+          break;
+        }
+        this.pushText(value);
+      }
+      this.endInput();
+    } catch (error) {
+      this.logger.error(error, 'Error reading deferred input stream');
+    } finally {
+      reader.releaseLock();
+      // Ensure output is closed when the stream ends
+      if (!this.#monitorMetricsTask) {
+        // No text was received, close the output directly
+        this.output.close();
+      }
+    }
   }
 
   protected async monitorMetrics() {
     const startTime = process.hrtime.bigint();
     let audioDuration = 0;
-    let ttfb: bigint | undefined;
+    let ttfb: bigint = BigInt(-1);
     let requestId = '';
 
     const emit = () => {
@@ -134,27 +238,32 @@ export abstract class SynthesizeStream
         const text = this.#metricsPendingTexts.shift()!;
         const duration = process.hrtime.bigint() - startTime;
         const metrics: TTSMetrics = {
+          type: 'tts_metrics',
           timestamp: Date.now(),
           requestId,
-          ttfb: Math.trunc(Number((ttfb || BigInt(0)) / BigInt(1000000))),
+          ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
           duration: Math.trunc(Number(duration / BigInt(1000000))),
           charactersCount: text.length,
           audioDuration,
-          cancelled: false, // XXX(nbsp)
-          label: this.label,
+          cancelled: this.abortController.signal.aborted,
+          label: this.#tts.label,
           streamed: false,
         };
-        this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);
+        this.#tts.emit('metrics_collected', metrics);
       }
     };
 
     for await (const audio of this.queue) {
+      if (this.abortController.signal.aborted) {
+        break;
+      }
       this.output.put(audio);
       if (audio === SynthesizeStream.END_OF_STREAM) continue;
       requestId = audio.requestId;
-      if (!ttfb) {
+      if (ttfb === BigInt(-1)) {
         ttfb = process.hrtime.bigint() - startTime;
       }
+      // TODO(AJS-102): use frame.durationMs once available in rtc-node
       audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
       if (audio.final) {
         emit();
@@ -164,13 +273,21 @@ export abstract class SynthesizeStream
     if (requestId) {
       emit();
     }
-    this.output.close();
+  }
+
+  protected abstract run(): Promise<void>;
+
+  updateInputStream(text: ReadableStream<string>) {
+    this.deferredInputStream.setSource(text);
   }
 
   /** Push a string of text to the TTS */
+  /** @deprecated Use `updateInputStream` instead */
   pushText(text: string) {
     if (!this.#monitorMetricsTask) {
       this.#monitorMetricsTask = this.monitorMetrics();
+      // Close output when metrics task completes
+      this.#monitorMetricsTask.finally(() => this.output.close());
     }
     this.#metricsText += text;
 
@@ -200,6 +317,7 @@ export abstract class SynthesizeStream
 
   /** Mark the input as ended and forbid additional pushes */
   endInput() {
+    this.flush();
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
@@ -215,9 +333,7 @@ export abstract class SynthesizeStream
 
   /** Close both the input and output of the TTS stream */
   close() {
-    this.input.close();
-    this.output.close();
-    this.closed = true;
+    this.abortController.abort();
   }
 
   [Symbol.asyncIterator](): SynthesizeStream {
@@ -246,12 +362,77 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
   abstract label: string;
   #text: string;
   #tts: TTS;
-
-  constructor(text: string, tts: TTS) {
+  private _connOptions: APIConnectOptions;
+  private logger = log();
+
+  constructor(
+    text: string,
+    tts: TTS,
+    connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+  ) {
     this.#text = text;
     this.#tts = tts;
+    this._connOptions = connOptions;
 
     this.monitorMetrics();
+
+    // this is a hack to immitate asyncio.create_task so that mainTask
+    // is run **after** the constructor has finished. Otherwise we get
+    // runtime error when trying to access class variables in the
+    // `run` method.
+    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));
+  }
+
+  private async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof APIStatusError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new APIConnectionError({
+              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false },
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { tts: this.#tts.label, attempt: i + 1, error },
+              `failed to generate TTS completion, retrying in ${retryInterval}s`,
+            );
+          }
+
+          if (retryInterval > 0) {
+            await delay(retryInterval);
+          }
+        } else {
+          this.emitError({ error: toError(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+
+  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
+    this.#tts.emit('error', {
+      type: 'tts_error',
+      timestamp: Date.now(),
+      label: this.#tts.label,
+      error,
+      recoverable,
+    });
+  }
+
+  protected abstract run(): Promise<void>;
+
+  get inputText(): string {
+    return this.#text;
   }
 
   protected async monitorMetrics() {
@@ -263,7 +444,7 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
     for await (const audio of this.queue) {
       this.output.put(audio);
       requestId = audio.requestId;
-      if (ttfb === BigInt(-1)) {
+      if (!ttfb) {
         ttfb = process.hrtime.bigint() - startTime;
       }
       audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
@@ -272,17 +453,18 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
 
     const duration = process.hrtime.bigint() - startTime;
     const metrics: TTSMetrics = {
+      type: 'tts_metrics',
       timestamp: Date.now(),
       requestId,
       ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
       duration: Math.trunc(Number(duration / BigInt(1000000))),
       charactersCount: this.#text.length,
       audioDuration,
-      cancelled: false, // XXX(nbsp)
-      label: this.label,
+      cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here
+      label: this.#tts.label,
       streamed: false,
     };
-    this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);
+    this.#tts.emit('metrics_collected', metrics);
   }
 
   /** Collect every frame into one in a single call */

package/src/types.ts

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export class APIConnectOptions {
+  /** Maximum number of retries to connect to the API. */
+  readonly maxRetry: number;
+  /** Interval between retries to connect to the API in milliseconds. */
+  readonly retryIntervalMs: number;
+  /** Timeout for connecting to the API in milliseconds. */
+  readonly timeoutMs: number;
+
+  constructor(options: Partial<APIConnectOptions> = {}) {
+    this.maxRetry = options.maxRetry ?? 3;
+    this.retryIntervalMs = options.retryIntervalMs ?? 2000;
+    this.timeoutMs = options.timeoutMs ?? 10000;
+
+    if (this.maxRetry < 0) {
+      throw new Error('maxRetry must be greater than or equal to 0');
+    }
+    if (this.retryIntervalMs < 0) {
+      throw new Error('retryIntervalMs must be greater than or equal to 0');
+    }
+    if (this.timeoutMs < 0) {
+      throw new Error('timeoutMs must be greater than or equal to 0');
+    }
+  }
+
+  /** @internal */
+  _intervalForRetry(numRetries: number): number {
+    /**
+     * Return the interval for the given number of retries.
+     *
+     * The first retry is immediate, and then uses specified retryIntervalMs
+     */
+    if (numRetries === 0) {
+      return 0.1;
+    }
+    return this.retryIntervalMs;
+  }
+}
+
+export const DEFAULT_API_CONNECT_OPTIONS = new APIConnectOptions();