@livekit/agents 1.1.0 → 1.2.0

package/src/tts/index.ts

@@ -10,3 +10,4 @@ export {
   ChunkedStream,
 } from './tts.js';
 export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+export { FallbackAdapter, type AvailabilityChangedEvent } from './fallback_adapter.js';

package/src/tts/stream_adapter.ts

@@ -2,7 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';
+import type { APIConnectOptions } from '../types.js';
+import { USERDATA_TIMED_TRANSCRIPT } from '../types.js';
 import { Task } from '../utils.js';
+import { createTimedString } from '../voice/io.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 
@@ -12,7 +15,7 @@ export class StreamAdapter extends TTS {
   label: string;
 
   constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
-    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
     this.label = this.#tts.label;
@@ -21,14 +24,21 @@ export class StreamAdapter extends TTS {
     this.#tts.on('metrics_collected', (metrics) => {
       this.emit('metrics_collected', metrics);
     });
+    this.#tts.on('error', (error) => {
+      this.emit('error', error);
+    });
   }
 
-  synthesize(text: string): ChunkedStream {
-    return this.#tts.synthesize(text);
+  synthesize(
+    text: string,
+    connOptions?: APIConnectOptions,
+    abortSignal?: AbortSignal,
+  ): ChunkedStream {
+    return this.#tts.synthesize(text, connOptions, abortSignal);
   }
 
-  stream(): StreamAdapterWrapper {
-    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options?.connOptions);
   }
 }
 
@@ -37,14 +47,16 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   #sentenceStream: SentenceStream;
   label: string;
 
-  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
-    super(tts);
+  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions) {
+    super(tts, connOptions);
     this.#tts = tts;
     this.#sentenceStream = sentenceTokenizer.stream();
     this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;
   }
 
   protected async run() {
+    let cumulativeDuration = 0;
+
     const forwardInput = async () => {
       for await (const input of this.input) {
         if (this.abortController.signal.aborted) break;
@@ -84,15 +96,33 @@ export class StreamAdapterWrapper extends SynthesizeStream {
       prevTask: Task<void> | undefined,
       controller: AbortController,
     ) => {
-      const audioStream = this.#tts.synthesize(token);
+      const audioStream = this.#tts.synthesize(token, this.connOptions, this.abortSignal);
 
       // wait for previous audio transcription to complete before starting
       // to queuing audio frames of the current token
       await prevTask?.result;
       if (controller.signal.aborted) return;
 
+      // Create a TimedString with the sentence text and current cumulative duration
+      const timedString = createTimedString({
+        text: token,
+        startTime: cumulativeDuration,
+      });
+
+      let isFirstFrame = true;
       for await (const audio of audioStream) {
         if (controller.signal.aborted) break;
+
+        // Attach the TimedString to the first frame of this sentence
+        if (isFirstFrame) {
+          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];
+          isFirstFrame = false;
+        }
+
+        // Track cumulative duration
+        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;
+        cumulativeDuration += frameDuration;
+
         this.queue.put(audio);
       }
     };

package/src/tts/tts.ts

@@ -3,16 +3,21 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { Span } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
-import { APIConnectionError, APIStatusError } from '../_exceptions.js';
+import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import type { TTSMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
-import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { recordException, traceTypes, tracer } from '../telemetry/index.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 
-/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
+/**
+ * SynthesizedAudio is a packet of speech synthesis as returned by the TTS.
+ */
 export interface SynthesizedAudio {
   /** Request ID (one segment could be made up of multiple requests) */
   requestId: string;
@@ -24,6 +29,10 @@ export interface SynthesizedAudio {
   deltaText?: string;
   /** Whether this is the last frame of the segment (streaming only) */
   final: boolean;
+  /**
+   * Timed transcripts associated with this audio packet (word-level timestamps).
+   */
+  timedTranscripts?: TimedString[];
 }
 
 /**
@@ -35,6 +44,8 @@ export interface SynthesizedAudio {
  */
 export interface TTSCapabilities {
   streaming: boolean;
+  // Whether this TTS supports aligned transcripts (word-level timestamps).
+  alignedTranscript?: boolean;
 }
 
 export interface TTSError {
@@ -85,15 +96,49 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCal
     return this.#numChannels;
   }
 
+  /**
+   * Get the model name/identifier for this TTS instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model(): string {
+    return 'unknown';
+  }
+
+  /**
+   * Get the provider name for this TTS instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /**
    * Receives text and returns synthesis in the form of a {@link ChunkedStream}
    */
-  abstract synthesize(text: string): ChunkedStream;
+  abstract synthesize(
+    text: string,
+    connOptions?: APIConnectOptions,
+    abortSignal?: AbortSignal,
+  ): ChunkedStream;
 
   /**
    * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+   *
+   * @param options - Optional configuration including connection options
    */
-  abstract stream(): SynthesizeStream;
+  abstract stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream;
+
+  async close(): Promise<void> {
+    return;
+  }
 }
 
 /**
@@ -123,12 +168,7 @@ export abstract class SynthesizeStream
     SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM
   >();
   protected closed = false;
-  abstract label: string;
-  #tts: TTS;
-  #metricsPendingTexts: string[] = [];
-  #metricsText = '';
-  #monitorMetricsTask?: Promise<void>;
-  private _connOptions: APIConnectOptions;
+  protected connOptions: APIConnectOptions;
   protected abortController = new AbortController();
 
   private deferredInputStream: DeferredReadableStream<
@@ -136,16 +176,27 @@ export abstract class SynthesizeStream
   >;
   private logger = log();
 
+  abstract label: string;
+
+  #tts: TTS;
+  #metricsPendingTexts: string[] = [];
+  #metricsText = '';
+  #monitorMetricsTask?: Promise<void>;
+  #ttsRequestSpan?: Span;
+  #inputTokens = 0;
+  #outputTokens = 0;
+
   constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {
     this.#tts = tts;
-    this._connOptions = connOptions;
+    this.connOptions = connOptions;
     this.deferredInputStream = new DeferredReadableStream();
     this.pumpInput();
+
     this.abortController.signal.addEventListener('abort', () => {
       this.deferredInputStream.detachSource();
       // TODO (AJS-36) clean this up when we refactor with streams
-      this.input.close();
-      this.output.close();
+      if (!this.input.closed) this.input.close();
+      if (!this.output.closed) this.output.close();
       this.closed = true;
     });
 
@@ -153,31 +204,49 @@ export abstract class SynthesizeStream
     // is run **after** the constructor has finished. Otherwise we get
     // runtime error when trying to access class variables in the
     // `run` method.
-    startSoon(() => this.mainTask().then(() => this.queue.close()));
+    startSoon(() => this.mainTask().finally(() => this.queue.close()));
   }
 
-  private async mainTask() {
-    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+  private _mainTaskImpl = async (span: Span) => {
+    this.#ttsRequestSpan = span;
+    span.setAttributes({
+      [traceTypes.ATTR_TTS_STREAMING]: true,
+      [traceTypes.ATTR_TTS_LABEL]: this.#tts.label,
+    });
+
+    for (let i = 0; i < this.connOptions.maxRetry + 1; i++) {
       try {
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'tts_request_run' },
+        );
       } catch (error) {
-        if (error instanceof APIStatusError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+        if (error instanceof APIError) {
+          const retryInterval = intervalForRetry(this.connOptions, i);
 
-          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+          if (this.connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
             throw error;
-          } else if (i === this._connOptions.maxRetry) {
+          } else if (i === this.connOptions.maxRetry) {
             this.emitError({ error, recoverable: false });
             throw new APIConnectionError({
-              message: `failed to generate TTS completion after ${this._connOptions.maxRetry + 1} attempts`,
+              message: `failed to generate TTS completion after ${this.connOptions.maxRetry + 1} attempts`,
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
-              `failed to synthesize speech, retrying in  ${retryInterval}s`,
+              `failed to synthesize speech, retrying in ${retryInterval}ms`,
             );
           }
 
@@ -190,7 +259,13 @@ export abstract class SynthesizeStream
         }
       }
     }
-  }
+  };
+
+  private mainTask = async () =>
+    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'tts_request',
+      endOnExit: false,
+    });
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
     this.#tts.emit('error', {
@@ -202,7 +277,16 @@ export abstract class SynthesizeStream
     });
   }
 
-  // TODO(AJS-37) Remove when refactoring TTS to use streams
+  // NOTE(AJS-37): The implementation below uses an AsyncIterableQueue (`this.input`)
+  // bridged from a DeferredReadableStream (`this.deferredInputStream`) rather than
+  // consuming the stream directly.
+  //
+  // A full refactor to native Web Streams was considered but is currently deferred.
+  // The primary reason is to maintain architectural parity with the Python SDK,
+  // which is a key design goal for the project. This ensures a consistent developer
+  // experience across both platforms.
+  //
+  // For more context, see the discussion in GitHub issue # 844.
   protected async pumpInput() {
     const reader = this.deferredInputStream.stream.getReader();
     try {
@@ -226,9 +310,21 @@ export abstract class SynthesizeStream
     }
   }
 
+  /**
+   * Set token usage for token-based TTS billing (e.g., OpenAI TTS).
+   * Plugins should call this method to report token usage.
+   */
+  protected setTokenUsage({
+    inputTokens = 0,
+    outputTokens = 0,
+  }: { inputTokens?: number; outputTokens?: number } = {}): void {
+    this.#inputTokens = inputTokens;
+    this.#outputTokens = outputTokens;
+  }
+
   protected async monitorMetrics() {
     const startTime = process.hrtime.bigint();
-    let audioDuration = 0;
+    let audioDurationMs = 0;
     let ttfb: bigint = BigInt(-1);
     let requestId = '';
 
@@ -236,19 +332,33 @@ export abstract class SynthesizeStream
       if (this.#metricsPendingTexts.length) {
         const text = this.#metricsPendingTexts.shift()!;
         const duration = process.hrtime.bigint() - startTime;
+        const roundedAudioDurationMs = Math.round(audioDurationMs);
         const metrics: TTSMetrics = {
           type: 'tts_metrics',
           timestamp: Date.now(),
           requestId,
-          ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
-          duration: Math.trunc(Number(duration / BigInt(1000000))),
+          ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
+          durationMs: Math.trunc(Number(duration / BigInt(1000000))),
           charactersCount: text.length,
-          audioDuration,
+          audioDurationMs: roundedAudioDurationMs,
           cancelled: this.abortController.signal.aborted,
           label: this.#tts.label,
-          streamed: false,
+          inputTokens: this.#inputTokens,
+          outputTokens: this.#outputTokens,
+          streamed: true,
+          metadata: {
+            modelProvider: this.#tts.provider,
+            modelName: this.#tts.model,
+          },
         };
+        if (this.#ttsRequestSpan) {
+          this.#ttsRequestSpan.setAttribute(traceTypes.ATTR_TTS_METRICS, JSON.stringify(metrics));
+        }
         this.#tts.emit('metrics_collected', metrics);
+
+        // Reset token usage after emitting metrics for the next segment
+        this.#inputTokens = 0;
+        this.#outputTokens = 0;
       }
     };
 
@@ -263,7 +373,7 @@ export abstract class SynthesizeStream
         ttfb = process.hrtime.bigint() - startTime;
       }
       // TODO(AJS-102): use frame.durationMs once available in rtc-node
-      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
+      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;
       if (audio.final) {
         emit();
       }
@@ -272,6 +382,11 @@ export abstract class SynthesizeStream
     if (requestId) {
       emit();
     }
+
+    if (this.#ttsRequestSpan) {
+      this.#ttsRequestSpan.end();
+      this.#ttsRequestSpan = undefined;
+    }
   }
 
   protected abstract run(): Promise<void>;
@@ -290,12 +405,11 @@ export abstract class SynthesizeStream
     }
     this.#metricsText += text;
 
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.put(text);
   }
 
@@ -305,24 +419,24 @@ export abstract class SynthesizeStream
       this.#metricsPendingTexts.push(this.#metricsText);
       this.#metricsText = '';
     }
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.put(SynthesizeStream.FLUSH_SENTINEL);
   }
 
   /** Mark the input as ended and forbid additional pushes */
   endInput() {
     this.flush();
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.close();
   }
 
@@ -330,6 +444,10 @@ export abstract class SynthesizeStream
     return this.output.next();
   }
 
+  get abortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
   /** Close both the input and output of the TTS stream */
   close() {
     this.abortController.abort();
@@ -361,34 +479,61 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
   abstract label: string;
   #text: string;
   #tts: TTS;
+  #ttsRequestSpan?: Span;
   private _connOptions: APIConnectOptions;
   private logger = log();
+  #inputTokens = 0;
+  #outputTokens = 0;
+
+  protected abortController = new AbortController();
 
   constructor(
     text: string,
     tts: TTS,
     connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    abortSignal?: AbortSignal,
   ) {
     this.#text = text;
     this.#tts = tts;
     this._connOptions = connOptions;
 
+    if (abortSignal) {
+      abortSignal.addEventListener('abort', () => this.abortController.abort(), { once: true });
+    }
+
     this.monitorMetrics();
 
     // this is a hack to immitate asyncio.create_task so that mainTask
     // is run **after** the constructor has finished. Otherwise we get
     // runtime error when trying to access class variables in the
     // `run` method.
-    Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));
+    Promise.resolve().then(() => this.mainTask().finally(() => this.queue.close()));
   }
 
-  private async mainTask() {
+  private _mainTaskImpl = async (span: Span) => {
+    this.#ttsRequestSpan = span;
+    span.setAttributes({
+      [traceTypes.ATTR_TTS_STREAMING]: false,
+      [traceTypes.ATTR_TTS_LABEL]: this.#tts.label,
+    });
+
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'tts_request_run' },
+        );
       } catch (error) {
-        if (error instanceof APIStatusError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+        if (error instanceof APIError) {
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -400,10 +545,11 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
-              `failed to generate TTS completion, retrying in ${retryInterval}s`,
+              `failed to generate TTS completion, retrying in ${retryInterval}ms`,
             );
           }
 
@@ -416,6 +562,13 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
         }
       }
     }
+  };
+
+  private async mainTask() {
+    return tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'tts_request',
+      endOnExit: false,
+    });
   }
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
@@ -434,9 +587,25 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
     return this.#text;
   }
 
+  get abortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
+  /**
+   * Set token usage for token-based TTS billing (e.g., OpenAI TTS).
+   * Plugins should call this method to report token usage.
+   */
+  protected setTokenUsage({
+    inputTokens = 0,
+    outputTokens = 0,
+  }: { inputTokens?: number; outputTokens?: number } = {}): void {
+    this.#inputTokens = inputTokens;
+    this.#outputTokens = outputTokens;
+  }
+
   protected async monitorMetrics() {
     const startTime = process.hrtime.bigint();
-    let audioDuration = 0;
+    let audioDurationMs = 0;
     let ttfb: bigint = BigInt(-1);
     let requestId = '';
 
@@ -446,7 +615,7 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
       if (ttfb === BigInt(-1)) {
         ttfb = process.hrtime.bigint() - startTime;
       }
-      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
+      audioDurationMs += (audio.frame.samplesPerChannel / audio.frame.sampleRate) * 1000;
     }
     this.output.close();
 
@@ -455,14 +624,27 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
       type: 'tts_metrics',
       timestamp: Date.now(),
       requestId,
-      ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
-      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      ttfbMs: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),
+      durationMs: Math.trunc(Number(duration / BigInt(1000000))),
       charactersCount: this.#text.length,
-      audioDuration,
+      audioDurationMs: Math.round(audioDurationMs),
       cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here
       label: this.#tts.label,
+      inputTokens: this.#inputTokens,
+      outputTokens: this.#outputTokens,
       streamed: false,
+      metadata: {
+        modelProvider: this.#tts.provider,
+        modelName: this.#tts.model,
+      },
     };
+
+    if (this.#ttsRequestSpan) {
+      this.#ttsRequestSpan.setAttribute(traceTypes.ATTR_TTS_METRICS, JSON.stringify(metrics));
+      this.#ttsRequestSpan.end();
+      this.#ttsRequestSpan = undefined;
+    }
+
     this.#tts.emit('metrics_collected', metrics);
   }
 
@@ -481,8 +663,9 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
 
   /** Close both the input and output of the TTS stream */
   close() {
-    this.queue.close();
-    this.output.close();
+    if (!this.queue.closed) this.queue.close();
+    if (!this.output.closed) this.output.close();
+    if (!this.abortController.signal.aborted) this.abortController.abort();
     this.closed = true;
   }