@livekit/agents 0.7.6 → 0.7.7

package/dist/tokenize/token_stream.d.cts

@@ -0,0 +1,39 @@
+import { AsyncIterableQueue } from '../utils.js';
+import type { TokenData } from './tokenizer.js';
+import { SentenceStream, WordStream } from './tokenizer.js';
+type TokenizeFunc = (x: string) => string[] | [string, number, number][];
+export declare class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
+    #private;
+    protected queue: AsyncIterableQueue<TokenData>;
+    protected closed: boolean;
+    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number);
+    /** Push a string of text into the token stream */
+    pushText(text: string): void;
+    /** Flush the stream, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<TokenData>>;
+    /** Close both the input and output of the token stream */
+    close(): void;
+    [Symbol.asyncIterator](): BufferedTokenStream;
+}
+export declare class BufferedSentenceStream extends SentenceStream {
+    #private;
+    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number);
+    pushText(text: string): void;
+    flush(): void;
+    close(): void;
+    next(): Promise<IteratorResult<TokenData>>;
+}
+export declare class BufferedWordStream extends WordStream {
+    #private;
+    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number);
+    pushText(text: string): void;
+    flush(): void;
+    endInput(): void;
+    close(): void;
+    next(): Promise<IteratorResult<TokenData>>;
+}
+export {};
+//# sourceMappingURL=token_stream.d.ts.map

package/dist/tokenize/tokenizer.d.cts

@@ -0,0 +1,55 @@
+import { AsyncIterableQueue } from '../utils.js';
+export declare const PUNCTUATIONS: string[];
+export interface TokenData {
+    segmentId: string;
+    token: string;
+}
+export declare abstract class SentenceTokenizer {
+    abstract tokenize(text: string, language?: string): string[];
+    /**
+     * Returns a {@link SentenceStream} that can be used to push strings and receive smaller segments.
+     */
+    abstract stream(): SentenceStream;
+}
+export declare abstract class SentenceStream {
+    #private;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    protected input: AsyncIterableQueue<string | typeof SentenceStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<TokenData>;
+    get closed(): boolean;
+    /** Push a string of text to the tokenizer */
+    pushText(text: string): void;
+    /** Flush the tokenizer, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<TokenData>>;
+    /** Close both the input and output of the tokenizer stream */
+    close(): void;
+    [Symbol.asyncIterator](): SentenceStream;
+}
+export declare abstract class WordTokenizer {
+    abstract tokenize(text: string, language?: string): string[];
+    /**
+     * Returns a {@link WordStream} that can be used to push words and receive smaller segments.
+     */
+    abstract stream(): WordStream;
+}
+export declare abstract class WordStream {
+    #private;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    protected input: AsyncIterableQueue<string | typeof WordStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<TokenData>;
+    get closed(): boolean;
+    /** Push a string of text to the tokenizer */
+    pushText(text: string): void;
+    /** Flush the tokenizer, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<TokenData>>;
+    /** Close both the input and output of the tokenizer stream */
+    close(): void;
+    [Symbol.asyncIterator](): WordStream;
+}
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/transcription.d.cts

@@ -0,0 +1,31 @@
+import { TranscriptionSegment } from '@livekit/protocol';
+import { AudioFrame } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { SentenceTokenizer } from './tokenize/tokenizer.js';
+export interface TextSyncOptions {
+    language: string;
+    speed: number;
+    newSentenceDelay: number;
+    sentenceTokenizer: SentenceTokenizer;
+    hyphenateWord: (word: string) => string[];
+    splitWords: (words: string) => [string, number, number][];
+}
+export declare const defaultTextSyncOptions: TextSyncOptions;
+type SyncCallbacks = {
+    textUpdated: (text: TranscriptionSegment) => void;
+};
+declare const TextAudioSynchronizer_base: new () => TypedEmitter<SyncCallbacks>;
+export declare class TextAudioSynchronizer extends TextAudioSynchronizer_base {
+    #private;
+    constructor(opts: TextSyncOptions);
+    pushAudio(frame: AudioFrame): void;
+    pushText(text: string): void;
+    markAudioSegmentEnd(): void;
+    markTextSegmentEnd(): void;
+    segmentPlayoutStarted(): void;
+    segmentPlayoutFinished(): void;
+    get playedText(): string;
+    close(interrupt: boolean): Promise<void>;
+}
+export {};
+//# sourceMappingURL=transcription.d.ts.map

package/dist/tts/index.d.cts

@@ -0,0 +1,3 @@
+export { type SynthesizedAudio, type TTSCapabilities, type TTSCallbacks, TTS, TTSEvent, SynthesizeStream, ChunkedStream, } from './tts.js';
+export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/tts/stream_adapter.d.cts

@@ -0,0 +1,17 @@
+import type { SentenceTokenizer } from '../tokenize/index.js';
+import type { ChunkedStream } from './tts.js';
+import { SynthesizeStream, TTS } from './tts.js';
+export declare class StreamAdapter extends TTS {
+    #private;
+    label: string;
+    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
+    synthesize(text: string): ChunkedStream;
+    stream(): StreamAdapterWrapper;
+}
+export declare class StreamAdapterWrapper extends SynthesizeStream {
+    #private;
+    label: string;
+    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
+    monitorMetrics(): Promise<void>;
+}
+//# sourceMappingURL=stream_adapter.d.ts.map

package/dist/tts/tts.cjs

@@ -79,7 +79,7 @@ class SynthesizeStream {
         const metrics = {
           timestamp: Date.now(),
           requestId,
-          ttfb: Math.trunc(Number(ttfb / BigInt(1e6))),
+          ttfb: Math.trunc(Number((ttfb || BigInt(0)) / BigInt(1e6))),
           duration: Math.trunc(Number(duration / BigInt(1e6))),
           charactersCount: text.length,
           audioDuration,
@@ -173,12 +173,12 @@ class ChunkedStream {
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
     let audioDuration = 0;
-    let ttfb;
+    let ttfb = BigInt(-1);
     let requestId = "";
     for await (const audio of this.queue) {
       this.output.put(audio);
       requestId = audio.requestId;
-      if (!ttfb) {
+      if (ttfb === BigInt(-1)) {
         ttfb = process.hrtime.bigint() - startTime;
       }
       audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
@@ -188,7 +188,7 @@ class ChunkedStream {
     const metrics = {
       timestamp: Date.now(),
       requestId,
-      ttfb: Math.trunc(Number(ttfb / BigInt(1e6))),
+      ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
       charactersCount: this.#text.length,
       audioDuration,

package/dist/tts/tts.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue, mergeFrames } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport enum TTSEvent {\n  METRICS_COLLECTED,\n}\n\nexport type TTSCallbacks = {\n  [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n\n  constructor(tts: TTS) {\n    this.#tts = tts;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const metrics: TTSMetrics = {\n          timestamp: Date.now(),\n          requestId,\n          ttfb: Math.trunc(Number(ttfb! / BigInt(1000000))),\n          duration: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDuration,\n          cancelled: false, // XXX(nbsp)\n          label: this.label,\n          streamed: false,\n        };\n        this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n    this.output.close();\n  }\n\n  /** Push a string of text to the TTS */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.input.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n\n  constructor(text: string, tts: TTS) {\n    this.#text = text;\n    this.#tts = tts;\n\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttfb: Math.trunc(Number(ttfb! / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDuration,\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      streamed: false,\n    };\n    this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,mBAAgD;AA2BzC,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAeL,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAEpB;AAAA,EACQ,SAAS,IAAI,gCAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,UAAsB;AAAA,UAC1B,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,UAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACvD,iBAAiB,KAAK;AAAA,UACtB;AAAA,UACA,WAAW;AAAA;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,MACpD;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAC7D,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAAA,IACjD;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,gCAAqC;AAAA,EACjD,SAAS,IAAI,gCAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EAEA,YAAY,MAAc,KAAU;AAClC,SAAK,QAAQ;AACb,SAAK,OAAO;AAEZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAAA,IAC/D;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,MAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,iBAAiB,KAAK,MAAM;AAAA,MAC5B;AAAA,MACA,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,eAAO,0BAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":["TTSEvent"]}
+{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue, mergeFrames } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport enum TTSEvent {\n  METRICS_COLLECTED,\n}\n\nexport type TTSCallbacks = {\n  [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n\n  constructor(tts: TTS) {\n    this.#tts = tts;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const metrics: TTSMetrics = {\n          timestamp: Date.now(),\n          requestId,\n          ttfb: Math.trunc(Number((ttfb || BigInt(0)) / BigInt(1000000))),\n          duration: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDuration,\n          cancelled: false, // XXX(nbsp)\n          label: this.label,\n          streamed: false,\n        };\n        this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n    this.output.close();\n  }\n\n  /** Push a string of text to the TTS */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.input.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n\n  constructor(text: string, tts: TTS) {\n    this.#text = text;\n    this.#tts = tts;\n\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDuration,\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      streamed: false,\n    };\n    this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,mBAAgD;AA2BzC,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAeL,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAEpB;AAAA,EACQ,SAAS,IAAI,gCAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,UAAsB;AAAA,UAC1B,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,MAAM,KAAK,MAAM,QAAQ,QAAQ,OAAO,CAAC,KAAK,OAAO,GAAO,CAAC,CAAC;AAAA,UAC9D,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACvD,iBAAiB,KAAK;AAAA,UACtB;AAAA,UACA,WAAW;AAAA;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,MACpD;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAC7D,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAAA,IACjD;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,gCAAqC;AAAA,EACjD,SAAS,IAAI,gCAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EAEA,YAAY,MAAc,KAAU;AAClC,SAAK,QAAQ;AACb,SAAK,OAAO;AAEZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAAA,IAC/D;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,iBAAiB,KAAK,MAAM;AAAA,MAC5B;AAAA,MACA,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,eAAO,0BAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":["TTSEvent"]}

package/dist/tts/tts.d.cts

@@ -0,0 +1,127 @@
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { TTSMetrics } from '../metrics/base.js';
+import { AsyncIterableQueue } from '../utils.js';
+/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
+export interface SynthesizedAudio {
+    /** Request ID (one segment could be made up of multiple requests) */
+    requestId: string;
+    /** Segment ID, each segment is separated by a flush */
+    segmentId: string;
+    /** Synthesized audio frame */
+    frame: AudioFrame;
+    /** Current segment of the synthesized audio */
+    deltaText?: string;
+    /** Whether this is the last frame of the segment (streaming only) */
+    final: boolean;
+}
+/**
+ * Describes the capabilities of the TTS provider.
+ *
+ * @remarks
+ * At present, only `streaming` is supplied to this interface, and the framework only supports
+ * providers that do have a streaming endpoint.
+ */
+export interface TTSCapabilities {
+    streaming: boolean;
+}
+export declare enum TTSEvent {
+    METRICS_COLLECTED = 0
+}
+export type TTSCallbacks = {
+    [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;
+};
+declare const TTS_base: new () => TypedEmitter<TTSCallbacks>;
+/**
+ * An instance of a text-to-speech adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child TTS class, which inherits this class's methods.
+ */
+export declare abstract class TTS extends TTS_base {
+    #private;
+    abstract label: string;
+    constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities);
+    /** Returns this TTS's capabilities */
+    get capabilities(): TTSCapabilities;
+    /** Returns the sample rate of audio frames returned by this TTS */
+    get sampleRate(): number;
+    /** Returns the channel count of audio frames returned by this TTS */
+    get numChannels(): number;
+    /**
+     * Receives text and returns synthesis in the form of a {@link ChunkedStream}
+     */
+    abstract synthesize(text: string): ChunkedStream;
+    /**
+     * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+     */
+    abstract stream(): SynthesizeStream;
+}
+/**
+ * An instance of a text-to-speech stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SynthesizeStream class, which inherits this class's methods.
+ */
+export declare abstract class SynthesizeStream implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM> {
+    #private;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    static readonly END_OF_STREAM: unique symbol;
+    protected input: AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>;
+    protected output: AsyncIterableQueue<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>;
+    protected closed: boolean;
+    abstract label: string;
+    constructor(tts: TTS);
+    protected monitorMetrics(): Promise<void>;
+    /** Push a string of text to the TTS */
+    pushText(text: string): void;
+    /** Flush the TTS, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>>;
+    /** Close both the input and output of the TTS stream */
+    close(): void;
+    [Symbol.asyncIterator](): SynthesizeStream;
+}
+/**
+ * An instance of a text-to-speech response, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child ChunkedStream class, which inherits this class's methods.
+ */
+export declare abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
+    #private;
+    protected queue: AsyncIterableQueue<SynthesizedAudio>;
+    protected output: AsyncIterableQueue<SynthesizedAudio>;
+    protected closed: boolean;
+    abstract label: string;
+    constructor(text: string, tts: TTS);
+    protected monitorMetrics(): Promise<void>;
+    /** Collect every frame into one in a single call */
+    collect(): Promise<AudioFrame>;
+    next(): Promise<IteratorResult<SynthesizedAudio>>;
+    /** Close both the input and output of the TTS stream */
+    close(): void;
+    [Symbol.asyncIterator](): ChunkedStream;
+}
+export {};
+//# sourceMappingURL=tts.d.ts.map

package/dist/tts/tts.js

@@ -53,7 +53,7 @@ class SynthesizeStream {
         const metrics = {
           timestamp: Date.now(),
           requestId,
-          ttfb: Math.trunc(Number(ttfb / BigInt(1e6))),
+          ttfb: Math.trunc(Number((ttfb || BigInt(0)) / BigInt(1e6))),
           duration: Math.trunc(Number(duration / BigInt(1e6))),
           charactersCount: text.length,
           audioDuration,
@@ -147,12 +147,12 @@ class ChunkedStream {
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
     let audioDuration = 0;
-    let ttfb;
+    let ttfb = BigInt(-1);
     let requestId = "";
     for await (const audio of this.queue) {
       this.output.put(audio);
       requestId = audio.requestId;
-      if (!ttfb) {
+      if (ttfb === BigInt(-1)) {
         ttfb = process.hrtime.bigint() - startTime;
       }
       audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;
@@ -162,7 +162,7 @@ class ChunkedStream {
     const metrics = {
       timestamp: Date.now(),
       requestId,
-      ttfb: Math.trunc(Number(ttfb / BigInt(1e6))),
+      ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
       charactersCount: this.#text.length,
       audioDuration,

package/dist/tts/tts.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue, mergeFrames } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport enum TTSEvent {\n  METRICS_COLLECTED,\n}\n\nexport type TTSCallbacks = {\n  [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n\n  constructor(tts: TTS) {\n    this.#tts = tts;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const metrics: TTSMetrics = {\n          timestamp: Date.now(),\n          requestId,\n          ttfb: Math.trunc(Number(ttfb! / BigInt(1000000))),\n          duration: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDuration,\n          cancelled: false, // XXX(nbsp)\n          label: this.label,\n          streamed: false,\n        };\n        this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n    this.output.close();\n  }\n\n  /** Push a string of text to the TTS */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.input.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n\n  constructor(text: string, tts: TTS) {\n    this.#text = text;\n    this.#tts = tts;\n\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttfb: Math.trunc(Number(ttfb! / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDuration,\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      streamed: false,\n    };\n    this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,mBAAmB;AA2BzC,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAeL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,mBAAoE;AAAA,EAChF,QAAQ,IAAI,mBAEpB;AAAA,EACQ,SAAS,IAAI,mBAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,UAAsB;AAAA,UAC1B,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,UAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACvD,iBAAiB,KAAK;AAAA,UACtB;AAAA,UACA,WAAW;AAAA;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,MACpD;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAC7D,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAAA,IACjD;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,mBAAqC;AAAA,EACjD,SAAS,IAAI,mBAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EAEA,YAAY,MAAc,KAAU;AAClC,SAAK,QAAQ;AACb,SAAK,OAAO;AAEZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAAA,IAC/D;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,MAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,iBAAiB,KAAK,MAAM;AAAA,MAC5B;AAAA,MACA,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,WAAO,YAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":["TTSEvent"]}
+{"version":3,"sources":["../../src/tts/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { TTSMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue, mergeFrames } from '../utils.js';\n\n/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */\nexport interface SynthesizedAudio {\n  /** Request ID (one segment could be made up of multiple requests) */\n  requestId: string;\n  /** Segment ID, each segment is separated by a flush */\n  segmentId: string;\n  /** Synthesized audio frame */\n  frame: AudioFrame;\n  /** Current segment of the synthesized audio */\n  deltaText?: string;\n  /** Whether this is the last frame of the segment (streaming only) */\n  final: boolean;\n}\n\n/**\n * Describes the capabilities of the TTS provider.\n *\n * @remarks\n * At present, only `streaming` is supplied to this interface, and the framework only supports\n * providers that do have a streaming endpoint.\n */\nexport interface TTSCapabilities {\n  streaming: boolean;\n}\n\nexport enum TTSEvent {\n  METRICS_COLLECTED,\n}\n\nexport type TTSCallbacks = {\n  [TTSEvent.METRICS_COLLECTED]: (metrics: TTSMetrics) => void;\n};\n\n/**\n * An instance of a text-to-speech adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child TTS class, which inherits this class's methods.\n */\nexport abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCallbacks>) {\n  #capabilities: TTSCapabilities;\n  #sampleRate: number;\n  #numChannels: number;\n  abstract label: string;\n\n  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n    this.#sampleRate = sampleRate;\n    this.#numChannels = numChannels;\n  }\n\n  /** Returns this TTS's capabilities */\n  get capabilities(): TTSCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Returns the sample rate of audio frames returned by this TTS */\n  get sampleRate(): number {\n    return this.#sampleRate;\n  }\n\n  /** Returns the channel count of audio frames returned by this TTS */\n  get numChannels(): number {\n    return this.#numChannels;\n  }\n\n  /**\n   * Receives text and returns synthesis in the form of a {@link ChunkedStream}\n   */\n  abstract synthesize(text: string): ChunkedStream;\n\n  /**\n   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data\n   */\n  abstract stream(): SynthesizeStream;\n}\n\n/**\n * An instance of a text-to-speech stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SynthesizeStream class, which inherits this class's methods.\n */\nexport abstract class SynthesizeStream\n  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>\n{\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');\n  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected output = new AsyncIterableQueue<\n    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM\n  >();\n  protected closed = false;\n  abstract label: string;\n  #tts: TTS;\n  #metricsPendingTexts: string[] = [];\n  #metricsText = '';\n  #monitorMetricsTask?: Promise<void>;\n\n  constructor(tts: TTS) {\n    this.#tts = tts;\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint | undefined;\n    let requestId = '';\n\n    const emit = () => {\n      if (this.#metricsPendingTexts.length) {\n        const text = this.#metricsPendingTexts.shift()!;\n        const duration = process.hrtime.bigint() - startTime;\n        const metrics: TTSMetrics = {\n          timestamp: Date.now(),\n          requestId,\n          ttfb: Math.trunc(Number((ttfb || BigInt(0)) / BigInt(1000000))),\n          duration: Math.trunc(Number(duration / BigInt(1000000))),\n          charactersCount: text.length,\n          audioDuration,\n          cancelled: false, // XXX(nbsp)\n          label: this.label,\n          streamed: false,\n        };\n        this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n      }\n    };\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      if (audio === SynthesizeStream.END_OF_STREAM) continue;\n      requestId = audio.requestId;\n      if (!ttfb) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n      if (audio.final) {\n        emit();\n      }\n    }\n\n    if (requestId) {\n      emit();\n    }\n    this.output.close();\n  }\n\n  /** Push a string of text to the TTS */\n  pushText(text: string) {\n    if (!this.#monitorMetricsTask) {\n      this.#monitorMetricsTask = this.monitorMetrics();\n    }\n    this.#metricsText += text;\n\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the TTS, causing it to process all pending text */\n  flush() {\n    if (this.#metricsText) {\n      this.#metricsPendingTexts.push(this.#metricsText);\n      this.#metricsText = '';\n    }\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SynthesizeStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.input.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SynthesizeStream {\n    return this;\n  }\n}\n\n/**\n * An instance of a text-to-speech response, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   await source.captureFrame(event.frame);\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child ChunkedStream class, which inherits this class's methods.\n */\nexport abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {\n  protected queue = new AsyncIterableQueue<SynthesizedAudio>();\n  protected output = new AsyncIterableQueue<SynthesizedAudio>();\n  protected closed = false;\n  abstract label: string;\n  #text: string;\n  #tts: TTS;\n\n  constructor(text: string, tts: TTS) {\n    this.#text = text;\n    this.#tts = tts;\n\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let audioDuration = 0;\n    let ttfb: bigint = BigInt(-1);\n    let requestId = '';\n\n    for await (const audio of this.queue) {\n      this.output.put(audio);\n      requestId = audio.requestId;\n      if (ttfb === BigInt(-1)) {\n        ttfb = process.hrtime.bigint() - startTime;\n      }\n      audioDuration += audio.frame.samplesPerChannel / audio.frame.sampleRate;\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: TTSMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttfb: ttfb === BigInt(-1) ? -1 : Math.trunc(Number(ttfb / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      charactersCount: this.#text.length,\n      audioDuration,\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      streamed: false,\n    };\n    this.#tts.emit(TTSEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** Collect every frame into one in a single call */\n  async collect(): Promise<AudioFrame> {\n    const frames = [];\n    for await (const event of this) {\n      frames.push(event.frame);\n    }\n    return mergeFrames(frames);\n  }\n\n  next(): Promise<IteratorResult<SynthesizedAudio>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the TTS stream */\n  close() {\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): ChunkedStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,mBAAmB;AA2BzC,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAeL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EACA;AAAA,EACA;AAAA,EAGA,YAAY,YAAoB,aAAqB,cAA+B;AAClF,UAAM;AACN,SAAK,gBAAgB;AACrB,SAAK,cAAc;AACnB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAWF;AAgBO,MAAe,iBAEtB;AAAA,EACE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EAClE,OAAgB,gBAAgB,OAAO,eAAe;AAAA,EAC5C,QAAQ,IAAI,mBAAoE;AAAA,EAChF,QAAQ,IAAI,mBAEpB;AAAA,EACQ,SAAS,IAAI,mBAErB;AAAA,EACQ,SAAS;AAAA,EAEnB;AAAA,EACA,uBAAiC,CAAC;AAAA,EAClC,eAAe;AAAA,EACf;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI;AACJ,QAAI,YAAY;AAEhB,UAAM,OAAO,MAAM;AACjB,UAAI,KAAK,qBAAqB,QAAQ;AACpC,cAAM,OAAO,KAAK,qBAAqB,MAAM;AAC7C,cAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,cAAM,UAAsB;AAAA,UAC1B,WAAW,KAAK,IAAI;AAAA,UACpB;AAAA,UACA,MAAM,KAAK,MAAM,QAAQ,QAAQ,OAAO,CAAC,KAAK,OAAO,GAAO,CAAC,CAAC;AAAA,UAC9D,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,UACvD,iBAAiB,KAAK;AAAA,UACtB;AAAA,UACA,WAAW;AAAA;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,UAAU;AAAA,QACZ;AACA,aAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,MACpD;AAAA,IACF;AAEA,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,UAAU,iBAAiB,cAAe;AAC9C,kBAAY,MAAM;AAClB,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAC7D,UAAI,MAAM,OAAO;AACf,aAAK;AAAA,MACP;AAAA,IACF;AAEA,QAAI,WAAW;AACb,WAAK;AAAA,IACP;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,CAAC,KAAK,qBAAqB;AAC7B,WAAK,sBAAsB,KAAK,eAAe;AAAA,IACjD;AACA,SAAK,gBAAgB;AAErB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,cAAc;AACrB,WAAK,qBAAqB,KAAK,KAAK,YAAY;AAChD,WAAK,eAAe;AAAA,IACtB;AACA,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,iBAAiB,cAAc;AAAA,EAChD;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA0F;AACxF,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAsB;AACzC,WAAO;AAAA,EACT;AACF;AAgBO,MAAe,cAAiE;AAAA,EAC3E,QAAQ,IAAI,mBAAqC;AAAA,EACjD,SAAS,IAAI,mBAAqC;AAAA,EAClD,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EAEA,YAAY,MAAc,KAAU;AAClC,SAAK,QAAQ;AACb,SAAK,OAAO;AAEZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,gBAAgB;AACpB,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAEhB,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,kBAAY,MAAM;AAClB,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,uBAAiB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAAA,IAC/D;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,iBAAiB,KAAK,MAAM;AAAA,MAC5B;AAAA,MACA,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,UAAU;AAAA,IACZ;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,MAAM,UAA+B;AACnC,UAAM,SAAS,CAAC;AAChB,qBAAiB,SAAS,MAAM;AAC9B,aAAO,KAAK,MAAM,KAAK;AAAA,IACzB;AACA,WAAO,YAAY,MAAM;AAAA,EAC3B;AAAA,EAEA,OAAkD;AAChD,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAmB;AACtC,WAAO;AAAA,EACT;AACF;","names":["TTSEvent"]}

package/dist/utils.d.cts

@@ -0,0 +1,72 @@
+import type { Room } from '@livekit/rtc-node';
+import { AudioFrame } from '@livekit/rtc-node';
+/** Union of a single and a list of {@link AudioFrame}s */
+export type AudioBuffer = AudioFrame[] | AudioFrame;
+/**
+ * Merge one or more {@link AudioFrame}s into a single one.
+ *
+ * @param buffer Either an {@link AudioFrame} or a list thereof
+ * @throws
+ * {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
+ * | TypeError} if sample rate or channel count are mismatched
+ */
+export declare const mergeFrames: (buffer: AudioBuffer) => AudioFrame;
+export declare const findMicroTrackId: (room: Room, identity: string) => string;
+/** @internal */
+export declare class Queue<T> {
+    #private;
+    /** @internal */
+    items: T[];
+    constructor(limit?: number);
+    get(): Promise<T>;
+    put(item: T): Promise<void>;
+}
+/** @internal */
+export declare class Future {
+    #private;
+    constructor();
+    get await(): Promise<void>;
+    get done(): boolean;
+    resolve(): void;
+    reject(error: Error): void;
+}
+/** @internal */
+export declare class CancellablePromise<T> {
+    #private;
+    constructor(executor: (resolve: (value: T | PromiseLike<T>) => void, reject: (reason?: any) => void, onCancel: (cancelFn: () => void) => void) => void);
+    get isCancelled(): boolean;
+    get error(): Error | null;
+    then<TResult1 = T, TResult2 = never>(onfulfilled?: ((value: T) => TResult1 | Promise<TResult1>) | null, onrejected?: ((reason: any) => TResult2 | Promise<TResult2>) | null): Promise<TResult1 | TResult2>;
+    catch<TResult = never>(onrejected?: ((reason: any) => TResult | Promise<TResult>) | null): Promise<T | TResult>;
+    finally(onfinally?: (() => void) | null): Promise<T>;
+    cancel(): void;
+    static from<T>(promise: Promise<T>): CancellablePromise<T>;
+}
+/** @internal */
+export declare function gracefullyCancel<T>(promise: CancellablePromise<T>): Promise<void>;
+/** @internal */
+export declare class AsyncIterableQueue<T> implements AsyncIterableIterator<T> {
+    #private;
+    private static readonly CLOSE_SENTINEL;
+    get closed(): boolean;
+    put(item: T): void;
+    close(): void;
+    next(): Promise<IteratorResult<T>>;
+    [Symbol.asyncIterator](): AsyncIterableQueue<T>;
+}
+/** @internal */
+export declare class ExpFilter {
+    #private;
+    constructor(alpha: number, max?: number);
+    reset(alpha?: number): void;
+    apply(exp: number, sample: number): number;
+    get filtered(): number | undefined;
+    set alpha(alpha: number);
+}
+/** @internal */
+export declare class AudioEnergyFilter {
+    #private;
+    constructor(cooldownSeconds?: number);
+    pushFrame(frame: AudioFrame): boolean;
+}
+//# sourceMappingURL=utils.d.ts.map

package/dist/vad.d.cts

@@ -0,0 +1,78 @@
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { VADMetrics } from './metrics/base.js';
+import { AsyncIterableQueue } from './utils.js';
+export declare enum VADEventType {
+    START_OF_SPEECH = 0,
+    INFERENCE_DONE = 1,
+    END_OF_SPEECH = 2,
+    METRICS_COLLECTED = 3
+}
+export interface VADEvent {
+    /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */
+    type: VADEventType;
+    /**
+     * Index of the audio sample where the event occurred, relative to the inference sample rate.
+     */
+    samplesIndex: number;
+    /** Timestamp when the event was fired. */
+    timestamp: number;
+    /** Duration of the speech segment. */
+    speechDuration: number;
+    /** Duration of the silence segment. */
+    silenceDuration: number;
+    /**
+     * List of audio frames associated with the speech.
+     *
+     * @remarks
+     * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.
+     * - For `inference_done` events, this contains the audio chunks that were processed.
+     * - For `end_of_speech` events, this contains the complete user speech.
+     */
+    frames: AudioFrame[];
+    /** Probability that speech is present (only for `INFERENCE_DONE` events). */
+    probability: number;
+    /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */
+    inferenceDuration: number;
+    /** Indicates whether speech was detected in the frames. */
+    speaking: boolean;
+    /** Threshold used to detect silence. */
+    rawAccumulatedSilence: number;
+    /** Threshold used to detect speech. */
+    rawAccumulatedSpeech: number;
+}
+export interface VADCapabilities {
+    updateInterval: number;
+}
+export type VADCallbacks = {
+    [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;
+};
+declare const VAD_base: new () => TypedEmitter<VADCallbacks>;
+export declare abstract class VAD extends VAD_base {
+    #private;
+    abstract label: string;
+    constructor(capabilities: VADCapabilities);
+    get capabilities(): VADCapabilities;
+    /**
+     * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
+     */
+    abstract stream(): VADStream;
+}
+export declare abstract class VADStream implements AsyncIterableIterator<VADEvent> {
+    #private;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    protected input: AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<VADEvent>;
+    protected output: AsyncIterableQueue<VADEvent>;
+    protected closed: boolean;
+    constructor(vad: VAD);
+    protected monitorMetrics(): Promise<void>;
+    pushFrame(frame: AudioFrame): void;
+    flush(): void;
+    endInput(): void;
+    next(): Promise<IteratorResult<VADEvent>>;
+    close(): void;
+    [Symbol.asyncIterator](): VADStream;
+}
+export {};
+//# sourceMappingURL=vad.d.ts.map

package/dist/version.d.cts

@@ -0,0 +1,2 @@
+export declare const version = "0.1.0";
+//# sourceMappingURL=version.d.ts.map

package/dist/worker.cjs

@@ -263,7 +263,8 @@ class Worker {
     this.#httpServer = new import_http_server.HTTPServer(opts.host, opts.port, () => ({
       agent_name: opts.agentName,
       worker_type: import_protocol.JobType[opts.workerType],
-      active_jobs: this.activeJobs.length
+      active_jobs: this.activeJobs.length,
+      sdk_version: import_version.version
     }));
   }
   /* @throws {@link WorkerError} if worker failed to connect or already running */