@livekit/agents 1.1.0 → 1.2.1

package/dist/tts/index.js

@@ -4,8 +4,10 @@ import {
   ChunkedStream
 } from "./tts.js";
 import { StreamAdapter, StreamAdapterWrapper } from "./stream_adapter.js";
+import { FallbackAdapter } from "./fallback_adapter.js";
 export {
   ChunkedStream,
+  FallbackAdapter,
   StreamAdapter,
   StreamAdapterWrapper,
   SynthesizeStream,

package/dist/tts/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  type TTSCallbacks,\n  TTS,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":"AAGA;AAAA,EAIE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;","names":[]}
+{"version":3,"sources":["../../src/tts/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  type SynthesizedAudio,\n  type TTSCapabilities,\n  type TTSCallbacks,\n  TTS,\n  SynthesizeStream,\n  ChunkedStream,\n} from './tts.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\nexport { FallbackAdapter, type AvailabilityChangedEvent } from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EAIE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;AACpD,SAAS,uBAAsD;","names":[]}

package/dist/tts/stream_adapter.cjs

@@ -22,14 +22,16 @@ __export(stream_adapter_exports, {
   StreamAdapterWrapper: () => StreamAdapterWrapper
 });
 module.exports = __toCommonJS(stream_adapter_exports);
+var import_types = require("../types.cjs");
 var import_utils = require("../utils.cjs");
+var import_io = require("../voice/io.cjs");
 var import_tts = require("./tts.cjs");
 class StreamAdapter extends import_tts.TTS {
   #tts;
   #sentenceTokenizer;
   label;
   constructor(tts, sentenceTokenizer) {
-    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
     this.label = this.#tts.label;
@@ -37,25 +39,29 @@ class StreamAdapter extends import_tts.TTS {
     this.#tts.on("metrics_collected", (metrics) => {
       this.emit("metrics_collected", metrics);
     });
+    this.#tts.on("error", (error) => {
+      this.emit("error", error);
+    });
   }
-  synthesize(text) {
-    return this.#tts.synthesize(text);
+  synthesize(text, connOptions, abortSignal) {
+    return this.#tts.synthesize(text, connOptions, abortSignal);
   }
-  stream() {
-    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+  stream(options) {
+    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options == null ? void 0 : options.connOptions);
   }
 }
 class StreamAdapterWrapper extends import_tts.SynthesizeStream {
   #tts;
   #sentenceStream;
   label;
-  constructor(tts, sentenceTokenizer) {
-    super(tts);
+  constructor(tts, sentenceTokenizer, connOptions) {
+    super(tts, connOptions);
     this.#tts = tts;
     this.#sentenceStream = sentenceTokenizer.stream();
     this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;
   }
   async run() {
+    let cumulativeDuration = 0;
     const forwardInput = async () => {
       for await (const input of this.input) {
         if (this.abortController.signal.aborted) break;
@@ -83,11 +89,22 @@ class StreamAdapterWrapper extends import_tts.SynthesizeStream {
       this.queue.put(import_tts.SynthesizeStream.END_OF_STREAM);
     };
     const synthesize = async (token, prevTask, controller) => {
-      const audioStream = this.#tts.synthesize(token);
+      const audioStream = this.#tts.synthesize(token, this.connOptions, this.abortSignal);
       await (prevTask == null ? void 0 : prevTask.result);
       if (controller.signal.aborted) return;
+      const timedString = (0, import_io.createTimedString)({
+        text: token,
+        startTime: cumulativeDuration
+      });
+      let isFirstFrame = true;
       for await (const audio of audioStream) {
         if (controller.signal.aborted) break;
+        if (isFirstFrame) {
+          audio.frame.userdata[import_types.USERDATA_TIMED_TRANSCRIPT] = [timedString];
+          isFirstFrame = false;
+        }
+        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;
+        cumulativeDuration += frameDuration;
         this.queue.put(audio);
       }
     };

package/dist/tts/stream_adapter.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport { Task } from '../utils.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n    this.label = this.#tts.label;\n    this.label = `tts.StreamAdapter<${this.#tts.label}>`;\n\n    this.#tts.on('metrics_collected', (metrics) => {\n      this.emit('metrics_collected', metrics);\n    });\n  }\n\n  synthesize(text: string): ChunkedStream {\n    return this.#tts.synthesize(text);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts);\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;\n  }\n\n  protected async run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (this.abortController.signal.aborted) break;\n\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesizeSentenceStream = async () => {\n      let task: Task<void> | undefined;\n      const tokenCompletionTasks: Task<void>[] = [];\n\n      for await (const ev of this.#sentenceStream) {\n        if (this.abortController.signal.aborted) break;\n\n        // this will enable non-blocking synthesis of the stream of tokens\n        task = Task.from(\n          (controller) => synthesize(ev.token, task, controller),\n          this.abortController,\n        );\n\n        tokenCompletionTasks.push(task);\n      }\n\n      await Promise.all(tokenCompletionTasks.map((t) => t.result));\n      this.queue.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    const synthesize = async (\n      token: string,\n      prevTask: Task<void> | undefined,\n      controller: AbortController,\n    ) => {\n      const audioStream = this.#tts.synthesize(token);\n\n      // wait for previous audio transcription to complete before starting\n      // to queuing audio frames of the current token\n      await prevTask?.result;\n      if (controller.signal.aborted) return;\n\n      for await (const audio of audioStream) {\n        if (controller.signal.aborted) break;\n        this.queue.put(audio);\n      }\n    };\n\n    await Promise.all([forwardInput(), synthesizeSentenceStream()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAqB;AAErB,iBAAsC;AAE/B,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,KAAK,CAAC;AAC1D,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAC1B,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,qBAAqB,CAAC,YAAY;AAC7C,WAAK,KAAK,qBAAqB,OAAO;AAAA,IACxC,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,MAA6B;AACtC,WAAO,KAAK,KAAK,WAAW,IAAI;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,kBAAkB;AAAA,EACpE;AACF;AAEO,MAAM,6BAA6B,4BAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAChD,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAAA,EAC1D;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAEzC,YAAI,UAAU,4BAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,2BAA2B,YAAY;AAC3C,UAAI;AACJ,YAAM,uBAAqC,CAAC;AAE5C,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAGzC,eAAO,kBAAK;AAAA,UACV,CAAC,eAAe,WAAW,GAAG,OAAO,MAAM,UAAU;AAAA,UACrD,KAAK;AAAA,QACP;AAEA,6BAAqB,KAAK,IAAI;AAAA,MAChC;AAEA,YAAM,QAAQ,IAAI,qBAAqB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAC3D,WAAK,MAAM,IAAI,4BAAiB,aAAa;AAAA,IAC/C;AAEA,UAAM,aAAa,OACjB,OACA,UACA,eACG;AACH,YAAM,cAAc,KAAK,KAAK,WAAW,KAAK;AAI9C,aAAM,qCAAU;AAChB,UAAI,WAAW,OAAO,QAAS;AAE/B,uBAAiB,SAAS,aAAa;AACrC,YAAI,WAAW,OAAO,QAAS;AAC/B,aAAK,MAAM,IAAI,KAAK;AAAA,MACtB;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,CAAC,aAAa,GAAG,yBAAyB,CAAC,CAAC;AAAA,EAChE;AACF;","names":[]}
+{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { USERDATA_TIMED_TRANSCRIPT } from '../types.js';\nimport { Task } from '../utils.js';\nimport { createTimedString } from '../voice/io.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n    this.label = this.#tts.label;\n    this.label = `tts.StreamAdapter<${this.#tts.label}>`;\n\n    this.#tts.on('metrics_collected', (metrics) => {\n      this.emit('metrics_collected', metrics);\n    });\n    this.#tts.on('error', (error) => {\n      this.emit('error', error);\n    });\n  }\n\n  synthesize(\n    text: string,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ): ChunkedStream {\n    return this.#tts.synthesize(text, connOptions, abortSignal);\n  }\n\n  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options?.connOptions);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions) {\n    super(tts, connOptions);\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;\n  }\n\n  protected async run() {\n    let cumulativeDuration = 0;\n\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (this.abortController.signal.aborted) break;\n\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesizeSentenceStream = async () => {\n      let task: Task<void> | undefined;\n      const tokenCompletionTasks: Task<void>[] = [];\n\n      for await (const ev of this.#sentenceStream) {\n        if (this.abortController.signal.aborted) break;\n\n        // this will enable non-blocking synthesis of the stream of tokens\n        task = Task.from(\n          (controller) => synthesize(ev.token, task, controller),\n          this.abortController,\n        );\n\n        tokenCompletionTasks.push(task);\n      }\n\n      await Promise.all(tokenCompletionTasks.map((t) => t.result));\n      this.queue.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    const synthesize = async (\n      token: string,\n      prevTask: Task<void> | undefined,\n      controller: AbortController,\n    ) => {\n      const audioStream = this.#tts.synthesize(token, this.connOptions, this.abortSignal);\n\n      // wait for previous audio transcription to complete before starting\n      // to queuing audio frames of the current token\n      await prevTask?.result;\n      if (controller.signal.aborted) return;\n\n      // Create a TimedString with the sentence text and current cumulative duration\n      const timedString = createTimedString({\n        text: token,\n        startTime: cumulativeDuration,\n      });\n\n      let isFirstFrame = true;\n      for await (const audio of audioStream) {\n        if (controller.signal.aborted) break;\n\n        // Attach the TimedString to the first frame of this sentence\n        if (isFirstFrame) {\n          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];\n          isFirstFrame = false;\n        }\n\n        // Track cumulative duration\n        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;\n        cumulativeDuration += frameDuration;\n\n        this.queue.put(audio);\n      }\n    };\n\n    await Promise.all([forwardInput(), synthesizeSentenceStream()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,mBAA0C;AAC1C,mBAAqB;AACrB,gBAAkC;AAElC,iBAAsC;AAE/B,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,MAAM,mBAAmB,KAAK,CAAC;AACnF,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAC1B,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,qBAAqB,CAAC,YAAY;AAC7C,WAAK,KAAK,qBAAqB,OAAO;AAAA,IACxC,CAAC;AACD,SAAK,KAAK,GAAG,SAAS,CAAC,UAAU;AAC/B,WAAK,KAAK,SAAS,KAAK;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,KAAK,KAAK,WAAW,MAAM,aAAa,WAAW;AAAA,EAC5D;AAAA,EAEA,OAAO,SAAqE;AAC1E,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,oBAAoB,mCAAS,WAAW;AAAA,EAC1F;AACF;AAEO,MAAM,6BAA6B,4BAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC,aAAiC;AAC3F,UAAM,KAAK,WAAW;AACtB,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAChD,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAAA,EAC1D;AAAA,EAEA,MAAgB,MAAM;AACpB,QAAI,qBAAqB;AAEzB,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAEzC,YAAI,UAAU,4BAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,2BAA2B,YAAY;AAC3C,UAAI;AACJ,YAAM,uBAAqC,CAAC;AAE5C,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAGzC,eAAO,kBAAK;AAAA,UACV,CAAC,eAAe,WAAW,GAAG,OAAO,MAAM,UAAU;AAAA,UACrD,KAAK;AAAA,QACP;AAEA,6BAAqB,KAAK,IAAI;AAAA,MAChC;AAEA,YAAM,QAAQ,IAAI,qBAAqB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAC3D,WAAK,MAAM,IAAI,4BAAiB,aAAa;AAAA,IAC/C;AAEA,UAAM,aAAa,OACjB,OACA,UACA,eACG;AACH,YAAM,cAAc,KAAK,KAAK,WAAW,OAAO,KAAK,aAAa,KAAK,WAAW;AAIlF,aAAM,qCAAU;AAChB,UAAI,WAAW,OAAO,QAAS;AAG/B,YAAM,kBAAc,6BAAkB;AAAA,QACpC,MAAM;AAAA,QACN,WAAW;AAAA,MACb,CAAC;AAED,UAAI,eAAe;AACnB,uBAAiB,SAAS,aAAa;AACrC,YAAI,WAAW,OAAO,QAAS;AAG/B,YAAI,cAAc;AAChB,gBAAM,MAAM,SAAS,sCAAyB,IAAI,CAAC,WAAW;AAC9D,yBAAe;AAAA,QACjB;AAGA,cAAM,gBAAgB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAClE,8BAAsB;AAEtB,aAAK,MAAM,IAAI,KAAK;AAAA,MACtB;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,CAAC,aAAa,GAAG,yBAAyB,CAAC,CAAC;AAAA,EAChE;AACF;","names":[]}

package/dist/tts/stream_adapter.d.cts

@@ -1,17 +1,20 @@
 import type { SentenceTokenizer } from '../tokenize/index.js';
+import type { APIConnectOptions } from '../types.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 export declare class StreamAdapter extends TTS {
     #private;
     label: string;
     constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
-    synthesize(text: string): ChunkedStream;
-    stream(): StreamAdapterWrapper;
+    synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream;
+    stream(options?: {
+        connOptions?: APIConnectOptions;
+    }): StreamAdapterWrapper;
 }
 export declare class StreamAdapterWrapper extends SynthesizeStream {
     #private;
     label: string;
-    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
+    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions);
     protected run(): Promise<void>;
 }
 //# sourceMappingURL=stream_adapter.d.ts.map

package/dist/tts/stream_adapter.d.ts

@@ -1,17 +1,20 @@
 import type { SentenceTokenizer } from '../tokenize/index.js';
+import type { APIConnectOptions } from '../types.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 export declare class StreamAdapter extends TTS {
     #private;
     label: string;
     constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
-    synthesize(text: string): ChunkedStream;
-    stream(): StreamAdapterWrapper;
+    synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream;
+    stream(options?: {
+        connOptions?: APIConnectOptions;
+    }): StreamAdapterWrapper;
 }
 export declare class StreamAdapterWrapper extends SynthesizeStream {
     #private;
     label: string;
-    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer);
+    constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions);
     protected run(): Promise<void>;
 }
 //# sourceMappingURL=stream_adapter.d.ts.map

package/dist/tts/stream_adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/tts/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAE9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEjD,qBAAa,aAAc,SAAQ,GAAG;;IAGpC,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;IAY1D,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAIvC,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,gBAAgB;;IAGxD,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;cAO1C,GAAG;CAuDpB"}
+{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/tts/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAIrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEjD,qBAAa,aAAc,SAAQ,GAAG;;IAGpC,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB;IAe1D,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW,GACxB,aAAa;IAIhB,MAAM,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,iBAAiB,CAAA;KAAE,GAAG,oBAAoB;CAG5E;AAED,qBAAa,oBAAqB,SAAQ,gBAAgB;;IAGxD,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,WAAW,CAAC,EAAE,iBAAiB;cAO3E,GAAG;CA2EpB"}

package/dist/tts/stream_adapter.js

@@ -1,11 +1,13 @@
+import { USERDATA_TIMED_TRANSCRIPT } from "../types.js";
 import { Task } from "../utils.js";
+import { createTimedString } from "../voice/io.js";
 import { SynthesizeStream, TTS } from "./tts.js";
 class StreamAdapter extends TTS {
   #tts;
   #sentenceTokenizer;
   label;
   constructor(tts, sentenceTokenizer) {
-    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
     this.label = this.#tts.label;
@@ -13,25 +15,29 @@ class StreamAdapter extends TTS {
     this.#tts.on("metrics_collected", (metrics) => {
       this.emit("metrics_collected", metrics);
     });
+    this.#tts.on("error", (error) => {
+      this.emit("error", error);
+    });
   }
-  synthesize(text) {
-    return this.#tts.synthesize(text);
+  synthesize(text, connOptions, abortSignal) {
+    return this.#tts.synthesize(text, connOptions, abortSignal);
   }
-  stream() {
-    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+  stream(options) {
+    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options == null ? void 0 : options.connOptions);
   }
 }
 class StreamAdapterWrapper extends SynthesizeStream {
   #tts;
   #sentenceStream;
   label;
-  constructor(tts, sentenceTokenizer) {
-    super(tts);
+  constructor(tts, sentenceTokenizer, connOptions) {
+    super(tts, connOptions);
     this.#tts = tts;
     this.#sentenceStream = sentenceTokenizer.stream();
     this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;
   }
   async run() {
+    let cumulativeDuration = 0;
     const forwardInput = async () => {
       for await (const input of this.input) {
         if (this.abortController.signal.aborted) break;
@@ -59,11 +65,22 @@ class StreamAdapterWrapper extends SynthesizeStream {
       this.queue.put(SynthesizeStream.END_OF_STREAM);
     };
     const synthesize = async (token, prevTask, controller) => {
-      const audioStream = this.#tts.synthesize(token);
+      const audioStream = this.#tts.synthesize(token, this.connOptions, this.abortSignal);
       await (prevTask == null ? void 0 : prevTask.result);
       if (controller.signal.aborted) return;
+      const timedString = createTimedString({
+        text: token,
+        startTime: cumulativeDuration
+      });
+      let isFirstFrame = true;
       for await (const audio of audioStream) {
         if (controller.signal.aborted) break;
+        if (isFirstFrame) {
+          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];
+          isFirstFrame = false;
+        }
+        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;
+        cumulativeDuration += frameDuration;
         this.queue.put(audio);
       }
     };

package/dist/tts/stream_adapter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport { Task } from '../utils.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n    this.label = this.#tts.label;\n    this.label = `tts.StreamAdapter<${this.#tts.label}>`;\n\n    this.#tts.on('metrics_collected', (metrics) => {\n      this.emit('metrics_collected', metrics);\n    });\n  }\n\n  synthesize(text: string): ChunkedStream {\n    return this.#tts.synthesize(text);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts);\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;\n  }\n\n  protected async run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (this.abortController.signal.aborted) break;\n\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesizeSentenceStream = async () => {\n      let task: Task<void> | undefined;\n      const tokenCompletionTasks: Task<void>[] = [];\n\n      for await (const ev of this.#sentenceStream) {\n        if (this.abortController.signal.aborted) break;\n\n        // this will enable non-blocking synthesis of the stream of tokens\n        task = Task.from(\n          (controller) => synthesize(ev.token, task, controller),\n          this.abortController,\n        );\n\n        tokenCompletionTasks.push(task);\n      }\n\n      await Promise.all(tokenCompletionTasks.map((t) => t.result));\n      this.queue.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    const synthesize = async (\n      token: string,\n      prevTask: Task<void> | undefined,\n      controller: AbortController,\n    ) => {\n      const audioStream = this.#tts.synthesize(token);\n\n      // wait for previous audio transcription to complete before starting\n      // to queuing audio frames of the current token\n      await prevTask?.result;\n      if (controller.signal.aborted) return;\n\n      for await (const audio of audioStream) {\n        if (controller.signal.aborted) break;\n        this.queue.put(audio);\n      }\n    };\n\n    await Promise.all([forwardInput(), synthesizeSentenceStream()]);\n  }\n}\n"],"mappings":"AAIA,SAAS,YAAY;AAErB,SAAS,kBAAkB,WAAW;AAE/B,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,KAAK,CAAC;AAC1D,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAC1B,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,qBAAqB,CAAC,YAAY;AAC7C,WAAK,KAAK,qBAAqB,OAAO;AAAA,IACxC,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,MAA6B;AACtC,WAAO,KAAK,KAAK,WAAW,IAAI;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,kBAAkB;AAAA,EACpE;AACF;AAEO,MAAM,6BAA6B,iBAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAChD,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAAA,EAC1D;AAAA,EAEA,MAAgB,MAAM;AACpB,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAEzC,YAAI,UAAU,iBAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,2BAA2B,YAAY;AAC3C,UAAI;AACJ,YAAM,uBAAqC,CAAC;AAE5C,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAGzC,eAAO,KAAK;AAAA,UACV,CAAC,eAAe,WAAW,GAAG,OAAO,MAAM,UAAU;AAAA,UACrD,KAAK;AAAA,QACP;AAEA,6BAAqB,KAAK,IAAI;AAAA,MAChC;AAEA,YAAM,QAAQ,IAAI,qBAAqB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAC3D,WAAK,MAAM,IAAI,iBAAiB,aAAa;AAAA,IAC/C;AAEA,UAAM,aAAa,OACjB,OACA,UACA,eACG;AACH,YAAM,cAAc,KAAK,KAAK,WAAW,KAAK;AAI9C,aAAM,qCAAU;AAChB,UAAI,WAAW,OAAO,QAAS;AAE/B,uBAAiB,SAAS,aAAa;AACrC,YAAI,WAAW,OAAO,QAAS;AAC/B,aAAK,MAAM,IAAI,KAAK;AAAA,MACtB;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,CAAC,aAAa,GAAG,yBAAyB,CAAC,CAAC;AAAA,EAChE;AACF;","names":[]}
+{"version":3,"sources":["../../src/tts/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { USERDATA_TIMED_TRANSCRIPT } from '../types.js';\nimport { Task } from '../utils.js';\nimport { createTimedString } from '../voice/io.js';\nimport type { ChunkedStream } from './tts.js';\nimport { SynthesizeStream, TTS } from './tts.js';\n\nexport class StreamAdapter extends TTS {\n  #tts: TTS;\n  #sentenceTokenizer: SentenceTokenizer;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {\n    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });\n    this.#tts = tts;\n    this.#sentenceTokenizer = sentenceTokenizer;\n    this.label = this.#tts.label;\n    this.label = `tts.StreamAdapter<${this.#tts.label}>`;\n\n    this.#tts.on('metrics_collected', (metrics) => {\n      this.emit('metrics_collected', metrics);\n    });\n    this.#tts.on('error', (error) => {\n      this.emit('error', error);\n    });\n  }\n\n  synthesize(\n    text: string,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ): ChunkedStream {\n    return this.#tts.synthesize(text, connOptions, abortSignal);\n  }\n\n  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer, options?.connOptions);\n  }\n}\n\nexport class StreamAdapterWrapper extends SynthesizeStream {\n  #tts: TTS;\n  #sentenceStream: SentenceStream;\n  label: string;\n\n  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer, connOptions?: APIConnectOptions) {\n    super(tts, connOptions);\n    this.#tts = tts;\n    this.#sentenceStream = sentenceTokenizer.stream();\n    this.label = `tts.StreamAdapterWrapper<${this.#tts.label}>`;\n  }\n\n  protected async run() {\n    let cumulativeDuration = 0;\n\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (this.abortController.signal.aborted) break;\n\n        if (input === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#sentenceStream.flush();\n        } else {\n          this.#sentenceStream.pushText(input);\n        }\n      }\n      this.#sentenceStream.endInput();\n      this.#sentenceStream.close();\n    };\n\n    const synthesizeSentenceStream = async () => {\n      let task: Task<void> | undefined;\n      const tokenCompletionTasks: Task<void>[] = [];\n\n      for await (const ev of this.#sentenceStream) {\n        if (this.abortController.signal.aborted) break;\n\n        // this will enable non-blocking synthesis of the stream of tokens\n        task = Task.from(\n          (controller) => synthesize(ev.token, task, controller),\n          this.abortController,\n        );\n\n        tokenCompletionTasks.push(task);\n      }\n\n      await Promise.all(tokenCompletionTasks.map((t) => t.result));\n      this.queue.put(SynthesizeStream.END_OF_STREAM);\n    };\n\n    const synthesize = async (\n      token: string,\n      prevTask: Task<void> | undefined,\n      controller: AbortController,\n    ) => {\n      const audioStream = this.#tts.synthesize(token, this.connOptions, this.abortSignal);\n\n      // wait for previous audio transcription to complete before starting\n      // to queuing audio frames of the current token\n      await prevTask?.result;\n      if (controller.signal.aborted) return;\n\n      // Create a TimedString with the sentence text and current cumulative duration\n      const timedString = createTimedString({\n        text: token,\n        startTime: cumulativeDuration,\n      });\n\n      let isFirstFrame = true;\n      for await (const audio of audioStream) {\n        if (controller.signal.aborted) break;\n\n        // Attach the TimedString to the first frame of this sentence\n        if (isFirstFrame) {\n          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];\n          isFirstFrame = false;\n        }\n\n        // Track cumulative duration\n        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;\n        cumulativeDuration += frameDuration;\n\n        this.queue.put(audio);\n      }\n    };\n\n    await Promise.all([forwardInput(), synthesizeSentenceStream()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,iCAAiC;AAC1C,SAAS,YAAY;AACrB,SAAS,yBAAyB;AAElC,SAAS,kBAAkB,WAAW;AAE/B,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC;AAC1D,UAAM,IAAI,YAAY,IAAI,aAAa,EAAE,WAAW,MAAM,mBAAmB,KAAK,CAAC;AACnF,SAAK,OAAO;AACZ,SAAK,qBAAqB;AAC1B,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,qBAAqB,CAAC,YAAY;AAC7C,WAAK,KAAK,qBAAqB,OAAO;AAAA,IACxC,CAAC;AACD,SAAK,KAAK,GAAG,SAAS,CAAC,UAAU;AAC/B,WAAK,KAAK,SAAS,KAAK;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,KAAK,KAAK,WAAW,MAAM,aAAa,WAAW;AAAA,EAC5D;AAAA,EAEA,OAAO,SAAqE;AAC1E,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,oBAAoB,mCAAS,WAAW;AAAA,EAC1F;AACF;AAEO,MAAM,6BAA6B,iBAAiB;AAAA,EACzD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,mBAAsC,aAAiC;AAC3F,UAAM,KAAK,WAAW;AACtB,SAAK,OAAO;AACZ,SAAK,kBAAkB,kBAAkB,OAAO;AAChD,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAAA,EAC1D;AAAA,EAEA,MAAgB,MAAM;AACpB,QAAI,qBAAqB;AAEzB,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAEzC,YAAI,UAAU,iBAAiB,gBAAgB;AAC7C,eAAK,gBAAgB,MAAM;AAAA,QAC7B,OAAO;AACL,eAAK,gBAAgB,SAAS,KAAK;AAAA,QACrC;AAAA,MACF;AACA,WAAK,gBAAgB,SAAS;AAC9B,WAAK,gBAAgB,MAAM;AAAA,IAC7B;AAEA,UAAM,2BAA2B,YAAY;AAC3C,UAAI;AACJ,YAAM,uBAAqC,CAAC;AAE5C,uBAAiB,MAAM,KAAK,iBAAiB;AAC3C,YAAI,KAAK,gBAAgB,OAAO,QAAS;AAGzC,eAAO,KAAK;AAAA,UACV,CAAC,eAAe,WAAW,GAAG,OAAO,MAAM,UAAU;AAAA,UACrD,KAAK;AAAA,QACP;AAEA,6BAAqB,KAAK,IAAI;AAAA,MAChC;AAEA,YAAM,QAAQ,IAAI,qBAAqB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAC3D,WAAK,MAAM,IAAI,iBAAiB,aAAa;AAAA,IAC/C;AAEA,UAAM,aAAa,OACjB,OACA,UACA,eACG;AACH,YAAM,cAAc,KAAK,KAAK,WAAW,OAAO,KAAK,aAAa,KAAK,WAAW;AAIlF,aAAM,qCAAU;AAChB,UAAI,WAAW,OAAO,QAAS;AAG/B,YAAM,cAAc,kBAAkB;AAAA,QACpC,MAAM;AAAA,QACN,WAAW;AAAA,MACb,CAAC;AAED,UAAI,eAAe;AACnB,uBAAiB,SAAS,aAAa;AACrC,YAAI,WAAW,OAAO,QAAS;AAG/B,YAAI,cAAc;AAChB,gBAAM,MAAM,SAAS,yBAAyB,IAAI,CAAC,WAAW;AAC9D,yBAAe;AAAA,QACjB;AAGA,cAAM,gBAAgB,MAAM,MAAM,oBAAoB,MAAM,MAAM;AAClE,8BAAsB;AAEtB,aAAK,MAAM,IAAI,KAAK;AAAA,MACtB;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,CAAC,aAAa,GAAG,yBAAyB,CAAC,CAAC;AAAA,EAChE;AACF;","names":[]}