@livekit/agents 0.4.3 → 0.4.4

package/dist/tts/stream_adapter.js

@@ -0,0 +1,50 @@
+import { SynthesizeStream, TTS } from './tts.js';
+export class StreamAdapter extends TTS {
+    #tts;
+    #sentenceTokenizer;
+    constructor(tts, sentenceTokenizer) {
+        super(tts.sampleRate, tts.numChannels, { streaming: true });
+        this.#tts = tts;
+        this.#sentenceTokenizer = sentenceTokenizer;
+    }
+    synthesize(text) {
+        return this.#tts.synthesize(text);
+    }
+    stream() {
+        return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+    }
+}
+export class StreamAdapterWrapper extends SynthesizeStream {
+    #tts;
+    #sentenceStream;
+    constructor(tts, sentenceTokenizer) {
+        super();
+        this.#tts = tts;
+        this.#sentenceStream = sentenceTokenizer.stream();
+        this.#run();
+    }
+    async #run() {
+        const forwardInput = async () => {
+            for await (const input of this.input) {
+                if (input === SynthesizeStream.FLUSH_SENTINEL) {
+                    this.#sentenceStream.flush();
+                }
+                else {
+                    this.#sentenceStream.pushText(input);
+                }
+            }
+            this.#sentenceStream.endInput();
+            this.#sentenceStream.close();
+        };
+        const synthesize = async () => {
+            for await (const ev of this.#sentenceStream) {
+                for await (const audio of this.#tts.synthesize(ev.token)) {
+                    this.queue.put(audio);
+                }
+            }
+            this.queue.put(SynthesizeStream.END_OF_STREAM);
+        };
+        Promise.all([forwardInput(), synthesize()]);
+    }
+}
+//# sourceMappingURL=stream_adapter.js.map

package/dist/tts/stream_adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream_adapter.js","sourceRoot":"","sources":["../../src/tts/stream_adapter.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEjD,MAAM,OAAO,aAAc,SAAQ,GAAG;IACpC,IAAI,CAAM;IACV,kBAAkB,CAAoB;IAEtC,YAAY,GAAQ,EAAE,iBAAoC;QACxD,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;IAC9C,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACtE,CAAC;CACF;AAED,MAAM,OAAO,oBAAqB,SAAQ,gBAAgB;IACxD,IAAI,CAAM;IACV,eAAe,CAAiB;IAEhC,YAAY,GAAQ,EAAE,iBAAoC;QACxD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,eAAe,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAElD,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;YAC9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrC,IAAI,KAAK,KAAK,gBAAgB,CAAC,cAAc,EAAE,CAAC;oBAC9C,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;gBAC/B,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC,CAAC;QAEF,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;YAC5B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC5C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF"}

package/dist/tts/tts.d.ts

@@ -37,6 +37,10 @@ export declare abstract class TTS {
     get sampleRate(): number;
     /** Returns the channel count of audio frames returned by this TTS */
     get numChannels(): number;
+    /**
+     * Receives text and returns synthesis in the form of a {@link ChunkedStream}
+     */
+    abstract synthesize(text: string): ChunkedStream;
     /**
      * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
      */
@@ -73,4 +77,28 @@ export declare abstract class SynthesizeStream implements AsyncIterableIterator<
     close(): void;
     [Symbol.asyncIterator](): SynthesizeStream;
 }
+/**
+ * An instance of a text-to-speech response, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child ChunkedStream class, which inherits this class's methods.
+ */
+export declare abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
+    protected queue: AsyncIterableQueue<SynthesizedAudio>;
+    protected closed: boolean;
+    /** Collect every frame into one in a single call */
+    collect(): Promise<AudioFrame>;
+    next(): Promise<IteratorResult<SynthesizedAudio>>;
+    /** Close both the input and output of the TTS stream */
+    close(): void;
+    [Symbol.asyncIterator](): ChunkedStream;
+}
 //# sourceMappingURL=tts.d.ts.map

package/dist/tts/tts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,CAAC;IAClB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,8BAAsB,GAAG;;gBAKX,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe;IAMlF,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,mEAAmE;IACnE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,qEAAqE;IACrE,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,gBAAgB;CACpC;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,gBACpB,YAAW,qBAAqB,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC;IAE1F,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,MAAM,CAAC,QAAQ,CAAC,aAAa,gBAA2B;IACxD,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,KAAK,+EAEX;IACJ,SAAS,CAAC,MAAM,UAAS;IAEzB,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM;IAUrB,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAIzF,wDAAwD;IACxD,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,gBAAgB;CAG3C"}
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AAE9D,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,CAAC;IAClB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,8BAAsB,GAAG;;gBAKX,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe;IAMlF,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,mEAAmE;IACnE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,qEAAqE;IACrE,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAEhD;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,gBAAgB;CACpC;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,gBACpB,YAAW,qBAAqB,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC;IAE1F,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,MAAM,CAAC,QAAQ,CAAC,aAAa,gBAA2B;IACxD,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,KAAK,+EAEX;IACJ,SAAS,CAAC,MAAM,UAAS;IAEzB,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM;IAUrB,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,GAAG,OAAO,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAIzF,wDAAwD;IACxD,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,gBAAgB;CAG3C;AAED;;;;;;;;;;;;;GAaG;AACH,8BAAsB,aAAc,YAAW,qBAAqB,CAAC,gBAAgB,CAAC;IACpF,SAAS,CAAC,KAAK,uCAA8C;IAC7D,SAAS,CAAC,MAAM,UAAS;IAEzB,oDAAoD;IAC9C,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC;IAQpC,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IAIjD,wDAAwD;IACxD,KAAK;IAKL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa;CAGxC"}

package/dist/tts/tts.js

@@ -1,4 +1,4 @@
-import { AsyncIterableQueue } from '../utils.js';
+import { AsyncIterableQueue, mergeFrames } from '../utils.js';
 /**
  * An instance of a text-to-speech adapter.
  *
@@ -91,4 +91,41 @@ export class SynthesizeStream {
         return this;
     }
 }
+/**
+ * An instance of a text-to-speech response, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child ChunkedStream class, which inherits this class's methods.
+ */
+export class ChunkedStream {
+    queue = new AsyncIterableQueue();
+    closed = false;
+    /** Collect every frame into one in a single call */
+    async collect() {
+        const frames = [];
+        for await (const event of this) {
+            frames.push(event.frame);
+        }
+        return mergeFrames(frames);
+    }
+    next() {
+        return this.queue.next();
+    }
+    /** Close both the input and output of the TTS stream */
+    close() {
+        this.queue.close();
+        this.closed = true;
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+}
 //# sourceMappingURL=tts.js.map

package/dist/tts/tts.js.map

@@ -1 +1 @@
-{"version":3,"file":"tts.js","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAyBjD;;;;;;GAMG;AACH,MAAM,OAAgB,GAAG;IACvB,aAAa,CAAkB;IAC/B,WAAW,CAAS;IACpB,YAAY,CAAS;IAErB,YAAY,UAAkB,EAAE,WAAmB,EAAE,YAA6B;QAChF,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,sCAAsC;IACtC,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,mEAAmE;IACnE,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;CAMF;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAgB,gBAAgB;IAG1B,MAAM,CAAU,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACpE,MAAM,CAAU,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;IAC9C,KAAK,GAAG,IAAI,kBAAkB,EAAmD,CAAC;IAClF,KAAK,GAAG,IAAI,kBAAkB,EAErC,CAAC;IACM,MAAM,GAAG,KAAK,CAAC;IAEzB,uCAAuC;IACvC,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,4DAA4D;IAC5D,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAClD,CAAC;IAED,2DAA2D;IAC3D,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,wDAAwD;IACxD,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC"}
+{"version":3,"file":"tts.js","sourceRoot":"","sources":["../../src/tts/tts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAyB9D;;;;;;GAMG;AACH,MAAM,OAAgB,GAAG;IACvB,aAAa,CAAkB;IAC/B,WAAW,CAAS;IACpB,YAAY,CAAS;IAErB,YAAY,UAAkB,EAAE,WAAmB,EAAE,YAA6B;QAChF,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,sCAAsC;IACtC,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,mEAAmE;IACnE,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;CAWF;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAgB,gBAAgB;IAG1B,MAAM,CAAU,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACpE,MAAM,CAAU,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;IAC9C,KAAK,GAAG,IAAI,kBAAkB,EAAmD,CAAC;IAClF,KAAK,GAAG,IAAI,kBAAkB,EAErC,CAAC;IACM,MAAM,GAAG,KAAK,CAAC;IAEzB,uCAAuC;IACvC,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,4DAA4D;IAC5D,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAClD,CAAC;IAED,2DAA2D;IAC3D,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,wDAAwD;IACxD,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;;AAGH;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAgB,aAAa;IACvB,KAAK,GAAG,IAAI,kBAAkB,EAAoB,CAAC;IACnD,MAAM,GAAG,KAAK,CAAC;IAEzB,oDAAoD;IACpD,KAAK,CAAC,OAAO;QACX,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,wDAAwD;IACxD,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/pipeline/agent_output.ts

@@ -134,7 +134,9 @@ const stringSynthesisTask = (text: string, handle: SynthesisHandle): Cancellable
     ttsStream.flush();
     ttsStream.endInput();
     for await (const audio of ttsStream) {
-      if (cancelled || audio === SynthesizeStream.END_OF_STREAM) break;
+      if (cancelled || audio === SynthesizeStream.END_OF_STREAM) {
+        break;
+      }
       handle.queue.put(audio.frame);
     }
     handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
@@ -156,18 +158,12 @@ const streamSynthesisTask = (
 
     const ttsStream = handle.tts.stream();
     const readGeneratedAudio = async () => {
-      let started = false;
       for await (const audio of ttsStream) {
         if (cancelled) break;
         if (audio === SynthesizeStream.END_OF_STREAM) {
-          if (started) {
-            break;
-          } else {
-            continue;
-          }
+          break;
         }
         handle.queue.put(audio.frame);
-        started = true;
       }
       handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
     };

package/src/pipeline/agent_playout.ts

@@ -148,7 +148,9 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
         });
 
         for await (const frame of handle.playoutSource) {
-          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) break;
+          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {
+            break;
+          }
           if (firstFrame) {
             this.#logger
               .child({ speechId: handle.speechId })

package/src/pipeline/index.ts

@@ -6,10 +6,10 @@ export {
   type AgentState,
   type BeforeTTSCallback,
   type BeforeLLMCallback,
-  type VPAEvent,
   type VPACallbacks,
   type AgentCallContext,
   type AgentTranscriptionOptions,
   type VPAOptions,
+  VPAEvent,
   VoicePipelineAgent,
 } from './pipeline_agent.js';

package/src/pipeline/pipeline_agent.ts

@@ -20,7 +20,7 @@ import type {
 import { LLMStream } from '../llm/index.js';
 import { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';
 import { log } from '../log.js';
-import type { STT } from '../stt/index.js';
+import { type STT, StreamAdapter as STTStreamAdapter } from '../stt/index.js';
 import {
   SentenceTokenizer as BasicSentenceTokenizer,
   WordTokenizer as BasicWordTokenizer,
@@ -28,6 +28,7 @@ import {
 } from '../tokenize/basic/index.js';
 import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
 import type { TTS } from '../tts/index.js';
+import { StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 import type { VAD, VADEvent } from '../vad.js';
 import type { SpeechSource, SynthesisHandle } from './agent_output.js';
@@ -252,6 +253,14 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
 
     this.#opts = { ...defaultVPAOptions, ...opts };
 
+    if (!stt.capabilities.streaming) {
+      stt = new STTStreamAdapter(stt, vad);
+    }
+
+    if (!tts.capabilities.streaming) {
+      tts = new TTSStreamAdapter(tts, new BasicSentenceTokenizer());
+    }
+
     this.#vad = vad;
     this.#stt = stt;
     this.#llm = llm;
@@ -593,6 +602,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.emit(VPAEvent.USER_SPEECH_COMMITTED, userMsg);
 
       this.#transcribedText = this.#transcribedText.slice(userQuestion.length);
+      handle.markUserCommitted();
     };
 
     // wait for the playHandle to finish and check every 1s if user question should be committed
@@ -618,7 +628,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     // if the answer is using tools, execute the functions and automatically generate
     // a response to the user question from the returned values
     if (isUsingTools && !interrupted) {
-      if (!userQuestion || handle.userCommitted) {
+      if (!userQuestion || !handle.userCommitted) {
         throw new Error('user speech should have been committed before using tools');
       }
       const llmStream = handle.source;
@@ -823,7 +833,7 @@ async function* llmStreamToStringIterable(
     if (firstFrame) {
       firstFrame = false;
       log()
-        .child({ speechId, elapsed: Math.round(Date.now() * 1000 - startTime) / 1000 })
+        .child({ speechId, elapsed: Math.round(Date.now() - startTime) })
         .debug('received first LLM token');
     }
     yield content;

package/src/stt/index.ts

@@ -10,3 +10,4 @@ export {
   STT,
   SpeechStream,
 } from './stt.js';
+export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';

package/src/stt/stream_adapter.ts

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { VAD, VADStream } from '../vad.js';
+import { VADEventType } from '../vad.js';
+import type { SpeechEvent } from './stt.js';
+import { STT, SpeechEventType, SpeechStream } from './stt.js';
+
+export class StreamAdapter extends STT {
+  #stt: STT;
+  #vad: VAD;
+
+  constructor(stt: STT, vad: VAD) {
+    super({ streaming: true, interimResults: false });
+    this.#stt = stt;
+    this.#vad = vad;
+  }
+
+  recognize(frame: AudioFrame): Promise<SpeechEvent> {
+    return this.#stt.recognize(frame);
+  }
+
+  stream(): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#stt, this.#vad);
+  }
+}
+
+export class StreamAdapterWrapper extends SpeechStream {
+  #stt: STT;
+  #vadStream: VADStream;
+
+  constructor(stt: STT, vad: VAD) {
+    super();
+    this.#stt = stt;
+    this.#vadStream = vad.stream();
+
+    this.#run();
+  }
+
+  async #run() {
+    const forwardInput = async () => {
+      for await (const input of this.input) {
+        if (input === SpeechStream.FLUSH_SENTINEL) {
+          this.#vadStream.flush();
+        } else {
+          this.#vadStream.pushFrame(input);
+        }
+      }
+      this.#vadStream.endInput();
+    };
+
+    const recognize = async () => {
+      for await (const ev of this.#vadStream) {
+        switch (ev.type) {
+          case VADEventType.START_OF_SPEECH:
+            this.queue.put({ type: SpeechEventType.START_OF_SPEECH, alternatives: [] });
+            break;
+          case VADEventType.END_OF_SPEECH:
+            this.queue.put({ type: SpeechEventType.END_OF_SPEECH, alternatives: [] });
+
+            const event = await this.#stt.recognize(ev.frames);
+            if (!event.alternatives.length || !event.alternatives[0].text) {
+              continue;
+            }
+
+            this.queue.put(event);
+            break;
+        }
+      }
+    };
+
+    Promise.all([forwardInput(), recognize()]);
+  }
+}

package/src/stt/stt.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
+import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue } from '../utils.js';
 
 /** Indicates start/middle/end of speech */
@@ -73,6 +74,9 @@ export abstract class STT {
     return this.#capabilities;
   }
 
+  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
+  abstract recognize(frame: AudioBuffer): Promise<SpeechEvent>;
+
   /**
    * Returns a {@link SpeechStream} that can be used to push audio frames and receive
    * transcriptions

package/src/tokenize/token_stream.ts

@@ -39,7 +39,7 @@ export class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
 
     while (true) {
       const tokens = this.#func(this.#inBuf);
-      if (tokens.length === 0) break;
+      if (tokens.length <= 1) break;
 
       if (this.#outBuf) this.#outBuf += ' ';
 
@@ -130,6 +130,10 @@ export class BufferedSentenceStream extends SentenceStream {
     this.#stream.pushText(text);
   }
 
+  flush() {
+    this.#stream.flush();
+  }
+
   close() {
     super.close();
     this.#stream.close();

package/src/tts/index.ts

@@ -1,4 +1,11 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-export { type SynthesizedAudio, type TTSCapabilities, TTS, SynthesizeStream } from './tts.js';
+export {
+  type SynthesizedAudio,
+  type TTSCapabilities,
+  TTS,
+  SynthesizeStream,
+  ChunkedStream,
+} from './tts.js';
+export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';

package/src/tts/stream_adapter.ts

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';
+import type { ChunkedStream } from './tts.js';
+import { SynthesizeStream, TTS } from './tts.js';
+
+export class StreamAdapter extends TTS {
+  #tts: TTS;
+  #sentenceTokenizer: SentenceTokenizer;
+
+  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
+    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    this.#tts = tts;
+    this.#sentenceTokenizer = sentenceTokenizer;
+  }
+
+  synthesize(text: string): ChunkedStream {
+    return this.#tts.synthesize(text);
+  }
+
+  stream(): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#tts, this.#sentenceTokenizer);
+  }
+}
+
+export class StreamAdapterWrapper extends SynthesizeStream {
+  #tts: TTS;
+  #sentenceStream: SentenceStream;
+
+  constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
+    super();
+    this.#tts = tts;
+    this.#sentenceStream = sentenceTokenizer.stream();
+
+    this.#run();
+  }
+
+  async #run() {
+    const forwardInput = async () => {
+      for await (const input of this.input) {
+        if (input === SynthesizeStream.FLUSH_SENTINEL) {
+          this.#sentenceStream.flush();
+        } else {
+          this.#sentenceStream.pushText(input);
+        }
+      }
+      this.#sentenceStream.endInput();
+      this.#sentenceStream.close();
+    };
+
+    const synthesize = async () => {
+      for await (const ev of this.#sentenceStream) {
+        for await (const audio of this.#tts.synthesize(ev.token)) {
+          this.queue.put(audio);
+        }
+      }
+      this.queue.put(SynthesizeStream.END_OF_STREAM);
+    };
+
+    Promise.all([forwardInput(), synthesize()]);
+  }
+}

package/src/tts/tts.ts

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
-import { AsyncIterableQueue } from '../utils.js';
+import { AsyncIterableQueue, mergeFrames } from '../utils.js';
 
 /** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
 export interface SynthesizedAudio {
@@ -60,6 +60,11 @@ export abstract class TTS {
     return this.#numChannels;
   }
 
+  /**
+   * Receives text and returns synthesis in the form of a {@link ChunkedStream}
+   */
+  abstract synthesize(text: string): ChunkedStream;
+
   /**
    * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
    */
@@ -139,3 +144,45 @@ export abstract class SynthesizeStream
     return this;
   }
 }
+
+/**
+ * An instance of a text-to-speech response, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child ChunkedStream class, which inherits this class's methods.
+ */
+export abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
+  protected queue = new AsyncIterableQueue<SynthesizedAudio>();
+  protected closed = false;
+
+  /** Collect every frame into one in a single call */
+  async collect(): Promise<AudioFrame> {
+    const frames = [];
+    for await (const event of this) {
+      frames.push(event.frame);
+    }
+    return mergeFrames(frames);
+  }
+
+  next(): Promise<IteratorResult<SynthesizedAudio>> {
+    return this.queue.next();
+  }
+
+  /** Close both the input and output of the TTS stream */
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+
+  [Symbol.asyncIterator](): ChunkedStream {
+    return this;
+  }
+}