@livekit/agents 0.4.3 → 0.4.5

package/src/tts/tts.ts

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
-import { AsyncIterableQueue } from '../utils.js';
+import { AsyncIterableQueue, mergeFrames } from '../utils.js';
 
 /** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
 export interface SynthesizedAudio {
@@ -60,6 +60,11 @@ export abstract class TTS {
     return this.#numChannels;
   }
 
+  /**
+   * Receives text and returns synthesis in the form of a {@link ChunkedStream}
+   */
+  abstract synthesize(text: string): ChunkedStream;
+
   /**
    * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
    */
@@ -139,3 +144,45 @@ export abstract class SynthesizeStream
     return this;
   }
 }
+
+/**
+ * An instance of a text-to-speech response, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child ChunkedStream class, which inherits this class's methods.
+ */
+export abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
+  protected queue = new AsyncIterableQueue<SynthesizedAudio>();
+  protected closed = false;
+
+  /** Collect every frame into one in a single call */
+  async collect(): Promise<AudioFrame> {
+    const frames = [];
+    for await (const event of this) {
+      frames.push(event.frame);
+    }
+    return mergeFrames(frames);
+  }
+
+  next(): Promise<IteratorResult<SynthesizedAudio>> {
+    return this.queue.next();
+  }
+
+  /** Close both the input and output of the TTS stream */
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+
+  [Symbol.asyncIterator](): ChunkedStream {
+    return this;
+  }
+}

package/src/utils.ts

@@ -28,8 +28,8 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
       throw new TypeError('buffer is empty');
     }
 
-    const sampleRate = buffer[0].sampleRate;
-    const channels = buffer[0].channels;
+    const sampleRate = buffer[0]!.sampleRate;
+    const channels = buffer[0]!.channels;
     let samplesPerChannel = 0;
     let data = new Int16Array();
 

package/src/worker.ts

@@ -89,8 +89,8 @@ const defaultCpuLoad = async (): Promise<number> => {
       let total = 0;
 
       for (let i = 0; i < cpus1.length; i++) {
-        const cpu1 = cpus1[i].times;
-        const cpu2 = cpus2[i].times;
+        const cpu1 = cpus1[i]!.times;
+        const cpu2 = cpus2[i]!.times;
 
         idle += cpu2.idle - cpu1.idle;
 
@@ -495,7 +495,7 @@ export class Worker {
           if (job.id in this.#pending) {
             const task = this.#pending[job.id];
             delete this.#pending[job.id];
-            task.resolve(msg.message.value);
+            task?.resolve(msg.message.value);
           } else {
             this.#logger.child({ job }).warn('received assignment for unknown job ' + job.id);
           }
@@ -610,17 +610,21 @@ export class Worker {
         this.#logger.child({ req }).warn(`assignment for job ${req.id} timed out`);
         return;
       }, ASSIGNMENT_TIMEOUT);
-      const asgn = await this.#pending[req.id].promise.then(async (asgn) => {
+      const asgn = await this.#pending[req.id]?.promise.then(async (asgn) => {
         clearTimeout(timer);
         return asgn;
       });
 
-      await this.#procPool.launchJob({
-        acceptArguments: args,
-        job: msg.job!,
-        url: asgn.url || this.#opts.wsURL,
-        token: asgn.token,
-      });
+      if (asgn) {
+        await this.#procPool.launchJob({
+          acceptArguments: args,
+          job: msg.job!,
+          url: asgn.url || this.#opts.wsURL,
+          token: asgn.token,
+        });
+      } else {
+        this.#logger.child({ requestId: req.id }).warn('pending assignment not found');
+      }
     };
 
     const req = new JobRequest(msg.job!, onReject, onAccept);