@livekit/agents 0.7.9 → 1.0.0-next.0

package/src/utils.ts

@@ -1,22 +1,29 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type {
-  LocalParticipant,
-  RemoteParticipant,
-  Room,
-  TrackPublication,
-} from '@livekit/rtc-node';
-import { AudioFrame, TrackSource } from '@livekit/rtc-node';
+import { AudioFrame, AudioResampler } from '@livekit/rtc-node';
+import { delay } from '@std/async';
 import { EventEmitter, once } from 'node:events';
+import type { ReadableStream } from 'node:stream/web';
+import { TransformStream, type TransformStreamDefaultController } from 'node:stream/web';
+import { v4 as uuidv4 } from 'uuid';
+import { log } from './log.js';
 
 /** Union of a single and a list of {@link AudioFrame}s */
 export type AudioBuffer = AudioFrame[] | AudioFrame;
 
+export const noop = () => {};
+
+export const isPending = async (promise: Promise<unknown>): Promise<boolean> => {
+  const sentinel = Symbol('sentinel');
+  const result = await Promise.race([promise, Promise.resolve(sentinel)]);
+  return result === sentinel;
+};
+
 /**
  * Merge one or more {@link AudioFrame}s into a single one.
  *
- * @param buffer Either an {@link AudioFrame} or a list thereof
+ * @param buffer - Either an {@link AudioFrame} or a list thereof
  * @throws
  * {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
  * | TypeError} if sample rate or channel count are mismatched
@@ -52,33 +59,6 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
   return buffer;
 };
 
-export const findMicroTrackId = (room: Room, identity: string): string => {
-  let p: RemoteParticipant | LocalParticipant | undefined = room.remoteParticipants.get(identity);
-
-  if (identity === room.localParticipant?.identity) {
-    p = room.localParticipant;
-  }
-
-  if (!p) {
-    throw new Error(`participant ${identity} not found`);
-  }
-
-  // find first micro track
-  let trackId: string | undefined;
-  p.trackPublications.forEach((track: TrackPublication) => {
-    if (track.source === TrackSource.SOURCE_MICROPHONE) {
-      trackId = track.sid;
-      return;
-    }
-  });
-
-  if (!trackId) {
-    throw new Error(`participant ${identity} does not have a microphone track`);
-  }
-
-  return trackId;
-};
-
 /** @internal */
 export class Queue<T> {
   /** @internal */
@@ -117,14 +97,14 @@ export class Queue<T> {
 }
 
 /** @internal */
-export class Future {
-  #await: Promise<void>;
-  #resolvePromise!: () => void;
+export class Future<T = void> {
+  #await: Promise<T>;
+  #resolvePromise!: (value: T) => void;
   #rejectPromise!: (error: Error) => void;
   #done: boolean = false;
 
   constructor() {
-    this.#await = new Promise<void>((resolve, reject) => {
+    this.#await = new Promise<T>((resolve, reject) => {
       this.#resolvePromise = resolve;
       this.#rejectPromise = reject;
     });
@@ -138,9 +118,9 @@ export class Future {
     return this.#done;
   }
 
-  resolve() {
+  resolve(value: T) {
     this.#done = true;
-    this.#resolvePromise();
+    this.#resolvePromise(value);
   }
 
   reject(error: Error) {
@@ -149,6 +129,48 @@ export class Future {
   }
 }
 
+/** @internal */
+export class Event {
+  #isSet = false;
+  #waiters: Array<() => void> = [];
+
+  async wait() {
+    if (this.#isSet) return true;
+
+    let resolve: () => void = noop;
+    const waiter = new Promise<void>((r) => {
+      resolve = r;
+      this.#waiters.push(resolve);
+    });
+
+    try {
+      await waiter;
+      return true;
+    } finally {
+      const index = this.#waiters.indexOf(resolve);
+      if (index !== -1) {
+        this.#waiters.splice(index, 1);
+      }
+    }
+  }
+
+  get isSet(): boolean {
+    return this.#isSet;
+  }
+
+  set(): void {
+    if (this.#isSet) return;
+
+    this.#isSet = true;
+    this.#waiters.forEach((resolve) => resolve());
+    this.#waiters = [];
+  }
+
+  clear(): void {
+    this.#isSet = false;
+  }
+}
+
 /** @internal */
 export class CancellablePromise<T> {
   #promise: Promise<T>;
@@ -159,7 +181,7 @@ export class CancellablePromise<T> {
   constructor(
     executor: (
       resolve: (value: T | PromiseLike<T>) => void,
-      reject: (reason?: any) => void,
+      reject: (reason?: unknown) => void,
       onCancel: (cancelFn: () => void) => void,
     ) => void,
   ) {
@@ -194,13 +216,13 @@ export class CancellablePromise<T> {
 
   then<TResult1 = T, TResult2 = never>(
     onfulfilled?: ((value: T) => TResult1 | Promise<TResult1>) | null,
-    onrejected?: ((reason: any) => TResult2 | Promise<TResult2>) | null,
+    onrejected?: ((reason: unknown) => TResult2 | Promise<TResult2>) | null,
   ): Promise<TResult1 | TResult2> {
     return this.#promise.then(onfulfilled, onrejected);
   }
 
   catch<TResult = never>(
-    onrejected?: ((reason: any) => TResult | Promise<TResult>) | null,
+    onrejected?: ((reason: unknown) => TResult | Promise<TResult>) | null,
   ): Promise<T | TResult> {
     return this.#promise.catch(onrejected);
   }
@@ -339,3 +361,312 @@ export class AudioEnergyFilter {
     return false;
   }
 }
+
+export const TASK_TIMEOUT_ERROR = new Error('Task cancellation timed out');
+
+export enum TaskResult {
+  Timeout = 'timeout',
+  Completed = 'completed',
+  Aborted = 'aborted',
+}
+
+/** @internal */
+/**
+ * A task that can be cancelled.
+ *
+ * We recommend using the `Task.from` method to create a task. When creating subtasks, pass the same controller to all subtasks.
+ *
+ * @example
+ * ```ts
+ * const parent = Task.from((controller) => {
+ *   const child1 = Task.from(() => { ... }, controller);
+ *   const child2 = Task.from(() => { ... }, controller);
+ * });
+ * parent.cancel();
+ * ```
+ *
+ * This will cancel all subtasks when the parent is cancelled.
+ *
+ * @param T - The type of the task result
+ */
+export class Task<T> {
+  private resultFuture: Future<T>;
+
+  #logger = log();
+
+  constructor(
+    private readonly fn: (controller: AbortController) => Promise<T>,
+    private readonly controller: AbortController,
+    readonly name?: string,
+  ) {
+    this.resultFuture = new Future();
+    this.runTask();
+  }
+
+  /**
+   * Creates a new task from a function.
+   *
+   * @param fn - The function to run
+   * @param controller - The abort controller to use
+   * @returns A new task
+   */
+  static from<T>(
+    fn: (controller: AbortController) => Promise<T>,
+    controller?: AbortController,
+    name?: string,
+  ) {
+    const abortController = controller ?? new AbortController();
+    return new Task(fn, abortController, name);
+  }
+
+  private async runTask() {
+    const run = async () => {
+      if (this.name) {
+        this.#logger.debug(`Task.runTask: task ${this.name} started`);
+      }
+      return await this.fn(this.controller);
+    };
+
+    return run()
+      .then((value) => {
+        this.resultFuture.resolve(value);
+        return value;
+      })
+      .catch((error) => {
+        this.resultFuture.reject(error);
+      })
+      .finally(() => {
+        if (this.name) {
+          this.#logger.debug(`Task.runTask: task ${this.name} done`);
+        }
+      });
+  }
+
+  /**
+   * Cancels the task.
+   */
+  cancel() {
+    this.controller.abort();
+  }
+
+  /**
+   * Cancels the task and waits for it to complete.
+   *
+   * @param timeout - The timeout in milliseconds
+   * @returns The result status of the task (timeout, completed, aborted)
+   */
+  async cancelAndWait(timeout?: number) {
+    this.cancel();
+
+    try {
+      // Race between task completion and timeout
+      const promises = [
+        this.result
+          .then(() => TaskResult.Completed)
+          .catch((error) => {
+            if (error.name === 'AbortError') {
+              return TaskResult.Aborted;
+            }
+            throw error;
+          }),
+      ];
+
+      if (timeout) {
+        promises.push(delay(timeout).then(() => TaskResult.Timeout));
+      }
+
+      const result = await Promise.race(promises);
+
+      // Check what happened
+      if (result === TaskResult.Timeout) {
+        throw TASK_TIMEOUT_ERROR;
+      }
+
+      return result;
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  /**
+   * The result of the task.
+   */
+  get result(): Promise<T> {
+    return this.resultFuture.await;
+  }
+
+  /**
+   * Whether the task has completed.
+   */
+  get done(): boolean {
+    return this.resultFuture.done;
+  }
+}
+
+export async function waitFor(tasks: Task<void>[]): Promise<void> {
+  await Promise.allSettled(tasks.map((task) => task.result));
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function cancelAndWait(tasks: Task<any>[], timeout?: number): Promise<void> {
+  await Promise.allSettled(tasks.map((task) => task.cancelAndWait(timeout)));
+}
+
+export function withResolvers<T = unknown>() {
+  let resolve!: (value: T | PromiseLike<T>) => void;
+  let reject!: (reason?: unknown) => void;
+
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+
+  return { promise, resolve, reject };
+}
+
+/**
+ * Generates a short UUID with a prefix. Mirrors the python agents implementation.
+ *
+ * @param prefix - The prefix to add to the UUID.
+ * @returns A short UUID with the prefix.
+ */
+export function shortuuid(prefix: string = ''): string {
+  return `${prefix}${uuidv4().slice(0, 12)}`;
+}
+
+const READONLY_SYMBOL = Symbol('Readonly');
+
+const MUTATION_METHODS = [
+  'push',
+  'pop',
+  'shift',
+  'unshift',
+  'splice',
+  'sort',
+  'reverse',
+  'fill',
+  'copyWithin',
+] as const;
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
+/**
+ * Creates a read-only proxy for an array.
+ * @param array - The array to make read-only.
+ * @param additionalErrorMessage - An additional error message to include in the error thrown when a mutation method is called.
+ * @returns A read-only proxy for the array.
+ */
+export function createImmutableArray<T>(array: T[], additionalErrorMessage: string = ''): T[] {
+  return new Proxy(array, {
+    get(target, key) {
+      if (key === READONLY_SYMBOL) {
+        return true;
+      }
+
+      // Intercept mutation methods
+      if (
+        typeof key === 'string' &&
+        MUTATION_METHODS.includes(key as (typeof MUTATION_METHODS)[number])
+      ) {
+        return function () {
+          throw new TypeError(
+            `Cannot call ${key}() on a read-only array. ${additionalErrorMessage}`.trim(),
+          );
+        };
+      }
+
+      return Reflect.get(target, key);
+    },
+    set(_, prop) {
+      throw new TypeError(
+        `Cannot assign to read-only array index "${String(prop)}". ${additionalErrorMessage}`.trim(),
+      );
+    },
+    deleteProperty(_, prop) {
+      throw new TypeError(
+        `Cannot delete read-only array index "${String(prop)}". ${additionalErrorMessage}`.trim(),
+      );
+    },
+    defineProperty(_, prop) {
+      throw new TypeError(
+        `Cannot define property "${String(prop)}" on a read-only array. ${additionalErrorMessage}`.trim(),
+      );
+    },
+    setPrototypeOf() {
+      throw new TypeError(
+        `Cannot change prototype of a read-only array. ${additionalErrorMessage}`.trim(),
+      );
+    },
+  });
+}
+
+export function isImmutableArray(array: unknown): boolean {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  return typeof array === 'object' && !!(array as any)[READONLY_SYMBOL];
+}
+
+/**
+ * Resamples an audio stream to a target sample rate.
+ *
+ * WARINING: The input stream will be locked until the resampled stream is closed.
+ *
+ * @param stream - The input stream to resample.
+ * @param outputRate - The target sample rate.
+ * @returns A new stream with the resampled audio.
+ */
+export function resampleStream({
+  stream,
+  outputRate,
+}: {
+  stream: ReadableStream<AudioFrame>;
+  outputRate: number;
+}): ReadableStream<AudioFrame> {
+  let resampler: AudioResampler | null = null;
+  const transformStream = new TransformStream<AudioFrame, AudioFrame>({
+    transform(chunk: AudioFrame, controller: TransformStreamDefaultController<AudioFrame>) {
+      if (!resampler) {
+        resampler = new AudioResampler(chunk.sampleRate, outputRate);
+      }
+      for (const frame of resampler.push(chunk)) {
+        controller.enqueue(frame);
+      }
+      for (const frame of resampler.flush()) {
+        controller.enqueue(frame);
+      }
+    },
+  });
+  return stream.pipeThrough(transformStream);
+}
+
+export class InvalidErrorType extends Error {
+  readonly error: unknown;
+
+  constructor(error: unknown) {
+    super(`Expected error, got ${error} (${typeof error})`);
+    this.error = error;
+    Error.captureStackTrace(this, InvalidErrorType);
+  }
+}
+
+/**
+ * In JS an error can be any arbitrary value.
+ * This function converts an unknown error to an Error and stores the original value in the error object.
+ *
+ * @param error - The error to convert.
+ * @returns An Error.
+ */
+export function toError(error: unknown): Error {
+  if (error instanceof Error) {
+    return error;
+  }
+  throw new InvalidErrorType(error);
+}
+
+/**
+ * This is a hack to immitate asyncio.create_task so that
+ * func will be run after the current event loop iteration.
+ *
+ * @param func - The function to run.
+ */
+export function startSoon(func: () => void) {
+  setTimeout(func, 0);
+}

package/src/vad.ts

@@ -4,8 +4,15 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { EventEmitter } from 'node:events';
+import type {
+  ReadableStream,
+  ReadableStreamDefaultReader,
+  WritableStreamDefaultWriter,
+} from 'node:stream/web';
+import { log } from './log.js';
 import type { VADMetrics } from './metrics/base.js';
-import { AsyncIterableQueue } from './utils.js';
+import { DeferredReadableStream } from './stream/deferred_stream.js';
+import { IdentityTransform } from './stream/identity_transform.js';
 
 export enum VADEventType {
   START_OF_SPEECH,
@@ -53,7 +60,7 @@ export interface VADCapabilities {
 }
 
 export type VADCallbacks = {
-  [VADEventType.METRICS_COLLECTED]: (metrics: VADMetrics) => void;
+  ['metrics_collected']: (metrics: VADMetrics) => void;
 };
 
 export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {
@@ -77,29 +84,75 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
   protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();
-  protected queue = new AsyncIterableQueue<VADEvent>();
-  protected output = new AsyncIterableQueue<VADEvent>();
+  protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();
+  protected output = new IdentityTransform<VADEvent>();
+  protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+  protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+  protected outputWriter: WritableStreamDefaultWriter<VADEvent>;
+  protected outputReader: ReadableStreamDefaultReader<VADEvent>;
   protected closed = false;
+  protected inputClosed = false;
+
   #vad: VAD;
   #lastActivityTime = BigInt(0);
+  private logger = log();
+  private deferredInputStream: DeferredReadableStream<AudioFrame>;
 
+  private metricsStream: ReadableStream<VADEvent>;
   constructor(vad: VAD) {
     this.#vad = vad;
+    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
+
+    this.inputWriter = this.input.writable.getWriter();
+    this.inputReader = this.input.readable.getReader();
+    this.outputWriter = this.output.writable.getWriter();
+
+    const [outputStream, metricsStream] = this.output.readable.tee();
+    this.metricsStream = metricsStream;
+    this.outputReader = outputStream.getReader();
+
+    this.pumpDeferredStream();
     this.monitorMetrics();
   }
 
+  /**
+   * Reads from the deferred input stream and forwards chunks to the input writer.
+   *
+   * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)
+   * because the inputWriter locks the this.input.writable stream. All writes must go through
+   * the inputWriter.
+   */
+  private async pumpDeferredStream() {
+    const reader = this.deferredInputStream.stream.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        await this.inputWriter.write(value);
+      }
+    } catch (e) {
+      this.logger.error(`Error pumping deferred stream: ${e}`);
+      throw e;
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
   protected async monitorMetrics() {
     let inferenceDurationTotal = 0;
     let inferenceCount = 0;
-
-    for await (const event of this.queue) {
-      this.output.put(event);
-      switch (event.type) {
+    const metricsReader = this.metricsStream.getReader();
+    while (true) {
+      const { done, value } = await metricsReader.read();
+      if (done) {
+        break;
+      }
+      switch (value.type) {
         case VADEventType.START_OF_SPEECH:
           inferenceCount++;
           if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
-            this.#vad.emit(VADEventType.METRICS_COLLECTED, {
+            this.#vad.emit('metrics_collected', {
+              type: 'vad_metrics',
               timestamp: Date.now(),
               idleTime: Math.trunc(
                 Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),
@@ -119,47 +172,62 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
           break;
       }
     }
-    this.output.close();
   }
 
+  updateInputStream(audioStream: ReadableStream<AudioFrame>) {
+    this.deferredInputStream.setSource(audioStream);
+  }
+
+  detachInputStream() {
+    this.deferredInputStream.detachSource();
+  }
+
+  /** @deprecated Use `updateInputStream` instead */
   pushFrame(frame: AudioFrame) {
-    if (this.input.closed) {
+    // TODO(AJS-395): remove this method
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.put(frame);
+    this.inputWriter.write(frame);
   }
 
   flush() {
-    if (this.input.closed) {
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.put(VADStream.FLUSH_SENTINEL);
+    this.inputWriter.write(VADStream.FLUSH_SENTINEL);
   }
 
   endInput() {
-    if (this.input.closed) {
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.close();
+    this.inputClosed = true;
+    this.input.writable.close();
   }
 
-  next(): Promise<IteratorResult<VADEvent>> {
-    return this.output.next();
+  async next(): Promise<IteratorResult<VADEvent>> {
+    return this.outputReader.read().then(({ done, value }) => {
+      if (done) {
+        return { done: true, value: undefined };
+      }
+      return { done: false, value };
+    });
   }
 
   close() {
-    this.input.close();
-    this.queue.close();
-    this.output.close();
+    this.outputWriter.releaseLock();
+    this.outputReader.cancel();
+    this.output.writable.close();
     this.closed = true;
   }