@livekit/agents 1.1.0 → 1.2.1

package/src/types.ts

@@ -1,42 +1,71 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-export class APIConnectOptions {
-  /** Maximum number of retries to connect to the API. */
-  readonly maxRetry: number;
-  /** Interval between retries to connect to the API in milliseconds. */
-  readonly retryIntervalMs: number;
-  /** Timeout for connecting to the API in milliseconds. */
-  readonly timeoutMs: number;
 
-  constructor(options: Partial<APIConnectOptions> = {}) {
-    this.maxRetry = options.maxRetry ?? 3;
-    this.retryIntervalMs = options.retryIntervalMs ?? 2000;
-    this.timeoutMs = options.timeoutMs ?? 10000;
+/**
+ * Key used to store timed transcripts in AudioFrame.userdata.
+ */
+export const USERDATA_TIMED_TRANSCRIPT = 'lk.timed_transcripts';
 
-    if (this.maxRetry < 0) {
-      throw new Error('maxRetry must be greater than or equal to 0');
-    }
-    if (this.retryIntervalMs < 0) {
-      throw new Error('retryIntervalMs must be greater than or equal to 0');
-    }
-    if (this.timeoutMs < 0) {
-      throw new Error('timeoutMs must be greater than or equal to 0');
-    }
-  }
+/**
+ * Connection options for API calls, controlling retry and timeout behavior.
+ */
+export interface APIConnectOptions {
+  /** Maximum number of retries to connect to the API. Default: 3 */
+  maxRetry: number;
+  /** Interval between retries to connect to the API in milliseconds. Default: 2000 */
+  retryIntervalMs: number;
+  /** Timeout for connecting to the API in milliseconds. Default: 10000 */
+  timeoutMs: number;
+}
 
-  /** @internal */
-  _intervalForRetry(numRetries: number): number {
-    /**
-     * Return the interval for the given number of retries.
-     *
-     * The first retry is immediate, and then uses specified retryIntervalMs
-     */
-    if (numRetries === 0) {
-      return 0.1;
-    }
-    return this.retryIntervalMs;
+export const DEFAULT_API_CONNECT_OPTIONS: APIConnectOptions = {
+  maxRetry: 3,
+  retryIntervalMs: 2000,
+  timeoutMs: 10000,
+};
+
+/**
+ * Return the interval for the given number of retries.
+ * The first retry is immediate, and then uses specified retryIntervalMs.
+ * @internal
+ */
+export function intervalForRetry(connOptions: APIConnectOptions, numRetries: number): number {
+  if (numRetries === 0) {
+    return 0.1;
   }
+  return connOptions.retryIntervalMs;
+}
+
+/**
+ * Connection options for the agent session, controlling retry and timeout behavior
+ * for STT, LLM, and TTS connections.
+ */
+export interface SessionConnectOptions {
+  /** Connection options for speech-to-text. */
+  sttConnOptions?: Partial<APIConnectOptions>;
+  /** Connection options for the language model. */
+  llmConnOptions?: Partial<APIConnectOptions>;
+  /** Connection options for text-to-speech. */
+  ttsConnOptions?: Partial<APIConnectOptions>;
+  /** Maximum number of consecutive unrecoverable errors from LLM or TTS before closing the session. Default: 3 */
+  maxUnrecoverableErrors?: number;
+}
+
+/**
+ * Resolved session connect options with all values populated.
+ * @internal
+ */
+export interface ResolvedSessionConnectOptions {
+  sttConnOptions: APIConnectOptions;
+  llmConnOptions: APIConnectOptions;
+  ttsConnOptions: APIConnectOptions;
+  maxUnrecoverableErrors: number;
 }
 
-export const DEFAULT_API_CONNECT_OPTIONS = new APIConnectOptions();
+export const DEFAULT_SESSION_CONNECT_OPTIONS: ResolvedSessionConnectOptions = {
+  sttConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  llmConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  ttsConnOptions: DEFAULT_API_CONNECT_OPTIONS,
+  maxUnrecoverableErrors: 3,
+};

package/src/utils.test.ts

@@ -5,15 +5,7 @@ import { AudioFrame } from '@livekit/rtc-node';
 import { ReadableStream } from 'node:stream/web';
 import { describe, expect, it } from 'vitest';
 import { initializeLogger } from '../src/log.js';
-import {
-  Event,
-  TASK_TIMEOUT_ERROR,
-  Task,
-  TaskResult,
-  delay,
-  isPending,
-  resampleStream,
-} from '../src/utils.js';
+import { Event, Task, TaskResult, delay, isPending, resampleStream } from '../src/utils.js';
 
 describe('utils', () => {
   // initialize logger
@@ -442,7 +434,8 @@ describe('utils', () => {
         await task.cancelAndWait(200);
         expect.fail('Task should have timed out');
       } catch (error: unknown) {
-        expect(error).toBe(TASK_TIMEOUT_ERROR);
+        expect(error).instanceof(Error);
+        expect((error as Error).message).toBe('Task cancellation timed out');
       }
     });
 
@@ -476,6 +469,93 @@ describe('utils', () => {
         expect((error as Error).name).toBe('TypeError');
       }
     });
+
+    it('should return undefined for Task.current outside task context', () => {
+      expect(Task.current()).toBeUndefined();
+    });
+
+    it('should preserve Task.current inside a task across awaits', async () => {
+      const task = Task.from(
+        async () => {
+          const currentAtStart = Task.current();
+          await delay(5);
+          const currentAfterAwait = Task.current();
+
+          expect(currentAtStart).toBeDefined();
+          expect(currentAfterAwait).toBe(currentAtStart);
+
+          return currentAtStart;
+        },
+        undefined,
+        'current-context-test',
+      );
+
+      const currentFromResult = await task.result;
+      expect(currentFromResult).toBe(task);
+    });
+
+    it('should isolate nested Task.current context and restore parent context', async () => {
+      const parentTask = Task.from(
+        async (controller) => {
+          const parentCurrent = Task.current();
+          expect(parentCurrent).toBeDefined();
+
+          const childTask = Task.from(
+            async () => {
+              const childCurrentStart = Task.current();
+              await delay(5);
+              const childCurrentAfterAwait = Task.current();
+
+              expect(childCurrentStart).toBeDefined();
+              expect(childCurrentAfterAwait).toBe(childCurrentStart);
+              expect(childCurrentStart).not.toBe(parentCurrent);
+
+              return childCurrentStart;
+            },
+            controller,
+            'child-current-context-test',
+          );
+
+          const childCurrent = await childTask.result;
+          const parentCurrentAfterChild = Task.current();
+
+          expect(parentCurrentAfterChild).toBe(parentCurrent);
+
+          return { parentCurrent, childCurrent };
+        },
+        undefined,
+        'parent-current-context-test',
+      );
+
+      const { parentCurrent, childCurrent } = await parentTask.result;
+      expect(parentCurrent).toBe(parentTask);
+      expect(childCurrent).not.toBe(parentCurrent);
+      expect(Task.current()).toBeUndefined();
+    });
+
+    it('should always expose Task.current for concurrent task callbacks', async () => {
+      const tasks = Array.from({ length: 25 }, (_, idx) =>
+        Task.from(
+          async () => {
+            const currentAtStart = Task.current();
+            await delay(1);
+            const currentAfterAwait = Task.current();
+
+            expect(currentAtStart).toBeDefined();
+            expect(currentAfterAwait).toBe(currentAtStart);
+
+            return currentAtStart;
+          },
+          undefined,
+          `current-context-stress-${idx}`,
+        ),
+      );
+
+      const currentTasks = await Promise.all(tasks.map((task) => task.result));
+      currentTasks.forEach((currentTask, idx) => {
+        expect(currentTask).toBe(tasks[idx]);
+      });
+    });
   });
 
   describe('Event', () => {

package/src/utils.ts

@@ -9,12 +9,30 @@ import type {
   TrackKind,
 } from '@livekit/rtc-node';
 import { AudioFrame, AudioResampler, RoomEvent } from '@livekit/rtc-node';
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter, once } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import { TransformStream, type TransformStreamDefaultController } from 'node:stream/web';
 import { v4 as uuidv4 } from 'uuid';
 import { log } from './log.js';
 
+/**
+ * Recursively expands all nested properties of a type,
+ * resolving aliases so as to inspect the real shape in IDE.
+ */
+// eslint-disable-next-line @typescript-eslint/ban-types
+export type Expand<T> = T extends Function
+  ? T
+  : T extends object
+    ? T extends Array<infer U>
+      ? Array<Expand<U>>
+      : T extends Map<infer K, infer V>
+        ? Map<Expand<K>, Expand<V>>
+        : T extends Set<infer M>
+          ? Set<Expand<M>>
+          : { [K in keyof T]: Expand<T[K]> }
+    : T;
+
 /** Union of a single and a list of {@link AudioFrame}s */
 export type AudioBuffer = AudioFrame[] | AudioFrame;
 
@@ -108,6 +126,9 @@ export class Future<T = void> {
   #resolvePromise!: (value: T) => void;
   #rejectPromise!: (error: Error) => void;
   #done: boolean = false;
+  #rejected: boolean = false;
+  #result: T | undefined = undefined;
+  #error: Error | undefined = undefined;
 
   constructor() {
     this.#await = new Promise<T>((resolve, reject) => {
@@ -124,14 +145,39 @@ export class Future<T = void> {
     return this.#done;
   }
 
+  get result(): T {
+    if (!this.#done) {
+      throw new Error('Future is not done');
+    }
+
+    if (this.#rejected) {
+      throw this.#error;
+    }
+
+    return this.#result!;
+  }
+
+  /** Whether the future was rejected (cancelled) */
+  get rejected() {
+    return this.#rejected;
+  }
+
   resolve(value: T) {
     this.#done = true;
+    this.#result = value;
     this.#resolvePromise(value);
   }
 
   reject(error: Error) {
     this.#done = true;
+    this.#rejected = true;
+    this.#error = error;
     this.#rejectPromise(error);
+    // Python calls Future.exception() right after set_exception() to silence
+    // "exception was never retrieved" warnings. In JS, consume the rejection
+    // immediately so Node does not emit unhandled-rejection noise before a
+    // later await/catch observes it.
+    void this.#await.catch(() => undefined);
   }
 }
 
@@ -368,8 +414,6 @@ export class AudioEnergyFilter {
   }
 }
 
-export const TASK_TIMEOUT_ERROR = new Error('Task cancellation timed out');
-
 export enum TaskResult {
   Timeout = 'timeout',
   Completed = 'completed',
@@ -396,7 +440,9 @@ export enum TaskResult {
  * @param T - The type of the task result
  */
 export class Task<T> {
+  private static readonly currentTaskStorage = new AsyncLocalStorage<Task<unknown>>();
   private resultFuture: Future<T>;
+  private doneCallbacks: Set<() => void> = new Set();
 
   #logger = log();
 
@@ -406,6 +452,21 @@ export class Task<T> {
     readonly name?: string,
   ) {
     this.resultFuture = new Future();
+    void this.resultFuture.await
+      .then(
+        () => undefined,
+        () => undefined,
+      )
+      .finally(() => {
+        for (const callback of this.doneCallbacks) {
+          try {
+            callback();
+          } catch (error) {
+            this.#logger.error({ error }, 'Task done callback failed');
+          }
+        }
+        this.doneCallbacks.clear();
+      });
     this.runTask();
   }
 
@@ -425,6 +486,13 @@ export class Task<T> {
     return new Task(fn, abortController, name);
   }
 
+  /**
+   * Returns the currently running task in this async context, if available.
+   */
+  static current(): Task<unknown> | undefined {
+    return Task.currentTaskStorage.getStore();
+  }
+
   private async runTask() {
     const run = async () => {
       if (this.name) {
@@ -433,7 +501,8 @@ export class Task<T> {
       return await this.fn(this.controller);
     };
 
-    return run()
+    return Task.currentTaskStorage
+      .run(this as Task<unknown>, run)
       .then((value) => {
         this.resultFuture.resolve(value);
         return value;
@@ -464,34 +533,30 @@ export class Task<T> {
   async cancelAndWait(timeout?: number) {
     this.cancel();
 
-    try {
-      // Race between task completion and timeout
-      const promises = [
-        this.result
-          .then(() => TaskResult.Completed)
-          .catch((error) => {
-            if (error.name === 'AbortError') {
-              return TaskResult.Aborted;
-            }
-            throw error;
-          }),
-      ];
-
-      if (timeout) {
-        promises.push(delay(timeout).then(() => TaskResult.Timeout));
-      }
-
-      const result = await Promise.race(promises);
+    // Race between task completion and timeout
+    const promises = [
+      this.result
+        .then(() => TaskResult.Completed)
+        .catch((error) => {
+          if (error.name === 'AbortError') {
+            return TaskResult.Aborted;
+          }
+          throw error;
+        }),
+    ];
+
+    if (timeout) {
+      promises.push(delay(timeout).then(() => TaskResult.Timeout));
+    }
 
-      // Check what happened
-      if (result === TaskResult.Timeout) {
-        throw TASK_TIMEOUT_ERROR;
-      }
+    const result = await Promise.race(promises);
 
-      return result;
-    } catch (error) {
-      throw error;
+    // Check what happened
+    if (result === TaskResult.Timeout) {
+      throw new Error('Task cancellation timed out');
     }
+
+    return result;
   }
 
   /**
@@ -509,7 +574,15 @@ export class Task<T> {
   }
 
   addDoneCallback(callback: () => void) {
-    this.resultFuture.await.finally(callback);
+    if (this.done) {
+      queueMicrotask(callback);
+      return;
+    }
+    this.doneCallbacks.add(callback);
+  }
+
+  removeDoneCallback(callback: () => void) {
+    this.doneCallbacks.delete(callback);
   }
 }
 
@@ -633,14 +706,22 @@ export function resampleStream({
   let resampler: AudioResampler | null = null;
   const transformStream = new TransformStream<AudioFrame, AudioFrame>({
     transform(chunk: AudioFrame, controller: TransformStreamDefaultController<AudioFrame>) {
+      if (chunk.samplesPerChannel === 0) {
+        controller.enqueue(chunk);
+        return;
+      }
       if (!resampler) {
         resampler = new AudioResampler(chunk.sampleRate, outputRate);
       }
       for (const frame of resampler.push(chunk)) {
         controller.enqueue(frame);
       }
-      for (const frame of resampler.flush()) {
-        controller.enqueue(frame);
+    },
+    flush(controller) {
+      if (resampler) {
+        for (const frame of resampler.flush()) {
+          controller.enqueue(frame);
+        }
       }
     },
   });
@@ -657,6 +738,20 @@ export class InvalidErrorType extends Error {
   }
 }
 
+/**
+ * Check if an error is a stream closed error that can be safely ignored during cleanup.
+ * This happens during handover/cleanup when close() is called while operations are still running.
+ *
+ * @param error - The error to check.
+ * @returns True if the error is a stream closed error.
+ */
+export function isStreamClosedError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    (error.message === 'Stream is closed' || error.message === 'Input is closed')
+  );
+}
+
 /**
  * In JS an error can be any arbitrary value.
  * This function converts an unknown error to an Error and stores the original value in the error object.
@@ -694,11 +789,11 @@ export type DelayOptions = {
  */
 export function delay(ms: number, options: DelayOptions = {}): Promise<void> {
   const { signal } = options;
-  if (signal?.aborted) return Promise.reject(signal.reason);
+  if (signal?.aborted) return Promise.reject(signal.reason ?? new Error('delay aborted'));
   return new Promise((resolve, reject) => {
     const abort = () => {
       clearTimeout(i);
-      reject(signal?.reason);
+      reject(signal?.reason ?? new Error('delay aborted'));
     };
     const done = () => {
       signal?.removeEventListener('abort', abort);
@@ -751,7 +846,12 @@ export async function waitForParticipant({
     }
   };
 
+  const onDisconnected = () => {
+    fut.reject(new Error('Got disconnected from room while waiting for participant'));
+  };
+
   room.on(RoomEvent.ParticipantConnected, onParticipantConnected);
+  room.on(RoomEvent.Disconnected, onDisconnected);
 
   try {
     for (const p of room.remoteParticipants.values()) {
@@ -764,6 +864,7 @@ export async function waitForParticipant({
     return await fut.await;
   } finally {
     room.off(RoomEvent.ParticipantConnected, onParticipantConnected);
+    room.off(RoomEvent.Disconnected, onDisconnected);
   }
 }
 
@@ -828,3 +929,47 @@ export async function waitForAbort(signal: AbortSignal) {
   signal.addEventListener('abort', handler, { once: true });
   return await abortFuture.await;
 }
+
+/**
+ * Combines two abort signals into a single abort signal.
+ * @param a - The first abort signal.
+ * @param b - The second abort signal.
+ * @returns A new abort signal that is aborted when either of the input signals is aborted.
+ */
+export const combineSignals = (a: AbortSignal, b: AbortSignal): AbortSignal => {
+  const c = new AbortController();
+  const abortFrom = (s: AbortSignal) => {
+    if (c.signal.aborted) return;
+    c.abort((s as any).reason);
+  };
+  if (a.aborted) {
+    abortFrom(a);
+  } else {
+    a.addEventListener('abort', () => abortFrom(a), { once: true });
+  }
+  if (b.aborted) {
+    abortFrom(b);
+  } else {
+    b.addEventListener('abort', () => abortFrom(b), { once: true });
+  }
+  return c.signal;
+};
+
+export const isCloud = (url: URL) => {
+  const hostname = url.hostname;
+  return hostname.endsWith('.livekit.cloud') || hostname.endsWith('.livekit.run');
+};
+
+/**
+ * Whether the agent is running in development mode (launched via `dev` or `connect`).
+ */
+export const isDevMode = (): boolean => {
+  return process.env.LIVEKIT_DEV_MODE === '1';
+};
+
+/**
+ * Whether the agent is hosted on LiveKit Cloud.
+ */
+export const isHosted = (): boolean => {
+  return process.env.LIVEKIT_REMOTE_EOT_URL !== undefined;
+};

package/src/vad.ts

@@ -30,9 +30,9 @@ export interface VADEvent {
   samplesIndex: number;
   /** Timestamp when the event was fired. */
   timestamp: number;
-  /** Duration of the speech segment. */
+  /** Duration of the speech segment in seconds. */
   speechDuration: number;
-  /** Duration of the silence segment. */
+  /** Duration of the silence segment in seconds. */
   silenceDuration: number;
   /**
    * List of audio frames associated with the speech.
@@ -56,6 +56,7 @@ export interface VADEvent {
 }
 
 export interface VADCapabilities {
+  /** Duration of each VAD inference window in milliseconds. Used to batch metrics emissions to roughly once per second. */
   updateInterval: number;
 }
 
@@ -80,6 +81,10 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
    * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
    */
   abstract stream(): VADStream;
+
+  async close(): Promise<void> {
+    return;
+  }
 }
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
@@ -93,14 +98,15 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
   protected closed = false;
   protected inputClosed = false;
 
-  #vad: VAD;
-  #lastActivityTime = BigInt(0);
-  private logger = log();
-  private deferredInputStream: DeferredReadableStream<AudioFrame>;
+  protected vad: VAD;
+  protected lastActivityTime = BigInt(0);
+  protected logger;
+  protected deferredInputStream: DeferredReadableStream<AudioFrame>;
 
   private metricsStream: ReadableStream<VADEvent>;
   constructor(vad: VAD) {
-    this.#vad = vad;
+    this.logger = log();
+    this.vad = vad;
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
 
     this.inputWriter = this.input.writable.getWriter();
@@ -139,7 +145,7 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
   }
 
   protected async monitorMetrics() {
-    let inferenceDurationTotal = 0;
+    let inferenceDurationTotalMs = 0;
     let inferenceCount = 0;
     const metricsReader = this.metricsStream.getReader();
     while (true) {
@@ -150,33 +156,51 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
       switch (value.type) {
         case VADEventType.START_OF_SPEECH:
           inferenceCount++;
-          if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
-            this.#vad.emit('metrics_collected', {
+          if (inferenceCount >= 1000 / this.vad.capabilities.updateInterval) {
+            this.vad.emit('metrics_collected', {
               type: 'vad_metrics',
               timestamp: Date.now(),
-              idleTime: Math.trunc(
-                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),
+              idleTimeMs: Math.trunc(
+                Number((process.hrtime.bigint() - this.lastActivityTime) / BigInt(1000000)),
               ),
-              inferenceDurationTotal,
+              inferenceDurationTotalMs,
               inferenceCount,
-              label: this.#vad.label,
+              label: this.vad.label,
             });
 
             inferenceCount = 0;
-            inferenceDurationTotal = 0;
+            inferenceDurationTotalMs = 0;
           }
           break;
         case VADEventType.INFERENCE_DONE:
-          inferenceDurationTotal += value.inferenceDuration;
-          this.#lastActivityTime = process.hrtime.bigint();
+          inferenceDurationTotalMs += Math.round(value.inferenceDuration);
+          this.lastActivityTime = process.hrtime.bigint();
           break;
         case VADEventType.END_OF_SPEECH:
-          this.#lastActivityTime = process.hrtime.bigint();
+          this.lastActivityTime = process.hrtime.bigint();
           break;
       }
     }
   }
 
+  /**
+   * Safely send a VAD event to the output stream, handling writer release errors during shutdown.
+   * @returns true if the event was sent, false if the stream is closing
+   * @throws Error if an unexpected error occurs
+   */
+  protected sendVADEvent(event: VADEvent): boolean {
+    if (this.closed) {
+      return false;
+    }
+
+    try {
+      this.outputWriter.write(event);
+      return true;
+    } catch (e) {
+      throw e;
+    }
+  }
+
   updateInputStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }

package/src/version.ts

@@ -2,4 +2,4 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export const version = '0.1.0';
+export const version = __PACKAGE_VERSION__;