@livekit/agents 0.7.8 → 1.0.0-next.0

package/src/_exceptions.ts

@@ -0,0 +1,137 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Raised when accepting a job but not receiving an assignment within the specified timeout.
+ * The server may have chosen another worker to handle this job.
+ */
+export class AssignmentTimeoutError extends Error {
+  constructor(message = 'Assignment timeout occurred') {
+    super(message);
+    this.name = 'AssignmentTimeoutError';
+    Error.captureStackTrace(this, AssignmentTimeoutError);
+  }
+}
+
+/**
+ * Interface for API error options
+ */
+interface APIErrorOptions {
+  body?: object | null;
+  retryable?: boolean;
+}
+
+const API_ERROR_SYMBOL = Symbol('APIError');
+
+/**
+ * Raised when an API request failed.
+ * This is used on our TTS/STT/LLM plugins.
+ */
+export class APIError extends Error {
+  readonly body: object | null;
+  readonly retryable: boolean;
+
+  constructor(message: string, { body = null, retryable = true }: APIErrorOptions = {}) {
+    super(message);
+    this.name = 'APIError';
+
+    this.body = body;
+    this.retryable = retryable;
+    Error.captureStackTrace(this, APIError);
+    Object.defineProperty(this, API_ERROR_SYMBOL, {
+      value: true,
+      writable: false,
+      enumerable: false,
+      configurable: false,
+    });
+  }
+
+  toString(): string {
+    return `${this.message} (body=${JSON.stringify(this.body)}, retryable=${this.retryable})`;
+  }
+}
+
+/**
+ * Interface for API status error options
+ */
+interface APIStatusErrorOptions extends APIErrorOptions {
+  statusCode?: number;
+  requestId?: string | null;
+}
+
+/**
+ * Raised when an API response has a status code of 4xx or 5xx.
+ */
+export class APIStatusError extends APIError {
+  readonly statusCode: number;
+  readonly requestId: string | null;
+
+  constructor({
+    message = 'API error.',
+    options = {},
+  }: {
+    message?: string;
+    options?: APIStatusErrorOptions;
+  }) {
+    const statusCode = options.statusCode ?? -1;
+    // 4xx errors are not retryable
+    const isRetryable = options.retryable ?? !(statusCode >= 400 && statusCode < 500);
+
+    super(message, { body: options.body, retryable: isRetryable });
+    this.name = 'APIStatusError';
+
+    this.statusCode = statusCode;
+    this.requestId = options.requestId ?? null;
+    Error.captureStackTrace(this, APIStatusError);
+  }
+
+  toString(): string {
+    return (
+      `${this.message} ` +
+      `(statusCode=${this.statusCode}, ` +
+      `requestId=${this.requestId}, ` +
+      `body=${JSON.stringify(this.body)}, ` +
+      `retryable=${this.retryable})`
+    );
+  }
+}
+
+/**
+ * Raised when an API request failed due to a connection error.
+ */
+export class APIConnectionError extends APIError {
+  constructor({
+    message = 'Connection error.',
+    options = {},
+  }: {
+    message?: string;
+    options?: APIErrorOptions;
+  }) {
+    super(message, { body: null, retryable: options.retryable ?? true });
+    this.name = 'APIConnectionError';
+    Error.captureStackTrace(this, APIConnectionError);
+  }
+}
+
+/**
+ * Raised when an API request timed out.
+ */
+export class APITimeoutError extends APIConnectionError {
+  constructor({
+    message = 'Request timed out.',
+    options = {},
+  }: {
+    message?: string;
+    options?: APIErrorOptions;
+  }) {
+    const retryable = options?.retryable ?? true;
+
+    super({ message, options: { retryable } });
+    this.name = 'APITimeoutError';
+    Error.captureStackTrace(this, APITimeoutError);
+  }
+}
+
+export function isAPIError(error: unknown): error is APIError {
+  return error !== null && typeof error === 'object' && API_ERROR_SYMBOL in error;
+}

package/src/audio.ts

@@ -3,6 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0
 import { AudioFrame } from '@livekit/rtc-node';
 import { log } from './log.js';
+import type { AudioBuffer } from './utils.js';
+
+export function calculateAudioDuration(frame: AudioBuffer) {
+  // TODO(AJS-102): use frame.durationMs once available in rtc-node
+  return Array.isArray(frame)
+    ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)
+    : frame.samplesPerChannel / frame.sampleRate;
+}
 
 /** AudioByteStream translates between LiveKit AudioFrame packets and raw byte data. */
 export class AudioByteStream {
@@ -51,7 +59,7 @@ export class AudioByteStream {
       return [];
     }
 
-    return [
+    const frames = [
       new AudioFrame(
         new Int16Array(this.#buf.buffer),
         this.#sampleRate,
@@ -59,5 +67,8 @@ export class AudioByteStream {
         this.#buf.length / 2,
       ),
     ];
+
+    this.#buf = new Int8Array(); // Clear buffer after flushing
+    return frames;
   }
 }

package/src/cli.ts

@@ -4,6 +4,7 @@
 import { Command, Option } from 'commander';
 import type { EventEmitter } from 'node:events';
 import { initializeLogger, log } from './log.js';
+import { Plugin } from './plugin.js';
 import { version } from './version.js';
 import { Worker, WorkerOptions } from './worker.js';
 
@@ -183,5 +184,41 @@ export const runApp = (opts: WorkerOptions) => {
       });
     });
 
+  program
+    .command('download-files')
+    .description('Download plugin dependency files')
+    .addOption(
+      new Option('--log-level <level>', 'Set the logging level')
+        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+        .default('debug')
+        .env('LOG_LEVEL'),
+    )
+    .action(() => {
+      const options = program.optsWithGlobals();
+      initializeLogger({ pretty: true, level: options.logLevel });
+      const logger = log();
+
+      const downloadFiles = async () => {
+        for (const plugin of Plugin.registeredPlugins) {
+          logger.info(`Downloading files for ${plugin.title}`);
+          try {
+            await plugin.downloadFiles();
+            logger.info(`Finished downloading files for ${plugin.title}`);
+          } catch (error) {
+            logger.error(`Failed to download files for ${plugin.title}: ${error}`);
+          }
+        }
+      };
+
+      downloadFiles()
+        .catch((error) => {
+          logger.fatal(`Error during file downloads: ${error}`);
+          process.exit(1);
+        })
+        .finally(() => {
+          process.exit(0);
+        });
+    });
+
   program.parse();
 };

package/src/constants.ts

@@ -4,4 +4,6 @@
 export const ATTRIBUTE_TRANSCRIPTION_TRACK_ID = 'lk.transcribed_track_id';
 export const ATTRIBUTE_TRANSCRIPTION_FINAL = 'lk.transcription_final';
 export const TOPIC_TRANSCRIPTION = 'lk.transcription';
+export const ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID = 'lk.segment_id';
+export const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf';
 export const TOPIC_CHAT = 'lk.chat';

package/src/http_server.ts

@@ -14,6 +14,7 @@ interface WorkerResponse {
   worker_type: string;
   active_jobs: number;
   sdk_version: string;
+  project_type: string;
 }
 
 export class HTTPServer {

package/src/index.ts

@@ -14,21 +14,24 @@ import * as ipc from './ipc/index.js';
 import * as llm from './llm/index.js';
 import * as metrics from './metrics/index.js';
 import * as multimodal from './multimodal/index.js';
-import * as pipeline from './pipeline/index.js';
+import * as stream from './stream/index.js';
 import * as stt from './stt/index.js';
 import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
+import * as voice from './voice/index.js';
 
-export * from './vad.js';
-export * from './plugin.js';
-export * from './version.js';
+export * from './audio.js';
+export * from './_exceptions.js';
+export * from './generator.js';
+export * from './inference_runner.js';
 export * from './job.js';
-export * from './worker.js';
-export * from './utils.js';
 export * from './log.js';
-export * from './generator.js';
-export * from './audio.js';
+export * from './plugin.js';
 export * from './transcription.js';
-export * from './inference_runner.js';
+export * from './types.js';
+export * from './utils.js';
+export * from './vad.js';
+export * from './version.js';
+export * from './worker.js';
 
-export { cli, stt, tts, llm, pipeline, multimodal, tokenize, metrics, ipc };
+export { cli, ipc, llm, metrics, multimodal, stream, stt, tokenize, tts, voice };

package/src/inference_runner.ts

@@ -3,8 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 /** @internal */
-export abstract class InferenceRunner {
-  static INFERENCE_METHOD: string;
+export abstract class InferenceRunner<InputType = unknown, OutputType = unknown> {
   static registeredRunners: { [id: string]: string } = {};
 
   static registerRunner(method: string, importPath: string) {
@@ -15,6 +14,6 @@ export abstract class InferenceRunner {
   }
 
   abstract initialize(): Promise<void>;
-  abstract run(data: unknown): Promise<unknown>;
+  abstract run(data: InputType): Promise<OutputType>;
   abstract close(): Promise<void>;
 }

package/src/ipc/inference_proc_executor.ts

@@ -3,8 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { ChildProcess } from 'node:child_process';
 import { fork } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
 import { log } from '../log.js';
+import { shortuuid } from '../utils.js';
 import type { InferenceExecutor } from './inference_executor.js';
 import type { IPCMessage } from './message.js';
 import { SupervisedProc } from './supervised_proc.js';
@@ -79,7 +79,7 @@ export class InferenceProcExecutor extends SupervisedProc implements InferenceEx
   }
 
   async doInference(method: string, data: unknown): Promise<unknown> {
-    const requestId = 'inference_req_' + randomUUID();
+    const requestId = shortuuid('inference_req_');
     const fut = new PendingInference();
     this.proc!.send({ case: 'inferenceRequest', value: { requestId, method, data } });
     this.#activeRequests[requestId] = fut;

package/src/ipc/job_executor.ts

@@ -5,7 +5,7 @@ import type { RunningJobInfo } from '../job.js';
 
 export interface JobExecutor {
   started: boolean;
-  userArguments: any;
+  userArguments: any; // eslint-disable-line @typescript-eslint/no-explicit-any
   runningJob: RunningJobInfo | undefined;
   status: JobStatus;
 

package/src/ipc/job_proc_executor.ts

@@ -12,7 +12,7 @@ import type { IPCMessage } from './message.js';
 import { SupervisedProc } from './supervised_proc.js';
 
 export class JobProcExecutor extends SupervisedProc implements JobExecutor {
-  #userArgs?: any;
+  #userArgs?: any; // eslint-disable-line @typescript-eslint/no-explicit-any
   #jobStatus?: JobStatus;
   #runningJob?: RunningJobInfo;
   #agent: string;

package/src/ipc/job_proc_lazy_main.ts

@@ -50,7 +50,7 @@ class InfClient implements InferenceExecutor {
   }
 
   async doInference(method: string, data: unknown): Promise<unknown> {
-    const requestId = 'inference_job_' + randomUUID();
+    const requestId = 'inference_job_' + randomUUID;
     process.send!({ case: 'inferenceRequest', value: { requestId, method, data } });
     this.#requests[requestId] = new PendingInference();
     const resp = await this.#requests[requestId]!.promise;

package/src/job.ts

@@ -26,6 +26,19 @@ export class CurrentJobContext {
   }
 }
 
+/**
+ * Returns the current job context.
+ *
+ * @throws {Error} if no job context is found
+ */
+export function getJobContext(): JobContext {
+  const ctx = CurrentJobContext.getCurrent();
+  if (!ctx) {
+    throw new Error('no job context found, are you running this code inside a job entrypoint?');
+  }
+  return ctx;
+}
+
 /** Which tracks, if any, should the agent automatically subscribe to? */
 export enum AutoSubscribe {
   SUBSCRIBE_ALL,
@@ -46,6 +59,7 @@ export type RunningJobInfo = {
   job: proto.Job;
   url: string;
   token: string;
+  workerId: string;
 };
 
 /** Attempted to add a function callback, but the function already exists. */
@@ -102,6 +116,10 @@ export class JobContext {
     return this.#info.job;
   }
 
+  get workerId(): string {
+    return this.#info.workerId;
+  }
+
   /** @returns The room the agent was called into */
   get room(): Room {
     return this.#room;