@livekit/agents 0.0.0-20260120144724

package/src/ipc/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './inference_executor.js';

package/src/ipc/inference_executor.ts

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface InferenceExecutor {
+  doInference(method: string, data: unknown): Promise<unknown>;
+}

package/src/ipc/inference_proc_executor.ts

@@ -0,0 +1,101 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import { extname } from 'node:path';
+import { log } from '../log.js';
+import { shortuuid } from '../utils.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { IPCMessage } from './message.js';
+import { SupervisedProc } from './supervised_proc.js';
+
+class PendingInference {
+  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {
+    this.resolve = resolve;
+  });
+  resolve(arg: { requestId: string; data: unknown; error?: Error }) {
+    arg;
+  }
+}
+
+const currentFileExtension = extname(import.meta.url);
+
+export class InferenceProcExecutor extends SupervisedProc implements InferenceExecutor {
+  #runners: { [id: string]: string };
+  #activeRequests: { [id: string]: PendingInference } = {};
+  #logger = log();
+
+  constructor({
+    runners,
+    initializeTimeout,
+    closeTimeout,
+    memoryWarnMB,
+    memoryLimitMB,
+    pingInterval,
+    pingTimeout,
+    highPingThreshold,
+  }: {
+    runners: { [id: string]: string };
+    initializeTimeout: number;
+    closeTimeout: number;
+    memoryWarnMB: number;
+    memoryLimitMB: number;
+    pingInterval: number;
+    pingTimeout: number;
+    highPingThreshold: number;
+  }) {
+    super(
+      initializeTimeout,
+      closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
+    );
+    this.#runners = runners;
+  }
+
+  createProcess(): ChildProcess {
+    const forkUrl = new URL(`./inference_proc_lazy_main${currentFileExtension}`, import.meta.url);
+
+    // When running via tsx/ts-node (TypeScript files), we need to inherit the parent's
+    // execArgv so the child process can also execute TypeScript with the same loader
+    const isTypeScript = currentFileExtension === '.ts';
+    const forkOptions = isTypeScript ? { execArgv: process.execArgv } : undefined;
+
+    return fork(forkUrl, [JSON.stringify(this.#runners)], forkOptions);
+  }
+
+  async mainTask(proc: ChildProcess) {
+    proc.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceResponse':
+          const res = this.#activeRequests[msg.value.requestId];
+          delete this.#activeRequests[msg.value.requestId];
+          if (!res) {
+            this.#logger
+              .child({ requestId: msg.value.requestId })
+              .warn('received unexpected inference response');
+            return;
+          }
+
+          res.resolve(msg.value);
+      }
+    });
+  }
+
+  async doInference(method: string, data: unknown): Promise<unknown> {
+    const requestId = shortuuid('inference_req_');
+    const fut = new PendingInference();
+    this.proc!.send({ case: 'inferenceRequest', value: { requestId, method, data } });
+    this.#activeRequests[requestId] = fut;
+
+    const res = await fut.promise;
+    if (res.error) {
+      throw new Error(`inference of ${method} failed: ${res.error}`);
+    }
+    return res.data;
+  }
+}

package/src/ipc/inference_proc_lazy_main.ts

@@ -0,0 +1,115 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { once } from 'node:events';
+import type { InferenceRunner } from '../inference_runner.js';
+import { initializeLogger, log } from '../log.js';
+import { Future } from '../utils.js';
+import type { IPCMessage } from './message.js';
+
+const ORPHANED_TIMEOUT = 15 * 1000;
+
+(async () => {
+  if (process.send) {
+    const join = new Future();
+
+    // don't do anything on C-c
+    // this is handled in cli, triggering a termination of all child processes at once.
+    process.on('SIGINT', () => {
+      logger.debug('SIGINT received in inference proc');
+    });
+
+    // don't do anything on SIGTERM
+    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed
+    process.on('SIGTERM', () => {
+      logger.debug('SIGTERM received in inference proc');
+    });
+
+    await once(process, 'message').then(([msg]: IPCMessage[]) => {
+      msg = msg!;
+      if (msg.case !== 'initializeRequest') {
+        throw new Error('first message must be InitializeRequest');
+      }
+      initializeLogger(msg.value.loggerOptions);
+    });
+    const logger = log().child({ pid: process.pid });
+
+    const runners: { [id: string]: InferenceRunner } = await Promise.all(
+      Object.entries(JSON.parse(process.argv[2]!)).map(async ([k, v]) => {
+        return [k, await import(v as string).then((m) => new m.default())];
+      }),
+    ).then(Object.fromEntries);
+
+    await Promise.all(
+      Object.entries(runners).map(async ([runner, v]) => {
+        logger.child({ runner }).debug('initializing inference runner');
+        await v.initialize();
+      }),
+    );
+    logger.debug('all inference runners initialized');
+    process.send({ case: 'initializeResponse' });
+
+    const orphanedTimeout = setTimeout(() => {
+      logger.warn('inference process orphaned, shutting down.');
+      process.exit();
+    }, ORPHANED_TIMEOUT);
+
+    const handleInferenceRequest = async ({
+      method,
+      requestId,
+      data,
+    }: {
+      method: string;
+      requestId: string;
+      data: unknown;
+    }) => {
+      if (!runners[method]) {
+        logger.child({ method }).warn('unknown inference method');
+      }
+
+      try {
+        const resp = await runners[method]!.run(data);
+        process.send!({ case: 'inferenceResponse', value: { requestId, data: resp } });
+      } catch (error) {
+        process.send!({ case: 'inferenceResponse', value: { requestId, error } });
+      }
+    };
+
+    const messageHandler = (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'pingRequest':
+          orphanedTimeout.refresh();
+          process.send!({
+            case: 'pongResponse',
+            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },
+          });
+          break;
+        case 'shutdownRequest':
+          logger.debug('inference process received shutdown request');
+          clearTimeout(orphanedTimeout);
+          // Remove our message handler to stop processing new messages
+          process.off('message', messageHandler);
+          Promise.all(Object.values(runners).map((r) => r.close()))
+            .then(() => {
+              logger.debug('Inference runners closed');
+              process.send!({ case: 'done' });
+              join.resolve();
+            })
+            .catch((err) => {
+              logger.error('Error closing inference runners:', err);
+            });
+          break;
+        case 'inferenceRequest':
+          handleInferenceRequest(msg.value);
+      }
+    };
+
+    process.on('message', messageHandler);
+
+    await join.await;
+
+    logger.debug('Inference process shutdown');
+
+    return process.exitCode;
+  }
+})();

package/src/ipc/job_executor.ts

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { RunningJobInfo } from '../job.js';
+
+export interface JobExecutor {
+  started: boolean;
+  userArguments: any; // eslint-disable-line @typescript-eslint/no-explicit-any
+  runningJob: RunningJobInfo | undefined;
+  status: JobStatus;
+
+  start(): Promise<void>;
+  join(): Promise<void>;
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+  launchJob(info: RunningJobInfo): Promise<void>;
+}
+
+export enum JobStatus {
+  RUNNING,
+  FAILED,
+  SUCCESS,
+}

package/src/ipc/job_proc_executor.ts

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import { extname } from 'node:path';
+import type { RunningJobInfo } from '../job.js';
+import { log } from '../log.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { JobExecutor } from './job_executor.js';
+import { JobStatus } from './job_executor.js';
+import type { IPCMessage } from './message.js';
+import { SupervisedProc } from './supervised_proc.js';
+
+const currentFileExtension = extname(import.meta.url);
+
+export class JobProcExecutor extends SupervisedProc implements JobExecutor {
+  #userArgs?: any; // eslint-disable-line @typescript-eslint/no-explicit-any
+  #jobStatus?: JobStatus;
+  #runningJob?: RunningJobInfo;
+  #agent: string;
+  #inferenceExecutor?: InferenceExecutor;
+  #inferenceTasks: Promise<void>[] = [];
+  #logger = log();
+
+  constructor(
+    agent: string,
+    inferenceExecutor: InferenceExecutor | undefined,
+    initializeTimeout: number,
+    closeTimeout: number,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
+    pingInterval: number,
+    pingTimeout: number,
+    highPingThreshold: number,
+  ) {
+    super(
+      initializeTimeout,
+      closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
+    );
+    this.#agent = agent;
+    this.#inferenceExecutor = inferenceExecutor;
+  }
+
+  get status(): JobStatus {
+    if (this.#jobStatus) {
+      return this.#jobStatus;
+    }
+    throw new Error('job status not available');
+  }
+
+  get userArguments(): any {
+    return this.#userArgs;
+  }
+
+  set userArguments(args: any) {
+    this.#userArgs = args;
+  }
+
+  get runningJob(): RunningJobInfo | undefined {
+    return this.#runningJob;
+  }
+
+  createProcess(): ChildProcess {
+    const forkUrl = new URL(`./job_proc_lazy_main${currentFileExtension}`, import.meta.url);
+
+    // When running via tsx/ts-node (TypeScript files), we need to inherit the parent's
+    // execArgv so the child process can also execute TypeScript with the same loader
+    const isTypeScript = currentFileExtension === '.ts';
+    const forkOptions = isTypeScript ? { execArgv: process.execArgv } : undefined;
+
+    return fork(forkUrl, [this.#agent], forkOptions);
+  }
+
+  async mainTask(proc: ChildProcess) {
+    proc.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceRequest':
+          this.#inferenceTasks.push(this.#doInferenceTask(proc, msg.value));
+      }
+    });
+  }
+
+  async #doInferenceTask(
+    proc: ChildProcess,
+    req: { method: string; requestId: string; data: unknown },
+  ) {
+    if (!this.#inferenceExecutor) {
+      this.#logger.warn('inference request received but no inference executor');
+      proc.send({
+        case: 'inferenceResponse',
+        value: { requestId: req.requestId, error: new Error('no inference executor') },
+      });
+      return;
+    }
+
+    try {
+      const data = await this.#inferenceExecutor.doInference(req.method, req.data);
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, data } });
+    } catch (error) {
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, error } });
+    }
+  }
+
+  async launchJob(info: RunningJobInfo) {
+    if (this.#runningJob) {
+      throw Error('process already has a running job');
+    }
+    if (!this.init.done) {
+      throw Error('process not initialized');
+    }
+    this.#jobStatus = JobStatus.RUNNING;
+    this.#runningJob = info;
+
+    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+  }
+}

package/src/ipc/job_proc_lazy_main.ts

@@ -0,0 +1,247 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { Room, RoomEvent } from '@livekit/rtc-node';
+import { EventEmitter, once } from 'node:events';
+import { pathToFileURL } from 'node:url';
+import type { Logger } from 'pino';
+import { type Agent, isAgent } from '../generator.js';
+import { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';
+import { initializeLogger, log } from '../log.js';
+import { Future, shortuuid } from '../utils.js';
+import { defaultInitializeProcessFunc } from '../worker.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { IPCMessage } from './message.js';
+
+const ORPHANED_TIMEOUT = 15 * 1000;
+
+type JobTask = {
+  ctx: JobContext;
+  task: Promise<void>;
+};
+
+class PendingInference {
+  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {
+    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
+  });
+  resolve(arg: { requestId: string; data: unknown; error?: Error }) {
+    arg; // useless call to counteract TypeScript E6133
+  }
+}
+
+class InfClient implements InferenceExecutor {
+  #requests: { [id: string]: PendingInference } = {};
+
+  constructor() {
+    process.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceResponse':
+          const fut = this.#requests[msg.value.requestId];
+          delete this.#requests[msg.value.requestId];
+          if (!fut) {
+            log().child({ resp: msg.value }).warn('received unexpected inference response');
+            return;
+          }
+          fut.resolve(msg.value);
+          break;
+      }
+    });
+  }
+
+  async doInference(method: string, data: unknown): Promise<unknown> {
+    const requestId = shortuuid('inference_job_');
+    process.send!({ case: 'inferenceRequest', value: { requestId, method, data } });
+    this.#requests[requestId] = new PendingInference();
+    const resp = await this.#requests[requestId]!.promise;
+    if (resp.error) {
+      throw new Error(`inference of ${method} failed: ${resp.error.message}`);
+    }
+    return resp.data;
+  }
+}
+
+const startJob = (
+  proc: JobProcess,
+  func: (ctx: JobContext) => Promise<void>,
+  info: RunningJobInfo,
+  closeEvent: EventEmitter,
+  logger: Logger,
+  joinFuture: Future,
+): JobTask => {
+  let connect = false;
+  let shutdown = false;
+
+  const room = new Room();
+  room.on(RoomEvent.Disconnected, () => {
+    if (!shutdown) {
+      closeEvent.emit('close', false);
+    }
+  });
+
+  const onConnect = () => {
+    connect = true;
+  };
+  const onShutdown = (reason: string) => {
+    shutdown = true;
+    closeEvent.emit('close', true, reason);
+  };
+
+  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());
+
+  const task = (async () => {
+    const unconnectedTimeout = setTimeout(() => {
+      if (!(connect || shutdown)) {
+        logger.warn(
+          'room not connect after job_entry was called after 10 seconds, ',
+          'did you forget to call ctx.connect()?',
+        );
+      }
+    }, 10000);
+
+    // Run the job function within the AsyncLocalStorage context
+    await runWithJobContextAsync(ctx, async () => {
+      const { tracer, traceTypes } = await import('../telemetry/index.js');
+      return tracer.startActiveSpan(
+        async (span) => {
+          span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);
+          span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);
+          span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');
+          return func(ctx);
+        },
+        { name: 'job_entrypoint' },
+      );
+    }).finally(() => {
+      clearTimeout(unconnectedTimeout);
+    });
+
+    await once(closeEvent, 'close').then((close) => {
+      logger.debug('shutting down');
+      shutdown = true;
+      process.send!({ case: 'exiting', value: { reason: close[1] } });
+    });
+
+    // Close the primary agent session if it exists
+    if (ctx._primaryAgentSession) {
+      await ctx._primaryAgentSession.close();
+    }
+
+    // Generate and save/upload session report
+    await ctx._onSessionEnd();
+
+    await room.disconnect();
+    logger.debug('disconnected from room');
+
+    const shutdownTasks = [];
+    for (const callback of ctx.shutdownCallbacks) {
+      shutdownTasks.push(callback());
+    }
+    await Promise.all(shutdownTasks).catch((error) =>
+      logger.error({ error }, 'error while shutting down the job'),
+    );
+
+    process.send!({ case: 'done' });
+    joinFuture.resolve();
+  })();
+
+  return { ctx, task };
+};
+
+(async () => {
+  if (process.send) {
+    const join = new Future();
+
+    // process.argv:
+    //   [0] `node'
+    //   [1] import.meta.filename
+    //   [2] import.meta.filename of function containing entry file
+    const moduleFile = process.argv[2];
+    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {
+      const agent = module.default;
+      if (agent === undefined || !isAgent(agent)) {
+        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);
+      }
+      return agent;
+    });
+    if (!agent.prewarm) {
+      agent.prewarm = defaultInitializeProcessFunc;
+    }
+
+    // don't do anything on C-c
+    // this is handled in cli, triggering a termination of all child processes at once.
+    process.on('SIGINT', () => {
+      logger.debug('SIGINT received in job proc');
+    });
+
+    // don't do anything on SIGTERM
+    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed
+    process.on('SIGTERM', () => {
+      logger.debug('SIGTERM received in job proc');
+    });
+
+    await once(process, 'message').then(([msg]: IPCMessage[]) => {
+      msg = msg!;
+      if (msg.case !== 'initializeRequest') {
+        throw new Error('first message must be InitializeRequest');
+      }
+      initializeLogger(msg.value.loggerOptions);
+    });
+    const proc = new JobProcess();
+    let logger = log().child({ pid: proc.pid });
+
+    process.on('unhandledRejection', (reason) => {
+      logger.debug({ error: reason }, 'Unhandled promise rejection');
+    });
+
+    logger.debug('initializing job runner');
+    await agent.prewarm(proc);
+    logger.debug('job runner initialized');
+    process.send({ case: 'initializeResponse' });
+
+    let job: JobTask | undefined = undefined;
+    const closeEvent = new EventEmitter();
+
+    const orphanedTimeout = setTimeout(() => {
+      logger.warn('job process orphaned, shutting down.');
+      join.resolve();
+    }, ORPHANED_TIMEOUT);
+
+    const messageHandler = (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'pingRequest': {
+          orphanedTimeout.refresh();
+          process.send!({
+            case: 'pongResponse',
+            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },
+          });
+          break;
+        }
+        case 'startJobRequest': {
+          if (job) {
+            throw new Error('job task already running');
+          }
+
+          logger = logger.child({ jobID: msg.value.runningJob.job.id });
+
+          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);
+          logger.debug('job started');
+          break;
+        }
+        case 'shutdownRequest': {
+          if (!job) {
+            join.resolve();
+          }
+          closeEvent.emit('close', 'shutdownRequest');
+          clearTimeout(orphanedTimeout);
+          process.off('message', messageHandler);
+        }
+      }
+    };
+
+    process.on('message', messageHandler);
+
+    await join.await;
+
+    logger.debug('Job process shutdown');
+    process.exit(0);
+  }
+})();

package/src/ipc/message.ts

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { RunningJobInfo } from '../job.js';
+import type { LoggerOptions } from '../log.js';
+
+export type IPCMessage =
+  | {
+      case: 'initializeRequest';
+      value: {
+        loggerOptions: LoggerOptions;
+        pingInterval?: number;
+        pingTimeout?: number;
+        highPingThreshold?: number;
+      };
+    }
+  | {
+      case: 'initializeResponse';
+      value: undefined;
+    }
+  | {
+      case: 'pingRequest';
+      value: { timestamp: number };
+    }
+  | {
+      case: 'pongResponse';
+      value: { lastTimestamp: number; timestamp: number };
+    }
+  | {
+      case: 'startJobRequest';
+      value: { runningJob: RunningJobInfo };
+    }
+  | {
+      case: 'shutdownRequest';
+      value: { reason?: string };
+    }
+  | {
+      case: 'inferenceRequest';
+      value: { method: string; requestId: string; data: unknown };
+    }
+  | {
+      case: 'inferenceResponse';
+      value: { requestId: string; data: unknown; error?: Error };
+    }
+  | {
+      case: 'exiting';
+      value: { reason?: string };
+    }
+  | {
+      case: 'done';
+      value: undefined;
+    };