@livekit/agents 0.6.4 → 0.7.1

package/src/ipc/job_proc_executor.ts

@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import type { RunningJobInfo } from '../job.js';
+import { log } from '../log.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { JobExecutor } from './job_executor.js';
+import { JobStatus } from './job_executor.js';
+import type { IPCMessage } from './message.js';
+import { SupervisedProc } from './supervised_proc.js';
+
+export class JobProcExecutor extends SupervisedProc implements JobExecutor {
+  #userArgs?: any;
+  #jobStatus?: JobStatus;
+  #runningJob?: RunningJobInfo;
+  #agent: string;
+  #inferenceExecutor?: InferenceExecutor;
+  #inferenceTasks: Promise<void>[] = [];
+  #logger = log();
+
+  constructor(
+    agent: string,
+    inferenceExecutor: InferenceExecutor | undefined,
+    initializeTimeout: number,
+    closeTimeout: number,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
+    pingInterval: number,
+    pingTimeout: number,
+    highPingThreshold: number,
+  ) {
+    super(
+      initializeTimeout,
+      closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
+    );
+    this.#agent = agent;
+    this.#inferenceExecutor = inferenceExecutor;
+  }
+
+  get status(): JobStatus {
+    if (this.#jobStatus) {
+      return this.#jobStatus;
+    }
+    throw new Error('job status not available');
+  }
+
+  get userArguments(): any {
+    return this.#userArgs;
+  }
+
+  set userArguments(args: any) {
+    this.#userArgs = args;
+  }
+
+  get runningJob(): RunningJobInfo | undefined {
+    return this.#runningJob;
+  }
+
+  createProcess(): ChildProcess {
+    return fork(new URL(import.meta.resolve('./job_proc_lazy_main.js')), [this.#agent]);
+  }
+
+  async mainTask(proc: ChildProcess) {
+    proc.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceRequest':
+          this.#inferenceTasks.push(this.#doInferenceTask(proc, msg.value));
+      }
+    });
+  }
+
+  async #doInferenceTask(
+    proc: ChildProcess,
+    req: { method: string; requestId: string; data: unknown },
+  ) {
+    if (!this.#inferenceExecutor) {
+      this.#logger.warn('inference request received but no inference executor');
+      proc.send({
+        case: 'inferenceResponse',
+        value: { requestId: req.requestId, error: new Error('no inference executor') },
+      });
+      return;
+    }
+
+    try {
+      const data = await this.#inferenceExecutor.doInference(req.method, req.data);
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, data } });
+    } catch (error) {
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, error } });
+    }
+  }
+
+  async launchJob(info: RunningJobInfo) {
+    if (this.#runningJob) {
+      throw Error('process already has a running job');
+    }
+    if (!this.init.done) {
+      throw Error('process not initialized');
+    }
+    this.#jobStatus = JobStatus.RUNNING;
+    this.#runningJob = info;
+
+    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+  }
+}

package/src/ipc/{job_main.ts → job_proc_lazy_main.ts}

@@ -2,34 +2,63 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { Room, RoomEvent } from '@livekit/rtc-node';
-import type { ChildProcess } from 'node:child_process';
-import { fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
 import { EventEmitter, once } from 'node:events';
 import { pathToFileURL } from 'node:url';
 import type { Logger } from 'pino';
 import { type Agent, isAgent } from '../generator.js';
-import type { RunningJobInfo } from '../job.js';
-import { JobContext } from '../job.js';
-import { JobProcess } from '../job.js';
+import { CurrentJobContext, JobContext, JobProcess, type RunningJobInfo } from '../job.js';
 import { initializeLogger, log } from '../log.js';
 import { defaultInitializeProcessFunc } from '../worker.js';
+import type { InferenceExecutor } from './inference_executor.js';
 import type { IPCMessage } from './message.js';
 
 const ORPHANED_TIMEOUT = 15 * 1000;
 
-type StartArgs = {
-  agentFile: string;
-  // userArguments: unknown;
-};
-
 type JobTask = {
   ctx: JobContext;
   task: Promise<void>;
 };
 
-export const runProcess = (args: StartArgs): ChildProcess => {
-  return fork(new URL(import.meta.url), [args.agentFile]);
-};
+class PendingInference {
+  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {
+    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
+  });
+  resolve(arg: { requestId: string; data: unknown; error?: Error }) {
+    arg; // useless call to counteract TypeScript E6133
+  }
+}
+
+class InfClient implements InferenceExecutor {
+  #requests: { [id: string]: PendingInference } = {};
+
+  constructor() {
+    process.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceResponse':
+          const fut = this.#requests[msg.value.requestId];
+          delete this.#requests[msg.value.requestId];
+          if (!fut) {
+            log().child({ resp: msg.value }).warn('received unexpected inference response');
+            return;
+          }
+          fut.resolve(msg.value);
+          break;
+      }
+    });
+  }
+
+  async doInference(method: string, data: unknown): Promise<unknown> {
+    const requestId = 'inference_job_' + randomUUID;
+    process.send!({ case: 'inferenceRequest', value: { requestId, method, data } });
+    this.#requests[requestId] = new PendingInference();
+    const resp = await this.#requests[requestId]!.promise;
+    if (resp.error) {
+      throw new Error(`inference of ${method} failed: ${resp.error.message}`);
+    }
+    return resp.data;
+  }
+}
 
 const startJob = (
   proc: JobProcess,
@@ -54,7 +83,8 @@ const startJob = (
     closeEvent.emit('close', true, reason);
   };
 
-  const ctx = new JobContext(proc, info, room, onConnect, onShutdown);
+  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());
+  new CurrentJobContext(ctx);
 
   const task = new Promise<void>(async () => {
     const unconnectedTimeout = setTimeout(() => {
@@ -110,6 +140,10 @@ const startJob = (
     // this is handled in cli, triggering a termination of all child processes at once.
     process.on('SIGINT', () => {});
 
+    // don't do anything on SIGTERM
+    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed
+    process.on('SIGTERM', () => {});
+
     await once(process, 'message').then(([msg]: IPCMessage[]) => {
       msg = msg!;
       if (msg.case !== 'initializeRequest') {
@@ -120,6 +154,10 @@ const startJob = (
     const proc = new JobProcess();
     let logger = log().child({ pid: proc.pid });
 
+    process.on('unhandledRejection', (reason) => {
+      logger.error(reason);
+    });
+
     logger.debug('initializing job runner');
     agent.prewarm(proc);
     logger.debug('job runner initialized');

package/src/ipc/message.ts

@@ -7,7 +7,12 @@ import type { LoggerOptions } from '../log.js';
 export type IPCMessage =
   | {
       case: 'initializeRequest';
-      value: { loggerOptions: LoggerOptions };
+      value: {
+        loggerOptions: LoggerOptions;
+        pingInterval?: number;
+        pingTimeout?: number;
+        highPingThreshold?: number;
+      };
     }
   | {
       case: 'initializeResponse';
@@ -29,6 +34,14 @@ export type IPCMessage =
       case: 'shutdownRequest';
       value: { reason?: string };
     }
+  | {
+      case: 'inferenceRequest';
+      value: { method: string; requestId: string; data: unknown };
+    }
+  | {
+      case: 'inferenceResponse';
+      value: { requestId: string; data: unknown; error?: Error };
+    }
   | {
       case: 'exiting';
       value: { reason?: string };

package/src/ipc/proc_pool.ts

@@ -4,8 +4,9 @@
 import { MultiMutex, Mutex } from '@livekit/mutex';
 import type { RunningJobInfo } from '../job.js';
 import { Queue } from '../utils.js';
+import type { InferenceExecutor } from './inference_executor.js';
 import type { JobExecutor } from './job_executor.js';
-import { ProcJobExecutor } from './proc_job_executor.js';
+import { JobProcExecutor } from './job_proc_executor.js';
 
 export class ProcPool {
   agent: string;
@@ -20,12 +21,18 @@ export class ProcPool {
   procMutex?: MultiMutex;
   procUnlock?: () => void;
   warmedProcQueue = new Queue<JobExecutor>();
+  inferenceExecutor?: InferenceExecutor;
+  memoryWarnMB: number;
+  memoryLimitMB: number;
 
   constructor(
     agent: string,
     numIdleProcesses: number,
     initializeTimeout: number,
     closeTimeout: number,
+    inferenceExecutor: InferenceExecutor | undefined,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
   ) {
     this.agent = agent;
     if (numIdleProcesses > 0) {
@@ -33,6 +40,9 @@ export class ProcPool {
     }
     this.initializeTimeout = initializeTimeout;
     this.closeTimeout = closeTimeout;
+    this.inferenceExecutor = inferenceExecutor;
+    this.memoryWarnMB = memoryWarnMB;
+    this.memoryLimitMB = memoryLimitMB;
   }
 
   get processes(): JobExecutor[] {
@@ -52,7 +62,17 @@ export class ProcPool {
         this.procUnlock = undefined;
       }
     } else {
-      proc = new ProcJobExecutor(this.agent, this.initializeTimeout, this.closeTimeout);
+      proc = new JobProcExecutor(
+        this.agent,
+        this.inferenceExecutor,
+        this.initializeTimeout,
+        this.closeTimeout,
+        this.memoryWarnMB,
+        this.memoryLimitMB,
+        2500,
+        60000,
+        500,
+      );
       this.executors.push(proc);
       await proc.start();
       await proc.initialize();
@@ -61,7 +81,17 @@ export class ProcPool {
   }
 
   async procWatchTask() {
-    const proc = new ProcJobExecutor(this.agent, this.initializeTimeout, this.closeTimeout);
+    const proc = new JobProcExecutor(
+      this.agent,
+      this.inferenceExecutor,
+      this.initializeTimeout,
+      this.closeTimeout,
+      this.memoryWarnMB,
+      this.memoryLimitMB,
+      2500,
+      60000,
+      500,
+    );
 
     try {
       this.executors.push(proc);

package/src/ipc/{proc_job_executor.ts → supervised_proc.ts}

@@ -6,31 +6,54 @@ import { once } from 'node:events';
 import type { RunningJobInfo } from '../job.js';
 import { log, loggerOptions } from '../log.js';
 import { Future } from '../utils.js';
-import type { ProcOpts } from './job_executor.js';
-import { JobExecutor } from './job_executor.js';
 import type { IPCMessage } from './message.js';
 
-export class ProcJobExecutor extends JobExecutor {
+export interface ProcOpts {
+  initializeTimeout: number;
+  closeTimeout: number;
+  memoryWarnMB: number;
+  memoryLimitMB: number;
+  pingInterval: number;
+  pingTimeout: number;
+  highPingThreshold: number;
+}
+
+export abstract class SupervisedProc {
   #opts: ProcOpts;
   #started = false;
   #closing = false;
   #runningJob?: RunningJobInfo = undefined;
-  #proc?: ChildProcess;
+  proc?: ChildProcess;
   #pingInterval?: ReturnType<typeof setInterval>;
+  #memoryWatch?: ReturnType<typeof setInterval>;
   #pongTimeout?: ReturnType<typeof setTimeout>;
-  #init = new Future();
+  protected init = new Future();
   #join = new Future();
   #logger = log().child({ runningJob: this.#runningJob });
 
-  constructor(agent: string, initializeTimeout: number, closeTimeout: number) {
-    super();
+  constructor(
+    initializeTimeout: number,
+    closeTimeout: number,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
+    pingInterval: number,
+    pingTimeout: number,
+    highPingThreshold: number,
+  ) {
     this.#opts = {
-      agent,
       initializeTimeout,
       closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
     };
   }
 
+  abstract createProcess(): ChildProcess;
+  abstract mainTask(child: ChildProcess): Promise<void>;
+
   get started(): boolean {
     return this.#started;
   }
@@ -46,36 +69,50 @@ export class ProcJobExecutor extends JobExecutor {
       throw new Error('runner is closed');
     }
 
-    this.#proc = await import('./job_main.js').then((m) =>
-      m.runProcess({
-        agentFile: this.#opts.agent,
-      }),
-    );
+    this.proc = this.createProcess();
 
     this.#started = true;
     this.run();
   }
 
   async run() {
-    await this.#init.await;
+    await this.init.await;
 
     this.#pingInterval = setInterval(() => {
-      this.#proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
-    }, this.PING_INTERVAL);
+      this.proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+    }, this.#opts.pingInterval);
 
     this.#pongTimeout = setTimeout(() => {
       this.#logger.warn('job is unresponsive');
       clearTimeout(this.#pongTimeout);
       clearInterval(this.#pingInterval);
-      this.#proc!.kill();
+      this.proc!.kill();
       this.#join.resolve();
-    }, this.PING_TIMEOUT);
+    }, this.#opts.pingTimeout);
+
+    this.#memoryWatch = setInterval(() => {
+      const memoryMB = process.memoryUsage().heapUsed / (1024 * 1024);
+      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {
+        this.#logger
+          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })
+          .error('process exceeded memory limit, killing process');
+        this.close();
+      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {
+        this.#logger
+          .child({
+            memoryUsageMB: memoryMB,
+            memoryWarnMB: this.#opts.memoryWarnMB,
+            memoryLimitMB: this.#opts.memoryLimitMB,
+          })
+          .error('process memory usage is high');
+      }
+    });
 
     const listener = (msg: IPCMessage) => {
       switch (msg.case) {
         case 'pongResponse': {
           const delay = Date.now() - msg.value.timestamp;
-          if (delay > this.HIGH_PING_THRESHOLD) {
+          if (delay > this.#opts.highPingThreshold) {
             this.#logger.child({ delay }).warn('job executor is unresponsive');
           }
           this.#pongTimeout?.refresh();
@@ -87,23 +124,26 @@ export class ProcJobExecutor extends JobExecutor {
         }
         case 'done': {
           this.#closing = true;
-          this.#proc!.off('message', listener);
+          this.proc!.off('message', listener);
           this.#join.resolve();
           break;
         }
       }
     };
-    this.#proc!.on('message', listener);
-    this.#proc!.on('error', (err) => {
+    this.proc!.on('message', listener);
+    this.proc!.on('error', (err) => {
       if (this.#closing) return;
       this.#logger
         .child({ err })
         .warn('job process exited unexpectedly; this likely means the error above caused a crash');
       clearTimeout(this.#pongTimeout);
       clearInterval(this.#pingInterval);
+      clearInterval(this.#memoryWatch);
       this.#join.resolve();
     });
 
+    this.mainTask(this.proc!);
+
     await this.#join.await;
   }
 
@@ -118,17 +158,25 @@ export class ProcJobExecutor extends JobExecutor {
   async initialize() {
     const timer = setTimeout(() => {
       const err = new Error('runner initialization timed out');
-      this.#init.reject(err);
+      this.init.reject(err);
       throw err;
     }, this.#opts.initializeTimeout);
-    this.#proc!.send({ case: 'initializeRequest', value: { loggerOptions } });
-    await once(this.#proc!, 'message').then(([msg]: IPCMessage[]) => {
+    this.proc!.send({
+      case: 'initializeRequest',
+      value: {
+        loggerOptions,
+        pingInterval: this.#opts.pingInterval,
+        pingTimeout: this.#opts.pingTimeout,
+        highPingThreshold: this.#opts.highPingThreshold,
+      },
+    });
+    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {
       clearTimeout(timer);
       if (msg!.case !== 'initializeResponse') {
         throw new Error('first message must be InitializeResponse');
       }
     });
-    this.#init.resolve();
+    this.init.resolve();
   }
 
   async close() {
@@ -138,11 +186,11 @@ export class ProcJobExecutor extends JobExecutor {
     this.#closing = true;
 
     if (!this.#runningJob) {
-      this.#proc!.kill();
+      this.proc!.kill();
       this.#join.resolve();
     }
 
-    this.#proc!.send({ case: 'shutdownRequest' });
+    this.proc!.send({ case: 'shutdownRequest' });
 
     const timer = setTimeout(() => {
       this.#logger.error('job shutdown is taking too much time');
@@ -159,6 +207,6 @@ export class ProcJobExecutor extends JobExecutor {
       throw new Error('executor already has a running job');
     }
     this.#runningJob = info;
-    this.#proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
   }
 }

package/src/job.ts

@@ -11,8 +11,21 @@ import type {
 } from '@livekit/rtc-node';
 import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
 import type { Logger } from 'pino';
+import type { InferenceExecutor } from './ipc/inference_executor.js';
 import { log } from './log.js';
 
+export class CurrentJobContext {
+  static #current: JobContext;
+
+  constructor(proc: JobContext) {
+    CurrentJobContext.#current = proc;
+  }
+
+  static getCurrent(): JobContext {
+    return CurrentJobContext.#current;
+  }
+}
+
 /** Which tracks, if any, should the agent automatically subscribe to? */
 export enum AutoSubscribe {
   SUBSCRIBE_ALL,
@@ -60,6 +73,7 @@ export class JobContext {
     };
   } = {};
   #logger: Logger;
+  #inferenceExecutor: InferenceExecutor;
 
   constructor(
     proc: JobProcess,
@@ -67,6 +81,7 @@ export class JobContext {
     room: Room,
     onConnect: () => void,
     onShutdown: (s: string) => void,
+    inferenceExecutor: InferenceExecutor,
   ) {
     this.#proc = proc;
     this.#info = info;
@@ -76,6 +91,7 @@ export class JobContext {
     this.onParticipantConnected = this.onParticipantConnected.bind(this);
     this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
     this.#logger = log().child({ info: this.#info });
+    this.#inferenceExecutor = inferenceExecutor;
   }
 
   get proc(): JobProcess {
@@ -96,6 +112,11 @@ export class JobContext {
     return this.#room.localParticipant;
   }
 
+  /** @returns The global inference executor */
+  get inferenceExecutor(): InferenceExecutor {
+    return this.#inferenceExecutor;
+  }
+
   /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
   addShutdownCallback(callback: () => Promise<void>) {
     this.shutdownCallbacks.push(callback);