@livekit/agents 0.6.3 → 0.7.0

package/src/ipc/inference_proc_executor.ts

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { log } from '../log.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { IPCMessage } from './message.js';
+import { SupervisedProc } from './supervised_proc.js';
+
+class PendingInference {
+  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {
+    this.resolve = resolve;
+  });
+  resolve(arg: { requestId: string; data: unknown; error?: Error }) {
+    arg;
+  }
+}
+
+export class InferenceProcExecutor extends SupervisedProc implements InferenceExecutor {
+  #runners: { [id: string]: string };
+  #activeRequests: { [id: string]: PendingInference } = {};
+  #logger = log();
+
+  constructor({
+    runners,
+    initializeTimeout,
+    closeTimeout,
+    memoryWarnMB,
+    memoryLimitMB,
+    pingInterval,
+    pingTimeout,
+    highPingThreshold,
+  }: {
+    runners: { [id: string]: string };
+    initializeTimeout: number;
+    closeTimeout: number;
+    memoryWarnMB: number;
+    memoryLimitMB: number;
+    pingInterval: number;
+    pingTimeout: number;
+    highPingThreshold: number;
+  }) {
+    super(
+      initializeTimeout,
+      closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
+    );
+    this.#runners = runners;
+  }
+
+  createProcess(): ChildProcess {
+    return fork(new URL(import.meta.resolve('./inference_proc_lazy_main.js')), [
+      JSON.stringify(this.#runners),
+    ]);
+  }
+
+  async mainTask(proc: ChildProcess) {
+    proc.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceResponse':
+          const res = this.#activeRequests[msg.value.requestId];
+          delete this.#activeRequests[msg.value.requestId];
+          if (!res) {
+            this.#logger
+              .child({ requestId: msg.value.requestId })
+              .warn('received unexpected inference response');
+            return;
+          }
+
+          res.resolve(msg.value);
+      }
+    });
+  }
+
+  async doInference(method: string, data: unknown): Promise<unknown> {
+    const requestId = 'inference_req_' + randomUUID();
+    const fut = new PendingInference();
+    this.proc!.send({ case: 'inferenceRequest', value: { requestId, method, data } });
+    this.#activeRequests[requestId] = fut;
+
+    const res = await fut.promise;
+    if (res.error) {
+      throw new Error(`inference of ${method} failed: ${res.error}`);
+    }
+    return res.data;
+  }
+}

package/src/ipc/inference_proc_lazy_main.ts

@@ -0,0 +1,86 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import EventEmitter, { once } from 'node:events';
+import type { InferenceRunner } from '../inference_runner.js';
+import { initializeLogger, log } from '../log.js';
+import type { IPCMessage } from './message.js';
+
+const ORPHANED_TIMEOUT = 15 * 1000;
+
+(async () => {
+  if (process.send) {
+    // don't do anything on C-c
+    // this is handled in cli, triggering a termination of all child processes at once.
+    process.on('SIGINT', () => {});
+
+    await once(process, 'message').then(([msg]: IPCMessage[]) => {
+      msg = msg!;
+      if (msg.case !== 'initializeRequest') {
+        throw new Error('first message must be InitializeRequest');
+      }
+      initializeLogger(msg.value.loggerOptions);
+    });
+    const logger = log().child({ pid: process.pid });
+
+    const runners: { [id: string]: InferenceRunner } = await Promise.all(
+      Object.entries(JSON.parse(process.argv[2]!)).map(async ([k, v]) => {
+        return [k, await import(v as string).then((m) => new m.default())];
+      }),
+    ).then(Object.fromEntries);
+
+    await Promise.all(
+      Object.entries(runners).map(async ([runner, v]) => {
+        logger.child({ runner }).debug('initializing inference runner');
+        await v.initialize();
+      }),
+    );
+    logger.debug('all inference runners initialized');
+    process.send({ case: 'initializeResponse' });
+
+    const closeEvent = new EventEmitter();
+
+    const orphanedTimeout = setTimeout(() => {
+      logger.warn('process orphaned, shutting down');
+      process.exit();
+    }, ORPHANED_TIMEOUT);
+
+    const handleInferenceRequest = async ({
+      method,
+      requestId,
+      data,
+    }: {
+      method: string;
+      requestId: string;
+      data: unknown;
+    }) => {
+      if (!runners[method]) {
+        logger.child({ method }).warn('unknown inference method');
+      }
+
+      try {
+        const resp = await runners[method]!.run(data);
+        process.send!({ case: 'inferenceResponse', value: { requestId, data: resp } });
+      } catch (error) {
+        process.send!({ case: 'inferenceResponse', value: { requestId, error } });
+      }
+    };
+
+    process.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'pingRequest':
+          orphanedTimeout.refresh();
+          process.send!({
+            case: 'pongResponse',
+            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },
+          });
+          break;
+        case 'shutdownRequest':
+          closeEvent.emit('close');
+          break;
+        case 'inferenceRequest':
+          handleInferenceRequest(msg.value);
+      }
+    });
+  }
+})();

package/src/ipc/job_executor.ts

@@ -3,23 +3,21 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { RunningJobInfo } from '../job.js';
 
-export interface ProcOpts {
-  agent: string;
-  initializeTimeout: number;
-  closeTimeout: number;
-}
-
-export abstract class JobExecutor {
-  PING_INTERVAL = 2.5 * 1000;
-  PING_TIMEOUT = 90 * 1000;
-  HIGH_PING_THRESHOLD = 0.5 * 1000;
+export interface JobExecutor {
+  started: boolean;
+  userArguments: any;
+  runningJob: RunningJobInfo | undefined;
+  status: JobStatus;
 
-  abstract get started(): boolean;
-  abstract get runningJob(): RunningJobInfo | undefined;
+  start(): Promise<void>;
+  join(): Promise<void>;
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+  launchJob(info: RunningJobInfo): Promise<void>;
+}
 
-  abstract start(): Promise<void>;
-  abstract join(): Promise<void>;
-  abstract initialize(): Promise<void>;
-  abstract close(): Promise<void>;
-  abstract launchJob(info: RunningJobInfo): Promise<void>;
+export enum JobStatus {
+  RUNNING,
+  FAILED,
+  SUCCESS,
 }

package/src/ipc/job_proc_executor.ts

@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import type { RunningJobInfo } from '../job.js';
+import { log } from '../log.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { JobExecutor } from './job_executor.js';
+import { JobStatus } from './job_executor.js';
+import type { IPCMessage } from './message.js';
+import { SupervisedProc } from './supervised_proc.js';
+
+export class JobProcExecutor extends SupervisedProc implements JobExecutor {
+  #userArgs?: any;
+  #jobStatus?: JobStatus;
+  #runningJob?: RunningJobInfo;
+  #agent: string;
+  #inferenceExecutor?: InferenceExecutor;
+  #inferenceTasks: Promise<void>[] = [];
+  #logger = log();
+
+  constructor(
+    agent: string,
+    inferenceExecutor: InferenceExecutor | undefined,
+    initializeTimeout: number,
+    closeTimeout: number,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
+    pingInterval: number,
+    pingTimeout: number,
+    highPingThreshold: number,
+  ) {
+    super(
+      initializeTimeout,
+      closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
+    );
+    this.#agent = agent;
+    this.#inferenceExecutor = inferenceExecutor;
+  }
+
+  get status(): JobStatus {
+    if (this.#jobStatus) {
+      return this.#jobStatus;
+    }
+    throw new Error('job status not available');
+  }
+
+  get userArguments(): any {
+    return this.#userArgs;
+  }
+
+  set userArguments(args: any) {
+    this.#userArgs = args;
+  }
+
+  get runningJob(): RunningJobInfo | undefined {
+    return this.#runningJob;
+  }
+
+  createProcess(): ChildProcess {
+    return fork(new URL(import.meta.resolve('./job_proc_lazy_main.js')), [this.#agent]);
+  }
+
+  async mainTask(proc: ChildProcess) {
+    proc.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceRequest':
+          this.#inferenceTasks.push(this.#doInferenceTask(proc, msg.value));
+      }
+    });
+  }
+
+  async #doInferenceTask(
+    proc: ChildProcess,
+    req: { method: string; requestId: string; data: unknown },
+  ) {
+    if (!this.#inferenceExecutor) {
+      this.#logger.warn('inference request received but no inference executor');
+      proc.send({
+        case: 'inferenceResponse',
+        value: { requestId: req.requestId, error: new Error('no inference executor') },
+      });
+      return;
+    }
+
+    try {
+      const data = await this.#inferenceExecutor.doInference(req.method, req.data);
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, data } });
+    } catch (error) {
+      proc.send({ case: 'inferenceResponse', value: { requestId: req.requestId, error } });
+    }
+  }
+
+  async launchJob(info: RunningJobInfo) {
+    if (this.#runningJob) {
+      throw Error('process already has a running job');
+    }
+    if (!this.init.done) {
+      throw Error('process not initialized');
+    }
+    this.#jobStatus = JobStatus.RUNNING;
+    this.#runningJob = info;
+
+    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+  }
+}

package/src/ipc/{job_main.ts → job_proc_lazy_main.ts}

@@ -2,34 +2,63 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { Room, RoomEvent } from '@livekit/rtc-node';
-import type { ChildProcess } from 'node:child_process';
-import { fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
 import { EventEmitter, once } from 'node:events';
 import { pathToFileURL } from 'node:url';
 import type { Logger } from 'pino';
 import { type Agent, isAgent } from '../generator.js';
-import type { RunningJobInfo } from '../job.js';
-import { JobContext } from '../job.js';
-import { JobProcess } from '../job.js';
+import { CurrentJobContext, JobContext, JobProcess, type RunningJobInfo } from '../job.js';
 import { initializeLogger, log } from '../log.js';
 import { defaultInitializeProcessFunc } from '../worker.js';
+import type { InferenceExecutor } from './inference_executor.js';
 import type { IPCMessage } from './message.js';
 
 const ORPHANED_TIMEOUT = 15 * 1000;
 
-type StartArgs = {
-  agentFile: string;
-  // userArguments: unknown;
-};
-
 type JobTask = {
   ctx: JobContext;
   task: Promise<void>;
 };
 
-export const runProcess = (args: StartArgs): ChildProcess => {
-  return fork(new URL(import.meta.url), [args.agentFile]);
-};
+class PendingInference {
+  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {
+    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
+  });
+  resolve(arg: { requestId: string; data: unknown; error?: Error }) {
+    arg; // useless call to counteract TypeScript E6133
+  }
+}
+
+class InfClient implements InferenceExecutor {
+  #requests: { [id: string]: PendingInference } = {};
+
+  constructor() {
+    process.on('message', (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'inferenceResponse':
+          const fut = this.#requests[msg.value.requestId];
+          delete this.#requests[msg.value.requestId];
+          if (!fut) {
+            log().child({ resp: msg.value }).warn('received unexpected inference response');
+            return;
+          }
+          fut.resolve(msg.value);
+          break;
+      }
+    });
+  }
+
+  async doInference(method: string, data: unknown): Promise<unknown> {
+    const requestId = 'inference_job_' + randomUUID;
+    process.send!({ case: 'inferenceRequest', value: { requestId, method, data } });
+    this.#requests[requestId] = new PendingInference();
+    const resp = await this.#requests[requestId]!.promise;
+    if (resp.error) {
+      throw new Error(`inference of ${method} failed: ${resp.error.message}`);
+    }
+    return resp.data;
+  }
+}
 
 const startJob = (
   proc: JobProcess,
@@ -54,7 +83,8 @@ const startJob = (
     closeEvent.emit('close', true, reason);
   };
 
-  const ctx = new JobContext(proc, info, room, onConnect, onShutdown);
+  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());
+  new CurrentJobContext(ctx);
 
   const task = new Promise<void>(async () => {
     const unconnectedTimeout = setTimeout(() => {

package/src/ipc/message.ts

@@ -7,7 +7,12 @@ import type { LoggerOptions } from '../log.js';
 export type IPCMessage =
   | {
       case: 'initializeRequest';
-      value: { loggerOptions: LoggerOptions };
+      value: {
+        loggerOptions: LoggerOptions;
+        pingInterval?: number;
+        pingTimeout?: number;
+        highPingThreshold?: number;
+      };
     }
   | {
       case: 'initializeResponse';
@@ -29,6 +34,14 @@ export type IPCMessage =
       case: 'shutdownRequest';
       value: { reason?: string };
     }
+  | {
+      case: 'inferenceRequest';
+      value: { method: string; requestId: string; data: unknown };
+    }
+  | {
+      case: 'inferenceResponse';
+      value: { requestId: string; data: unknown; error?: Error };
+    }
   | {
       case: 'exiting';
       value: { reason?: string };

package/src/ipc/proc_pool.ts

@@ -4,8 +4,9 @@
 import { MultiMutex, Mutex } from '@livekit/mutex';
 import type { RunningJobInfo } from '../job.js';
 import { Queue } from '../utils.js';
+import type { InferenceExecutor } from './inference_executor.js';
 import type { JobExecutor } from './job_executor.js';
-import { ProcJobExecutor } from './proc_job_executor.js';
+import { JobProcExecutor } from './job_proc_executor.js';
 
 export class ProcPool {
   agent: string;
@@ -20,12 +21,18 @@ export class ProcPool {
   procMutex?: MultiMutex;
   procUnlock?: () => void;
   warmedProcQueue = new Queue<JobExecutor>();
+  inferenceExecutor?: InferenceExecutor;
+  memoryWarnMB: number;
+  memoryLimitMB: number;
 
   constructor(
     agent: string,
     numIdleProcesses: number,
     initializeTimeout: number,
     closeTimeout: number,
+    inferenceExecutor: InferenceExecutor | undefined,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
   ) {
     this.agent = agent;
     if (numIdleProcesses > 0) {
@@ -33,6 +40,9 @@ export class ProcPool {
     }
     this.initializeTimeout = initializeTimeout;
     this.closeTimeout = closeTimeout;
+    this.inferenceExecutor = inferenceExecutor;
+    this.memoryWarnMB = memoryWarnMB;
+    this.memoryLimitMB = memoryLimitMB;
   }
 
   get processes(): JobExecutor[] {
@@ -52,7 +62,17 @@ export class ProcPool {
         this.procUnlock = undefined;
       }
     } else {
-      proc = new ProcJobExecutor(this.agent, this.initializeTimeout, this.closeTimeout);
+      proc = new JobProcExecutor(
+        this.agent,
+        this.inferenceExecutor,
+        this.initializeTimeout,
+        this.closeTimeout,
+        this.memoryWarnMB,
+        this.memoryLimitMB,
+        2500,
+        60000,
+        500,
+      );
       this.executors.push(proc);
       await proc.start();
       await proc.initialize();
@@ -61,7 +81,17 @@ export class ProcPool {
   }
 
   async procWatchTask() {
-    const proc = new ProcJobExecutor(this.agent, this.initializeTimeout, this.closeTimeout);
+    const proc = new JobProcExecutor(
+      this.agent,
+      this.inferenceExecutor,
+      this.initializeTimeout,
+      this.closeTimeout,
+      this.memoryWarnMB,
+      this.memoryLimitMB,
+      2500,
+      60000,
+      500,
+    );
 
     try {
       this.executors.push(proc);