@livekit/agents 0.1.0 → 0.2.0

package/src/worker.ts

@@ -1,45 +1,91 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type {
+  JobAssignment,
+  JobTermination,
+  ParticipantInfo,
+  TrackSource,
+} from '@livekit/protocol';
 import {
   type AvailabilityRequest,
-  type Job,
-  type JobAssignment,
   JobType,
   ParticipantPermission,
   ServerMessage,
   WorkerMessage,
+  WorkerStatus,
 } from '@livekit/protocol';
 import { EventEmitter } from 'events';
-import { AccessToken } from 'livekit-server-sdk';
+import { AccessToken, RoomServiceClient } from 'livekit-server-sdk';
 import os from 'os';
 import { WebSocket } from 'ws';
 import { HTTPServer } from './http_server.js';
-import { JobProcess } from './ipc/job_process.js';
-import { type AvailRes, JobRequest } from './job_request.js';
-import type { AcceptData } from './job_request.js';
+import { ProcPool } from './ipc/proc_pool.js';
+import type { JobAcceptArguments, JobProcess, RunningJobInfo } from './job.js';
+import { JobRequest } from './job.js';
 import { log } from './log.js';
+import { Future } from './utils.js';
 import { version } from './version.js';
 
 const MAX_RECONNECT_ATTEMPTS = 10;
-const ASSIGNMENT_TIMEOUT = 15 * 1000;
-const LOAD_INTERVAL = 5 * 1000;
-
-const cpuLoad = (): number =>
-  (os
-    .cpus()
-    .reduce(
-      (acc, x) => acc + x.times.idle / Object.values(x.times).reduce((acc, x) => acc + x, 0),
-      0,
-    ) /
-    os.cpus().length) *
-  100;
-
-class WorkerPermissions {
+const ASSIGNMENT_TIMEOUT = 7.5 * 1000;
+const UPDATE_LOAD_INTERVAL = 2.5 * 1000;
+
+/** Necessary credentials not provided and not found in an appropriate environmental variable. */
+export class MissingCredentialsError extends Error {
+  constructor(msg?: string) {
+    super(msg);
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+/** Worker did not run as expected. */
+export class WorkerError extends Error {
+  constructor(msg?: string) {
+    super(msg);
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+/** @internal */
+export const defaultInitializeProcessFunc = (_: JobProcess) => _;
+const defaultRequestFunc = async (ctx: JobRequest) => {
+  await ctx.accept();
+};
+const defaultCpuLoad = async (): Promise<number> => {
+  return new Promise((resolve) => {
+    const cpus1 = os.cpus();
+
+    setTimeout(() => {
+      const cpus2 = os.cpus();
+
+      let idle = 0;
+      let total = 0;
+
+      for (let i = 0; i < cpus1.length; i++) {
+        const cpu1 = cpus1[i].times;
+        const cpu2 = cpus2[i].times;
+
+        idle += cpu2.idle - cpu1.idle;
+
+        const total1 = Object.values(cpu1).reduce((acc, i) => acc + i, 0);
+        const total2 = Object.values(cpu2).reduce((acc, i) => acc + i, 0);
+
+        total += total2 - total1;
+      }
+
+      resolve(+(1 - idle / total).toFixed(2));
+    }, UPDATE_LOAD_INTERVAL);
+  });
+};
+
+/** Participant permissions to pass to every agent spun up by this worker. */
+export class WorkerPermissions {
   canPublish: boolean;
   canSubscribe: boolean;
   canPublishData: boolean;
   canUpdateMetadata: boolean;
+  canPublishSources: TrackSource[];
   hidden: boolean;
 
   constructor(
@@ -47,21 +93,37 @@ class WorkerPermissions {
     canSubscribe = true,
     canPublishData = true,
     canUpdateMetadata = true,
+    canPublishSources: TrackSource[] = [],
     hidden = false,
   ) {
     this.canPublish = canPublish;
     this.canSubscribe = canSubscribe;
     this.canPublishData = canPublishData;
     this.canUpdateMetadata = canUpdateMetadata;
+    this.canPublishSources = canPublishSources;
     this.hidden = hidden;
   }
 }
 
+/**
+ * Data class describing worker behaviour.
+ *
+ * @remarks
+ * The Agents framework provides sane worker defaults, and works out-of-the-box with no tweaking
+ * necessary. The only mandatory parameter is `agent`, which points to the entry function.
+ *
+ * This class is mostly useful in conjunction with {@link cli.runApp}.
+ */
 export class WorkerOptions {
-  requestFunc: (arg: JobRequest) => Promise<void>;
-  cpuLoadFunc: () => number;
-  namespace: string;
+  agent: string;
+  requestFunc: (job: JobRequest) => Promise<void>;
+  loadFunc: () => Promise<number>;
+  loadThreshold: number;
+  numIdleProcesses: number;
+  shutdownProcessTimeout: number;
+  initializeProcessTimeout: number;
   permissions: WorkerPermissions;
+  agentName: string;
   workerType: JobType;
   maxRetry: number;
   wsURL: string;
@@ -69,24 +131,43 @@ export class WorkerOptions {
   apiSecret?: string;
   host: string;
   port: number;
+  logLevel: string;
 
+  /** @param options */
   constructor({
-    requestFunc,
-    cpuLoadFunc = cpuLoad,
-    namespace = 'default',
+    agent,
+    requestFunc = defaultRequestFunc,
+    loadFunc = defaultCpuLoad,
+    loadThreshold = 0.65,
+    numIdleProcesses = 3,
+    shutdownProcessTimeout = 60 * 1000,
+    initializeProcessTimeout = 10 * 1000,
     permissions = new WorkerPermissions(),
-    workerType = JobType.JT_PUBLISHER,
+    agentName = '',
+    workerType = JobType.JT_ROOM,
     maxRetry = MAX_RECONNECT_ATTEMPTS,
     wsURL = 'ws://localhost:7880',
     apiKey = undefined,
     apiSecret = undefined,
     host = 'localhost',
     port = 8081,
+    logLevel = 'info',
   }: {
-    requestFunc: (arg: JobRequest) => Promise<void>;
-    cpuLoadFunc?: () => number;
-    namespace?: string;
+    /**
+     * Path to a file that has {@link Agent} as a default export, dynamically imported later for
+     * entrypoint and prewarm functions
+     */
+    agent: string;
+    requestFunc?: (job: JobRequest) => Promise<void>;
+    /** Called to determine the current load of the worker. Should return a value between 0 and 1. */
+    loadFunc?: () => Promise<number>;
+    /** When the load exceeds this threshold, the worker will be marked as unavailable. */
+    loadThreshold?: number;
+    numIdleProcesses?: number;
+    shutdownProcessTimeout?: number;
+    initializeProcessTimeout?: number;
     permissions?: WorkerPermissions;
+    agentName?: string;
     workerType?: JobType;
     maxRetry?: number;
     wsURL?: string;
@@ -94,11 +175,17 @@ export class WorkerOptions {
     apiSecret?: string;
     host?: string;
     port?: number;
+    logLevel?: string;
   }) {
+    this.agent = agent;
     this.requestFunc = requestFunc;
-    this.cpuLoadFunc = cpuLoadFunc;
-    this.namespace = namespace;
+    this.loadFunc = loadFunc;
+    this.loadThreshold = loadThreshold;
+    this.numIdleProcesses = numIdleProcesses;
+    this.shutdownProcessTimeout = shutdownProcessTimeout;
+    this.initializeProcessTimeout = initializeProcessTimeout;
     this.permissions = permissions;
+    this.agentName = agentName;
     this.workerType = workerType;
     this.maxRetry = maxRetry;
     this.wsURL = wsURL;
@@ -106,101 +193,119 @@ export class WorkerOptions {
     this.apiSecret = apiSecret;
     this.host = host;
     this.port = port;
+    this.logLevel = logLevel;
   }
 }
 
-class ActiveJob {
-  job: Job;
-  acceptData: AcceptData;
-
-  constructor(job: Job, acceptData: AcceptData) {
-    this.job = job;
-    this.acceptData = acceptData;
-  }
-}
-
-type AssignmentPair = {
-  // this string is the JSON string version of the JobAssignment.
-  // we keep it around to unpack it again in the child, because we can't pass Job directly.
-  raw: string;
-  asgn: JobAssignment;
-};
-
 class PendingAssignment {
-  promise = new Promise<AssignmentPair>((resolve) => {
-    this.resolve = resolve; // oh, JavaScript.
+  promise = new Promise<JobAssignment>((resolve) => {
+    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
   });
-  resolve(arg: AssignmentPair) {
-    arg;
+  resolve(arg: JobAssignment) {
+    arg; // useless call to counteract TypeScript E6133
   }
 }
 
+/**
+ * Central orchestrator for all processes and job requests.
+ *
+ * @remarks
+ * For most usecases, Worker should not be initialized or handled directly; you should instead call
+ * for its creation through {@link cli.runApp}. This could, however, be useful in situations where
+ * you don't have access to a command line, such as a headless program, or one that uses Agents
+ * behind a wrapper.
+ */
 export class Worker {
-  opts: WorkerOptions;
+  #opts: WorkerOptions;
+  #procPool: ProcPool;
+
   #id = 'unregistered';
-  session: WebSocket | undefined = undefined;
-  closed = false;
-  httpServer: HTTPServer;
-  logger = log.child({ version });
+  #closed = true;
+  #draining = false;
+  #connecting = false;
+  #tasks: Promise<void>[] = [];
+  #pending: { [id: string]: PendingAssignment } = {};
+  #close = new Future();
+
   event = new EventEmitter();
-  pending: { [id: string]: { value: PendingAssignment } } = {};
-  processes: { [id: string]: { proc: JobProcess; activeJob: ActiveJob } } = {};
+  #session: WebSocket | undefined = undefined;
+  #httpServer: HTTPServer;
+  #logger = log().child({ version });
 
+  /* @throws {@link MissingCredentialsError} if URL, API key or API secret are missing */
   constructor(opts: WorkerOptions) {
     opts.wsURL = opts.wsURL || process.env.LIVEKIT_URL || '';
     opts.apiKey = opts.apiKey || process.env.LIVEKIT_API_KEY || '';
     opts.apiSecret = opts.apiSecret || process.env.LIVEKIT_API_SECRET || '';
 
-    this.opts = opts;
-    this.httpServer = new HTTPServer(opts.host, opts.port);
-  }
+    if (opts.wsURL === '')
+      throw new MissingCredentialsError('--url is required, or set LIVEKIT_URL env var');
+    if (opts.apiKey === '')
+      throw new MissingCredentialsError('--api-key is required, or set LIVEKIT_API_KEY env var');
+    if (opts.apiSecret === '')
+      throw new MissingCredentialsError(
+        '--api-secret is required, or set LIVEKIT_API_SECRET env var',
+      );
 
-  get id(): string {
-    return this.#id;
+    this.#procPool = new ProcPool(
+      opts.agent,
+      opts.numIdleProcesses,
+      opts.initializeProcessTimeout,
+      opts.shutdownProcessTimeout,
+    );
+
+    this.#opts = opts;
+    this.#httpServer = new HTTPServer(opts.host, opts.port);
   }
 
+  /* @throws {@link WorkerError} if worker failed to connect or already running */
   async run() {
-    this.logger.info('starting worker');
+    if (!this.#closed) {
+      throw new WorkerError('worker is already running');
+    }
 
-    if (this.opts.wsURL === '') throw new Error('--url is required, or set LIVEKIT_URL env var');
-    if (this.opts.apiKey === '')
-      throw new Error('--api-key is required, or set LIVEKIT_API_KEY env var');
-    if (this.opts.apiSecret === '')
-      throw new Error('--api-secret is required, or set LIVEKIT_API_SECRET env var');
+    this.#logger.info('starting worker');
+    this.#closed = false;
+    this.#procPool.start();
 
     const workerWS = async () => {
       let retries = 0;
-      while (!this.closed) {
-        const token = new AccessToken(this.opts.apiKey, this.opts.apiSecret);
-        token.addGrant({ agent: true });
-        const jwt = await token.toJwt();
+      this.#connecting = true;
 
-        const url = new URL(this.opts.wsURL);
+      while (!this.#closed) {
+        const url = new URL(this.#opts.wsURL);
         url.protocol = url.protocol.replace('http', 'ws');
-        this.session = new WebSocket(url + 'agent', {
+        const token = new AccessToken(this.#opts.apiKey, this.#opts.apiSecret);
+        token.addGrant({ agent: true });
+        const jwt = await token.toJwt();
+        this.#session = new WebSocket(url + 'agent', {
           headers: { authorization: 'Bearer ' + jwt },
         });
 
         try {
           await new Promise((resolve, reject) => {
-            this.session!.on('open', resolve);
-            this.session!.on('error', (error) => reject(error));
-            this.session!.on('close', (code) => reject(`WebSocket returned ${code}`));
+            this.#session!.on('open', resolve);
+            this.#session!.on('error', (error) => reject(error));
+            this.#session!.on('close', (code) => reject(`WebSocket returned ${code}`));
           });
 
-          this.runWS(this.session!);
+          retries = 0;
+          this.#logger.debug('connected to LiveKit server');
+          this.#runWS(this.#session);
           return;
         } catch (e) {
-          if (this.closed) return;
-          if (retries >= this.opts.maxRetry) {
-            throw new Error(`failed to connect to LiveKit server after ${retries} attempts: ${e}`);
+          if (this.#closed) return;
+          if (retries >= this.#opts.maxRetry) {
+            throw new WorkerError(
+              `failed to connect to LiveKit server after ${retries} attempts: ${e}`,
+            );
           }
 
           retries++;
           const delay = Math.min(retries * 2, 10);
 
-          this.logger.warn(
-            `failed to connect to LiveKit server, retrying in ${delay} seconds: ${e} (${retries}/${this.opts.maxRetry})`,
+          this.#logger.warn(
+            `failed to connect to LiveKit server, retrying in ${delay} seconds: ${e} (${retries}/${this.#opts.maxRetry})`,
           );
 
           await new Promise((resolve) => setTimeout(resolve, delay * 1000));
@@ -208,24 +313,85 @@ export class Worker {
       }
     };
 
-    await Promise.all([workerWS(), this.httpServer.run()]);
+    await Promise.all([workerWS(), this.#httpServer.run()]);
+    this.#close.resolve();
   }
 
-  startProcess(job: Job, acceptData: AcceptData, raw: string) {
-    const proc = new JobProcess(job, acceptData, raw, this.opts.wsURL);
-    this.processes[job.id] = { proc, activeJob: new ActiveJob(job, acceptData) };
-    proc
-      .run()
-      .catch((e) => {
-        proc.logger.error(`error running job process ${proc.job.id}: ${e}`);
-      })
-      .finally(() => {
-        proc.clear();
-        delete this.processes[job.id];
-      });
+  get id(): string {
+    return this.#id;
+  }
+
+  get activeJobs(): RunningJobInfo[] {
+    return this.#procPool.processes
+      .filter((proc) => proc.runningJob)
+      .map((proc) => proc.runningJob!);
   }
 
-  runWS(ws: WebSocket) {
+  /* @throws {@link WorkerError} if worker did not drain in time */
+  async drain(timeout?: number) {
+    if (this.#draining) {
+      return;
+    }
+
+    this.#logger.info('draining worker');
+    this.#draining = true;
+
+    this.event.emit(
+      'worker_msg',
+      new WorkerMessage({
+        message: {
+          case: 'updateWorker',
+          value: {
+            status: WorkerStatus.WS_FULL,
+          },
+        },
+      }),
+    );
+
+    const joinJobs = async () => {
+      return Promise.all(
+        this.#procPool.processes.map((proc) => {
+          if (!proc.runningJob) {
+            proc.close();
+          }
+          return proc.join();
+        }),
+      );
+    };
+
+    const timer = setTimeout(() => {
+      throw new WorkerError('timed out draining');
+    }, timeout);
+    if (timeout === undefined) clearTimeout(timer);
+    await joinJobs().then(() => {
+      clearTimeout(timer);
+    });
+  }
+
+  async simulateJob(roomName: string, participantIdentity?: string) {
+    const client = new RoomServiceClient(this.#opts.wsURL, this.#opts.apiKey, this.#opts.apiSecret);
+    const room = await client.createRoom({ name: roomName });
+    let participant: ParticipantInfo | undefined = undefined;
+    if (participantIdentity) {
+      participant = await client.getParticipant(roomName, participantIdentity);
+    }
+
+    this.event!.emit(
+      'worker_msg',
+      new WorkerMessage({
+        message: {
+          case: 'simulateJob',
+          value: {
+            type: JobType.JT_PUBLISHER,
+            room,
+            participant,
+          },
+        },
+      }),
+    );
+  }
+
+  #runWS(ws: WebSocket) {
     let closingWS = false;
 
     const send = (msg: WorkerMessage) => {
@@ -239,44 +405,62 @@ export class Worker {
 
     ws.addEventListener('close', () => {
       closingWS = true;
-      this.logger.error('worker connection closed unexpectedly');
+      this.#logger.error('worker connection closed unexpectedly');
       this.close();
     });
 
     ws.addEventListener('message', (event) => {
       if (event.type !== 'message') {
-        this.logger.warn('unexpected message type: ' + event.type);
+        this.#logger.warn('unexpected message type: ' + event.type);
         return;
       }
 
       const msg = new ServerMessage();
       msg.fromBinary(event.data as Uint8Array);
+
+      // register is the only valid first message, and it is only valid as the
+      // first message
+      if (this.#connecting && msg.message.case !== 'register') {
+        throw new WorkerError('expected register response as first message');
+      }
+
       switch (msg.message.case) {
         case 'register': {
           this.#id = msg.message.value.workerId;
-          log
+          this.#logger
             .child({ id: this.id, server_info: msg.message.value.serverInfo })
             .info('registered worker');
+          this.event.emit(
+            'worker_registered',
+            msg.message.value.workerId,
+            msg.message.value.serverInfo!,
+          );
+          this.#connecting = false;
           break;
         }
         case 'availability': {
-          this.availability(msg.message.value);
+          const task = this.#availability(msg.message.value);
+          this.#tasks.push(task);
+          task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
           break;
         }
         case 'assignment': {
           const job = msg.message.value.job!;
-          if (job.id in this.pending) {
-            const task = this.pending[job.id];
-            delete this.pending[job.id];
-            task.value.resolve({
-              asgn: msg.message.value,
-              raw: msg.toJsonString(),
-            });
+          if (job.id in this.#pending) {
+            const task = this.#pending[job.id];
+            delete this.#pending[job.id];
+            task.resolve(msg.message.value);
           } else {
-            log.child({ job }).warn('received assignment for unknown job ' + job.id);
+            this.#logger.child({ job }).warn('received assignment for unknown job ' + job.id);
           }
           break;
         }
+        case 'termination': {
+          const task = this.#termination(msg.message.value);
+          this.#tasks.push(task);
+          task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+          break;
+        }
       }
     });
 
@@ -286,13 +470,14 @@ export class Worker {
         message: {
           case: 'register',
           value: {
-            type: this.opts.workerType,
-            namespace: this.opts.namespace,
+            type: this.#opts.workerType,
+            agentName: this.#opts.agentName,
             allowedPermissions: new ParticipantPermission({
-              canPublish: this.opts.permissions.canPublish,
-              canSubscribe: this.opts.permissions.canSubscribe,
-              canPublishData: this.opts.permissions.canPublishData,
-              hidden: this.opts.permissions.hidden,
+              canPublish: this.#opts.permissions.canPublish,
+              canSubscribe: this.#opts.permissions.canSubscribe,
+              canPublishData: this.#opts.permissions.canPublishData,
+              canUpdateMetadata: this.#opts.permissions.canUpdateMetadata,
+              hidden: this.#opts.permissions.hidden,
               agent: true,
             }),
             version,
@@ -301,84 +486,148 @@ export class Worker {
       }),
     );
 
+    let currentStatus = WorkerStatus.WS_AVAILABLE;
     const loadMonitor = setInterval(() => {
       if (closingWS) clearInterval(loadMonitor);
+
+      const oldStatus = currentStatus;
+      this.#opts.loadFunc().then((currentLoad: number) => {
+        const isFull = currentLoad >= this.#opts.loadThreshold;
+        const currentlyAvailable = !isFull;
+        currentStatus = currentlyAvailable ? WorkerStatus.WS_AVAILABLE : WorkerStatus.WS_FULL;
+
+        if (oldStatus != currentStatus) {
+          const extra = { load: currentLoad, loadThreshold: this.#opts.loadThreshold };
+          if (isFull) {
+            this.#logger.child(extra).info('worker is at full capacity, marking as unavailable');
+          } else {
+            this.#logger.child(extra).info('worker is below capacity, marking as available');
+          }
+        }
+
+        this.event.emit(
+          'worker_msg',
+          new WorkerMessage({
+            message: {
+              case: 'updateWorker',
+              value: {
+                load: currentLoad,
+                status: currentStatus,
+              },
+            },
+          }),
+        );
+      });
+    }, UPDATE_LOAD_INTERVAL);
+  }
+
+  async #availability(msg: AvailabilityRequest) {
+    let answered = false;
+
+    const onReject = async () => {
+      answered = true;
       this.event.emit(
         'worker_msg',
         new WorkerMessage({
           message: {
-            case: 'updateWorker',
+            case: 'availability',
             value: {
-              load: cpuLoad(),
+              jobId: msg.job!.id,
+              available: false,
             },
           },
         }),
       );
-    }, LOAD_INTERVAL);
-  }
+    };
 
-  async availability(msg: AvailabilityRequest) {
-    const tx = new EventEmitter();
-    const req = new JobRequest(msg.job!, tx);
+    const onAccept = async (args: JobAcceptArguments) => {
+      answered = true;
 
-    tx.on('recv', async (av: AvailRes) => {
-      const msg = new WorkerMessage({
-        message: {
-          case: 'availability',
-          value: {
-            available: av.avail,
-            jobId: req.id,
-            participantIdentity: av.data?.identity,
-            participantName: av.data?.name,
-            participantMetadata: av.data?.metadata,
+      this.event.emit(
+        'worker_msg',
+        new WorkerMessage({
+          message: {
+            case: 'availability',
+            value: {
+              jobId: msg.job!.id,
+              available: true,
+              participantIdentity: args.identity,
+              participantName: args.name,
+              participantMetadata: args.metadata,
+            },
           },
-        },
-      });
-
-      this.pending[req.id] = { value: new PendingAssignment() };
-      this.event.emit('worker_msg', msg);
-      if (!av.avail) return;
+        }),
+      );
 
+      this.#pending[req.id] = new PendingAssignment();
       const timer = setTimeout(() => {
-        log.child({ req }).warn(`assignment for job ${req.id} timed out`);
+        this.#logger.child({ req }).warn(`assignment for job ${req.id} timed out`);
         return;
       }, ASSIGNMENT_TIMEOUT);
-      this.pending[req.id].value.promise.then(({ asgn, raw }) => {
+      const asgn = await this.#pending[req.id].promise.then(async (asgn) => {
         clearTimeout(timer);
-        this.startProcess(asgn!.job!, av.data!, raw);
+        return asgn;
       });
-    });
 
-    try {
-      this.opts.requestFunc(req);
-    } catch (e) {
-      log.child({ req }).error(`user request handler for job ${req.id} failed`);
-    } finally {
-      if (!req.answered) {
-        log.child({ req }).error(`no answer for job ${req.id}, automatically rejecting the job`);
-        this.event.emit(
-          'worker_msg',
-          new WorkerMessage({
-            message: {
-              case: 'availability',
-              value: {
-                available: false,
-              },
-            },
-          }),
-        );
+      await this.#procPool.launchJob({
+        acceptArguments: args,
+        job: msg.job!,
+        url: asgn.url || this.#opts.wsURL,
+        token: asgn.token,
+      });
+    };
+
+    const req = new JobRequest(msg.job!, onReject, onAccept);
+    this.#logger
+      .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+      .info('received job request');
+
+    const jobRequestTask = async () => {
+      try {
+        await this.#opts.requestFunc(req);
+      } catch (e) {
+        this.#logger
+          .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+          .info('jobRequestFunc failed');
+        await onReject();
+      }
+
+      if (!answered) {
+        this.#logger
+          .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+          .info('no answer was given inside the jobRequestFunc, automatically rejecting the job');
       }
+    };
+
+    const task = jobRequestTask();
+    this.#tasks.push(task);
+    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+  }
+
+  async #termination(msg: JobTermination) {
+    const proc = this.#procPool.getByJobId(msg.jobId);
+    if (proc === null) {
+      // safe to ignore
+      return;
     }
+    await proc.close();
   }
 
   async close() {
-    if (this.closed) return;
-    this.closed = true;
-    this.logger.debug('shutting down worker');
-    await this.httpServer.close();
-    for await (const value of Object.values(this.processes)) {
-      await value.proc.close();
+    if (this.#closed) {
+      await this.#close.await;
+      return;
     }
-    this.session?.close();
+
+    this.#logger.info('shutting down worker');
+
+    this.#closed = true;
+
+    await this.#procPool.close();
+    await this.#httpServer.close();
+    await Promise.allSettled(this.#tasks);
+
+    this.#session?.close();
+    await this.#close.await;
   }
 }