@livekit/agents 1.2.1 → 1.2.2

package/dist/ipc/job_proc_lazy_main.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            log().child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      throw new Error('IPC channel closed');\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":"AAGA,SAAS,MAAM,WAAW,eAAe;AACzC,SAAS,cAAc,YAAY;AACnC,SAAS,qBAAqB;AAE9B,SAAqB,eAAe;AACpC,SAAS,YAAY,YAAiC,8BAA8B;AACpF,SAAS,kBAAkB,WAAW;AACtC,SAAS,QAAQ,iBAAiB;AAClC,SAAS,oCAAoC;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,UAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,UAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EAEjD,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,gBAAI,EAAE,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AAC9E;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,YAAY,UAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,KAAK;AACtB,OAAK,GAAG,UAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,WAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,YAAM,uBAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AA/H1B;AAgIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,YAAM,KAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,OAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,OAAO,cAAc,UAAW,EAAE,UAAU,KAAK,CAAC,WAAW;AA5L5F;AA8LM,YAAMA,SACJ,OAAO,OAAO,YAAY,cAAc,QAAQ,OAAO,OAAO,IAC1D,OAAO,WACP,YAAO,YAAP,mBAAgB;AACtB,UAAIA,WAAU,UAAa,CAAC,QAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,UAAM,KAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uBAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,WAAW;AAC5B,QAAI,SAAS,IAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,aAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,YAAM,QAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["agent"]}
+{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n  #logger = log();\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            this.#logger.child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      this.#logger.debug(\n        { method, requestId },\n        'IPC channel closed during inference, aborting gracefully',\n      );\n      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":"AAGA,SAAS,MAAM,WAAW,eAAe;AACzC,SAAS,cAAc,YAAY;AACnC,SAAS,qBAAqB;AAE9B,SAAqB,eAAe;AACpC,SAAS,YAAY,YAAiC,8BAA8B;AACpF,SAAS,kBAAkB,WAAW;AACtC,SAAS,QAAQ,iBAAiB;AAClC,SAAS,oCAAoC;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,UAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,UAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EACjD,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AACrF;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,YAAY,UAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,WAAK,QAAQ;AAAA,QACX,EAAE,QAAQ,UAAU;AAAA,QACpB;AAAA,MACF;AACA,YAAM,IAAI,MAAM,aAAa,MAAM,yDAAyD;AAAA,IAC9F;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,KAAK;AACtB,OAAK,GAAG,UAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,WAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,YAAM,uBAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AApI1B;AAqIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,YAAM,KAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,OAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,OAAO,cAAc,UAAW,EAAE,UAAU,KAAK,CAAC,WAAW;AAjM5F;AAmMM,YAAMA,SACJ,OAAO,OAAO,YAAY,cAAc,QAAQ,OAAO,OAAO,IAC1D,OAAO,WACP,YAAO,YAAP,mBAAgB;AACtB,UAAIA,WAAU,UAAa,CAAC,QAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,UAAM,KAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uBAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,WAAW;AAC5B,QAAI,SAAS,IAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,aAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,YAAM,QAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["agent"]}

package/dist/ipc/supervised_proc.cjs

@@ -76,7 +76,10 @@ class SupervisedProc {
     }
     this.proc = this.createProcess();
     this.#started = true;
-    this.run();
+    this.run().catch((err) => {
+      this.#logger.child({ err }).warn("supervised process run failed");
+      this.#join.resolve();
+    });
   }
   async run() {
     await this.init.await;

package/dist/ipc/supervised_proc.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run();\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAAqB;AACrB,sBAAqB;AAErB,iBAAmC;AACnC,mBAAuB;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,oBAAO;AAAA,EAC5B,QAAQ,IAAI,oBAAO;AAAA,EACnB,cAAU,gBAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI;AAAA,EACX;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AA5F3C;AA6FM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AA5H1C;AA6HM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AA7KrB;AA8KI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,mBAAe,0BAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,cAAM,yBAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AAvMhB;AAwMI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AA3NxC;AA4NI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AAtOzD;AAuOI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":["pidusage"]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run().catch((err) => {\n      this.#logger.child({ err }).warn('supervised process run failed');\n      // Note: we intentionally do NOT kill the child process here. Killing it\n      // would race with initialize()'s `once(proc, 'message')`, causing\n      // initialize() to hang forever and deadlocking the caller (proc_pool).\n      // The child process is cleaned up when the pool shuts down.\n      this.#join.resolve();\n    });\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAAqB;AACrB,sBAAqB;AAErB,iBAAmC;AACnC,mBAAuB;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,oBAAO;AAAA,EAC5B,QAAQ,IAAI,oBAAO;AAAA,EACnB,cAAU,gBAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI,EAAE,MAAM,CAAC,QAAQ;AACxB,WAAK,QAAQ,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,+BAA+B;AAKhE,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AAnG3C;AAoGM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AAnI1C;AAoIM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AApLrB;AAqLI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,mBAAe,0BAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,cAAM,yBAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AA9MhB;AA+MI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AAlOxC;AAmOI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AA7OzD;AA8OI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":["pidusage"]}

package/dist/ipc/supervised_proc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"supervised_proc.d.ts","sourceRoot":"","sources":["../../src/ipc/supervised_proc.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,QAAQ;IACvB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,8BAAsB,cAAc;;IAKlC,IAAI,CAAC,EAAE,YAAY,CAAC;IAIpB,SAAS,CAAC,IAAI,eAAgB;gBAK5B,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,iBAAiB,EAAE,MAAM;IAa3B,QAAQ,CAAC,aAAa,IAAI,YAAY;IACtC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAErD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,UAAU,IAAI,cAAc,GAAG,SAAS,CAE3C;IAEK,KAAK;IAaL,GAAG;IA4EH,IAAI;IAQJ,UAAU;IA0BV,KAAK;IAoBL,SAAS,CAAC,IAAI,EAAE,cAAc;YAWtB,qBAAqB;IAiBnC,OAAO,CAAC,WAAW;CAKpB"}
+{"version":3,"file":"supervised_proc.d.ts","sourceRoot":"","sources":["../../src/ipc/supervised_proc.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,QAAQ;IACvB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,8BAAsB,cAAc;;IAKlC,IAAI,CAAC,EAAE,YAAY,CAAC;IAIpB,SAAS,CAAC,IAAI,eAAgB;gBAK5B,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,iBAAiB,EAAE,MAAM;IAa3B,QAAQ,CAAC,aAAa,IAAI,YAAY;IACtC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAErD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,UAAU,IAAI,cAAc,GAAG,SAAS,CAE3C;IAEK,KAAK;IAoBL,GAAG;IA4EH,IAAI;IAQJ,UAAU;IA0BV,KAAK;IAoBL,SAAS,CAAC,IAAI,EAAE,cAAc;YAWtB,qBAAqB;IAiBnC,OAAO,CAAC,WAAW;CAKpB"}

package/dist/ipc/supervised_proc.js

@@ -43,7 +43,10 @@ class SupervisedProc {
     }
     this.proc = this.createProcess();
     this.#started = true;
-    this.run();
+    this.run().catch((err) => {
+      this.#logger.child({ err }).warn("supervised process run failed");
+      this.#join.resolve();
+    });
   }
   async run() {
     await this.init.await;

package/dist/ipc/supervised_proc.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run();\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":"AAIA,SAAS,YAAY;AACrB,OAAO,cAAc;AAErB,SAAS,KAAK,qBAAqB;AACnC,SAAS,cAAc;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,OAAO;AAAA,EAC5B,QAAQ,IAAI,OAAO;AAAA,EACnB,UAAU,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI;AAAA,EACX;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AA5F3C;AA6FM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AA5H1C;AA6HM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AA7KrB;AA8KI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,eAAe,cAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,UAAM,KAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AAvMhB;AAwMI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AA3NxC;AA4NI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AAtOzD;AAuOI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run().catch((err) => {\n      this.#logger.child({ err }).warn('supervised process run failed');\n      // Note: we intentionally do NOT kill the child process here. Killing it\n      // would race with initialize()'s `once(proc, 'message')`, causing\n      // initialize() to hang forever and deadlocking the caller (proc_pool).\n      // The child process is cleaned up when the pool shuts down.\n      this.#join.resolve();\n    });\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":"AAIA,SAAS,YAAY;AACrB,OAAO,cAAc;AAErB,SAAS,KAAK,qBAAqB;AACnC,SAAS,cAAc;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,OAAO;AAAA,EAC5B,QAAQ,IAAI,OAAO;AAAA,EACnB,UAAU,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI,EAAE,MAAM,CAAC,QAAQ;AACxB,WAAK,QAAQ,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,+BAA+B;AAKhE,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AAnG3C;AAoGM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AAnI1C;AAoIM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AApLrB;AAqLI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,eAAe,cAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,UAAM,KAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AA9MhB;AA+MI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AAlOxC;AAmOI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AA7OzD;AA8OI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":[]}

package/dist/ipc/supervised_proc.test.cjs

@@ -119,6 +119,88 @@ async function getChildMemoryUsageMB(pid) {
     (0, import_vitest.expect)(skipped).toBeGreaterThan(0);
   });
 });
+(0, import_vitest.describe)("init timeout rejection handling", () => {
+  (0, import_vitest.it)("does not produce unhandled rejection when init times out", async () => {
+    var _a;
+    const unhandled = [];
+    const handler = (reason) => unhandled.push(reason);
+    process.on("unhandledRejection", handler);
+    const slowScript = (0, import_node_path.join)((0, import_node_os.tmpdir)(), "test_slow_init_child.mjs");
+    (0, import_node_fs.writeFileSync)(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`
+    );
+    const { SupervisedProc } = await import("./supervised_proc.cjs");
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return (0, import_node_child_process.fork)(slowScript, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
+      }
+      async mainTask() {
+      }
+    }
+    const proc = new TestProc(
+      50,
+      // initializeTimeout — fires before child responds at 200ms
+      1e3,
+      // closeTimeout
+      0,
+      // memoryWarnMB
+      0,
+      // memoryLimitMB
+      5e3,
+      // pingInterval
+      6e4,
+      // pingTimeout
+      2500
+      // highPingThreshold
+    );
+    await proc.start();
+    await proc.initialize();
+    await new Promise((r) => setTimeout(r, 100));
+    process.off("unhandledRejection", handler);
+    (_a = proc.proc) == null ? void 0 : _a.kill();
+    try {
+      (0, import_node_fs.unlinkSync)(slowScript);
+    } catch {
+    }
+    (0, import_vitest.expect)(unhandled).toEqual([]);
+  });
+  (0, import_vitest.it)("join() resolves after init timeout instead of hanging forever", async () => {
+    var _a;
+    const slowScript = (0, import_node_path.join)((0, import_node_os.tmpdir)(), "test_slow_init_child_join.mjs");
+    (0, import_node_fs.writeFileSync)(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`
+    );
+    const { SupervisedProc } = await import("./supervised_proc.cjs");
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return (0, import_node_child_process.fork)(slowScript, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
+      }
+      async mainTask() {
+      }
+    }
+    const proc = new TestProc(50, 1e3, 0, 0, 5e3, 6e4, 2500);
+    await proc.start();
+    await proc.initialize();
+    const result = await Promise.race([
+      proc.join().then(() => "resolved"),
+      new Promise((r) => setTimeout(() => r("timeout"), 2e3))
+    ]);
+    (_a = proc.proc) == null ? void 0 : _a.kill();
+    try {
+      (0, import_node_fs.unlinkSync)(slowScript);
+    } catch {
+    }
+    (0, import_vitest.expect)(result).toBe("resolved");
+  });
+});
 (0, import_vitest.describe)("timer cleanup", () => {
   (0, import_vitest.it)("clearInterval stops the interval", async () => {
     let count = 0;

package/dist/ipc/supervised_proc.test.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { fork, spawn } from 'node:child_process';\nimport { unlinkSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport { join } from 'node:path';\nimport pidusage from 'pidusage';\nimport { afterAll, beforeAll, describe, expect, it } from 'vitest';\n\nconst childScript = join(tmpdir(), 'test_child.mjs');\n\nbeforeAll(() => {\n  writeFileSync(\n    childScript,\n    `process.on('message', (msg) => process.send?.({ echo: msg }));\n     setInterval(() => {}, 1000);`,\n  );\n});\n\nafterAll(() => {\n  try {\n    unlinkSync(childScript);\n  } catch {}\n});\n\nasync function getChildMemoryUsageMB(pid: number | undefined): Promise<number> {\n  if (!pid) return 0;\n  try {\n    const stats = await pidusage(pid);\n    return stats.memory / (1024 * 1024);\n  } catch (err) {\n    const code = (err as NodeJS.ErrnoException).code;\n    if (code === 'ENOENT' || code === 'ESRCH') {\n      return 0;\n    }\n    throw err;\n  }\n}\n\ndescribe('pidusage on dead process', () => {\n  it('raw pidusage throws on dead pid', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    await expect(pidusage(pid)).rejects.toThrow();\n  });\n\n  it('fixed version returns 0 instead of crashing', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    const mem = await getChildMemoryUsageMB(pid);\n    expect(mem).toBe(0);\n  });\n\n  it('handles concurrent calls on dying process', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    child.kill('SIGKILL');\n\n    const results = await Promise.all([\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n    ]);\n\n    await exitPromise;\n    expect(results.every((r) => r === 0)).toBe(true);\n  });\n});\n\ndescribe('IPC send on dead process', () => {\n  it('child.connected becomes false when child dies', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    await new Promise((r) => setTimeout(r, 50));\n    expect(child.connected).toBe(true);\n\n    child.kill('SIGKILL');\n    await exitPromise;\n\n    expect(child.connected).toBe(false);\n  });\n\n  it('checking connected before send prevents crash', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    // Suppress EPIPE errors that can occur due to race conditions between\n    // child.connected check and the actual pipe state\n    child.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code !== 'EPIPE') throw err;\n    });\n\n    let sent = 0;\n    let skipped = 0;\n\n    const interval = setInterval(() => {\n      if (child.connected) {\n        child.send({ ping: Date.now() });\n        sent++;\n      } else {\n        skipped++;\n      }\n    }, 20);\n\n    await new Promise((r) => setTimeout(r, 60));\n    child.kill('SIGKILL');\n    await exitPromise;\n    await new Promise((r) => setTimeout(r, 80));\n    clearInterval(interval);\n\n    expect(sent).toBeGreaterThan(0);\n    expect(skipped).toBeGreaterThan(0);\n  });\n});\n\ndescribe('timer cleanup', () => {\n  it('clearInterval stops the interval', async () => {\n    let count = 0;\n    const interval = setInterval(() => count++, 30);\n\n    await new Promise((r) => setTimeout(r, 80));\n    const countAtClear = count;\n    clearInterval(interval);\n\n    await new Promise((r) => setTimeout(r, 80));\n    expect(count).toBe(countAtClear);\n  });\n\n  it('double clear is safe', () => {\n    const interval = setInterval(() => {}, 100);\n    const timeout = setTimeout(() => {}, 1000);\n\n    clearInterval(interval);\n    clearTimeout(timeout);\n\n    expect(() => {\n      clearInterval(interval);\n      clearTimeout(timeout);\n    }).not.toThrow();\n  });\n});\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGA,gCAA4B;AAC5B,qBAA0C;AAC1C,qBAAuB;AACvB,uBAAqB;AACrB,sBAAqB;AACrB,oBAA0D;AAE1D,MAAM,kBAAc,2BAAK,uBAAO,GAAG,gBAAgB;AAAA,IAEnD,yBAAU,MAAM;AACd;AAAA,IACE;AAAA,IACA;AAAA;AAAA,EAEF;AACF,CAAC;AAAA,IAED,wBAAS,MAAM;AACb,MAAI;AACF,mCAAW,WAAW;AAAA,EACxB,QAAQ;AAAA,EAAC;AACX,CAAC;AAED,eAAe,sBAAsB,KAA0C;AAC7E,MAAI,CAAC,IAAK,QAAO;AACjB,MAAI;AACF,UAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,WAAO,MAAM,UAAU,OAAO;AAAA,EAChC,SAAS,KAAK;AACZ,UAAM,OAAQ,IAA8B;AAC5C,QAAI,SAAS,YAAY,SAAS,SAAS;AACzC,aAAO;AAAA,IACT;AACA,UAAM;AAAA,EACR;AACF;AAAA,IAEA,wBAAS,4BAA4B,MAAM;AACzC,wBAAG,mCAAmC,YAAY;AAChD,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,cAAM,0BAAO,gBAAAA,SAAS,GAAG,CAAC,EAAE,QAAQ,QAAQ;AAAA,EAC9C,CAAC;AAED,wBAAG,+CAA+C,YAAY;AAC5D,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,MAAM,MAAM,sBAAsB,GAAG;AAC3C,8BAAO,GAAG,EAAE,KAAK,CAAC;AAAA,EACpB,CAAC;AAED,wBAAG,6CAA6C,YAAY;AAC1D,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAClB,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,KAAK,SAAS;AAEpB,UAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAChC,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,IAC3B,CAAC;AAED,UAAM;AACN,8BAAO,QAAQ,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,4BAA4B,MAAM;AACzC,wBAAG,iDAAiD,YAAY;AAC9D,UAAM,YAAQ,gCAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,8BAAO,MAAM,SAAS,EAAE,KAAK,IAAI;AAEjC,UAAM,KAAK,SAAS;AACpB,UAAM;AAEN,8BAAO,MAAM,SAAS,EAAE,KAAK,KAAK;AAAA,EACpC,CAAC;AAED,wBAAG,iDAAiD,YAAY;AAC9D,UAAM,YAAQ,gCAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAIhE,UAAM,GAAG,SAAS,CAAC,QAA+B;AAChD,UAAI,IAAI,SAAS,QAAS,OAAM;AAAA,IAClC,CAAC;AAED,QAAI,OAAO;AACX,QAAI,UAAU;AAEd,UAAM,WAAW,YAAY,MAAM;AACjC,UAAI,MAAM,WAAW;AACnB,cAAM,KAAK,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;AAC/B;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF,GAAG,EAAE;AAEL,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,KAAK,SAAS;AACpB,UAAM;AACN,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,kBAAc,QAAQ;AAEtB,8BAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,8BAAO,OAAO,EAAE,gBAAgB,CAAC;AAAA,EACnC,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,iBAAiB,MAAM;AAC9B,wBAAG,oCAAoC,YAAY;AACjD,QAAI,QAAQ;AACZ,UAAM,WAAW,YAAY,MAAM,SAAS,EAAE;AAE9C,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,eAAe;AACrB,kBAAc,QAAQ;AAEtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,8BAAO,KAAK,EAAE,KAAK,YAAY;AAAA,EACjC,CAAC;AAED,wBAAG,wBAAwB,MAAM;AAC/B,UAAM,WAAW,YAAY,MAAM;AAAA,IAAC,GAAG,GAAG;AAC1C,UAAM,UAAU,WAAW,MAAM;AAAA,IAAC,GAAG,GAAI;AAEzC,kBAAc,QAAQ;AACtB,iBAAa,OAAO;AAEpB,8BAAO,MAAM;AACX,oBAAc,QAAQ;AACtB,mBAAa,OAAO;AAAA,IACtB,CAAC,EAAE,IAAI,QAAQ;AAAA,EACjB,CAAC;AACH,CAAC;","names":["pidusage"]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { fork, spawn } from 'node:child_process';\nimport { unlinkSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport { join } from 'node:path';\nimport pidusage from 'pidusage';\nimport { afterAll, beforeAll, describe, expect, it } from 'vitest';\n\nconst childScript = join(tmpdir(), 'test_child.mjs');\n\nbeforeAll(() => {\n  writeFileSync(\n    childScript,\n    `process.on('message', (msg) => process.send?.({ echo: msg }));\n     setInterval(() => {}, 1000);`,\n  );\n});\n\nafterAll(() => {\n  try {\n    unlinkSync(childScript);\n  } catch {}\n});\n\nasync function getChildMemoryUsageMB(pid: number | undefined): Promise<number> {\n  if (!pid) return 0;\n  try {\n    const stats = await pidusage(pid);\n    return stats.memory / (1024 * 1024);\n  } catch (err) {\n    const code = (err as NodeJS.ErrnoException).code;\n    if (code === 'ENOENT' || code === 'ESRCH') {\n      return 0;\n    }\n    throw err;\n  }\n}\n\ndescribe('pidusage on dead process', () => {\n  it('raw pidusage throws on dead pid', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    await expect(pidusage(pid)).rejects.toThrow();\n  });\n\n  it('fixed version returns 0 instead of crashing', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    const mem = await getChildMemoryUsageMB(pid);\n    expect(mem).toBe(0);\n  });\n\n  it('handles concurrent calls on dying process', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    child.kill('SIGKILL');\n\n    const results = await Promise.all([\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n    ]);\n\n    await exitPromise;\n    expect(results.every((r) => r === 0)).toBe(true);\n  });\n});\n\ndescribe('IPC send on dead process', () => {\n  it('child.connected becomes false when child dies', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    await new Promise((r) => setTimeout(r, 50));\n    expect(child.connected).toBe(true);\n\n    child.kill('SIGKILL');\n    await exitPromise;\n\n    expect(child.connected).toBe(false);\n  });\n\n  it('checking connected before send prevents crash', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    // Suppress EPIPE errors that can occur due to race conditions between\n    // child.connected check and the actual pipe state\n    child.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code !== 'EPIPE') throw err;\n    });\n\n    let sent = 0;\n    let skipped = 0;\n\n    const interval = setInterval(() => {\n      if (child.connected) {\n        child.send({ ping: Date.now() });\n        sent++;\n      } else {\n        skipped++;\n      }\n    }, 20);\n\n    await new Promise((r) => setTimeout(r, 60));\n    child.kill('SIGKILL');\n    await exitPromise;\n    await new Promise((r) => setTimeout(r, 80));\n    clearInterval(interval);\n\n    expect(sent).toBeGreaterThan(0);\n    expect(skipped).toBeGreaterThan(0);\n  });\n});\n\ndescribe('init timeout rejection handling', () => {\n  it('does not produce unhandled rejection when init times out', async () => {\n    // Regression test: before the fix, run() was called without await in start().\n    // When init timed out, the rejection in run()'s `await this.init.await` escaped\n    // as an unhandled rejection — crashing the Node.js process.\n    const unhandled: unknown[] = [];\n    const handler = (reason: unknown) => unhandled.push(reason);\n    process.on('unhandledRejection', handler);\n\n    // Child that responds AFTER the timeout — simulates slow init under CPU pressure.\n    // Timeout fires at 50ms (init.reject), child responds at 200ms (once() resolves).\n    // Before the fix, init.reject caused an unhandled rejection in run().\n    const slowScript = join(tmpdir(), 'test_slow_init_child.mjs');\n    writeFileSync(\n      slowScript,\n      `process.on('message', () => {\n        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);\n      });\n      setInterval(() => {}, 1000);`,\n    );\n\n    const { SupervisedProc } = await import('./supervised_proc.js');\n    class TestProc extends SupervisedProc {\n      createProcess() {\n        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n      }\n      async mainTask() {}\n    }\n\n    const proc = new TestProc(\n      50, // initializeTimeout — fires before child responds at 200ms\n      1000, // closeTimeout\n      0, // memoryWarnMB\n      0, // memoryLimitMB\n      5000, // pingInterval\n      60000, // pingTimeout\n      2500, // highPingThreshold\n    );\n\n    await proc.start();\n    // initialize() returns normally: child responds at 200ms, once() resolves,\n    // but init was already rejected at 50ms — run() gets the rejection.\n    await proc.initialize();\n\n    // Give the event loop a tick for any unhandled rejection to surface\n    await new Promise((r) => setTimeout(r, 100));\n\n    process.off('unhandledRejection', handler);\n    proc.proc?.kill();\n    try {\n      unlinkSync(slowScript);\n    } catch {}\n\n    expect(unhandled).toEqual([]);\n  });\n\n  it('join() resolves after init timeout instead of hanging forever', async () => {\n    // When run() fails early (before registering proc event handlers),\n    // #join must still resolve so that join() and close() don't hang.\n    const slowScript = join(tmpdir(), 'test_slow_init_child_join.mjs');\n    writeFileSync(\n      slowScript,\n      `process.on('message', () => {\n        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);\n      });\n      setInterval(() => {}, 1000);`,\n    );\n\n    const { SupervisedProc } = await import('./supervised_proc.js');\n    class TestProc extends SupervisedProc {\n      createProcess() {\n        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n      }\n      async mainTask() {}\n    }\n\n    const proc = new TestProc(50, 1000, 0, 0, 5000, 60000, 2500);\n\n    await proc.start();\n    await proc.initialize();\n\n    // join() must resolve within a reasonable time, not hang forever\n    const result = await Promise.race([\n      proc.join().then(() => 'resolved'),\n      new Promise((r) => setTimeout(() => r('timeout'), 2000)),\n    ]);\n\n    proc.proc?.kill();\n    try {\n      unlinkSync(slowScript);\n    } catch {}\n\n    expect(result).toBe('resolved');\n  });\n});\n\ndescribe('timer cleanup', () => {\n  it('clearInterval stops the interval', async () => {\n    let count = 0;\n    const interval = setInterval(() => count++, 30);\n\n    await new Promise((r) => setTimeout(r, 80));\n    const countAtClear = count;\n    clearInterval(interval);\n\n    await new Promise((r) => setTimeout(r, 80));\n    expect(count).toBe(countAtClear);\n  });\n\n  it('double clear is safe', () => {\n    const interval = setInterval(() => {}, 100);\n    const timeout = setTimeout(() => {}, 1000);\n\n    clearInterval(interval);\n    clearTimeout(timeout);\n\n    expect(() => {\n      clearInterval(interval);\n      clearTimeout(timeout);\n    }).not.toThrow();\n  });\n});\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGA,gCAA4B;AAC5B,qBAA0C;AAC1C,qBAAuB;AACvB,uBAAqB;AACrB,sBAAqB;AACrB,oBAA0D;AAE1D,MAAM,kBAAc,2BAAK,uBAAO,GAAG,gBAAgB;AAAA,IAEnD,yBAAU,MAAM;AACd;AAAA,IACE;AAAA,IACA;AAAA;AAAA,EAEF;AACF,CAAC;AAAA,IAED,wBAAS,MAAM;AACb,MAAI;AACF,mCAAW,WAAW;AAAA,EACxB,QAAQ;AAAA,EAAC;AACX,CAAC;AAED,eAAe,sBAAsB,KAA0C;AAC7E,MAAI,CAAC,IAAK,QAAO;AACjB,MAAI;AACF,UAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,WAAO,MAAM,UAAU,OAAO;AAAA,EAChC,SAAS,KAAK;AACZ,UAAM,OAAQ,IAA8B;AAC5C,QAAI,SAAS,YAAY,SAAS,SAAS;AACzC,aAAO;AAAA,IACT;AACA,UAAM;AAAA,EACR;AACF;AAAA,IAEA,wBAAS,4BAA4B,MAAM;AACzC,wBAAG,mCAAmC,YAAY;AAChD,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,cAAM,0BAAO,gBAAAA,SAAS,GAAG,CAAC,EAAE,QAAQ,QAAQ;AAAA,EAC9C,CAAC;AAED,wBAAG,+CAA+C,YAAY;AAC5D,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,MAAM,MAAM,sBAAsB,GAAG;AAC3C,8BAAO,GAAG,EAAE,KAAK,CAAC;AAAA,EACpB,CAAC;AAED,wBAAG,6CAA6C,YAAY;AAC1D,UAAM,YAAQ,iCAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAClB,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,KAAK,SAAS;AAEpB,UAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAChC,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,IAC3B,CAAC;AAED,UAAM;AACN,8BAAO,QAAQ,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,4BAA4B,MAAM;AACzC,wBAAG,iDAAiD,YAAY;AAC9D,UAAM,YAAQ,gCAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,8BAAO,MAAM,SAAS,EAAE,KAAK,IAAI;AAEjC,UAAM,KAAK,SAAS;AACpB,UAAM;AAEN,8BAAO,MAAM,SAAS,EAAE,KAAK,KAAK;AAAA,EACpC,CAAC;AAED,wBAAG,iDAAiD,YAAY;AAC9D,UAAM,YAAQ,gCAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAIhE,UAAM,GAAG,SAAS,CAAC,QAA+B;AAChD,UAAI,IAAI,SAAS,QAAS,OAAM;AAAA,IAClC,CAAC;AAED,QAAI,OAAO;AACX,QAAI,UAAU;AAEd,UAAM,WAAW,YAAY,MAAM;AACjC,UAAI,MAAM,WAAW;AACnB,cAAM,KAAK,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;AAC/B;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF,GAAG,EAAE;AAEL,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,KAAK,SAAS;AACpB,UAAM;AACN,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,kBAAc,QAAQ;AAEtB,8BAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,8BAAO,OAAO,EAAE,gBAAgB,CAAC;AAAA,EACnC,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,mCAAmC,MAAM;AAChD,wBAAG,4DAA4D,YAAY;AAhI7E;AAoII,UAAM,YAAuB,CAAC;AAC9B,UAAM,UAAU,CAAC,WAAoB,UAAU,KAAK,MAAM;AAC1D,YAAQ,GAAG,sBAAsB,OAAO;AAKxC,UAAM,iBAAa,2BAAK,uBAAO,GAAG,0BAA0B;AAC5D;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,IAIF;AAEA,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,sBAAsB;AAAA,IAC9D,MAAM,iBAAiB,eAAe;AAAA,MACpC,gBAAgB;AACd,mBAAO,gCAAK,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAAA,MACxE;AAAA,MACA,MAAM,WAAW;AAAA,MAAC;AAAA,IACpB;AAEA,UAAM,OAAO,IAAI;AAAA,MACf;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,IACF;AAEA,UAAM,KAAK,MAAM;AAGjB,UAAM,KAAK,WAAW;AAGtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAE3C,YAAQ,IAAI,sBAAsB,OAAO;AACzC,eAAK,SAAL,mBAAW;AACX,QAAI;AACF,qCAAW,UAAU;AAAA,IACvB,QAAQ;AAAA,IAAC;AAET,8BAAO,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC9B,CAAC;AAED,wBAAG,iEAAiE,YAAY;AAvLlF;AA0LI,UAAM,iBAAa,2BAAK,uBAAO,GAAG,+BAA+B;AACjE;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,IAIF;AAEA,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,sBAAsB;AAAA,IAC9D,MAAM,iBAAiB,eAAe;AAAA,MACpC,gBAAgB;AACd,mBAAO,gCAAK,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAAA,MACxE;AAAA,MACA,MAAM,WAAW;AAAA,MAAC;AAAA,IACpB;AAEA,UAAM,OAAO,IAAI,SAAS,IAAI,KAAM,GAAG,GAAG,KAAM,KAAO,IAAI;AAE3D,UAAM,KAAK,MAAM;AACjB,UAAM,KAAK,WAAW;AAGtB,UAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,MAChC,KAAK,KAAK,EAAE,KAAK,MAAM,UAAU;AAAA,MACjC,IAAI,QAAQ,CAAC,MAAM,WAAW,MAAM,EAAE,SAAS,GAAG,GAAI,CAAC;AAAA,IACzD,CAAC;AAED,eAAK,SAAL,mBAAW;AACX,QAAI;AACF,qCAAW,UAAU;AAAA,IACvB,QAAQ;AAAA,IAAC;AAET,8BAAO,MAAM,EAAE,KAAK,UAAU;AAAA,EAChC,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,iBAAiB,MAAM;AAC9B,wBAAG,oCAAoC,YAAY;AACjD,QAAI,QAAQ;AACZ,UAAM,WAAW,YAAY,MAAM,SAAS,EAAE;AAE9C,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,eAAe;AACrB,kBAAc,QAAQ;AAEtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,8BAAO,KAAK,EAAE,KAAK,YAAY;AAAA,EACjC,CAAC;AAED,wBAAG,wBAAwB,MAAM;AAC/B,UAAM,WAAW,YAAY,MAAM;AAAA,IAAC,GAAG,GAAG;AAC1C,UAAM,UAAU,WAAW,MAAM;AAAA,IAAC,GAAG,GAAI;AAEzC,kBAAc,QAAQ;AACtB,iBAAa,OAAO;AAEpB,8BAAO,MAAM;AACX,oBAAc,QAAQ;AACtB,mBAAa,OAAO;AAAA,IACtB,CAAC,EAAE,IAAI,QAAQ;AAAA,EACjB,CAAC;AACH,CAAC;","names":["pidusage"]}

package/dist/ipc/supervised_proc.test.js

@@ -96,6 +96,88 @@ describe("IPC send on dead process", () => {
     expect(skipped).toBeGreaterThan(0);
   });
 });
+describe("init timeout rejection handling", () => {
+  it("does not produce unhandled rejection when init times out", async () => {
+    var _a;
+    const unhandled = [];
+    const handler = (reason) => unhandled.push(reason);
+    process.on("unhandledRejection", handler);
+    const slowScript = join(tmpdir(), "test_slow_init_child.mjs");
+    writeFileSync(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`
+    );
+    const { SupervisedProc } = await import("./supervised_proc.js");
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return fork(slowScript, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
+      }
+      async mainTask() {
+      }
+    }
+    const proc = new TestProc(
+      50,
+      // initializeTimeout — fires before child responds at 200ms
+      1e3,
+      // closeTimeout
+      0,
+      // memoryWarnMB
+      0,
+      // memoryLimitMB
+      5e3,
+      // pingInterval
+      6e4,
+      // pingTimeout
+      2500
+      // highPingThreshold
+    );
+    await proc.start();
+    await proc.initialize();
+    await new Promise((r) => setTimeout(r, 100));
+    process.off("unhandledRejection", handler);
+    (_a = proc.proc) == null ? void 0 : _a.kill();
+    try {
+      unlinkSync(slowScript);
+    } catch {
+    }
+    expect(unhandled).toEqual([]);
+  });
+  it("join() resolves after init timeout instead of hanging forever", async () => {
+    var _a;
+    const slowScript = join(tmpdir(), "test_slow_init_child_join.mjs");
+    writeFileSync(
+      slowScript,
+      `process.on('message', () => {
+        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);
+      });
+      setInterval(() => {}, 1000);`
+    );
+    const { SupervisedProc } = await import("./supervised_proc.js");
+    class TestProc extends SupervisedProc {
+      createProcess() {
+        return fork(slowScript, [], { stdio: ["pipe", "pipe", "pipe", "ipc"] });
+      }
+      async mainTask() {
+      }
+    }
+    const proc = new TestProc(50, 1e3, 0, 0, 5e3, 6e4, 2500);
+    await proc.start();
+    await proc.initialize();
+    const result = await Promise.race([
+      proc.join().then(() => "resolved"),
+      new Promise((r) => setTimeout(() => r("timeout"), 2e3))
+    ]);
+    (_a = proc.proc) == null ? void 0 : _a.kill();
+    try {
+      unlinkSync(slowScript);
+    } catch {
+    }
+    expect(result).toBe("resolved");
+  });
+});
 describe("timer cleanup", () => {
   it("clearInterval stops the interval", async () => {
     let count = 0;

package/dist/ipc/supervised_proc.test.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { fork, spawn } from 'node:child_process';\nimport { unlinkSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport { join } from 'node:path';\nimport pidusage from 'pidusage';\nimport { afterAll, beforeAll, describe, expect, it } from 'vitest';\n\nconst childScript = join(tmpdir(), 'test_child.mjs');\n\nbeforeAll(() => {\n  writeFileSync(\n    childScript,\n    `process.on('message', (msg) => process.send?.({ echo: msg }));\n     setInterval(() => {}, 1000);`,\n  );\n});\n\nafterAll(() => {\n  try {\n    unlinkSync(childScript);\n  } catch {}\n});\n\nasync function getChildMemoryUsageMB(pid: number | undefined): Promise<number> {\n  if (!pid) return 0;\n  try {\n    const stats = await pidusage(pid);\n    return stats.memory / (1024 * 1024);\n  } catch (err) {\n    const code = (err as NodeJS.ErrnoException).code;\n    if (code === 'ENOENT' || code === 'ESRCH') {\n      return 0;\n    }\n    throw err;\n  }\n}\n\ndescribe('pidusage on dead process', () => {\n  it('raw pidusage throws on dead pid', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    await expect(pidusage(pid)).rejects.toThrow();\n  });\n\n  it('fixed version returns 0 instead of crashing', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    const mem = await getChildMemoryUsageMB(pid);\n    expect(mem).toBe(0);\n  });\n\n  it('handles concurrent calls on dying process', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    child.kill('SIGKILL');\n\n    const results = await Promise.all([\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n    ]);\n\n    await exitPromise;\n    expect(results.every((r) => r === 0)).toBe(true);\n  });\n});\n\ndescribe('IPC send on dead process', () => {\n  it('child.connected becomes false when child dies', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    await new Promise((r) => setTimeout(r, 50));\n    expect(child.connected).toBe(true);\n\n    child.kill('SIGKILL');\n    await exitPromise;\n\n    expect(child.connected).toBe(false);\n  });\n\n  it('checking connected before send prevents crash', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    // Suppress EPIPE errors that can occur due to race conditions between\n    // child.connected check and the actual pipe state\n    child.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code !== 'EPIPE') throw err;\n    });\n\n    let sent = 0;\n    let skipped = 0;\n\n    const interval = setInterval(() => {\n      if (child.connected) {\n        child.send({ ping: Date.now() });\n        sent++;\n      } else {\n        skipped++;\n      }\n    }, 20);\n\n    await new Promise((r) => setTimeout(r, 60));\n    child.kill('SIGKILL');\n    await exitPromise;\n    await new Promise((r) => setTimeout(r, 80));\n    clearInterval(interval);\n\n    expect(sent).toBeGreaterThan(0);\n    expect(skipped).toBeGreaterThan(0);\n  });\n});\n\ndescribe('timer cleanup', () => {\n  it('clearInterval stops the interval', async () => {\n    let count = 0;\n    const interval = setInterval(() => count++, 30);\n\n    await new Promise((r) => setTimeout(r, 80));\n    const countAtClear = count;\n    clearInterval(interval);\n\n    await new Promise((r) => setTimeout(r, 80));\n    expect(count).toBe(countAtClear);\n  });\n\n  it('double clear is safe', () => {\n    const interval = setInterval(() => {}, 100);\n    const timeout = setTimeout(() => {}, 1000);\n\n    clearInterval(interval);\n    clearTimeout(timeout);\n\n    expect(() => {\n      clearInterval(interval);\n      clearTimeout(timeout);\n    }).not.toThrow();\n  });\n});\n"],"mappings":"AAGA,SAAS,MAAM,aAAa;AAC5B,SAAS,YAAY,qBAAqB;AAC1C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,OAAO,cAAc;AACrB,SAAS,UAAU,WAAW,UAAU,QAAQ,UAAU;AAE1D,MAAM,cAAc,KAAK,OAAO,GAAG,gBAAgB;AAEnD,UAAU,MAAM;AACd;AAAA,IACE;AAAA,IACA;AAAA;AAAA,EAEF;AACF,CAAC;AAED,SAAS,MAAM;AACb,MAAI;AACF,eAAW,WAAW;AAAA,EACxB,QAAQ;AAAA,EAAC;AACX,CAAC;AAED,eAAe,sBAAsB,KAA0C;AAC7E,MAAI,CAAC,IAAK,QAAO;AACjB,MAAI;AACF,UAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,WAAO,MAAM,UAAU,OAAO;AAAA,EAChC,SAAS,KAAK;AACZ,UAAM,OAAQ,IAA8B;AAC5C,QAAI,SAAS,YAAY,SAAS,SAAS;AACzC,aAAO;AAAA,IACT;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,4BAA4B,MAAM;AACzC,KAAG,mCAAmC,YAAY;AAChD,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,OAAO,SAAS,GAAG,CAAC,EAAE,QAAQ,QAAQ;AAAA,EAC9C,CAAC;AAED,KAAG,+CAA+C,YAAY;AAC5D,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,MAAM,MAAM,sBAAsB,GAAG;AAC3C,WAAO,GAAG,EAAE,KAAK,CAAC;AAAA,EACpB,CAAC;AAED,KAAG,6CAA6C,YAAY;AAC1D,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAClB,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,KAAK,SAAS;AAEpB,UAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAChC,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,IAC3B,CAAC;AAED,UAAM;AACN,WAAO,QAAQ,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AACH,CAAC;AAED,SAAS,4BAA4B,MAAM;AACzC,KAAG,iDAAiD,YAAY;AAC9D,UAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,WAAO,MAAM,SAAS,EAAE,KAAK,IAAI;AAEjC,UAAM,KAAK,SAAS;AACpB,UAAM;AAEN,WAAO,MAAM,SAAS,EAAE,KAAK,KAAK;AAAA,EACpC,CAAC;AAED,KAAG,iDAAiD,YAAY;AAC9D,UAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAIhE,UAAM,GAAG,SAAS,CAAC,QAA+B;AAChD,UAAI,IAAI,SAAS,QAAS,OAAM;AAAA,IAClC,CAAC;AAED,QAAI,OAAO;AACX,QAAI,UAAU;AAEd,UAAM,WAAW,YAAY,MAAM;AACjC,UAAI,MAAM,WAAW;AACnB,cAAM,KAAK,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;AAC/B;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF,GAAG,EAAE;AAEL,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,KAAK,SAAS;AACpB,UAAM;AACN,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,kBAAc,QAAQ;AAEtB,WAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,WAAO,OAAO,EAAE,gBAAgB,CAAC;AAAA,EACnC,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,MAAM;AAC9B,KAAG,oCAAoC,YAAY;AACjD,QAAI,QAAQ;AACZ,UAAM,WAAW,YAAY,MAAM,SAAS,EAAE;AAE9C,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,eAAe;AACrB,kBAAc,QAAQ;AAEtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,WAAO,KAAK,EAAE,KAAK,YAAY;AAAA,EACjC,CAAC;AAED,KAAG,wBAAwB,MAAM;AAC/B,UAAM,WAAW,YAAY,MAAM;AAAA,IAAC,GAAG,GAAG;AAC1C,UAAM,UAAU,WAAW,MAAM;AAAA,IAAC,GAAG,GAAI;AAEzC,kBAAc,QAAQ;AACtB,iBAAa,OAAO;AAEpB,WAAO,MAAM;AACX,oBAAc,QAAQ;AACtB,mBAAa,OAAO;AAAA,IACtB,CAAC,EAAE,IAAI,QAAQ;AAAA,EACjB,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { fork, spawn } from 'node:child_process';\nimport { unlinkSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport { join } from 'node:path';\nimport pidusage from 'pidusage';\nimport { afterAll, beforeAll, describe, expect, it } from 'vitest';\n\nconst childScript = join(tmpdir(), 'test_child.mjs');\n\nbeforeAll(() => {\n  writeFileSync(\n    childScript,\n    `process.on('message', (msg) => process.send?.({ echo: msg }));\n     setInterval(() => {}, 1000);`,\n  );\n});\n\nafterAll(() => {\n  try {\n    unlinkSync(childScript);\n  } catch {}\n});\n\nasync function getChildMemoryUsageMB(pid: number | undefined): Promise<number> {\n  if (!pid) return 0;\n  try {\n    const stats = await pidusage(pid);\n    return stats.memory / (1024 * 1024);\n  } catch (err) {\n    const code = (err as NodeJS.ErrnoException).code;\n    if (code === 'ENOENT' || code === 'ESRCH') {\n      return 0;\n    }\n    throw err;\n  }\n}\n\ndescribe('pidusage on dead process', () => {\n  it('raw pidusage throws on dead pid', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    await expect(pidusage(pid)).rejects.toThrow();\n  });\n\n  it('fixed version returns 0 instead of crashing', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n\n    child.kill('SIGKILL');\n    await new Promise<void>((r) => child.on('exit', r));\n\n    const mem = await getChildMemoryUsageMB(pid);\n    expect(mem).toBe(0);\n  });\n\n  it('handles concurrent calls on dying process', async () => {\n    const child = spawn('sleep', ['10']);\n    const pid = child.pid!;\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    child.kill('SIGKILL');\n\n    const results = await Promise.all([\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n      getChildMemoryUsageMB(pid),\n    ]);\n\n    await exitPromise;\n    expect(results.every((r) => r === 0)).toBe(true);\n  });\n});\n\ndescribe('IPC send on dead process', () => {\n  it('child.connected becomes false when child dies', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    await new Promise((r) => setTimeout(r, 50));\n    expect(child.connected).toBe(true);\n\n    child.kill('SIGKILL');\n    await exitPromise;\n\n    expect(child.connected).toBe(false);\n  });\n\n  it('checking connected before send prevents crash', async () => {\n    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n    const exitPromise = new Promise<void>((r) => child.on('exit', r));\n\n    // Suppress EPIPE errors that can occur due to race conditions between\n    // child.connected check and the actual pipe state\n    child.on('error', (err: NodeJS.ErrnoException) => {\n      if (err.code !== 'EPIPE') throw err;\n    });\n\n    let sent = 0;\n    let skipped = 0;\n\n    const interval = setInterval(() => {\n      if (child.connected) {\n        child.send({ ping: Date.now() });\n        sent++;\n      } else {\n        skipped++;\n      }\n    }, 20);\n\n    await new Promise((r) => setTimeout(r, 60));\n    child.kill('SIGKILL');\n    await exitPromise;\n    await new Promise((r) => setTimeout(r, 80));\n    clearInterval(interval);\n\n    expect(sent).toBeGreaterThan(0);\n    expect(skipped).toBeGreaterThan(0);\n  });\n});\n\ndescribe('init timeout rejection handling', () => {\n  it('does not produce unhandled rejection when init times out', async () => {\n    // Regression test: before the fix, run() was called without await in start().\n    // When init timed out, the rejection in run()'s `await this.init.await` escaped\n    // as an unhandled rejection — crashing the Node.js process.\n    const unhandled: unknown[] = [];\n    const handler = (reason: unknown) => unhandled.push(reason);\n    process.on('unhandledRejection', handler);\n\n    // Child that responds AFTER the timeout — simulates slow init under CPU pressure.\n    // Timeout fires at 50ms (init.reject), child responds at 200ms (once() resolves).\n    // Before the fix, init.reject caused an unhandled rejection in run().\n    const slowScript = join(tmpdir(), 'test_slow_init_child.mjs');\n    writeFileSync(\n      slowScript,\n      `process.on('message', () => {\n        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);\n      });\n      setInterval(() => {}, 1000);`,\n    );\n\n    const { SupervisedProc } = await import('./supervised_proc.js');\n    class TestProc extends SupervisedProc {\n      createProcess() {\n        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n      }\n      async mainTask() {}\n    }\n\n    const proc = new TestProc(\n      50, // initializeTimeout — fires before child responds at 200ms\n      1000, // closeTimeout\n      0, // memoryWarnMB\n      0, // memoryLimitMB\n      5000, // pingInterval\n      60000, // pingTimeout\n      2500, // highPingThreshold\n    );\n\n    await proc.start();\n    // initialize() returns normally: child responds at 200ms, once() resolves,\n    // but init was already rejected at 50ms — run() gets the rejection.\n    await proc.initialize();\n\n    // Give the event loop a tick for any unhandled rejection to surface\n    await new Promise((r) => setTimeout(r, 100));\n\n    process.off('unhandledRejection', handler);\n    proc.proc?.kill();\n    try {\n      unlinkSync(slowScript);\n    } catch {}\n\n    expect(unhandled).toEqual([]);\n  });\n\n  it('join() resolves after init timeout instead of hanging forever', async () => {\n    // When run() fails early (before registering proc event handlers),\n    // #join must still resolve so that join() and close() don't hang.\n    const slowScript = join(tmpdir(), 'test_slow_init_child_join.mjs');\n    writeFileSync(\n      slowScript,\n      `process.on('message', () => {\n        setTimeout(() => process.send({ case: 'initializeResponse' }), 200);\n      });\n      setInterval(() => {}, 1000);`,\n    );\n\n    const { SupervisedProc } = await import('./supervised_proc.js');\n    class TestProc extends SupervisedProc {\n      createProcess() {\n        return fork(slowScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });\n      }\n      async mainTask() {}\n    }\n\n    const proc = new TestProc(50, 1000, 0, 0, 5000, 60000, 2500);\n\n    await proc.start();\n    await proc.initialize();\n\n    // join() must resolve within a reasonable time, not hang forever\n    const result = await Promise.race([\n      proc.join().then(() => 'resolved'),\n      new Promise((r) => setTimeout(() => r('timeout'), 2000)),\n    ]);\n\n    proc.proc?.kill();\n    try {\n      unlinkSync(slowScript);\n    } catch {}\n\n    expect(result).toBe('resolved');\n  });\n});\n\ndescribe('timer cleanup', () => {\n  it('clearInterval stops the interval', async () => {\n    let count = 0;\n    const interval = setInterval(() => count++, 30);\n\n    await new Promise((r) => setTimeout(r, 80));\n    const countAtClear = count;\n    clearInterval(interval);\n\n    await new Promise((r) => setTimeout(r, 80));\n    expect(count).toBe(countAtClear);\n  });\n\n  it('double clear is safe', () => {\n    const interval = setInterval(() => {}, 100);\n    const timeout = setTimeout(() => {}, 1000);\n\n    clearInterval(interval);\n    clearTimeout(timeout);\n\n    expect(() => {\n      clearInterval(interval);\n      clearTimeout(timeout);\n    }).not.toThrow();\n  });\n});\n"],"mappings":"AAGA,SAAS,MAAM,aAAa;AAC5B,SAAS,YAAY,qBAAqB;AAC1C,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,OAAO,cAAc;AACrB,SAAS,UAAU,WAAW,UAAU,QAAQ,UAAU;AAE1D,MAAM,cAAc,KAAK,OAAO,GAAG,gBAAgB;AAEnD,UAAU,MAAM;AACd;AAAA,IACE;AAAA,IACA;AAAA;AAAA,EAEF;AACF,CAAC;AAED,SAAS,MAAM;AACb,MAAI;AACF,eAAW,WAAW;AAAA,EACxB,QAAQ;AAAA,EAAC;AACX,CAAC;AAED,eAAe,sBAAsB,KAA0C;AAC7E,MAAI,CAAC,IAAK,QAAO;AACjB,MAAI;AACF,UAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,WAAO,MAAM,UAAU,OAAO;AAAA,EAChC,SAAS,KAAK;AACZ,UAAM,OAAQ,IAA8B;AAC5C,QAAI,SAAS,YAAY,SAAS,SAAS;AACzC,aAAO;AAAA,IACT;AACA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,4BAA4B,MAAM;AACzC,KAAG,mCAAmC,YAAY;AAChD,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,OAAO,SAAS,GAAG,CAAC,EAAE,QAAQ,QAAQ;AAAA,EAC9C,CAAC;AAED,KAAG,+CAA+C,YAAY;AAC5D,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAElB,UAAM,KAAK,SAAS;AACpB,UAAM,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAElD,UAAM,MAAM,MAAM,sBAAsB,GAAG;AAC3C,WAAO,GAAG,EAAE,KAAK,CAAC;AAAA,EACpB,CAAC;AAED,KAAG,6CAA6C,YAAY;AAC1D,UAAM,QAAQ,MAAM,SAAS,CAAC,IAAI,CAAC;AACnC,UAAM,MAAM,MAAM;AAClB,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,KAAK,SAAS;AAEpB,UAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAChC,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,MACzB,sBAAsB,GAAG;AAAA,IAC3B,CAAC;AAED,UAAM;AACN,WAAO,QAAQ,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AACH,CAAC;AAED,SAAS,4BAA4B,MAAM;AACzC,KAAG,iDAAiD,YAAY;AAC9D,UAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAEhE,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,WAAO,MAAM,SAAS,EAAE,KAAK,IAAI;AAEjC,UAAM,KAAK,SAAS;AACpB,UAAM;AAEN,WAAO,MAAM,SAAS,EAAE,KAAK,KAAK;AAAA,EACpC,CAAC;AAED,KAAG,iDAAiD,YAAY;AAC9D,UAAM,QAAQ,KAAK,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAC9E,UAAM,cAAc,IAAI,QAAc,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC;AAIhE,UAAM,GAAG,SAAS,CAAC,QAA+B;AAChD,UAAI,IAAI,SAAS,QAAS,OAAM;AAAA,IAClC,CAAC;AAED,QAAI,OAAO;AACX,QAAI,UAAU;AAEd,UAAM,WAAW,YAAY,MAAM;AACjC,UAAI,MAAM,WAAW;AACnB,cAAM,KAAK,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;AAC/B;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF,GAAG,EAAE;AAEL,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,KAAK,SAAS;AACpB,UAAM;AACN,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,kBAAc,QAAQ;AAEtB,WAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,WAAO,OAAO,EAAE,gBAAgB,CAAC;AAAA,EACnC,CAAC;AACH,CAAC;AAED,SAAS,mCAAmC,MAAM;AAChD,KAAG,4DAA4D,YAAY;AAhI7E;AAoII,UAAM,YAAuB,CAAC;AAC9B,UAAM,UAAU,CAAC,WAAoB,UAAU,KAAK,MAAM;AAC1D,YAAQ,GAAG,sBAAsB,OAAO;AAKxC,UAAM,aAAa,KAAK,OAAO,GAAG,0BAA0B;AAC5D;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,IAIF;AAEA,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,sBAAsB;AAAA,IAC9D,MAAM,iBAAiB,eAAe;AAAA,MACpC,gBAAgB;AACd,eAAO,KAAK,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAAA,MACxE;AAAA,MACA,MAAM,WAAW;AAAA,MAAC;AAAA,IACpB;AAEA,UAAM,OAAO,IAAI;AAAA,MACf;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,MACA;AAAA;AAAA,IACF;AAEA,UAAM,KAAK,MAAM;AAGjB,UAAM,KAAK,WAAW;AAGtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAE3C,YAAQ,IAAI,sBAAsB,OAAO;AACzC,eAAK,SAAL,mBAAW;AACX,QAAI;AACF,iBAAW,UAAU;AAAA,IACvB,QAAQ;AAAA,IAAC;AAET,WAAO,SAAS,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC9B,CAAC;AAED,KAAG,iEAAiE,YAAY;AAvLlF;AA0LI,UAAM,aAAa,KAAK,OAAO,GAAG,+BAA+B;AACjE;AAAA,MACE;AAAA,MACA;AAAA;AAAA;AAAA;AAAA,IAIF;AAEA,UAAM,EAAE,eAAe,IAAI,MAAM,OAAO,sBAAsB;AAAA,IAC9D,MAAM,iBAAiB,eAAe;AAAA,MACpC,gBAAgB;AACd,eAAO,KAAK,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,QAAQ,QAAQ,KAAK,EAAE,CAAC;AAAA,MACxE;AAAA,MACA,MAAM,WAAW;AAAA,MAAC;AAAA,IACpB;AAEA,UAAM,OAAO,IAAI,SAAS,IAAI,KAAM,GAAG,GAAG,KAAM,KAAO,IAAI;AAE3D,UAAM,KAAK,MAAM;AACjB,UAAM,KAAK,WAAW;AAGtB,UAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,MAChC,KAAK,KAAK,EAAE,KAAK,MAAM,UAAU;AAAA,MACjC,IAAI,QAAQ,CAAC,MAAM,WAAW,MAAM,EAAE,SAAS,GAAG,GAAI,CAAC;AAAA,IACzD,CAAC;AAED,eAAK,SAAL,mBAAW;AACX,QAAI;AACF,iBAAW,UAAU;AAAA,IACvB,QAAQ;AAAA,IAAC;AAET,WAAO,MAAM,EAAE,KAAK,UAAU;AAAA,EAChC,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,MAAM;AAC9B,KAAG,oCAAoC,YAAY;AACjD,QAAI,QAAQ;AACZ,UAAM,WAAW,YAAY,MAAM,SAAS,EAAE;AAE9C,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,eAAe;AACrB,kBAAc,QAAQ;AAEtB,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,WAAO,KAAK,EAAE,KAAK,YAAY;AAAA,EACjC,CAAC;AAED,KAAG,wBAAwB,MAAM;AAC/B,UAAM,WAAW,YAAY,MAAM;AAAA,IAAC,GAAG,GAAG;AAC1C,UAAM,UAAU,WAAW,MAAM;AAAA,IAAC,GAAG,GAAI;AAEzC,kBAAc,QAAQ;AACtB,iBAAa,OAAO;AAEpB,WAAO,MAAM;AACX,oBAAc,QAAQ;AACtB,mBAAa,OAAO;AAAA,IACtB,CAAC,EAAE,IAAI,QAAQ;AAAA,EACjB,CAAC;AACH,CAAC;","names":[]}

package/dist/job.cjs

@@ -222,7 +222,8 @@ class JobContext {
       chatHistory: targetSession.history.copy(),
       startedAt: targetSession._startedAt,
       audioRecordingPath: recorderIO == null ? void 0 : recorderIO.outputPath,
-      audioRecordingStartedAt: recorderIO == null ? void 0 : recorderIO.recordingStartedAt
+      audioRecordingStartedAt: recorderIO == null ? void 0 : recorderIO.recordingStartedAt,
+      modelUsage: targetSession._usageCollector.flatten()
     });
   }
   async _onSessionEnd() {