@livekit/agents 1.2.0 → 1.2.2

package/dist/inference/tts.test.js

@@ -229,4 +229,76 @@ describe("TTS constructor fallback and connOptions", () => {
     expect(tts["opts"].connOptions.retryIntervalMs).toBe(2e3);
   });
 });
+describe("TTS provider modelOptions parity", () => {
+  it("preserves ElevenLabs inference model options", () => {
+    const modelOptions = {
+      speed: 1.2,
+      stability: 0.5,
+      similarity_boost: 0.8,
+      enable_logging: false
+    };
+    const tts = new TTS({
+      model: "elevenlabs/eleven_flash_v2_5",
+      apiKey: "test-key",
+      apiSecret: "test-secret",
+      baseURL: "https://example.livekit.cloud",
+      modelOptions
+    });
+    expect(tts["opts"].modelOptions).toEqual(modelOptions);
+  });
+  it("accepts expanded Cartesia inference model options", () => {
+    const modelOptions = {
+      speed: 1.15,
+      emotion: "curious",
+      add_timestamps: true
+    };
+    const tts = new TTS({
+      model: "cartesia/sonic",
+      apiKey: "test-key",
+      apiSecret: "test-secret",
+      baseURL: "https://example.livekit.cloud",
+      modelOptions
+    });
+    expect(tts["opts"].modelOptions).toEqual(modelOptions);
+  });
+  it("accepts Deepgram inference model options", () => {
+    const modelOptions = { mip_opt_out: true };
+    const tts = new TTS({
+      model: "deepgram/aura-2",
+      apiKey: "test-key",
+      apiSecret: "test-secret",
+      baseURL: "https://example.livekit.cloud",
+      modelOptions
+    });
+    expect(tts["opts"].modelOptions).toEqual(modelOptions);
+  });
+  it("accepts Rime inference model options", () => {
+    const modelOptions = {
+      speed_alpha: 0.9,
+      pause_between_brackets: true
+    };
+    const tts = new TTS({
+      model: "rime/mistv2",
+      apiKey: "test-key",
+      apiSecret: "test-secret",
+      baseURL: "https://example.livekit.cloud",
+      modelOptions
+    });
+    expect(tts["opts"].modelOptions).toEqual(modelOptions);
+  });
+  it("accepts Inworld inference model options", () => {
+    const modelOptions = {
+      timestamp_type: "WORD",
+      apply_text_normalization: "ON"
+    };
+    const tts = new TTS({
+      model: "inworld/inworld-tts-1",
+      apiKey: "test-key",
+      apiSecret: "test-secret",
+      baseURL: "https://example.livekit.cloud",
+      modelOptions
+    });
+    expect(tts["opts"].modelOptions).toEqual(modelOptions);
+  });
+});
 //# sourceMappingURL=tts.test.js.map

package/dist/inference/tts.test.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/inference/tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { beforeAll, describe, expect, it } from 'vitest';\nimport { normalizeLanguage } from '../language.js';\nimport { initializeLogger } from '../log.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { TTS, type TTSFallbackModel, normalizeTTSFallback, parseTTSModelString } from './tts.js';\n\nbeforeAll(() => {\n  initializeLogger({ level: 'silent', pretty: false });\n});\n\n/** Helper to create TTS with required credentials. */\nfunction makeTts(overrides: Record<string, unknown> = {}) {\n  const defaults = {\n    model: 'cartesia/sonic' as const,\n    apiKey: 'test-key',\n    apiSecret: 'test-secret',\n    baseURL: 'https://example.livekit.cloud',\n  };\n  return new TTS({ ...defaults, ...overrides });\n}\n\ndescribe('parseTTSModelString', () => {\n  it('simple model without voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia');\n    expect(model).toBe('cartesia');\n    expect(voice).toBeUndefined();\n  });\n\n  it('model with voice suffix', () => {\n    const [model, voice] = parseTTSModelString('cartesia:my-voice-id');\n    expect(model).toBe('cartesia');\n    expect(voice).toBe('my-voice-id');\n  });\n\n  it('provider/model format without voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBeUndefined();\n  });\n\n  it('provider/model format with voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic:my-voice-id');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBe('my-voice-id');\n  });\n\n  it.each([\n    ['elevenlabs/eleven_flash_v2:voice123', 'elevenlabs/eleven_flash_v2', 'voice123'],\n    ['rime:speaker-a', 'rime', 'speaker-a'],\n    ['rime/mist:narrator', 'rime/mist', 'narrator'],\n    ['inworld/inworld-tts-1:character', 'inworld/inworld-tts-1', 'character'],\n    ['cartesia/sonic-turbo:deep-voice', 'cartesia/sonic-turbo', 'deep-voice'],\n  ])('various providers and voices: %s', (modelStr, expectedModel, expectedVoice) => {\n    const [model, voice] = parseTTSModelString(modelStr);\n    expect(model).toBe(expectedModel);\n    expect(voice).toBe(expectedVoice);\n  });\n\n  it('empty voice after colon', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic:');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBe('');\n  });\n});\n\ndescribe('normalizeTTSFallback', () => {\n  it('single string model', () => {\n    const result = normalizeTTSFallback('cartesia/sonic');\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);\n  });\n\n  it('single string model with voice', () => {\n    const result = normalizeTTSFallback('cartesia/sonic:my-voice');\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);\n  });\n\n  it('single FallbackModel dict', () => {\n    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: 'narrator' };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);\n  });\n\n  it('list of string models', () => {\n    const result = normalizeTTSFallback(['cartesia/sonic', 'elevenlabs/eleven_flash_v2']);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: '' },\n      { model: 'elevenlabs/eleven_flash_v2', voice: '' },\n    ]);\n  });\n\n  it('list of string models with voices', () => {\n    const result = normalizeTTSFallback(['cartesia/sonic:voice1', 'elevenlabs:voice2']);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs', voice: 'voice2' },\n    ]);\n  });\n\n  it('list of FallbackModel dicts', () => {\n    const fallbacks: TTSFallbackModel[] = [\n      { model: 'cartesia/sonic', voice: 'narrator' },\n      { model: 'elevenlabs', voice: '' },\n    ];\n    const result = normalizeTTSFallback(fallbacks);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'narrator' },\n      { model: 'elevenlabs', voice: '' },\n    ]);\n  });\n\n  it('mixed list of strings and dicts', () => {\n    const result = normalizeTTSFallback([\n      'cartesia/sonic:voice1',\n      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' } as TTSFallbackModel,\n      'rime/mist',\n    ]);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' },\n      { model: 'rime/mist', voice: '' },\n    ]);\n  });\n\n  it('FallbackModel with extraKwargs is preserved', () => {\n    const fallback: TTSFallbackModel = {\n      model: 'cartesia/sonic',\n      voice: 'narrator',\n      extraKwargs: { duration: 30.0, speed: 'fast' },\n    };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([\n      {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    ]);\n  });\n\n  it('list with extraKwargs preserved', () => {\n    const result = normalizeTTSFallback([\n      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } } as TTSFallbackModel,\n      'elevenlabs:voice2',\n      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } } as TTSFallbackModel,\n    ]);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } },\n      { model: 'elevenlabs', voice: 'voice2' },\n      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } },\n    ]);\n  });\n\n  it('empty list returns empty list', () => {\n    const result = normalizeTTSFallback([]);\n    expect(result).toEqual([]);\n  });\n\n  it('FallbackModel with empty voice', () => {\n    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: '' };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);\n  });\n});\n\ndescribe('TTS constructor fallback and connOptions', () => {\n  it('normalizes language in constructor', () => {\n    const tts = makeTts({ language: 'english' });\n    expect(tts['opts'].language).toBe('en');\n  });\n\n  it('normalizes updated language values', () => {\n    const tts = makeTts();\n    tts.updateOptions({ language: 'en_US' });\n    expect(tts['opts'].language).toBe(normalizeLanguage('en_US'));\n  });\n\n  it('fallback not given defaults to undefined', () => {\n    const tts = makeTts();\n    expect(tts['opts'].fallback).toBeUndefined();\n  });\n\n  it('fallback single string is normalized', () => {\n    const tts = makeTts({ fallback: 'elevenlabs/eleven_flash_v2' });\n    expect(tts['opts'].fallback).toEqual([{ model: 'elevenlabs/eleven_flash_v2', voice: '' }]);\n  });\n\n  it('fallback single string with voice is normalized', () => {\n    const tts = makeTts({ fallback: 'cartesia/sonic:my-voice' });\n    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);\n  });\n\n  it('fallback list of strings is normalized', () => {\n    const tts = makeTts({ fallback: ['cartesia/sonic', 'elevenlabs'] });\n    expect(tts['opts'].fallback).toEqual([\n      { model: 'cartesia/sonic', voice: '' },\n      { model: 'elevenlabs', voice: '' },\n    ]);\n  });\n\n  it('fallback single FallbackModel is normalized to list', () => {\n    const tts = makeTts({ fallback: { model: 'cartesia/sonic', voice: 'narrator' } });\n    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);\n  });\n\n  it('fallback with extraKwargs is preserved', () => {\n    const tts = makeTts({\n      fallback: {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    });\n    expect(tts['opts'].fallback).toEqual([\n      {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    ]);\n  });\n\n  it('fallback mixed list is normalized', () => {\n    const tts = makeTts({\n      fallback: [\n        'cartesia/sonic:voice1',\n        { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },\n        'rime/mist',\n      ],\n    });\n    expect(tts['opts'].fallback).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },\n      { model: 'rime/mist', voice: '' },\n    ]);\n  });\n\n  it('connOptions not given uses default', () => {\n    const tts = makeTts();\n    expect(tts['opts'].connOptions).toEqual(DEFAULT_API_CONNECT_OPTIONS);\n  });\n\n  it('connOptions custom timeout', () => {\n    const custom: APIConnectOptions = { timeoutMs: 30000, maxRetry: 3, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.timeoutMs).toBe(30000);\n  });\n\n  it('connOptions custom maxRetry', () => {\n    const custom: APIConnectOptions = { timeoutMs: 10000, maxRetry: 5, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.maxRetry).toBe(5);\n  });\n\n  it('connOptions full custom', () => {\n    const custom: APIConnectOptions = { timeoutMs: 60000, maxRetry: 10, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.timeoutMs).toBe(60000);\n    expect(tts['opts'].connOptions!.maxRetry).toBe(10);\n    expect(tts['opts'].connOptions!.retryIntervalMs).toBe(2000);\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW,UAAU,QAAQ,UAAU;AAChD,SAAS,yBAAyB;AAClC,SAAS,wBAAwB;AACjC,SAAiC,mCAAmC;AACpE,SAAS,KAA4B,sBAAsB,2BAA2B;AAEtF,UAAU,MAAM;AACd,mBAAiB,EAAE,OAAO,UAAU,QAAQ,MAAM,CAAC;AACrD,CAAC;AAGD,SAAS,QAAQ,YAAqC,CAAC,GAAG;AACxD,QAAM,WAAW;AAAA,IACf,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,SAAS;AAAA,EACX;AACA,SAAO,IAAI,IAAI,EAAE,GAAG,UAAU,GAAG,UAAU,CAAC;AAC9C;AAEA,SAAS,uBAAuB,MAAM;AACpC,KAAG,8BAA8B,MAAM;AACrC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,UAAU;AACrD,WAAO,KAAK,EAAE,KAAK,UAAU;AAC7B,WAAO,KAAK,EAAE,cAAc;AAAA,EAC9B,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,sBAAsB;AACjE,WAAO,KAAK,EAAE,KAAK,UAAU;AAC7B,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,uCAAuC,MAAM;AAC9C,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,gBAAgB;AAC3D,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,cAAc;AAAA,EAC9B,CAAC;AAED,KAAG,oCAAoC,MAAM;AAC3C,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,4BAA4B;AACvE,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,KAAK;AAAA,IACN,CAAC,uCAAuC,8BAA8B,UAAU;AAAA,IAChF,CAAC,kBAAkB,QAAQ,WAAW;AAAA,IACtC,CAAC,sBAAsB,aAAa,UAAU;AAAA,IAC9C,CAAC,mCAAmC,yBAAyB,WAAW;AAAA,IACxE,CAAC,mCAAmC,wBAAwB,YAAY;AAAA,EAC1E,CAAC,EAAE,oCAAoC,CAAC,UAAU,eAAe,kBAAkB;AACjF,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,QAAQ;AACnD,WAAO,KAAK,EAAE,KAAK,aAAa;AAChC,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,iBAAiB;AAC5D,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,KAAK,EAAE;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,MAAM;AACrC,KAAG,uBAAuB,MAAM;AAC9B,UAAM,SAAS,qBAAqB,gBAAgB;AACpD,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,GAAG,CAAC,CAAC;AAAA,EACjE,CAAC;AAED,KAAG,kCAAkC,MAAM;AACzC,UAAM,SAAS,qBAAqB,yBAAyB;AAC7D,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACzE,CAAC;AAED,KAAG,6BAA6B,MAAM;AACpC,UAAM,WAA6B,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAChF,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACzE,CAAC;AAED,KAAG,yBAAyB,MAAM;AAChC,UAAM,SAAS,qBAAqB,CAAC,kBAAkB,4BAA4B,CAAC;AACpF,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,GAAG;AAAA,MACrC,EAAE,OAAO,8BAA8B,OAAO,GAAG;AAAA,IACnD,CAAC;AAAA,EACH,CAAC;AAED,KAAG,qCAAqC,MAAM;AAC5C,UAAM,SAAS,qBAAqB,CAAC,yBAAyB,mBAAmB,CAAC;AAClF,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,cAAc,OAAO,SAAS;AAAA,IACzC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,YAAgC;AAAA,MACpC,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAAA,MAC7C,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC;AACA,UAAM,SAAS,qBAAqB,SAAS;AAC7C,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAAA,MAC7C,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,mCAAmC,MAAM;AAC1C,UAAM,SAAS,qBAAqB;AAAA,MAClC;AAAA,MACA,EAAE,OAAO,8BAA8B,OAAO,SAAS;AAAA,MACvD;AAAA,IACF,CAAC;AACD,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,8BAA8B,OAAO,SAAS;AAAA,MACvD,EAAE,OAAO,aAAa,OAAO,GAAG;AAAA,IAClC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,+CAA+C,MAAM;AACtD,UAAM,WAA6B;AAAA,MACjC,OAAO;AAAA,MACP,OAAO;AAAA,MACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,IAC/C;AACA,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB;AAAA,QACE,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,KAAG,mCAAmC,MAAM;AAC1C,UAAM,SAAS,qBAAqB;AAAA,MAClC,EAAE,OAAO,kBAAkB,OAAO,MAAM,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE;AAAA,MACA,EAAE,OAAO,aAAa,OAAO,IAAI,aAAa,EAAE,QAAQ,KAAK,EAAE;AAAA,IACjE,CAAC;AACD,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,MAAM,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE,EAAE,OAAO,cAAc,OAAO,SAAS;AAAA,MACvC,EAAE,OAAO,aAAa,OAAO,IAAI,aAAa,EAAE,QAAQ,KAAK,EAAE;AAAA,IACjE,CAAC;AAAA,EACH,CAAC;AAED,KAAG,iCAAiC,MAAM;AACxC,UAAM,SAAS,qBAAqB,CAAC,CAAC;AACtC,WAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC3B,CAAC;AAED,KAAG,kCAAkC,MAAM;AACzC,UAAM,WAA6B,EAAE,OAAO,kBAAkB,OAAO,GAAG;AACxE,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,GAAG,CAAC,CAAC;AAAA,EACjE,CAAC;AACH,CAAC;AAED,SAAS,4CAA4C,MAAM;AACzD,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ,EAAE,UAAU,UAAU,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,KAAK,IAAI;AAAA,EACxC,CAAC;AAED,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ;AACpB,QAAI,cAAc,EAAE,UAAU,QAAQ,CAAC;AACvC,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,KAAK,kBAAkB,OAAO,CAAC;AAAA,EAC9D,CAAC;AAED,KAAG,4CAA4C,MAAM;AACnD,UAAM,MAAM,QAAQ;AACpB,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,cAAc;AAAA,EAC7C,CAAC;AAED,KAAG,wCAAwC,MAAM;AAC/C,UAAM,MAAM,QAAQ,EAAE,UAAU,6BAA6B,CAAC;AAC9D,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,8BAA8B,OAAO,GAAG,CAAC,CAAC;AAAA,EAC3F,CAAC;AAED,KAAG,mDAAmD,MAAM;AAC1D,UAAM,MAAM,QAAQ,EAAE,UAAU,0BAA0B,CAAC;AAC3D,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACvF,CAAC;AAED,KAAG,0CAA0C,MAAM;AACjD,UAAM,MAAM,QAAQ,EAAE,UAAU,CAAC,kBAAkB,YAAY,EAAE,CAAC;AAClE,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC,EAAE,OAAO,kBAAkB,OAAO,GAAG;AAAA,MACrC,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,uDAAuD,MAAM;AAC9D,UAAM,MAAM,QAAQ,EAAE,UAAU,EAAE,OAAO,kBAAkB,OAAO,WAAW,EAAE,CAAC;AAChF,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACvF,CAAC;AAED,KAAG,0CAA0C,MAAM;AACjD,UAAM,MAAM,QAAQ;AAAA,MAClB,UAAU;AAAA,QACR,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AACD,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC;AAAA,QACE,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,KAAG,qCAAqC,MAAM;AAC5C,UAAM,MAAM,QAAQ;AAAA,MAClB,UAAU;AAAA,QACR;AAAA,QACA,EAAE,OAAO,cAAc,OAAO,UAAU,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,QACvE;AAAA,MACF;AAAA,IACF,CAAC;AACD,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,cAAc,OAAO,UAAU,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE,EAAE,OAAO,aAAa,OAAO,GAAG;AAAA,IAClC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ;AACpB,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,2BAA2B;AAAA,EACrE,CAAC;AAED,KAAG,8BAA8B,MAAM;AACrC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,GAAG,iBAAiB,IAAK;AACzF,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,SAAS,EAAE,KAAK,GAAK;AAAA,EACvD,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,GAAG,iBAAiB,IAAK;AACzF,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,QAAQ,EAAE,KAAK,CAAC;AAAA,EAClD,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,IAAI,iBAAiB,IAAK;AAC1F,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,SAAS,EAAE,KAAK,GAAK;AACrD,WAAO,IAAI,MAAM,EAAE,YAAa,QAAQ,EAAE,KAAK,EAAE;AACjD,WAAO,IAAI,MAAM,EAAE,YAAa,eAAe,EAAE,KAAK,GAAI;AAAA,EAC5D,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/inference/tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { beforeAll, describe, expect, it } from 'vitest';\nimport { normalizeLanguage } from '../language.js';\nimport { initializeLogger } from '../log.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { TTS, type TTSFallbackModel, normalizeTTSFallback, parseTTSModelString } from './tts.js';\n\nbeforeAll(() => {\n  initializeLogger({ level: 'silent', pretty: false });\n});\n\n/** Helper to create TTS with required credentials. */\nfunction makeTts(overrides: Record<string, unknown> = {}) {\n  const defaults = {\n    model: 'cartesia/sonic' as const,\n    apiKey: 'test-key',\n    apiSecret: 'test-secret',\n    baseURL: 'https://example.livekit.cloud',\n  };\n  return new TTS({ ...defaults, ...overrides });\n}\n\ndescribe('parseTTSModelString', () => {\n  it('simple model without voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia');\n    expect(model).toBe('cartesia');\n    expect(voice).toBeUndefined();\n  });\n\n  it('model with voice suffix', () => {\n    const [model, voice] = parseTTSModelString('cartesia:my-voice-id');\n    expect(model).toBe('cartesia');\n    expect(voice).toBe('my-voice-id');\n  });\n\n  it('provider/model format without voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBeUndefined();\n  });\n\n  it('provider/model format with voice', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic:my-voice-id');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBe('my-voice-id');\n  });\n\n  it.each([\n    ['elevenlabs/eleven_flash_v2:voice123', 'elevenlabs/eleven_flash_v2', 'voice123'],\n    ['rime:speaker-a', 'rime', 'speaker-a'],\n    ['rime/mist:narrator', 'rime/mist', 'narrator'],\n    ['inworld/inworld-tts-1:character', 'inworld/inworld-tts-1', 'character'],\n    ['cartesia/sonic-turbo:deep-voice', 'cartesia/sonic-turbo', 'deep-voice'],\n  ])('various providers and voices: %s', (modelStr, expectedModel, expectedVoice) => {\n    const [model, voice] = parseTTSModelString(modelStr);\n    expect(model).toBe(expectedModel);\n    expect(voice).toBe(expectedVoice);\n  });\n\n  it('empty voice after colon', () => {\n    const [model, voice] = parseTTSModelString('cartesia/sonic:');\n    expect(model).toBe('cartesia/sonic');\n    expect(voice).toBe('');\n  });\n});\n\ndescribe('normalizeTTSFallback', () => {\n  it('single string model', () => {\n    const result = normalizeTTSFallback('cartesia/sonic');\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);\n  });\n\n  it('single string model with voice', () => {\n    const result = normalizeTTSFallback('cartesia/sonic:my-voice');\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);\n  });\n\n  it('single FallbackModel dict', () => {\n    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: 'narrator' };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);\n  });\n\n  it('list of string models', () => {\n    const result = normalizeTTSFallback(['cartesia/sonic', 'elevenlabs/eleven_flash_v2']);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: '' },\n      { model: 'elevenlabs/eleven_flash_v2', voice: '' },\n    ]);\n  });\n\n  it('list of string models with voices', () => {\n    const result = normalizeTTSFallback(['cartesia/sonic:voice1', 'elevenlabs:voice2']);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs', voice: 'voice2' },\n    ]);\n  });\n\n  it('list of FallbackModel dicts', () => {\n    const fallbacks: TTSFallbackModel[] = [\n      { model: 'cartesia/sonic', voice: 'narrator' },\n      { model: 'elevenlabs', voice: '' },\n    ];\n    const result = normalizeTTSFallback(fallbacks);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'narrator' },\n      { model: 'elevenlabs', voice: '' },\n    ]);\n  });\n\n  it('mixed list of strings and dicts', () => {\n    const result = normalizeTTSFallback([\n      'cartesia/sonic:voice1',\n      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' } as TTSFallbackModel,\n      'rime/mist',\n    ]);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs/eleven_flash_v2', voice: 'custom' },\n      { model: 'rime/mist', voice: '' },\n    ]);\n  });\n\n  it('FallbackModel with extraKwargs is preserved', () => {\n    const fallback: TTSFallbackModel = {\n      model: 'cartesia/sonic',\n      voice: 'narrator',\n      extraKwargs: { duration: 30.0, speed: 'fast' },\n    };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([\n      {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    ]);\n  });\n\n  it('list with extraKwargs preserved', () => {\n    const result = normalizeTTSFallback([\n      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } } as TTSFallbackModel,\n      'elevenlabs:voice2',\n      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } } as TTSFallbackModel,\n    ]);\n    expect(result).toEqual([\n      { model: 'cartesia/sonic', voice: 'v1', extraKwargs: { speed: 'slow' } },\n      { model: 'elevenlabs', voice: 'voice2' },\n      { model: 'rime/mist', voice: '', extraKwargs: { custom: true } },\n    ]);\n  });\n\n  it('empty list returns empty list', () => {\n    const result = normalizeTTSFallback([]);\n    expect(result).toEqual([]);\n  });\n\n  it('FallbackModel with empty voice', () => {\n    const fallback: TTSFallbackModel = { model: 'cartesia/sonic', voice: '' };\n    const result = normalizeTTSFallback(fallback);\n    expect(result).toEqual([{ model: 'cartesia/sonic', voice: '' }]);\n  });\n});\n\ndescribe('TTS constructor fallback and connOptions', () => {\n  it('normalizes language in constructor', () => {\n    const tts = makeTts({ language: 'english' });\n    expect(tts['opts'].language).toBe('en');\n  });\n\n  it('normalizes updated language values', () => {\n    const tts = makeTts();\n    tts.updateOptions({ language: 'en_US' });\n    expect(tts['opts'].language).toBe(normalizeLanguage('en_US'));\n  });\n\n  it('fallback not given defaults to undefined', () => {\n    const tts = makeTts();\n    expect(tts['opts'].fallback).toBeUndefined();\n  });\n\n  it('fallback single string is normalized', () => {\n    const tts = makeTts({ fallback: 'elevenlabs/eleven_flash_v2' });\n    expect(tts['opts'].fallback).toEqual([{ model: 'elevenlabs/eleven_flash_v2', voice: '' }]);\n  });\n\n  it('fallback single string with voice is normalized', () => {\n    const tts = makeTts({ fallback: 'cartesia/sonic:my-voice' });\n    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'my-voice' }]);\n  });\n\n  it('fallback list of strings is normalized', () => {\n    const tts = makeTts({ fallback: ['cartesia/sonic', 'elevenlabs'] });\n    expect(tts['opts'].fallback).toEqual([\n      { model: 'cartesia/sonic', voice: '' },\n      { model: 'elevenlabs', voice: '' },\n    ]);\n  });\n\n  it('fallback single FallbackModel is normalized to list', () => {\n    const tts = makeTts({ fallback: { model: 'cartesia/sonic', voice: 'narrator' } });\n    expect(tts['opts'].fallback).toEqual([{ model: 'cartesia/sonic', voice: 'narrator' }]);\n  });\n\n  it('fallback with extraKwargs is preserved', () => {\n    const tts = makeTts({\n      fallback: {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    });\n    expect(tts['opts'].fallback).toEqual([\n      {\n        model: 'cartesia/sonic',\n        voice: 'narrator',\n        extraKwargs: { duration: 30.0, speed: 'fast' },\n      },\n    ]);\n  });\n\n  it('fallback mixed list is normalized', () => {\n    const tts = makeTts({\n      fallback: [\n        'cartesia/sonic:voice1',\n        { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },\n        'rime/mist',\n      ],\n    });\n    expect(tts['opts'].fallback).toEqual([\n      { model: 'cartesia/sonic', voice: 'voice1' },\n      { model: 'elevenlabs', voice: 'custom', extraKwargs: { speed: 'slow' } },\n      { model: 'rime/mist', voice: '' },\n    ]);\n  });\n\n  it('connOptions not given uses default', () => {\n    const tts = makeTts();\n    expect(tts['opts'].connOptions).toEqual(DEFAULT_API_CONNECT_OPTIONS);\n  });\n\n  it('connOptions custom timeout', () => {\n    const custom: APIConnectOptions = { timeoutMs: 30000, maxRetry: 3, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.timeoutMs).toBe(30000);\n  });\n\n  it('connOptions custom maxRetry', () => {\n    const custom: APIConnectOptions = { timeoutMs: 10000, maxRetry: 5, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.maxRetry).toBe(5);\n  });\n\n  it('connOptions full custom', () => {\n    const custom: APIConnectOptions = { timeoutMs: 60000, maxRetry: 10, retryIntervalMs: 2000 };\n    const tts = makeTts({ connOptions: custom });\n    expect(tts['opts'].connOptions).toEqual(custom);\n    expect(tts['opts'].connOptions!.timeoutMs).toBe(60000);\n    expect(tts['opts'].connOptions!.maxRetry).toBe(10);\n    expect(tts['opts'].connOptions!.retryIntervalMs).toBe(2000);\n  });\n});\n\ndescribe('TTS provider modelOptions parity', () => {\n  it('preserves ElevenLabs inference model options', () => {\n    const modelOptions = {\n      speed: 1.2,\n      stability: 0.5,\n      similarity_boost: 0.8,\n      enable_logging: false,\n    };\n\n    const tts = new TTS({\n      model: 'elevenlabs/eleven_flash_v2_5' as const,\n      apiKey: 'test-key',\n      apiSecret: 'test-secret',\n      baseURL: 'https://example.livekit.cloud',\n      modelOptions,\n    });\n\n    expect(tts['opts'].modelOptions).toEqual(modelOptions);\n  });\n\n  it('accepts expanded Cartesia inference model options', () => {\n    const modelOptions = {\n      speed: 1.15,\n      emotion: 'curious',\n      add_timestamps: true,\n    };\n\n    const tts = new TTS({\n      model: 'cartesia/sonic' as const,\n      apiKey: 'test-key',\n      apiSecret: 'test-secret',\n      baseURL: 'https://example.livekit.cloud',\n      modelOptions,\n    });\n\n    expect(tts['opts'].modelOptions).toEqual(modelOptions);\n  });\n\n  it('accepts Deepgram inference model options', () => {\n    const modelOptions = { mip_opt_out: true };\n\n    const tts = new TTS({\n      model: 'deepgram/aura-2' as const,\n      apiKey: 'test-key',\n      apiSecret: 'test-secret',\n      baseURL: 'https://example.livekit.cloud',\n      modelOptions,\n    });\n\n    expect(tts['opts'].modelOptions).toEqual(modelOptions);\n  });\n\n  it('accepts Rime inference model options', () => {\n    const modelOptions = {\n      speed_alpha: 0.9,\n      pause_between_brackets: true,\n    };\n\n    const tts = new TTS({\n      model: 'rime/mistv2' as const,\n      apiKey: 'test-key',\n      apiSecret: 'test-secret',\n      baseURL: 'https://example.livekit.cloud',\n      modelOptions,\n    });\n\n    expect(tts['opts'].modelOptions).toEqual(modelOptions);\n  });\n\n  it('accepts Inworld inference model options', () => {\n    const modelOptions = {\n      timestamp_type: 'WORD' as const,\n      apply_text_normalization: 'ON' as const,\n    };\n\n    const tts = new TTS({\n      model: 'inworld/inworld-tts-1' as const,\n      apiKey: 'test-key',\n      apiSecret: 'test-secret',\n      baseURL: 'https://example.livekit.cloud',\n      modelOptions,\n    });\n\n    expect(tts['opts'].modelOptions).toEqual(modelOptions);\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW,UAAU,QAAQ,UAAU;AAChD,SAAS,yBAAyB;AAClC,SAAS,wBAAwB;AACjC,SAAiC,mCAAmC;AACpE,SAAS,KAA4B,sBAAsB,2BAA2B;AAEtF,UAAU,MAAM;AACd,mBAAiB,EAAE,OAAO,UAAU,QAAQ,MAAM,CAAC;AACrD,CAAC;AAGD,SAAS,QAAQ,YAAqC,CAAC,GAAG;AACxD,QAAM,WAAW;AAAA,IACf,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,SAAS;AAAA,EACX;AACA,SAAO,IAAI,IAAI,EAAE,GAAG,UAAU,GAAG,UAAU,CAAC;AAC9C;AAEA,SAAS,uBAAuB,MAAM;AACpC,KAAG,8BAA8B,MAAM;AACrC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,UAAU;AACrD,WAAO,KAAK,EAAE,KAAK,UAAU;AAC7B,WAAO,KAAK,EAAE,cAAc;AAAA,EAC9B,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,sBAAsB;AACjE,WAAO,KAAK,EAAE,KAAK,UAAU;AAC7B,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,uCAAuC,MAAM;AAC9C,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,gBAAgB;AAC3D,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,cAAc;AAAA,EAC9B,CAAC;AAED,KAAG,oCAAoC,MAAM;AAC3C,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,4BAA4B;AACvE,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,KAAK;AAAA,IACN,CAAC,uCAAuC,8BAA8B,UAAU;AAAA,IAChF,CAAC,kBAAkB,QAAQ,WAAW;AAAA,IACtC,CAAC,sBAAsB,aAAa,UAAU;AAAA,IAC9C,CAAC,mCAAmC,yBAAyB,WAAW;AAAA,IACxE,CAAC,mCAAmC,wBAAwB,YAAY;AAAA,EAC1E,CAAC,EAAE,oCAAoC,CAAC,UAAU,eAAe,kBAAkB;AACjF,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,QAAQ;AACnD,WAAO,KAAK,EAAE,KAAK,aAAa;AAChC,WAAO,KAAK,EAAE,KAAK,aAAa;AAAA,EAClC,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,iBAAiB;AAC5D,WAAO,KAAK,EAAE,KAAK,gBAAgB;AACnC,WAAO,KAAK,EAAE,KAAK,EAAE;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,MAAM;AACrC,KAAG,uBAAuB,MAAM;AAC9B,UAAM,SAAS,qBAAqB,gBAAgB;AACpD,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,GAAG,CAAC,CAAC;AAAA,EACjE,CAAC;AAED,KAAG,kCAAkC,MAAM;AACzC,UAAM,SAAS,qBAAqB,yBAAyB;AAC7D,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACzE,CAAC;AAED,KAAG,6BAA6B,MAAM;AACpC,UAAM,WAA6B,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAChF,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACzE,CAAC;AAED,KAAG,yBAAyB,MAAM;AAChC,UAAM,SAAS,qBAAqB,CAAC,kBAAkB,4BAA4B,CAAC;AACpF,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,GAAG;AAAA,MACrC,EAAE,OAAO,8BAA8B,OAAO,GAAG;AAAA,IACnD,CAAC;AAAA,EACH,CAAC;AAED,KAAG,qCAAqC,MAAM;AAC5C,UAAM,SAAS,qBAAqB,CAAC,yBAAyB,mBAAmB,CAAC;AAClF,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,cAAc,OAAO,SAAS;AAAA,IACzC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,YAAgC;AAAA,MACpC,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAAA,MAC7C,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC;AACA,UAAM,SAAS,qBAAqB,SAAS;AAC7C,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,WAAW;AAAA,MAC7C,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,mCAAmC,MAAM;AAC1C,UAAM,SAAS,qBAAqB;AAAA,MAClC;AAAA,MACA,EAAE,OAAO,8BAA8B,OAAO,SAAS;AAAA,MACvD;AAAA,IACF,CAAC;AACD,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,8BAA8B,OAAO,SAAS;AAAA,MACvD,EAAE,OAAO,aAAa,OAAO,GAAG;AAAA,IAClC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,+CAA+C,MAAM;AACtD,UAAM,WAA6B;AAAA,MACjC,OAAO;AAAA,MACP,OAAO;AAAA,MACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,IAC/C;AACA,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB;AAAA,QACE,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,KAAG,mCAAmC,MAAM;AAC1C,UAAM,SAAS,qBAAqB;AAAA,MAClC,EAAE,OAAO,kBAAkB,OAAO,MAAM,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE;AAAA,MACA,EAAE,OAAO,aAAa,OAAO,IAAI,aAAa,EAAE,QAAQ,KAAK,EAAE;AAAA,IACjE,CAAC;AACD,WAAO,MAAM,EAAE,QAAQ;AAAA,MACrB,EAAE,OAAO,kBAAkB,OAAO,MAAM,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE,EAAE,OAAO,cAAc,OAAO,SAAS;AAAA,MACvC,EAAE,OAAO,aAAa,OAAO,IAAI,aAAa,EAAE,QAAQ,KAAK,EAAE;AAAA,IACjE,CAAC;AAAA,EACH,CAAC;AAED,KAAG,iCAAiC,MAAM;AACxC,UAAM,SAAS,qBAAqB,CAAC,CAAC;AACtC,WAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC3B,CAAC;AAED,KAAG,kCAAkC,MAAM;AACzC,UAAM,WAA6B,EAAE,OAAO,kBAAkB,OAAO,GAAG;AACxE,UAAM,SAAS,qBAAqB,QAAQ;AAC5C,WAAO,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,GAAG,CAAC,CAAC;AAAA,EACjE,CAAC;AACH,CAAC;AAED,SAAS,4CAA4C,MAAM;AACzD,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ,EAAE,UAAU,UAAU,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,KAAK,IAAI;AAAA,EACxC,CAAC;AAED,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ;AACpB,QAAI,cAAc,EAAE,UAAU,QAAQ,CAAC;AACvC,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,KAAK,kBAAkB,OAAO,CAAC;AAAA,EAC9D,CAAC;AAED,KAAG,4CAA4C,MAAM;AACnD,UAAM,MAAM,QAAQ;AACpB,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,cAAc;AAAA,EAC7C,CAAC;AAED,KAAG,wCAAwC,MAAM;AAC/C,UAAM,MAAM,QAAQ,EAAE,UAAU,6BAA6B,CAAC;AAC9D,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,8BAA8B,OAAO,GAAG,CAAC,CAAC;AAAA,EAC3F,CAAC;AAED,KAAG,mDAAmD,MAAM;AAC1D,UAAM,MAAM,QAAQ,EAAE,UAAU,0BAA0B,CAAC;AAC3D,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACvF,CAAC;AAED,KAAG,0CAA0C,MAAM;AACjD,UAAM,MAAM,QAAQ,EAAE,UAAU,CAAC,kBAAkB,YAAY,EAAE,CAAC;AAClE,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC,EAAE,OAAO,kBAAkB,OAAO,GAAG;AAAA,MACrC,EAAE,OAAO,cAAc,OAAO,GAAG;AAAA,IACnC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,uDAAuD,MAAM;AAC9D,UAAM,MAAM,QAAQ,EAAE,UAAU,EAAE,OAAO,kBAAkB,OAAO,WAAW,EAAE,CAAC;AAChF,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,OAAO,kBAAkB,OAAO,WAAW,CAAC,CAAC;AAAA,EACvF,CAAC;AAED,KAAG,0CAA0C,MAAM;AACjD,UAAM,MAAM,QAAQ;AAAA,MAClB,UAAU;AAAA,QACR,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AACD,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC;AAAA,QACE,OAAO;AAAA,QACP,OAAO;AAAA,QACP,aAAa,EAAE,UAAU,IAAM,OAAO,OAAO;AAAA,MAC/C;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,KAAG,qCAAqC,MAAM;AAC5C,UAAM,MAAM,QAAQ;AAAA,MAClB,UAAU;AAAA,QACR;AAAA,QACA,EAAE,OAAO,cAAc,OAAO,UAAU,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,QACvE;AAAA,MACF;AAAA,IACF,CAAC;AACD,WAAO,IAAI,MAAM,EAAE,QAAQ,EAAE,QAAQ;AAAA,MACnC,EAAE,OAAO,kBAAkB,OAAO,SAAS;AAAA,MAC3C,EAAE,OAAO,cAAc,OAAO,UAAU,aAAa,EAAE,OAAO,OAAO,EAAE;AAAA,MACvE,EAAE,OAAO,aAAa,OAAO,GAAG;AAAA,IAClC,CAAC;AAAA,EACH,CAAC;AAED,KAAG,sCAAsC,MAAM;AAC7C,UAAM,MAAM,QAAQ;AACpB,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,2BAA2B;AAAA,EACrE,CAAC;AAED,KAAG,8BAA8B,MAAM;AACrC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,GAAG,iBAAiB,IAAK;AACzF,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,SAAS,EAAE,KAAK,GAAK;AAAA,EACvD,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,GAAG,iBAAiB,IAAK;AACzF,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,QAAQ,EAAE,KAAK,CAAC;AAAA,EAClD,CAAC;AAED,KAAG,2BAA2B,MAAM;AAClC,UAAM,SAA4B,EAAE,WAAW,KAAO,UAAU,IAAI,iBAAiB,IAAK;AAC1F,UAAM,MAAM,QAAQ,EAAE,aAAa,OAAO,CAAC;AAC3C,WAAO,IAAI,MAAM,EAAE,WAAW,EAAE,QAAQ,MAAM;AAC9C,WAAO,IAAI,MAAM,EAAE,YAAa,SAAS,EAAE,KAAK,GAAK;AACrD,WAAO,IAAI,MAAM,EAAE,YAAa,QAAQ,EAAE,KAAK,EAAE;AACjD,WAAO,IAAI,MAAM,EAAE,YAAa,eAAe,EAAE,KAAK,GAAI;AAAA,EAC5D,CAAC;AACH,CAAC;AAED,SAAS,oCAAoC,MAAM;AACjD,KAAG,gDAAgD,MAAM;AACvD,UAAM,eAAe;AAAA,MACnB,OAAO;AAAA,MACP,WAAW;AAAA,MACX,kBAAkB;AAAA,MAClB,gBAAgB;AAAA,IAClB;AAEA,UAAM,MAAM,IAAI,IAAI;AAAA,MAClB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAED,WAAO,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,YAAY;AAAA,EACvD,CAAC;AAED,KAAG,qDAAqD,MAAM;AAC5D,UAAM,eAAe;AAAA,MACnB,OAAO;AAAA,MACP,SAAS;AAAA,MACT,gBAAgB;AAAA,IAClB;AAEA,UAAM,MAAM,IAAI,IAAI;AAAA,MAClB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAED,WAAO,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,YAAY;AAAA,EACvD,CAAC;AAED,KAAG,4CAA4C,MAAM;AACnD,UAAM,eAAe,EAAE,aAAa,KAAK;AAEzC,UAAM,MAAM,IAAI,IAAI;AAAA,MAClB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAED,WAAO,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,YAAY;AAAA,EACvD,CAAC;AAED,KAAG,wCAAwC,MAAM;AAC/C,UAAM,eAAe;AAAA,MACnB,aAAa;AAAA,MACb,wBAAwB;AAAA,IAC1B;AAEA,UAAM,MAAM,IAAI,IAAI;AAAA,MAClB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAED,WAAO,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,YAAY;AAAA,EACvD,CAAC;AAED,KAAG,2CAA2C,MAAM;AAClD,UAAM,eAAe;AAAA,MACnB,gBAAgB;AAAA,MAChB,0BAA0B;AAAA,IAC5B;AAEA,UAAM,MAAM,IAAI,IAAI;AAAA,MAClB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,SAAS;AAAA,MACT;AAAA,IACF,CAAC;AAED,WAAO,IAAI,MAAM,EAAE,YAAY,EAAE,QAAQ,YAAY;AAAA,EACvD,CAAC;AACH,CAAC;","names":[]}

package/dist/ipc/job_proc_lazy_main.cjs

@@ -56,6 +56,7 @@ class PendingInference {
 }
 class InfClient {
   #requests = {};
+  #logger = (0, import_log.log)();
   constructor() {
     process.on("message", (msg) => {
       switch (msg.case) {
@@ -63,7 +64,7 @@ class InfClient {
           const fut = this.#requests[msg.value.requestId];
           delete this.#requests[msg.value.requestId];
           if (!fut) {
-            (0, import_log.log)().child({ resp: msg.value }).warn("received unexpected inference response");
+            this.#logger.child({ resp: msg.value }).warn("received unexpected inference response");
             return;
           }
           fut.resolve(msg.value);
@@ -74,7 +75,11 @@ class InfClient {
   async doInference(method, data) {
     const requestId = (0, import_utils.shortuuid)("inference_job_");
     if (!safeSend({ case: "inferenceRequest", value: { requestId, method, data } })) {
-      throw new Error("IPC channel closed");
+      this.#logger.debug(
+        { method, requestId },
+        "IPC channel closed during inference, aborting gracefully"
+      );
+      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);
     }
     this.#requests[requestId] = new PendingInference();
     const resp = await this.#requests[requestId].promise;

package/dist/ipc/job_proc_lazy_main.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            log().child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      throw new Error('IPC channel closed');\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGA,sBAAyC;AACzC,yBAAmC;AACnC,sBAA8B;AAE9B,uBAAoC;AACpC,iBAAoF;AACpF,iBAAsC;AACtC,mBAAkC;AAClC,oBAA6C;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,0BAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,0BAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EAEjD,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,gCAAI,EAAE,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AAC9E;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,gBAAY,wBAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,qBAAK;AACtB,OAAK,GAAG,0BAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,sBAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,gBAAM,mCAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AA/H1B;AAgIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,gBAAM,yBAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,oBAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,WAAO,+BAAc,UAAW,EAAE,UAAU,KAAK,CAACA,YAAW;AA5L5F;AA8LM,YAAMC,SACJ,OAAOD,QAAO,YAAY,kBAAc,0BAAQA,QAAO,OAAO,IAC1DA,QAAO,WACP,KAAAA,QAAO,YAAP,mBAAgB;AACtB,UAAIC,WAAU,UAAa,KAAC,0BAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,cAAM,yBAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uCAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,sBAAW;AAC5B,QAAI,aAAS,gBAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,gCAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,gBAAM,yBAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["module","agent"]}
+{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n  #logger = log();\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            this.#logger.child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      this.#logger.debug(\n        { method, requestId },\n        'IPC channel closed during inference, aborting gracefully',\n      );\n      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGA,sBAAyC;AACzC,yBAAmC;AACnC,sBAA8B;AAE9B,uBAAoC;AACpC,iBAAoF;AACpF,iBAAsC;AACtC,mBAAkC;AAClC,oBAA6C;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,0BAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,0BAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EACjD,cAAU,gBAAI;AAAA,EAEd,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AACrF;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,gBAAY,wBAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,WAAK,QAAQ;AAAA,QACX,EAAE,QAAQ,UAAU;AAAA,QACpB;AAAA,MACF;AACA,YAAM,IAAI,MAAM,aAAa,MAAM,yDAAyD;AAAA,IAC9F;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,qBAAK;AACtB,OAAK,GAAG,0BAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,sBAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,gBAAM,mCAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AApI1B;AAqIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,gBAAM,yBAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,oBAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,WAAO,+BAAc,UAAW,EAAE,UAAU,KAAK,CAACA,YAAW;AAjM5F;AAmMM,YAAMC,SACJ,OAAOD,QAAO,YAAY,kBAAc,0BAAQA,QAAO,OAAO,IAC1DA,QAAO,WACP,KAAAA,QAAO,YAAP,mBAAgB;AACtB,UAAIC,WAAU,UAAa,KAAC,0BAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,cAAM,yBAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uCAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,sBAAW;AAC5B,QAAI,aAAS,gBAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,gCAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,gBAAM,yBAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["module","agent"]}

package/dist/ipc/job_proc_lazy_main.js

@@ -33,6 +33,7 @@ class PendingInference {
 }
 class InfClient {
   #requests = {};
+  #logger = log();
   constructor() {
     process.on("message", (msg) => {
       switch (msg.case) {
@@ -40,7 +41,7 @@ class InfClient {
           const fut = this.#requests[msg.value.requestId];
           delete this.#requests[msg.value.requestId];
           if (!fut) {
-            log().child({ resp: msg.value }).warn("received unexpected inference response");
+            this.#logger.child({ resp: msg.value }).warn("received unexpected inference response");
             return;
           }
           fut.resolve(msg.value);
@@ -51,7 +52,11 @@ class InfClient {
   async doInference(method, data) {
     const requestId = shortuuid("inference_job_");
     if (!safeSend({ case: "inferenceRequest", value: { requestId, method, data } })) {
-      throw new Error("IPC channel closed");
+      this.#logger.debug(
+        { method, requestId },
+        "IPC channel closed during inference, aborting gracefully"
+      );
+      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);
     }
     this.#requests[requestId] = new PendingInference();
     const resp = await this.#requests[requestId].promise;

package/dist/ipc/job_proc_lazy_main.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            log().child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      throw new Error('IPC channel closed');\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":"AAGA,SAAS,MAAM,WAAW,eAAe;AACzC,SAAS,cAAc,YAAY;AACnC,SAAS,qBAAqB;AAE9B,SAAqB,eAAe;AACpC,SAAS,YAAY,YAAiC,8BAA8B;AACpF,SAAS,kBAAkB,WAAW;AACtC,SAAS,QAAQ,iBAAiB;AAClC,SAAS,oCAAoC;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,UAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,UAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EAEjD,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,gBAAI,EAAE,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AAC9E;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,YAAY,UAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,KAAK;AACtB,OAAK,GAAG,UAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,WAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,YAAM,uBAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AA/H1B;AAgIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,YAAM,KAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,OAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,OAAO,cAAc,UAAW,EAAE,UAAU,KAAK,CAAC,WAAW;AA5L5F;AA8LM,YAAMA,SACJ,OAAO,OAAO,YAAY,cAAc,QAAQ,OAAO,OAAO,IAC1D,OAAO,WACP,YAAO,YAAP,mBAAgB;AACtB,UAAIA,WAAU,UAAa,CAAC,QAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,UAAM,KAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uBAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,WAAW;AAC5B,QAAI,SAAS,IAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,aAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,YAAM,QAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["agent"]}
+{"version":3,"sources":["../../src/ipc/job_proc_lazy_main.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Room, RoomEvent, dispose } from '@livekit/rtc-node';\nimport { EventEmitter, once } from 'node:events';\nimport { pathToFileURL } from 'node:url';\nimport type { Logger } from 'pino';\nimport { type Agent, isAgent } from '../generator.js';\nimport { JobContext, JobProcess, type RunningJobInfo, runWithJobContextAsync } from '../job.js';\nimport { initializeLogger, log } from '../log.js';\nimport { Future, shortuuid } from '../utils.js';\nimport { defaultInitializeProcessFunc } from '../worker.js';\nimport type { InferenceExecutor } from './inference_executor.js';\nimport type { IPCMessage } from './message.js';\n\nconst ORPHANED_TIMEOUT = 15 * 1000;\n\nconst safeSend = (msg: IPCMessage): boolean => {\n  try {\n    if (process.connected && process.send) {\n      process.send(msg);\n      return true;\n    }\n    return false;\n  } catch (error) {\n    // Channel closed is expected during graceful shutdown\n    // Log at debug level to avoid noise in production logs\n    if (error instanceof Error && error.message.includes('Channel closed')) {\n      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');\n    } else {\n      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');\n    }\n    return false;\n  }\n};\n\ntype JobTask = {\n  ctx: JobContext;\n  task: Promise<void>;\n};\n\nclass PendingInference {\n  promise = new Promise<{ requestId: string; data: unknown; error?: Error }>((resolve) => {\n    this.resolve = resolve; // this is how JavaScript lets you resolve promises externally\n  });\n  resolve(arg: { requestId: string; data: unknown; error?: Error }) {\n    arg; // useless call to counteract TypeScript E6133\n  }\n}\n\nclass InfClient implements InferenceExecutor {\n  #requests: { [id: string]: PendingInference } = {};\n  #logger = log();\n\n  constructor() {\n    process.on('message', (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'inferenceResponse':\n          const fut = this.#requests[msg.value.requestId];\n          delete this.#requests[msg.value.requestId];\n          if (!fut) {\n            this.#logger.child({ resp: msg.value }).warn('received unexpected inference response');\n            return;\n          }\n          fut.resolve(msg.value);\n          break;\n      }\n    });\n  }\n\n  async doInference(method: string, data: unknown): Promise<unknown> {\n    const requestId = shortuuid('inference_job_');\n    if (!safeSend({ case: 'inferenceRequest', value: { requestId, method, data } })) {\n      this.#logger.debug(\n        { method, requestId },\n        'IPC channel closed during inference, aborting gracefully',\n      );\n      throw new Error(`Inference ${method} aborted: IPC channel closed (expected during shutdown)`);\n    }\n\n    this.#requests[requestId] = new PendingInference();\n    const resp = await this.#requests[requestId]!.promise;\n    if (resp.error) {\n      throw new Error(`inference of ${method} failed: ${resp.error.message}`);\n    }\n    return resp.data;\n  }\n}\n\nconst startJob = (\n  proc: JobProcess,\n  func: (ctx: JobContext) => Promise<void>,\n  info: RunningJobInfo,\n  closeEvent: EventEmitter,\n  logger: Logger,\n  joinFuture: Future,\n): JobTask => {\n  let connect = false;\n  let shutdown = false;\n\n  const room = new Room();\n  room.on(RoomEvent.Disconnected, () => {\n    if (!shutdown) {\n      closeEvent.emit('close', false);\n    }\n  });\n\n  const onConnect = () => {\n    connect = true;\n  };\n  const onShutdown = (reason: string) => {\n    shutdown = true;\n    closeEvent.emit('close', true, reason);\n  };\n\n  const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());\n\n  const task = (async () => {\n    const unconnectedTimeout = setTimeout(() => {\n      if (!(connect || shutdown)) {\n        logger.warn(\n          'room not connect after job_entry was called after 10 seconds, ',\n          'did you forget to call ctx.connect()?',\n        );\n      }\n    }, 10000);\n\n    try {\n      // Run the job function within the AsyncLocalStorage context\n      await runWithJobContextAsync(ctx, async () => {\n        const { tracer, traceTypes } = await import('../telemetry/index.js');\n        return tracer.startActiveSpan(\n          async (span) => {\n            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);\n            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);\n            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');\n            return func(ctx);\n          },\n          { name: 'job_entrypoint' },\n        );\n      }).finally(() => {\n        clearTimeout(unconnectedTimeout);\n      });\n\n      await once(closeEvent, 'close').then((close) => {\n        logger.debug('shutting down');\n        shutdown = true;\n        safeSend({ case: 'exiting', value: { reason: close[1] } });\n      });\n    } catch (error) {\n      logger.error({ error }, 'error in entry function');\n      shutdown = true;\n      safeSend({\n        case: 'exiting',\n        value: { reason: error instanceof Error ? error.message : String(error) },\n      });\n    }\n\n    // Close the primary agent session if it exists\n    if (ctx._primaryAgentSession) {\n      await ctx._primaryAgentSession.close();\n    }\n\n    // Generate and save/upload session report\n    await ctx._onSessionEnd();\n\n    await room.disconnect();\n    logger.debug('disconnected from room');\n\n    const shutdownTasks = [];\n    for (const callback of ctx.shutdownCallbacks) {\n      shutdownTasks.push(callback());\n    }\n    await Promise.all(shutdownTasks).catch((error) =>\n      logger.error({ error }, 'error while shutting down the job'),\n    );\n\n    safeSend({ case: 'done', value: undefined });\n    joinFuture.resolve();\n  })();\n\n  return { ctx, task };\n};\n\n(async () => {\n  if (process.send) {\n    const join = new Future();\n\n    // process.argv:\n    //   [0] `node'\n    //   [1] import.meta.filename\n    //   [2] import.meta.filename of function containing entry file\n    const moduleFile = process.argv[2];\n    const agent: Agent = await import(pathToFileURL(moduleFile!).pathname).then((module) => {\n      // Handle both ESM (module.default is the agent) and CJS (module.default.default is the agent)\n      const agent =\n        typeof module.default === 'function' || isAgent(module.default)\n          ? module.default\n          : module.default?.default;\n      if (agent === undefined || !isAgent(agent)) {\n        throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);\n      }\n      return agent;\n    });\n    if (!agent.prewarm) {\n      agent.prewarm = defaultInitializeProcessFunc;\n    }\n\n    // don't do anything on C-c\n    // this is handled in cli, triggering a termination of all child processes at once.\n    process.on('SIGINT', () => {\n      logger.debug('SIGINT received in job proc');\n    });\n\n    // don't do anything on SIGTERM\n    // Render uses SIGTERM in autoscale, this ensures the processes are properly drained if needed\n    process.on('SIGTERM', () => {\n      logger.debug('SIGTERM received in job proc');\n    });\n\n    await once(process, 'message').then(([msg]: IPCMessage[]) => {\n      msg = msg!;\n      if (msg.case !== 'initializeRequest') {\n        throw new Error('first message must be InitializeRequest');\n      }\n      initializeLogger(msg.value.loggerOptions);\n    });\n    const proc = new JobProcess();\n    let logger = log().child({ pid: proc.pid });\n\n    process.on('unhandledRejection', (reason) => {\n      logger.debug({ error: reason }, 'Unhandled promise rejection');\n    });\n\n    logger.debug('initializing job runner');\n    await agent.prewarm(proc);\n    logger.debug('job runner initialized');\n    safeSend({ case: 'initializeResponse', value: undefined });\n\n    let job: JobTask | undefined = undefined;\n    const closeEvent = new EventEmitter();\n\n    const orphanedTimeout = setTimeout(() => {\n      logger.warn('job process orphaned, shutting down.');\n      join.resolve();\n    }, ORPHANED_TIMEOUT);\n\n    const messageHandler = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pingRequest': {\n          orphanedTimeout.refresh();\n          safeSend({\n            case: 'pongResponse',\n            value: { lastTimestamp: msg.value.timestamp, timestamp: Date.now() },\n          });\n          break;\n        }\n        case 'startJobRequest': {\n          if (job) {\n            throw new Error('job task already running');\n          }\n\n          logger = logger.child({ jobID: msg.value.runningJob.job.id });\n\n          job = startJob(proc, agent.entry, msg.value.runningJob, closeEvent, logger, join);\n          logger.debug('job started');\n          break;\n        }\n        case 'shutdownRequest': {\n          if (!job) {\n            join.resolve();\n          }\n          closeEvent.emit('close', 'shutdownRequest');\n          clearTimeout(orphanedTimeout);\n          process.off('message', messageHandler);\n        }\n      }\n    };\n\n    process.on('message', messageHandler);\n\n    await join.await;\n\n    // Dispose native FFI resources (Rust FfiServer, tokio runtimes, libwebrtc)\n    // before process.exit() to prevent libc++abi mutex crash during teardown.\n    // Without this, process.exit() can kill the process while native threads are\n    // still running, causing: \"mutex lock failed: Invalid argument\"\n    // See: https://github.com/livekit/node-sdks/issues/564\n    try {\n      await dispose();\n      logger.debug('native resources disposed');\n    } catch (error) {\n      logger.warn({ error }, 'failed to dispose native resources');\n    }\n\n    logger.debug('Job process shutdown');\n    process.exit(0);\n  }\n})();\n"],"mappings":"AAGA,SAAS,MAAM,WAAW,eAAe;AACzC,SAAS,cAAc,YAAY;AACnC,SAAS,qBAAqB;AAE9B,SAAqB,eAAe;AACpC,SAAS,YAAY,YAAiC,8BAA8B;AACpF,SAAS,kBAAkB,WAAW;AACtC,SAAS,QAAQ,iBAAiB;AAClC,SAAS,oCAAoC;AAI7C,MAAM,mBAAmB,KAAK;AAE9B,MAAM,WAAW,CAAC,QAA6B;AAC7C,MAAI;AACF,QAAI,QAAQ,aAAa,QAAQ,MAAM;AACrC,cAAQ,KAAK,GAAG;AAChB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AAGd,QAAI,iBAAiB,SAAS,MAAM,QAAQ,SAAS,gBAAgB,GAAG;AACtE,UAAI,EAAE,MAAM,EAAE,SAAS,IAAI,KAAK,GAAG,sCAAsC;AAAA,IAC3E,OAAO;AACL,UAAI,EAAE,MAAM,EAAE,OAAO,SAAS,IAAI,KAAK,GAAG,8BAA8B;AAAA,IAC1E;AACA,WAAO;AAAA,EACT;AACF;AAOA,MAAM,iBAAiB;AAAA,EACrB,UAAU,IAAI,QAA6D,CAAC,YAAY;AACtF,SAAK,UAAU;AAAA,EACjB,CAAC;AAAA,EACD,QAAQ,KAA0D;AAChE;AAAA,EACF;AACF;AAEA,MAAM,UAAuC;AAAA,EAC3C,YAAgD,CAAC;AAAA,EACjD,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,YAAQ,GAAG,WAAW,CAAC,QAAoB;AACzC,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK;AACH,gBAAM,MAAM,KAAK,UAAU,IAAI,MAAM,SAAS;AAC9C,iBAAO,KAAK,UAAU,IAAI,MAAM,SAAS;AACzC,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,EAAE,KAAK,wCAAwC;AACrF;AAAA,UACF;AACA,cAAI,QAAQ,IAAI,KAAK;AACrB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,QAAgB,MAAiC;AACjE,UAAM,YAAY,UAAU,gBAAgB;AAC5C,QAAI,CAAC,SAAS,EAAE,MAAM,oBAAoB,OAAO,EAAE,WAAW,QAAQ,KAAK,EAAE,CAAC,GAAG;AAC/E,WAAK,QAAQ;AAAA,QACX,EAAE,QAAQ,UAAU;AAAA,QACpB;AAAA,MACF;AACA,YAAM,IAAI,MAAM,aAAa,MAAM,yDAAyD;AAAA,IAC9F;AAEA,SAAK,UAAU,SAAS,IAAI,IAAI,iBAAiB;AACjD,UAAM,OAAO,MAAM,KAAK,UAAU,SAAS,EAAG;AAC9C,QAAI,KAAK,OAAO;AACd,YAAM,IAAI,MAAM,gBAAgB,MAAM,YAAY,KAAK,MAAM,OAAO,EAAE;AAAA,IACxE;AACA,WAAO,KAAK;AAAA,EACd;AACF;AAEA,MAAM,WAAW,CACf,MACA,MACA,MACA,YACA,QACA,eACY;AACZ,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,QAAM,OAAO,IAAI,KAAK;AACtB,OAAK,GAAG,UAAU,cAAc,MAAM;AACpC,QAAI,CAAC,UAAU;AACb,iBAAW,KAAK,SAAS,KAAK;AAAA,IAChC;AAAA,EACF,CAAC;AAED,QAAM,YAAY,MAAM;AACtB,cAAU;AAAA,EACZ;AACA,QAAM,aAAa,CAAC,WAAmB;AACrC,eAAW;AACX,eAAW,KAAK,SAAS,MAAM,MAAM;AAAA,EACvC;AAEA,QAAM,MAAM,IAAI,WAAW,MAAM,MAAM,MAAM,WAAW,YAAY,IAAI,UAAU,CAAC;AAEnF,QAAM,QAAQ,YAAY;AACxB,UAAM,qBAAqB,WAAW,MAAM;AAC1C,UAAI,EAAE,WAAW,WAAW;AAC1B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF,GAAG,GAAK;AAER,QAAI;AAEF,YAAM,uBAAuB,KAAK,YAAY;AAC5C,cAAM,EAAE,QAAQ,WAAW,IAAI,MAAM,OAAO,uBAAuB;AACnE,eAAO,OAAO;AAAA,UACZ,OAAO,SAAS;AApI1B;AAqIY,iBAAK,aAAa,WAAW,aAAa,KAAK,IAAI,EAAE;AACrD,iBAAK,aAAa,WAAW,iBAAiB,KAAK,IAAI,SAAS;AAChE,iBAAK,aAAa,WAAW,kBAAgB,UAAK,IAAI,SAAT,mBAAe,SAAQ,EAAE;AACtE,mBAAO,KAAK,GAAG;AAAA,UACjB;AAAA,UACA,EAAE,MAAM,iBAAiB;AAAA,QAC3B;AAAA,MACF,CAAC,EAAE,QAAQ,MAAM;AACf,qBAAa,kBAAkB;AAAA,MACjC,CAAC;AAED,YAAM,KAAK,YAAY,OAAO,EAAE,KAAK,CAAC,UAAU;AAC9C,eAAO,MAAM,eAAe;AAC5B,mBAAW;AACX,iBAAS,EAAE,MAAM,WAAW,OAAO,EAAE,QAAQ,MAAM,CAAC,EAAE,EAAE,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,MAAM,EAAE,MAAM,GAAG,yBAAyB;AACjD,iBAAW;AACX,eAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;AAAA,MAC1E,CAAC;AAAA,IACH;AAGA,QAAI,IAAI,sBAAsB;AAC5B,YAAM,IAAI,qBAAqB,MAAM;AAAA,IACvC;AAGA,UAAM,IAAI,cAAc;AAExB,UAAM,KAAK,WAAW;AACtB,WAAO,MAAM,wBAAwB;AAErC,UAAM,gBAAgB,CAAC;AACvB,eAAW,YAAY,IAAI,mBAAmB;AAC5C,oBAAc,KAAK,SAAS,CAAC;AAAA,IAC/B;AACA,UAAM,QAAQ,IAAI,aAAa,EAAE;AAAA,MAAM,CAAC,UACtC,OAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,IAC7D;AAEA,aAAS,EAAE,MAAM,QAAQ,OAAO,OAAU,CAAC;AAC3C,eAAW,QAAQ;AAAA,EACrB,GAAG;AAEH,SAAO,EAAE,KAAK,KAAK;AACrB;AAAA,CAEC,YAAY;AACX,MAAI,QAAQ,MAAM;AAChB,UAAM,OAAO,IAAI,OAAO;AAMxB,UAAM,aAAa,QAAQ,KAAK,CAAC;AACjC,UAAM,QAAe,MAAM,OAAO,cAAc,UAAW,EAAE,UAAU,KAAK,CAAC,WAAW;AAjM5F;AAmMM,YAAMA,SACJ,OAAO,OAAO,YAAY,cAAc,QAAQ,OAAO,OAAO,IAC1D,OAAO,WACP,YAAO,YAAP,mBAAgB;AACtB,UAAIA,WAAU,UAAa,CAAC,QAAQA,MAAK,GAAG;AAC1C,cAAM,IAAI,MAAM,8DAA8D,UAAU,EAAE;AAAA,MAC5F;AACA,aAAOA;AAAA,IACT,CAAC;AACD,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,UAAU;AAAA,IAClB;AAIA,YAAQ,GAAG,UAAU,MAAM;AACzB,aAAO,MAAM,6BAA6B;AAAA,IAC5C,CAAC;AAID,YAAQ,GAAG,WAAW,MAAM;AAC1B,aAAO,MAAM,8BAA8B;AAAA,IAC7C,CAAC;AAED,UAAM,KAAK,SAAS,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC3D,YAAM;AACN,UAAI,IAAI,SAAS,qBAAqB;AACpC,cAAM,IAAI,MAAM,yCAAyC;AAAA,MAC3D;AACA,uBAAiB,IAAI,MAAM,aAAa;AAAA,IAC1C,CAAC;AACD,UAAM,OAAO,IAAI,WAAW;AAC5B,QAAI,SAAS,IAAI,EAAE,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;AAE1C,YAAQ,GAAG,sBAAsB,CAAC,WAAW;AAC3C,aAAO,MAAM,EAAE,OAAO,OAAO,GAAG,6BAA6B;AAAA,IAC/D,CAAC;AAED,WAAO,MAAM,yBAAyB;AACtC,UAAM,MAAM,QAAQ,IAAI;AACxB,WAAO,MAAM,wBAAwB;AACrC,aAAS,EAAE,MAAM,sBAAsB,OAAO,OAAU,CAAC;AAEzD,QAAI,MAA2B;AAC/B,UAAM,aAAa,IAAI,aAAa;AAEpC,UAAM,kBAAkB,WAAW,MAAM;AACvC,aAAO,KAAK,sCAAsC;AAClD,WAAK,QAAQ;AAAA,IACf,GAAG,gBAAgB;AAEnB,UAAM,iBAAiB,CAAC,QAAoB;AAC1C,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,eAAe;AAClB,0BAAgB,QAAQ;AACxB,mBAAS;AAAA,YACP,MAAM;AAAA,YACN,OAAO,EAAE,eAAe,IAAI,MAAM,WAAW,WAAW,KAAK,IAAI,EAAE;AAAA,UACrE,CAAC;AACD;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,KAAK;AACP,kBAAM,IAAI,MAAM,0BAA0B;AAAA,UAC5C;AAEA,mBAAS,OAAO,MAAM,EAAE,OAAO,IAAI,MAAM,WAAW,IAAI,GAAG,CAAC;AAE5D,gBAAM,SAAS,MAAM,MAAM,OAAO,IAAI,MAAM,YAAY,YAAY,QAAQ,IAAI;AAChF,iBAAO,MAAM,aAAa;AAC1B;AAAA,QACF;AAAA,QACA,KAAK,mBAAmB;AACtB,cAAI,CAAC,KAAK;AACR,iBAAK,QAAQ;AAAA,UACf;AACA,qBAAW,KAAK,SAAS,iBAAiB;AAC1C,uBAAa,eAAe;AAC5B,kBAAQ,IAAI,WAAW,cAAc;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,GAAG,WAAW,cAAc;AAEpC,UAAM,KAAK;AAOX,QAAI;AACF,YAAM,QAAQ;AACd,aAAO,MAAM,2BAA2B;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,KAAK,EAAE,MAAM,GAAG,oCAAoC;AAAA,IAC7D;AAEA,WAAO,MAAM,sBAAsB;AACnC,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,GAAG;","names":["agent"]}

package/dist/ipc/supervised_proc.cjs

@@ -76,7 +76,10 @@ class SupervisedProc {
     }
     this.proc = this.createProcess();
     this.#started = true;
-    this.run();
+    this.run().catch((err) => {
+      this.#logger.child({ err }).warn("supervised process run failed");
+      this.#join.resolve();
+    });
   }
   async run() {
     await this.init.await;

package/dist/ipc/supervised_proc.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run();\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAAqB;AACrB,sBAAqB;AAErB,iBAAmC;AACnC,mBAAuB;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,oBAAO;AAAA,EAC5B,QAAQ,IAAI,oBAAO;AAAA,EACnB,cAAU,gBAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI;AAAA,EACX;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AA5F3C;AA6FM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AA5H1C;AA6HM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AA7KrB;AA8KI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,mBAAe,0BAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,cAAM,yBAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AAvMhB;AAwMI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AA3NxC;AA4NI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AAtOzD;AAuOI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":["pidusage"]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run().catch((err) => {\n      this.#logger.child({ err }).warn('supervised process run failed');\n      // Note: we intentionally do NOT kill the child process here. Killing it\n      // would race with initialize()'s `once(proc, 'message')`, causing\n      // initialize() to hang forever and deadlocking the caller (proc_pool).\n      // The child process is cleaned up when the pool shuts down.\n      this.#join.resolve();\n    });\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAAqB;AACrB,sBAAqB;AAErB,iBAAmC;AACnC,mBAAuB;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,oBAAO;AAAA,EAC5B,QAAQ,IAAI,oBAAO;AAAA,EACnB,cAAU,gBAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI,EAAE,MAAM,CAAC,QAAQ;AACxB,WAAK,QAAQ,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,+BAA+B;AAKhE,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AAnG3C;AAoGM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AAnI1C;AAoIM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AApLrB;AAqLI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,mBAAe,0BAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,cAAM,yBAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AA9MhB;AA+MI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AAlOxC;AAmOI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AA7OzD;AA8OI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,UAAM,gBAAAA,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":["pidusage"]}

package/dist/ipc/supervised_proc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"supervised_proc.d.ts","sourceRoot":"","sources":["../../src/ipc/supervised_proc.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,QAAQ;IACvB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,8BAAsB,cAAc;;IAKlC,IAAI,CAAC,EAAE,YAAY,CAAC;IAIpB,SAAS,CAAC,IAAI,eAAgB;gBAK5B,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,iBAAiB,EAAE,MAAM;IAa3B,QAAQ,CAAC,aAAa,IAAI,YAAY;IACtC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAErD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,UAAU,IAAI,cAAc,GAAG,SAAS,CAE3C;IAEK,KAAK;IAaL,GAAG;IA4EH,IAAI;IAQJ,UAAU;IA0BV,KAAK;IAoBL,SAAS,CAAC,IAAI,EAAE,cAAc;YAWtB,qBAAqB;IAiBnC,OAAO,CAAC,WAAW;CAKpB"}
+{"version":3,"file":"supervised_proc.d.ts","sourceRoot":"","sources":["../../src/ipc/supervised_proc.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,QAAQ;IACvB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,8BAAsB,cAAc;;IAKlC,IAAI,CAAC,EAAE,YAAY,CAAC;IAIpB,SAAS,CAAC,IAAI,eAAgB;gBAK5B,iBAAiB,EAAE,MAAM,EACzB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,iBAAiB,EAAE,MAAM;IAa3B,QAAQ,CAAC,aAAa,IAAI,YAAY;IACtC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAErD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,UAAU,IAAI,cAAc,GAAG,SAAS,CAE3C;IAEK,KAAK;IAoBL,GAAG;IA4EH,IAAI;IAQJ,UAAU;IA0BV,KAAK;IAoBL,SAAS,CAAC,IAAI,EAAE,cAAc;YAWtB,qBAAqB;IAiBnC,OAAO,CAAC,WAAW;CAKpB"}

package/dist/ipc/supervised_proc.js

@@ -43,7 +43,10 @@ class SupervisedProc {
     }
     this.proc = this.createProcess();
     this.#started = true;
-    this.run();
+    this.run().catch((err) => {
+      this.#logger.child({ err }).warn("supervised process run failed");
+      this.#join.resolve();
+    });
   }
   async run() {
     await this.init.await;

package/dist/ipc/supervised_proc.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run();\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":"AAIA,SAAS,YAAY;AACrB,OAAO,cAAc;AAErB,SAAS,KAAK,qBAAqB;AACnC,SAAS,cAAc;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,OAAO;AAAA,EAC5B,QAAQ,IAAI,OAAO;AAAA,EACnB,UAAU,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI;AAAA,EACX;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AA5F3C;AA6FM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AA5H1C;AA6HM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AA7KrB;AA8KI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,eAAe,cAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,UAAM,KAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AAvMhB;AAwMI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AA3NxC;AA4NI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AAtOzD;AAuOI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  protected init = new Future();\n  #join = new Future();\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(\n    initializeTimeout: number,\n    closeTimeout: number,\n    memoryWarnMB: number,\n    memoryLimitMB: number,\n    pingInterval: number,\n    pingTimeout: number,\n    highPingThreshold: number,\n  ) {\n    this.#opts = {\n      initializeTimeout,\n      closeTimeout,\n      memoryWarnMB,\n      memoryLimitMB,\n      pingInterval,\n      pingTimeout,\n      highPingThreshold,\n    };\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.run().catch((err) => {\n      this.#logger.child({ err }).warn('supervised process run failed');\n      // Note: we intentionally do NOT kill the child process here. Killing it\n      // would race with initialize()'s `once(proc, 'message')`, causing\n      // initialize() to hang forever and deadlocking the caller (proc_pool).\n      // The child process is cleaned up when the pool shuts down.\n      this.#join.resolve();\n    });\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill();\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    this.#memoryMonitorInterval = setInterval(async () => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })\n          .error('process exceeded memory limit, killing process');\n        this.close();\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        this.#logger\n          .child({\n            memoryUsageMB: memoryMB,\n            memoryWarnMB: this.#opts.memoryWarnMB,\n            memoryLimitMB: this.#opts.memoryLimitMB,\n          })\n          .warn('process memory usage is high');\n      }\n    }, 5000);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'done': {\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (err) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ err })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize() {\n    const timer = setTimeout(() => {\n      this.init.reject(new Error('runner initialization timed out'));\n    }, this.#opts.initializeTimeout);\n    if (!this.proc?.connected) {\n      this.init.reject(new Error('process not connected'));\n      return;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n      },\n    });\n    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {\n      clearTimeout(timer);\n      if (msg!.case !== 'initializeResponse') {\n        throw new Error('first message must be InitializeResponse');\n      }\n    });\n    this.init.resolve();\n  }\n\n  async close() {\n    if (!this.#started) {\n      return;\n    }\n    this.#closing = true;\n\n    if (this.proc?.connected) {\n      this.proc.send({ case: 'shutdownRequest' });\n    }\n\n    const timer = setTimeout(() => {\n      this.#logger.error('job shutdown is taking too much time');\n      this.proc!.kill();\n    }, this.#opts.closeTimeout);\n    await this.#join.await.then(() => {\n      clearTimeout(timer);\n      this.clearTimers();\n    });\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":"AAIA,SAAS,YAAY;AACrB,OAAO,cAAc;AAErB,SAAS,KAAK,qBAAqB;AACnC,SAAS,cAAc;AAoBhB,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACU,OAAO,IAAI,OAAO;AAAA,EAC5B,QAAQ,IAAI,OAAO;AAAA,EACnB,UAAU,IAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YACE,mBACA,cACA,cACA,eACA,cACA,aACA,mBACA;AACA,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAKA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AApEzB;AAqEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,IAAI,EAAE,MAAM,CAAC,QAAQ;AACxB,WAAK,QAAQ,MAAM,EAAE,IAAI,CAAC,EAAE,KAAK,+BAA+B;AAKhE,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AAnG3C;AAoGM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK;AAChB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,SAAK,yBAAyB,YAAY,YAAY;AACpD,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,EAAE,eAAe,UAAU,eAAe,KAAK,MAAM,cAAc,CAAC,EAC1E,MAAM,gDAAgD;AACzD,aAAK,MAAM;AAAA,MACb,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,aAAK,QACF,MAAM;AAAA,UACL,eAAe;AAAA,UACf,cAAc,KAAK,MAAM;AAAA,UACzB,eAAe,KAAK,MAAM;AAAA,QAC5B,CAAC,EACA,KAAK,8BAA8B;AAAA,MACxC;AAAA,IACF,GAAG,GAAI;AAEP,UAAM,WAAW,CAAC,QAAoB;AAnI1C;AAoIM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,QAAQ;AAC9B,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,IAAI,CAAC,EACb,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,aAAa;AApLrB;AAqLI,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,KAAK,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,IAC/D,GAAG,KAAK,MAAM,iBAAiB;AAC/B,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,WAAK,KAAK,OAAO,IAAI,MAAM,uBAAuB,CAAC;AACnD;AAAA,IACF;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,eAAe,cAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,MAChC;AAAA,IACF,CAAC;AACD,UAAM,KAAK,KAAK,MAAO,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,MAAoB;AAC9D,mBAAa,KAAK;AAClB,UAAI,IAAK,SAAS,sBAAsB;AACtC,cAAM,IAAI,MAAM,0CAA0C;AAAA,MAC5D;AAAA,IACF,CAAC;AACD,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AA9MhB;AA+MI,QAAI,CAAC,KAAK,UAAU;AAClB;AAAA,IACF;AACA,SAAK,WAAW;AAEhB,SAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,WAAK,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAAA,IAC5C;AAEA,UAAM,QAAQ,WAAW,MAAM;AAC7B,WAAK,QAAQ,MAAM,sCAAsC;AACzD,WAAK,KAAM,KAAK;AAAA,IAClB,GAAG,KAAK,MAAM,YAAY;AAC1B,UAAM,KAAK,MAAM,MAAM,KAAK,MAAM;AAChC,mBAAa,KAAK;AAClB,WAAK,YAAY;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,MAAsB;AAlOxC;AAmOI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AA7OzD;AA8OI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":[]}