@omnitron-dev/titan-pm 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/backoff.d.ts +12 -0
  2. package/dist/backoff.d.ts.map +1 -0
  3. package/dist/backoff.js +44 -0
  4. package/dist/backoff.js.map +1 -0
  5. package/dist/child-contract.d.ts +58 -0
  6. package/dist/child-contract.d.ts.map +1 -0
  7. package/dist/child-contract.js +27 -0
  8. package/dist/child-contract.js.map +1 -0
  9. package/dist/confirm-death.d.ts +18 -0
  10. package/dist/confirm-death.d.ts.map +1 -0
  11. package/dist/confirm-death.js +49 -0
  12. package/dist/confirm-death.js.map +1 -0
  13. package/dist/decorators.d.ts +1 -10
  14. package/dist/decorators.d.ts.map +1 -1
  15. package/dist/decorators.js +43 -99
  16. package/dist/decorators.js.map +1 -1
  17. package/dist/discovery.d.ts +24 -0
  18. package/dist/discovery.d.ts.map +1 -0
  19. package/dist/discovery.js +162 -0
  20. package/dist/discovery.js.map +1 -0
  21. package/dist/fork-worker.js +64 -27
  22. package/dist/fork-worker.js.map +1 -1
  23. package/dist/index.d.ts +6 -1
  24. package/dist/index.d.ts.map +1 -1
  25. package/dist/index.js +5 -0
  26. package/dist/index.js.map +1 -1
  27. package/dist/last-resort-handlers.d.ts +5 -0
  28. package/dist/last-resort-handlers.d.ts.map +1 -0
  29. package/dist/last-resort-handlers.js +40 -0
  30. package/dist/last-resort-handlers.js.map +1 -0
  31. package/dist/lifecycle-queue.d.ts +20 -0
  32. package/dist/lifecycle-queue.d.ts.map +1 -0
  33. package/dist/lifecycle-queue.js +110 -0
  34. package/dist/lifecycle-queue.js.map +1 -0
  35. package/dist/liveness.d.ts +5 -0
  36. package/dist/liveness.d.ts.map +1 -0
  37. package/dist/liveness.js +34 -0
  38. package/dist/liveness.js.map +1 -0
  39. package/dist/netron-client.d.ts +10 -9
  40. package/dist/netron-client.d.ts.map +1 -1
  41. package/dist/netron-client.js +32 -21
  42. package/dist/netron-client.js.map +1 -1
  43. package/dist/parent-death-watchdog.d.ts +13 -0
  44. package/dist/parent-death-watchdog.d.ts.map +1 -0
  45. package/dist/parent-death-watchdog.js +58 -0
  46. package/dist/parent-death-watchdog.js.map +1 -0
  47. package/dist/pm.module.d.ts.map +1 -1
  48. package/dist/pm.module.js.map +1 -1
  49. package/dist/process-health.d.ts +2 -0
  50. package/dist/process-health.d.ts.map +1 -1
  51. package/dist/process-health.js +49 -13
  52. package/dist/process-health.js.map +1 -1
  53. package/dist/process-manager.d.ts +6 -0
  54. package/dist/process-manager.d.ts.map +1 -1
  55. package/dist/process-manager.js +111 -4
  56. package/dist/process-manager.js.map +1 -1
  57. package/dist/process-metrics.d.ts +1 -0
  58. package/dist/process-metrics.d.ts.map +1 -1
  59. package/dist/process-metrics.js +18 -19
  60. package/dist/process-metrics.js.map +1 -1
  61. package/dist/process-pool.d.ts +12 -0
  62. package/dist/process-pool.d.ts.map +1 -1
  63. package/dist/process-pool.js +340 -147
  64. package/dist/process-pool.js.map +1 -1
  65. package/dist/process-registry.d.ts.map +1 -1
  66. package/dist/process-spawner.d.ts +27 -5
  67. package/dist/process-spawner.d.ts.map +1 -1
  68. package/dist/process-spawner.js +397 -70
  69. package/dist/process-spawner.js.map +1 -1
  70. package/dist/process-supervisor.d.ts +7 -0
  71. package/dist/process-supervisor.d.ts.map +1 -1
  72. package/dist/process-supervisor.js +141 -17
  73. package/dist/process-supervisor.js.map +1 -1
  74. package/dist/process-workflow.d.ts.map +1 -1
  75. package/dist/process-workflow.js +11 -4
  76. package/dist/process-workflow.js.map +1 -1
  77. package/dist/service-proxy.d.ts.map +1 -1
  78. package/dist/service-proxy.js +33 -3
  79. package/dist/service-proxy.js.map +1 -1
  80. package/dist/shutdown-windows.d.ts +19 -0
  81. package/dist/shutdown-windows.d.ts.map +1 -0
  82. package/dist/shutdown-windows.js +35 -0
  83. package/dist/shutdown-windows.js.map +1 -0
  84. package/dist/thread-pool.d.ts +7 -0
  85. package/dist/thread-pool.d.ts.map +1 -0
  86. package/dist/thread-pool.js +11 -0
  87. package/dist/thread-pool.js.map +1 -0
  88. package/dist/types.d.ts +91 -37
  89. package/dist/types.d.ts.map +1 -1
  90. package/dist/types.js +35 -39
  91. package/dist/types.js.map +1 -1
  92. package/dist/utils/deque.d.ts.map +1 -1
  93. package/dist/worker-health.d.ts +8 -0
  94. package/dist/worker-health.d.ts.map +1 -0
  95. package/dist/worker-health.js +63 -0
  96. package/dist/worker-health.js.map +1 -0
  97. package/dist/worker-runtime.js +25 -52
  98. package/dist/worker-runtime.js.map +1 -1
  99. package/dist/worker-traffic.d.ts +8 -0
  100. package/dist/worker-traffic.d.ts.map +1 -0
  101. package/dist/worker-traffic.js +56 -0
  102. package/dist/worker-traffic.js.map +1 -0
  103. package/package.json +11 -10
@@ -1,3 +1,4 @@
1
+ import { EventEmitter } from '@omnitron-dev/eventemitter';
1
2
  import { Worker } from 'worker_threads';
2
3
  import { fork } from 'node:child_process';
3
4
  import { promises as fs } from 'node:fs';
@@ -6,16 +7,79 @@ import path from 'node:path';
6
7
  import os from 'node:os';
7
8
  import { fileURLToPath } from 'node:url';
8
9
  import { createInterface } from 'node:readline';
10
+ import net from 'node:net';
9
11
  import { Errors } from '@omnitron-dev/titan/errors';
12
+ import { isAlive as isPidAlive } from './liveness.js';
10
13
  import { ProcessStatus } from './types.js';
11
14
  import { NetronClient } from './netron-client.js';
12
15
  import { ServiceProxyHandler } from './service-proxy.js';
13
16
  import { getAvailablePort } from '@omnitron-dev/titan/utils';
14
17
  import { generateUuidV7 } from '@omnitron-dev/titan/utils';
18
+ import { shutdownLadder, childDrainMs, DEFAULT_SHUTDOWN_BUDGET_MS } from './shutdown-windows.js';
19
+ import { confirmDeath } from './confirm-death.js';
20
+ import { threadPoolSizeFor } from './thread-pool.js';
15
21
  let MockProcessSpawnerClass = null;
16
22
  const SPAWNER_DIR = path.dirname(fileURLToPath(import.meta.url));
17
- export class WorkerHandle {
18
- constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig) {
23
+ async function isSocketRefused(socketPath, timeoutMs = 250) {
24
+ return new Promise((resolve) => {
25
+ let settled = false;
26
+ const done = (refused) => {
27
+ if (settled)
28
+ return;
29
+ settled = true;
30
+ clearTimeout(timer);
31
+ socket.destroy();
32
+ resolve(refused);
33
+ };
34
+ const socket = net.connect({ path: socketPath });
35
+ const timer = setTimeout(() => done(false), timeoutMs);
36
+ socket.once('connect', () => done(false));
37
+ socket.once('error', (error) => done(error.code === 'ECONNREFUSED'));
38
+ });
39
+ }
40
+ const STALE_SOCKET_MIN_AGE_MS = 10 * 60 * 1000;
41
+ const EARLY_LOG_MAX_BYTES = 1024 * 1024;
42
+ export function hasExited(child) {
43
+ if (child.exitCode !== null || child.signalCode !== null)
44
+ return true;
45
+ return child.pid == null ? true : !isPidAlive(child.pid);
46
+ }
47
+ export async function killChildAndVerify(child, logger, context, termGraceMs = 3000) {
48
+ if (hasExited(child))
49
+ return true;
50
+ const exited = new Promise((resolve) => {
51
+ const done = () => resolve(true);
52
+ child.once('exit', done);
53
+ setTimeout(() => {
54
+ child.off('exit', done);
55
+ resolve(false);
56
+ }, termGraceMs).unref();
57
+ });
58
+ try {
59
+ child.kill('SIGTERM');
60
+ }
61
+ catch {
62
+ return true;
63
+ }
64
+ if (await exited)
65
+ return true;
66
+ if (hasExited(child))
67
+ return true;
68
+ logger.warn({ ...context, pid: child.pid }, 'Child ignored SIGTERM during cleanup — escalating to SIGKILL');
69
+ try {
70
+ child.kill('SIGKILL');
71
+ }
72
+ catch {
73
+ return true;
74
+ }
75
+ await new Promise((r) => {
76
+ setTimeout(r, 200).unref();
77
+ });
78
+ return hasExited(child);
79
+ }
80
+ export class WorkerHandle extends EventEmitter {
81
+ constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true) {
82
+ super();
19
83
  this.id = id;
20
84
  this.transportUrl = transportUrl;
21
85
  this.serviceName = serviceName;
@@ -26,11 +90,43 @@ export class WorkerHandle {
26
90
  this.transportConfig = transportConfig;
27
91
  this.messageHandlers = new Map();
28
92
  this.logHandlers = new Set();
93
+ this.earlyLogs = [];
94
+ this.earlyLogBytes = 0;
95
+ this.earlyLogsDropped = 0;
96
+ this.exitEmitted = false;
29
97
  this._status = initialStatus;
30
98
  this.worker = worker;
31
99
  this.netronClient = netronClient;
32
100
  this.setupMessageHandlers();
33
- this.setupDefaultLogForwarding();
101
+ if (forwardChildLogs)
102
+ this.setupDefaultLogForwarding();
103
+ }
104
+ emitExit(code, signal) {
105
+ if (this.exitEmitted)
106
+ return;
107
+ this.exitEmitted = true;
108
+ const expected = this._status === ProcessStatus.STOPPING || this._status === ProcessStatus.STOPPED;
109
+ this._status = ProcessStatus.STOPPED;
110
+ const forced = expected && signal === 'SIGKILL';
111
+ const info = {
112
+ workerId: this.id,
113
+ serviceName: this.serviceName,
114
+ code,
115
+ signal,
116
+ expected,
117
+ forced,
118
+ };
119
+ this.logger[expected && !forced ? 'info' : 'warn'](info, !expected
120
+ ? 'Worker exited unexpectedly'
121
+ : forced
122
+ ? 'Worker had to be killed — it did not exit within its shutdown window'
123
+ : 'Worker exited cleanly');
124
+ try {
125
+ this.emit('exit', info);
126
+ }
127
+ catch (err) {
128
+ this.logger.error({ err, workerId: this.id }, 'exit listener threw');
129
+ }
34
130
  }
35
131
  get status() {
36
132
  return this._status;
@@ -40,9 +136,15 @@ export class WorkerHandle {
40
136
  return undefined;
41
137
  return this.worker.pid;
42
138
  }
43
- async terminate() {
44
- try {
139
+ onExit(handler) {
140
+ this.on('exit', handler);
141
+ return () => this.off('exit', handler);
142
+ }
143
+ async terminate(opts) {
144
+ if (this._status !== ProcessStatus.STOPPED) {
45
145
  this._status = ProcessStatus.STOPPING;
146
+ }
147
+ try {
46
148
  if (this.netronClient?.isConnected()) {
47
149
  await this.netronClient.disconnect();
48
150
  }
@@ -52,7 +154,7 @@ export class WorkerHandle {
52
154
  }
53
155
  else {
54
156
  const child = this.worker;
55
- await this.terminateChildProcess(child);
157
+ await this.terminateChildProcess(child, opts?.shutdownTimeout);
56
158
  }
57
159
  if (this.transportConfig?.type === 'unix' && this.transportConfig.path) {
58
160
  try {
@@ -70,16 +172,32 @@ export class WorkerHandle {
70
172
  throw error;
71
173
  }
72
174
  }
73
- async terminateChildProcess(child) {
74
- const GRACEFUL_TIMEOUT = 2000;
75
- const SIGTERM_TIMEOUT = 2000;
76
- const SIGKILL_TIMEOUT = 1000;
175
+ async terminateChildProcess(child, totalDeadlineMs) {
176
+ const ladder = shutdownLadder(totalDeadlineMs);
177
+ const GRACEFUL_TIMEOUT = ladder.gracefulMs;
178
+ const SIGTERM_TIMEOUT = ladder.sigtermMs;
179
+ const SIGKILL_TIMEOUT = ladder.sigkillMs;
77
180
  return new Promise((resolve, reject) => {
78
181
  let resolved = false;
182
+ let termTimer = null;
183
+ let killTimer = null;
184
+ let finalTimer = null;
79
185
  const cleanup = () => {
80
186
  resolved = true;
81
187
  child.off('exit', exitHandler);
82
188
  child.off('error', errorHandler);
189
+ if (termTimer) {
190
+ clearTimeout(termTimer);
191
+ termTimer = null;
192
+ }
193
+ if (killTimer) {
194
+ clearTimeout(killTimer);
195
+ killTimer = null;
196
+ }
197
+ if (finalTimer) {
198
+ clearTimeout(finalTimer);
199
+ finalTimer = null;
200
+ }
83
201
  };
84
202
  const exitHandler = () => {
85
203
  if (!resolved) {
@@ -98,7 +216,7 @@ export class WorkerHandle {
98
216
  };
99
217
  child.once('exit', exitHandler);
100
218
  child.once('error', errorHandler);
101
- if (child.exitCode !== null || child.killed) {
219
+ if (hasExited(child)) {
102
220
  cleanup();
103
221
  resolve();
104
222
  return;
@@ -111,8 +229,8 @@ export class WorkerHandle {
111
229
  catch (_e) {
112
230
  this.logger.debug({ workerId: this.id }, 'IPC channel closed, proceeding with SIGTERM');
113
231
  }
114
- setTimeout(() => {
115
- if (resolved || child.exitCode !== null || child.killed)
232
+ termTimer = setTimeout(() => {
233
+ if (resolved || hasExited(child))
116
234
  return;
117
235
  this.logger.debug({ workerId: this.id }, 'Graceful shutdown timeout, sending SIGTERM');
118
236
  try {
@@ -121,8 +239,8 @@ export class WorkerHandle {
121
239
  catch (_e) {
122
240
  }
123
241
  }, GRACEFUL_TIMEOUT);
124
- setTimeout(() => {
125
- if (resolved || child.exitCode !== null || child.killed)
242
+ killTimer = setTimeout(() => {
243
+ if (resolved || hasExited(child))
126
244
  return;
127
245
  this.logger.warn({ workerId: this.id }, 'SIGTERM timeout, sending SIGKILL');
128
246
  try {
@@ -131,12 +249,28 @@ export class WorkerHandle {
131
249
  catch (_e) {
132
250
  }
133
251
  }, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT);
134
- setTimeout(() => {
135
- if (!resolved) {
136
- cleanup();
137
- this.logger.error({ workerId: this.id }, 'Process termination timeout - process may be zombie');
252
+ finalTimer = setTimeout(() => {
253
+ if (resolved)
254
+ return;
255
+ cleanup();
256
+ if (child.pid == null) {
138
257
  resolve();
258
+ return;
139
259
  }
260
+ void confirmDeath(child).then((verdict) => {
261
+ if (verdict.dead) {
262
+ resolve();
263
+ return;
264
+ }
265
+ this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process survived SIGKILL — escalating one more SIGKILL and logging the leak');
266
+ try {
267
+ if (child.pid != null)
268
+ process.kill(child.pid, 'SIGKILL');
269
+ }
270
+ catch {
271
+ }
272
+ resolve();
273
+ });
140
274
  }, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT + SIGKILL_TIMEOUT);
141
275
  });
142
276
  }
@@ -147,29 +281,70 @@ export class WorkerHandle {
147
281
  }
148
282
  else {
149
283
  const child = this.worker;
150
- return !child.killed && child.exitCode === null;
284
+ return !hasExited(child);
151
285
  }
152
286
  }
153
287
  async send(message) {
154
288
  if (this.isWorkerThread) {
155
289
  const worker = this.worker;
290
+ if (this._status !== ProcessStatus.RUNNING && this._status !== ProcessStatus.STARTING) {
291
+ throw Errors.unavailable(`Worker thread ${this.id} is ${this._status}; cannot post message`);
292
+ }
156
293
  worker.postMessage(message);
157
294
  }
158
295
  else {
159
296
  const child = this.worker;
160
- if (child.send) {
161
- child.send(message);
297
+ if (!child.send || !child.connected) {
298
+ throw Errors.unavailable(`IPC channel to child ${this.id} is closed; cannot send`);
162
299
  }
300
+ child.send(message);
163
301
  }
164
302
  }
165
303
  onMessage(handler) {
166
304
  const id = generateUuidV7();
167
305
  this.messageHandlers.set(id, handler);
168
306
  }
307
+ seedStartupOutput(output) {
308
+ if (!output)
309
+ return;
310
+ for (const [text, stream] of [
311
+ [output.stdout, 'stdout'],
312
+ [output.stderr, 'stderr'],
313
+ ]) {
314
+ for (const line of text.split('\n')) {
315
+ if (line.trim())
316
+ this.emitLog(line, stream);
317
+ }
318
+ }
319
+ }
169
320
  onLog(handler) {
321
+ const first = this.logHandlers.size === 0;
170
322
  this.logHandlers.add(handler);
323
+ if (!first)
324
+ return;
325
+ const buffered = this.earlyLogs;
326
+ this.earlyLogs = null;
327
+ if (!buffered)
328
+ return;
329
+ if (this.earlyLogsDropped > 0) {
330
+ handler(`[titan-pm] ${this.earlyLogsDropped} earlier line(s) from this process were dropped: ` +
331
+ 'it printed more than the pre-capture buffer holds', 'stderr');
332
+ }
333
+ for (const entry of buffered) {
334
+ handler(entry.line, entry.stream);
335
+ }
171
336
  }
172
337
  emitLog(line, stream) {
338
+ if (this.logHandlers.size === 0 && this.earlyLogs) {
339
+ this.earlyLogs.push({ line, stream });
340
+ this.earlyLogBytes += line.length;
341
+ while (this.earlyLogBytes > EARLY_LOG_MAX_BYTES && this.earlyLogs.length > 1) {
342
+ const dropped = this.earlyLogs.shift();
343
+ this.earlyLogBytes -= dropped.line.length;
344
+ this.earlyLogsDropped++;
345
+ }
346
+ return;
347
+ }
173
348
  for (const handler of this.logHandlers) {
174
349
  handler(line, stream);
175
350
  }
@@ -185,11 +360,12 @@ export class WorkerHandle {
185
360
  });
186
361
  worker.on('error', (error) => {
187
362
  this.logger.error({ error, workerId: this.id }, 'Worker error');
188
- this._status = ProcessStatus.FAILED;
363
+ if (this._status !== ProcessStatus.STOPPING && this._status !== ProcessStatus.STOPPED) {
364
+ this._status = ProcessStatus.FAILED;
365
+ }
189
366
  });
190
367
  worker.on('exit', (code) => {
191
- this.logger.debug({ workerId: this.id, code }, 'Worker exited');
192
- this._status = ProcessStatus.STOPPED;
368
+ this.emitExit(code, null);
193
369
  });
194
370
  }
195
371
  else {
@@ -202,11 +378,12 @@ export class WorkerHandle {
202
378
  });
203
379
  child.on('error', (error) => {
204
380
  this.logger.error({ error, workerId: this.id }, 'Child process error');
205
- this._status = ProcessStatus.FAILED;
381
+ if (this._status !== ProcessStatus.STOPPING && this._status !== ProcessStatus.STOPPED) {
382
+ this._status = ProcessStatus.FAILED;
383
+ }
206
384
  });
207
385
  child.on('exit', (code, signal) => {
208
- this.logger.debug({ workerId: this.id, code, signal }, 'Child process exited');
209
- this._status = ProcessStatus.STOPPED;
386
+ this.emitExit(code, signal);
210
387
  });
211
388
  this.setupLogCapture(child);
212
389
  }
@@ -285,13 +462,27 @@ function sanitizeEnv(env) {
285
462
  }
286
463
  return result;
287
464
  }
465
+ function childEnv(extra = {}) {
466
+ const base = sanitizeEnv(process.env);
467
+ return {
468
+ ...base,
469
+ UV_THREADPOOL_SIZE: threadPoolSizeFor(base, { parallelism: os.availableParallelism() }),
470
+ ...extra,
471
+ };
472
+ }
288
473
  export class ProcessSpawner {
289
474
  constructor(logger, config = {}) {
290
475
  this.logger = logger;
291
476
  this.config = config;
477
+ this.ownSockets = new Set();
478
+ this.forkedPids = new Set();
479
+ this.staleSweep = null;
292
480
  let currentDir = SPAWNER_DIR;
293
- if (currentDir.includes('/src/') || currentDir.includes('\\src\\')) {
294
- currentDir = currentDir.replace(/[/\\]src[/\\]/, path.sep + 'dist' + path.sep);
481
+ const segments = currentDir.split(path.sep);
482
+ const srcIndex = segments.lastIndexOf('src');
483
+ if (srcIndex !== -1) {
484
+ segments[srcIndex] = 'dist';
485
+ currentDir = segments.join(path.sep);
295
486
  }
296
487
  this.workerRuntimePath = path.join(currentDir, 'worker-runtime.js');
297
488
  this.forkWorkerPath = path.join(currentDir, 'fork-worker.js');
@@ -325,12 +516,22 @@ export class ProcessSpawner {
325
516
  options: options,
326
517
  dependencies: options.dependencies,
327
518
  };
328
- const isolation = options.isolation || this.config.isolation || 'worker';
329
- const useWorkerThreads = isolation === 'worker' && (!options.execArgv || options.execArgv.length === 0);
519
+ const declaredIsolation = options.isolation;
520
+ const legacySandbox = declaredIsolation === 'vm' || declaredIsolation === 'container' ? declaredIsolation : undefined;
521
+ const strategy = (options.isolation === 'none' || options.isolation === 'worker' || options.isolation === 'child'
522
+ ? options.isolation
523
+ : undefined) ??
524
+ this.config.isolation ??
525
+ 'worker';
526
+ const sandbox = options.sandbox ?? legacySandbox;
527
+ const sandboxUnimplemented = sandbox === 'vm' || sandbox === 'container';
528
+ const useWorkerThreads = strategy === 'worker' && !sandboxUnimplemented && (!options.execArgv || options.execArgv.length === 0);
330
529
  let worker;
331
530
  let netronClient = null;
332
531
  try {
333
- if (isolation === 'vm' || isolation === 'container') {
532
+ if (sandboxUnimplemented) {
533
+ this.logger.warn({ processId, isolation: sandbox, actual: 'child' }, `Process isolation '${sandbox}' is not implemented — spawning an ordinary child process. ` +
534
+ 'It has the same filesystem and network access as its parent; do not rely on it as a security boundary.');
334
535
  worker = await this.spawnChildProcess(context, options.execArgv);
335
536
  }
336
537
  else if (useWorkerThreads) {
@@ -343,8 +544,10 @@ export class ProcessSpawner {
343
544
  const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout);
344
545
  const actualServiceName = readyInfo.serviceName || serviceName;
345
546
  const actualServiceVersion = readyInfo.serviceVersion || serviceVersion;
346
- if (isolation !== 'none') {
347
- netronClient = new NetronClient(processId, this.logger);
547
+ if (strategy !== 'none') {
548
+ netronClient = new NetronClient(processId, this.logger, {
549
+ ...(options.requestTimeout !== undefined && { requestTimeout: options.requestTimeout }),
550
+ });
348
551
  await netronClient.start();
349
552
  await netronClient.connect(transport.url);
350
553
  const qualifiedServiceName = actualServiceVersion
@@ -352,10 +555,14 @@ export class ProcessSpawner {
352
555
  : actualServiceName;
353
556
  const proxyHandler = new ServiceProxyHandler(processId, netronClient, qualifiedServiceName, this.logger);
354
557
  const proxy = proxyHandler.createProxy();
355
- return new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport);
558
+ const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
559
+ handle.seedStartupOutput(readyInfo.startupOutput);
560
+ return handle;
356
561
  }
357
562
  else {
358
- return new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport);
563
+ const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
564
+ handle.seedStartupOutput(readyInfo.startupOutput);
565
+ return handle;
359
566
  }
360
567
  }
361
568
  catch (error) {
@@ -367,13 +574,13 @@ export class ProcessSpawner {
367
574
  }
368
575
  else {
369
576
  const child = worker;
370
- if (child.pid && !child.killed) {
371
- child.kill('SIGTERM');
372
- setTimeout(() => {
373
- if (!child.killed) {
374
- child.kill('SIGKILL');
375
- }
376
- }, 3000).unref();
577
+ if (child.pid) {
578
+ const gone = await killChildAndVerify(child, this.logger, { serviceName, processId });
579
+ if (!gone) {
580
+ this.logger.error({ serviceName, processId, pid: child.pid }, 'Child survived SIGKILL after a failed spawn — it still holds whatever it held ' +
581
+ '(ports, locks). It is parented to this process, so the orphan sweep will not ' +
582
+ 'see it: it needs the abandoned-child sweep.');
583
+ }
377
584
  }
378
585
  }
379
586
  }
@@ -387,25 +594,55 @@ export class ProcessSpawner {
387
594
  }
388
595
  throw error;
389
596
  }
597
+ finally {
598
+ this.releaseForkClaim(worker?.pid);
599
+ }
390
600
  }
391
601
  async cleanup() {
392
- try {
393
- const entries = await fs.readdir(this.tempDir);
394
- for (const entry of entries) {
395
- if (entry.endsWith('.sock')) {
396
- try {
397
- await fs.unlink(path.join(this.tempDir, entry));
398
- }
399
- catch { }
400
- }
602
+ for (const socketPath of this.ownSockets) {
603
+ try {
604
+ await fs.unlink(socketPath);
605
+ }
606
+ catch {
401
607
  }
402
608
  }
609
+ this.ownSockets.clear();
610
+ try {
611
+ await fs.rmdir(this.tempDir);
612
+ }
403
613
  catch {
404
614
  }
615
+ }
616
+ async sweepStaleSockets() {
617
+ let entries;
405
618
  try {
406
- await fs.rmdir(this.tempDir);
619
+ entries = await fs.readdir(this.tempDir);
407
620
  }
408
621
  catch {
622
+ return;
623
+ }
624
+ const cutoff = Date.now() - STALE_SOCKET_MIN_AGE_MS;
625
+ for (const entry of entries) {
626
+ if (!entry.endsWith('.sock'))
627
+ continue;
628
+ const socketPath = path.join(this.tempDir, entry);
629
+ if (this.ownSockets.has(socketPath))
630
+ continue;
631
+ try {
632
+ const stat = await fs.stat(socketPath);
633
+ if (stat.mtimeMs > cutoff)
634
+ continue;
635
+ }
636
+ catch {
637
+ continue;
638
+ }
639
+ if (!(await isSocketRefused(socketPath)))
640
+ continue;
641
+ try {
642
+ await fs.unlink(socketPath);
643
+ }
644
+ catch {
645
+ }
409
646
  }
410
647
  }
411
648
  async setupTransport(processId, options) {
@@ -425,8 +662,12 @@ export class ProcessSpawner {
425
662
  }
426
663
  case 'unix':
427
664
  default: {
428
- await fs.mkdir(this.tempDir, { recursive: true });
665
+ await fs.mkdir(this.tempDir, { recursive: true, mode: 0o700 });
666
+ await fs.chmod(this.tempDir, 0o700).catch(() => {
667
+ });
429
668
  const socketPath = path.join(this.tempDir, `${processId}.sock`);
669
+ this.ownSockets.add(socketPath);
670
+ this.staleSweep ??= this.sweepStaleSockets().catch(() => undefined);
430
671
  return {
431
672
  type: 'unix',
432
673
  path: socketPath,
@@ -458,7 +699,7 @@ export class ProcessSpawner {
458
699
  const serializableContext = this.makeSerializable(context);
459
700
  const worker = new Worker(this.workerRuntimePath, {
460
701
  workerData: serializableContext,
461
- env: sanitizeEnv(process.env),
702
+ env: childEnv(),
462
703
  });
463
704
  return worker;
464
705
  }
@@ -484,39 +725,112 @@ export class ProcessSpawner {
484
725
  if (contextJson.length > 65536) {
485
726
  throw new Error(`TITAN_WORKER_CONTEXT exceeds 64KB (${contextJson.length} bytes). Reduce dependencies.`);
486
727
  }
728
+ const windowMs = shutdownLadder(context.options?.shutdownTimeout ?? DEFAULT_SHUTDOWN_BUDGET_MS).childWindowMs;
487
729
  const child = fork(this.forkWorkerPath, [], {
488
730
  env: {
489
- ...sanitizeEnv(process.env),
490
- ...(context.options?.env ?? {}),
731
+ ...childEnv(context.options?.env ?? {}),
491
732
  TITAN_WORKER_CONTEXT: contextJson,
733
+ TITAN_SHUTDOWN_TIMEOUT_MS: String(windowMs),
734
+ TITAN_DRAIN_TIMEOUT_MS: context.options?.env?.['TITAN_DRAIN_TIMEOUT_MS'] || String(childDrainMs(windowMs)),
492
735
  },
493
736
  execArgv: execArgv || [],
494
737
  silent: true,
495
738
  ...(context.options?.cwd ? { cwd: context.options.cwd } : {}),
496
739
  });
740
+ if (typeof child.pid === 'number') {
741
+ const pid = child.pid;
742
+ this.forkedPids.add(pid);
743
+ child.once('exit', () => this.forkedPids.delete(pid));
744
+ }
497
745
  return child;
498
746
  }
747
+ getForkedPids() {
748
+ return this.forkedPids;
749
+ }
750
+ releaseForkClaim(pid) {
751
+ if (typeof pid === 'number')
752
+ this.forkedPids.delete(pid);
753
+ }
499
754
  async waitForReady(worker, isWorkerThread, timeout = 5000) {
500
755
  return new Promise((resolve, reject) => {
501
- const stderrChunks = [];
756
+ const STDERR_CAP_BYTES = 64 * 1024;
757
+ const stderrBuf = [];
758
+ const stdoutBuf = [];
759
+ const stderrBytes = 0;
760
+ const stdoutBytes = 0;
761
+ const appendBounded = (target, bytesRef, chunk) => {
762
+ target.push(chunk);
763
+ bytesRef.v += chunk.length;
764
+ while (bytesRef.v > STDERR_CAP_BYTES && target.length > 1) {
765
+ const dropped = target.shift();
766
+ bytesRef.v -= dropped.length;
767
+ }
768
+ };
769
+ const stderrRef = { v: stderrBytes };
770
+ const stdoutRef = { v: stdoutBytes };
771
+ const LAG_PROBE_INTERVAL_MS = 1_000;
772
+ const LAG_SIGNIFICANT_MS = 2_000;
773
+ let maxLagMs = 0;
774
+ let expectedTickAt = Date.now() + LAG_PROBE_INTERVAL_MS;
775
+ const lagProbe = setInterval(() => {
776
+ const now = Date.now();
777
+ const lag = now - expectedTickAt;
778
+ expectedTickAt = now + LAG_PROBE_INTERVAL_MS;
779
+ if (lag > maxLagMs)
780
+ maxLagMs = lag;
781
+ }, LAG_PROBE_INTERVAL_MS);
782
+ lagProbe.unref?.();
502
783
  if (!isWorkerThread) {
503
784
  const child = worker;
504
785
  child.stderr?.on('data', (chunk) => {
505
- const line = chunk.toString().trim();
506
- if (line)
507
- stderrChunks.push(line);
508
- if (stderrChunks.length > 20)
509
- stderrChunks.shift();
786
+ appendBounded(stderrBuf, stderrRef, chunk.toString());
787
+ });
788
+ child.stdout?.on('data', (chunk) => {
789
+ appendBounded(stdoutBuf, stdoutRef, chunk.toString());
510
790
  });
511
791
  }
792
+ const buildDiagnostics = () => {
793
+ const stderr = stderrBuf.join('').trimEnd();
794
+ const stdout = stdoutBuf.join('').trimEnd();
795
+ return {
796
+ stderr: stderr || undefined,
797
+ stdout: stdout || undefined,
798
+ stderrBytes: stderrRef.v,
799
+ stdoutBytes: stdoutRef.v,
800
+ };
801
+ };
512
802
  const timer = setTimeout(() => {
803
+ clearInterval(lagProbe);
513
804
  const pid = !isWorkerThread ? worker.pid : undefined;
514
805
  const pidHint = pid ? ` (pid: ${pid})` : '';
515
- const stderrHint = stderrChunks.length > 0 ? `. Last stderr: ${stderrChunks.slice(-3).join(' | ')}` : '';
516
- reject(Errors.timeout(`Worker startup${pidHint}${stderrHint}`, timeout));
806
+ const diag = buildDiagnostics();
807
+ const stderrTail = diag.stderr
808
+ ? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-15).join('\n')
809
+ : '';
810
+ const supervisorNote = maxLagMs >= LAG_SIGNIFICANT_MS
811
+ ? `\n--- this timeout is ours, not the child's ---\n` +
812
+ `The supervisor's own 1s timer ran ${Math.round(maxLagMs / 1000)}s late ` +
813
+ `during this window, so it was not running to read the child or to ` +
814
+ `receive its ready message. Check free memory and swap on this host ` +
815
+ `before reading the application.`
816
+ : '';
817
+ const err = Errors.timeout(`Worker startup${pidHint}`, timeout);
818
+ if (supervisorNote) {
819
+ err.message += supervisorNote;
820
+ }
821
+ if (stderrTail) {
822
+ err.message += stderrTail;
823
+ }
824
+ err.details = {
825
+ ...(err.details ?? {}),
826
+ ...diag,
827
+ supervisorMaxLagMs: maxLagMs,
828
+ };
829
+ reject(err);
517
830
  }, timeout);
518
831
  const cleanup = () => {
519
832
  clearTimeout(timer);
833
+ clearInterval(lagProbe);
520
834
  if (isWorkerThread) {
521
835
  worker.off('message', messageHandler);
522
836
  }
@@ -532,19 +846,32 @@ export class ProcessSpawner {
532
846
  resolve({
533
847
  serviceName: data.serviceName,
534
848
  serviceVersion: data.serviceVersion,
849
+ startupOutput: { stdout: stdoutBuf.join(''), stderr: stderrBuf.join('') },
535
850
  });
536
851
  }
537
852
  else if (data.type === 'error') {
538
853
  cleanup();
539
854
  const errMsg = data.error?.message || data.message || 'Worker initialization failed';
540
- reject(new Error(errMsg));
855
+ const err = new Error(errMsg);
856
+ err.details = buildDiagnostics();
857
+ if (data.error?.stack)
858
+ err.childStack = data.error.stack;
859
+ reject(err);
541
860
  }
542
861
  }
543
862
  };
544
- const exitHandler = (code) => {
863
+ const exitHandler = (code, signal) => {
545
864
  cleanup();
546
- const stderrHint = stderrChunks.length > 0 ? `. Last stderr: ${stderrChunks.slice(-3).join(' | ')}` : '';
547
- reject(new Error(`Worker exited during startup with code ${code}${stderrHint}`));
865
+ const diag = buildDiagnostics();
866
+ const stderrTail = diag.stderr
867
+ ? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-20).join('\n')
868
+ : '';
869
+ const how = signal
870
+ ? `after being killed by ${signal} — it did not choose to exit, so its own logs will not explain this`
871
+ : `with code ${code}`;
872
+ const err = new Error(`Worker exited during startup ${how}${stderrTail}`);
873
+ err.details = { ...diag, exitCode: code, exitSignal: signal };
874
+ reject(err);
548
875
  };
549
876
  if (isWorkerThread) {
550
877
  worker.on('message', messageHandler);