@omnitron-dev/titan-pm 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/backoff.d.ts +12 -0
  2. package/dist/backoff.d.ts.map +1 -0
  3. package/dist/backoff.js +44 -0
  4. package/dist/backoff.js.map +1 -0
  5. package/dist/child-contract.d.ts +58 -0
  6. package/dist/child-contract.d.ts.map +1 -0
  7. package/dist/child-contract.js +27 -0
  8. package/dist/child-contract.js.map +1 -0
  9. package/dist/confirm-death.d.ts +18 -0
  10. package/dist/confirm-death.d.ts.map +1 -0
  11. package/dist/confirm-death.js +49 -0
  12. package/dist/confirm-death.js.map +1 -0
  13. package/dist/decorators.d.ts +1 -10
  14. package/dist/decorators.d.ts.map +1 -1
  15. package/dist/decorators.js +43 -99
  16. package/dist/decorators.js.map +1 -1
  17. package/dist/discovery.d.ts +24 -0
  18. package/dist/discovery.d.ts.map +1 -0
  19. package/dist/discovery.js +162 -0
  20. package/dist/discovery.js.map +1 -0
  21. package/dist/fork-worker.js +61 -45
  22. package/dist/fork-worker.js.map +1 -1
  23. package/dist/index.d.ts +6 -1
  24. package/dist/index.d.ts.map +1 -1
  25. package/dist/index.js +5 -0
  26. package/dist/index.js.map +1 -1
  27. package/dist/last-resort-handlers.d.ts +5 -0
  28. package/dist/last-resort-handlers.d.ts.map +1 -0
  29. package/dist/last-resort-handlers.js +40 -0
  30. package/dist/last-resort-handlers.js.map +1 -0
  31. package/dist/lifecycle-queue.d.ts +20 -0
  32. package/dist/lifecycle-queue.d.ts.map +1 -0
  33. package/dist/lifecycle-queue.js +110 -0
  34. package/dist/lifecycle-queue.js.map +1 -0
  35. package/dist/liveness.d.ts +5 -0
  36. package/dist/liveness.d.ts.map +1 -0
  37. package/dist/liveness.js +34 -0
  38. package/dist/liveness.js.map +1 -0
  39. package/dist/netron-client.d.ts +10 -8
  40. package/dist/netron-client.d.ts.map +1 -1
  41. package/dist/netron-client.js +16 -10
  42. package/dist/netron-client.js.map +1 -1
  43. package/dist/parent-death-watchdog.d.ts +13 -0
  44. package/dist/parent-death-watchdog.d.ts.map +1 -0
  45. package/dist/parent-death-watchdog.js +58 -0
  46. package/dist/parent-death-watchdog.js.map +1 -0
  47. package/dist/pm.module.d.ts.map +1 -1
  48. package/dist/pm.module.js.map +1 -1
  49. package/dist/process-health.d.ts.map +1 -1
  50. package/dist/process-health.js +20 -4
  51. package/dist/process-health.js.map +1 -1
  52. package/dist/process-manager.d.ts +4 -0
  53. package/dist/process-manager.d.ts.map +1 -1
  54. package/dist/process-manager.js +51 -4
  55. package/dist/process-manager.js.map +1 -1
  56. package/dist/process-metrics.d.ts.map +1 -1
  57. package/dist/process-metrics.js +10 -19
  58. package/dist/process-metrics.js.map +1 -1
  59. package/dist/process-pool.d.ts +4 -0
  60. package/dist/process-pool.d.ts.map +1 -1
  61. package/dist/process-pool.js +112 -29
  62. package/dist/process-pool.js.map +1 -1
  63. package/dist/process-registry.d.ts.map +1 -1
  64. package/dist/process-spawner.d.ts +18 -2
  65. package/dist/process-spawner.d.ts.map +1 -1
  66. package/dist/process-spawner.js +289 -62
  67. package/dist/process-spawner.js.map +1 -1
  68. package/dist/process-supervisor.d.ts +1 -0
  69. package/dist/process-supervisor.d.ts.map +1 -1
  70. package/dist/process-supervisor.js +59 -45
  71. package/dist/process-supervisor.js.map +1 -1
  72. package/dist/process-workflow.d.ts.map +1 -1
  73. package/dist/service-proxy.d.ts.map +1 -1
  74. package/dist/service-proxy.js +33 -3
  75. package/dist/service-proxy.js.map +1 -1
  76. package/dist/shutdown-windows.d.ts +19 -0
  77. package/dist/shutdown-windows.d.ts.map +1 -0
  78. package/dist/shutdown-windows.js +35 -0
  79. package/dist/shutdown-windows.js.map +1 -0
  80. package/dist/thread-pool.d.ts +7 -0
  81. package/dist/thread-pool.d.ts.map +1 -0
  82. package/dist/thread-pool.js +11 -0
  83. package/dist/thread-pool.js.map +1 -0
  84. package/dist/types.d.ts +79 -37
  85. package/dist/types.d.ts.map +1 -1
  86. package/dist/types.js +35 -39
  87. package/dist/types.js.map +1 -1
  88. package/dist/utils/deque.d.ts.map +1 -1
  89. package/dist/worker-health.d.ts +8 -0
  90. package/dist/worker-health.d.ts.map +1 -0
  91. package/dist/worker-health.js +63 -0
  92. package/dist/worker-health.js.map +1 -0
  93. package/dist/worker-runtime.js +17 -53
  94. package/dist/worker-runtime.js.map +1 -1
  95. package/dist/worker-traffic.d.ts +8 -0
  96. package/dist/worker-traffic.d.ts.map +1 -0
  97. package/dist/worker-traffic.js +56 -0
  98. package/dist/worker-traffic.js.map +1 -0
  99. package/package.json +9 -8
@@ -7,16 +7,78 @@ import path from 'node:path';
7
7
  import os from 'node:os';
8
8
  import { fileURLToPath } from 'node:url';
9
9
  import { createInterface } from 'node:readline';
10
+ import net from 'node:net';
10
11
  import { Errors } from '@omnitron-dev/titan/errors';
12
+ import { isAlive as isPidAlive } from './liveness.js';
11
13
  import { ProcessStatus } from './types.js';
12
14
  import { NetronClient } from './netron-client.js';
13
15
  import { ServiceProxyHandler } from './service-proxy.js';
14
16
  import { getAvailablePort } from '@omnitron-dev/titan/utils';
15
17
  import { generateUuidV7 } from '@omnitron-dev/titan/utils';
18
+ import { shutdownLadder, childDrainMs, DEFAULT_SHUTDOWN_BUDGET_MS } from './shutdown-windows.js';
19
+ import { confirmDeath } from './confirm-death.js';
20
+ import { threadPoolSizeFor } from './thread-pool.js';
16
21
  let MockProcessSpawnerClass = null;
17
22
  const SPAWNER_DIR = path.dirname(fileURLToPath(import.meta.url));
23
+ async function isSocketRefused(socketPath, timeoutMs = 250) {
24
+ return new Promise((resolve) => {
25
+ let settled = false;
26
+ const done = (refused) => {
27
+ if (settled)
28
+ return;
29
+ settled = true;
30
+ clearTimeout(timer);
31
+ socket.destroy();
32
+ resolve(refused);
33
+ };
34
+ const socket = net.connect({ path: socketPath });
35
+ const timer = setTimeout(() => done(false), timeoutMs);
36
+ socket.once('connect', () => done(false));
37
+ socket.once('error', (error) => done(error.code === 'ECONNREFUSED'));
38
+ });
39
+ }
40
+ const STALE_SOCKET_MIN_AGE_MS = 10 * 60 * 1000;
41
+ const EARLY_LOG_MAX_BYTES = 1024 * 1024;
42
+ export function hasExited(child) {
43
+ if (child.exitCode !== null || child.signalCode !== null)
44
+ return true;
45
+ return child.pid == null ? true : !isPidAlive(child.pid);
46
+ }
47
+ export async function killChildAndVerify(child, logger, context, termGraceMs = 3000) {
48
+ if (hasExited(child))
49
+ return true;
50
+ const exited = new Promise((resolve) => {
51
+ const done = () => resolve(true);
52
+ child.once('exit', done);
53
+ setTimeout(() => {
54
+ child.off('exit', done);
55
+ resolve(false);
56
+ }, termGraceMs).unref();
57
+ });
58
+ try {
59
+ child.kill('SIGTERM');
60
+ }
61
+ catch {
62
+ return true;
63
+ }
64
+ if (await exited)
65
+ return true;
66
+ if (hasExited(child))
67
+ return true;
68
+ logger.warn({ ...context, pid: child.pid }, 'Child ignored SIGTERM during cleanup — escalating to SIGKILL');
69
+ try {
70
+ child.kill('SIGKILL');
71
+ }
72
+ catch {
73
+ return true;
74
+ }
75
+ await new Promise((r) => {
76
+ setTimeout(r, 200).unref();
77
+ });
78
+ return hasExited(child);
79
+ }
18
80
  export class WorkerHandle extends EventEmitter {
19
- constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig) {
81
+ constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true) {
20
82
  super();
21
83
  this.id = id;
22
84
  this.transportUrl = transportUrl;
@@ -28,12 +90,16 @@ export class WorkerHandle extends EventEmitter {
28
90
  this.transportConfig = transportConfig;
29
91
  this.messageHandlers = new Map();
30
92
  this.logHandlers = new Set();
93
+ this.earlyLogs = [];
94
+ this.earlyLogBytes = 0;
95
+ this.earlyLogsDropped = 0;
31
96
  this.exitEmitted = false;
32
97
  this._status = initialStatus;
33
98
  this.worker = worker;
34
99
  this.netronClient = netronClient;
35
100
  this.setupMessageHandlers();
36
- this.setupDefaultLogForwarding();
101
+ if (forwardChildLogs)
102
+ this.setupDefaultLogForwarding();
37
103
  }
38
104
  emitExit(code, signal) {
39
105
  if (this.exitEmitted)
@@ -41,14 +107,20 @@ export class WorkerHandle extends EventEmitter {
41
107
  this.exitEmitted = true;
42
108
  const expected = this._status === ProcessStatus.STOPPING || this._status === ProcessStatus.STOPPED;
43
109
  this._status = ProcessStatus.STOPPED;
110
+ const forced = expected && signal === 'SIGKILL';
44
111
  const info = {
45
112
  workerId: this.id,
46
113
  serviceName: this.serviceName,
47
114
  code,
48
115
  signal,
49
116
  expected,
117
+ forced,
50
118
  };
51
- this.logger[expected ? 'info' : 'warn'](info, expected ? 'Worker exited cleanly' : 'Worker exited unexpectedly');
119
+ this.logger[expected && !forced ? 'info' : 'warn'](info, !expected
120
+ ? 'Worker exited unexpectedly'
121
+ : forced
122
+ ? 'Worker had to be killed — it did not exit within its shutdown window'
123
+ : 'Worker exited cleanly');
52
124
  try {
53
125
  this.emit('exit', info);
54
126
  }
@@ -101,21 +173,10 @@ export class WorkerHandle extends EventEmitter {
101
173
  }
102
174
  }
103
175
  async terminateChildProcess(child, totalDeadlineMs) {
104
- let GRACEFUL_TIMEOUT = 2000;
105
- let SIGTERM_TIMEOUT = 2000;
106
- let SIGKILL_TIMEOUT = 1000;
107
- if (totalDeadlineMs !== undefined) {
108
- if (totalDeadlineMs === 0) {
109
- GRACEFUL_TIMEOUT = 0;
110
- SIGTERM_TIMEOUT = 0;
111
- SIGKILL_TIMEOUT = 500;
112
- }
113
- else {
114
- GRACEFUL_TIMEOUT = Math.floor(totalDeadlineMs * 0.4);
115
- SIGTERM_TIMEOUT = Math.floor(totalDeadlineMs * 0.4);
116
- SIGKILL_TIMEOUT = totalDeadlineMs - GRACEFUL_TIMEOUT - SIGTERM_TIMEOUT;
117
- }
118
- }
176
+ const ladder = shutdownLadder(totalDeadlineMs);
177
+ const GRACEFUL_TIMEOUT = ladder.gracefulMs;
178
+ const SIGTERM_TIMEOUT = ladder.sigtermMs;
179
+ const SIGKILL_TIMEOUT = ladder.sigkillMs;
119
180
  return new Promise((resolve, reject) => {
120
181
  let resolved = false;
121
182
  let termTimer = null;
@@ -155,7 +216,7 @@ export class WorkerHandle extends EventEmitter {
155
216
  };
156
217
  child.once('exit', exitHandler);
157
218
  child.once('error', errorHandler);
158
- if (child.exitCode !== null || child.killed) {
219
+ if (hasExited(child)) {
159
220
  cleanup();
160
221
  resolve();
161
222
  return;
@@ -169,7 +230,7 @@ export class WorkerHandle extends EventEmitter {
169
230
  this.logger.debug({ workerId: this.id }, 'IPC channel closed, proceeding with SIGTERM');
170
231
  }
171
232
  termTimer = setTimeout(() => {
172
- if (resolved || child.exitCode !== null || child.killed)
233
+ if (resolved || hasExited(child))
173
234
  return;
174
235
  this.logger.debug({ workerId: this.id }, 'Graceful shutdown timeout, sending SIGTERM');
175
236
  try {
@@ -179,7 +240,7 @@ export class WorkerHandle extends EventEmitter {
179
240
  }
180
241
  }, GRACEFUL_TIMEOUT);
181
242
  killTimer = setTimeout(() => {
182
- if (resolved || child.exitCode !== null || child.killed)
243
+ if (resolved || hasExited(child))
183
244
  return;
184
245
  this.logger.warn({ workerId: this.id }, 'SIGTERM timeout, sending SIGKILL');
185
246
  try {
@@ -189,11 +250,27 @@ export class WorkerHandle extends EventEmitter {
189
250
  }
190
251
  }, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT);
191
252
  finalTimer = setTimeout(() => {
192
- if (!resolved) {
193
- cleanup();
194
- this.logger.error({ workerId: this.id }, 'Process termination timeout - process may be zombie');
253
+ if (resolved)
254
+ return;
255
+ cleanup();
256
+ if (child.pid == null) {
195
257
  resolve();
258
+ return;
196
259
  }
260
+ void confirmDeath(child).then((verdict) => {
261
+ if (verdict.dead) {
262
+ resolve();
263
+ return;
264
+ }
265
+ this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process survived SIGKILL — escalating one more SIGKILL and logging the leak');
266
+ try {
267
+ if (child.pid != null)
268
+ process.kill(child.pid, 'SIGKILL');
269
+ }
270
+ catch {
271
+ }
272
+ resolve();
273
+ });
197
274
  }, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT + SIGKILL_TIMEOUT);
198
275
  });
199
276
  }
@@ -204,7 +281,7 @@ export class WorkerHandle extends EventEmitter {
204
281
  }
205
282
  else {
206
283
  const child = this.worker;
207
- return !child.killed && child.exitCode === null;
284
+ return !hasExited(child);
208
285
  }
209
286
  }
210
287
  async send(message) {
@@ -227,10 +304,47 @@ export class WorkerHandle extends EventEmitter {
227
304
  const id = generateUuidV7();
228
305
  this.messageHandlers.set(id, handler);
229
306
  }
307
+ seedStartupOutput(output) {
308
+ if (!output)
309
+ return;
310
+ for (const [text, stream] of [
311
+ [output.stdout, 'stdout'],
312
+ [output.stderr, 'stderr'],
313
+ ]) {
314
+ for (const line of text.split('\n')) {
315
+ if (line.trim())
316
+ this.emitLog(line, stream);
317
+ }
318
+ }
319
+ }
230
320
  onLog(handler) {
321
+ const first = this.logHandlers.size === 0;
231
322
  this.logHandlers.add(handler);
323
+ if (!first)
324
+ return;
325
+ const buffered = this.earlyLogs;
326
+ this.earlyLogs = null;
327
+ if (!buffered)
328
+ return;
329
+ if (this.earlyLogsDropped > 0) {
330
+ handler(`[titan-pm] ${this.earlyLogsDropped} earlier line(s) from this process were dropped: ` +
331
+ 'it printed more than the pre-capture buffer holds', 'stderr');
332
+ }
333
+ for (const entry of buffered) {
334
+ handler(entry.line, entry.stream);
335
+ }
232
336
  }
233
337
  emitLog(line, stream) {
338
+ if (this.logHandlers.size === 0 && this.earlyLogs) {
339
+ this.earlyLogs.push({ line, stream });
340
+ this.earlyLogBytes += line.length;
341
+ while (this.earlyLogBytes > EARLY_LOG_MAX_BYTES && this.earlyLogs.length > 1) {
342
+ const dropped = this.earlyLogs.shift();
343
+ this.earlyLogBytes -= dropped.line.length;
344
+ this.earlyLogsDropped++;
345
+ }
346
+ return;
347
+ }
234
348
  for (const handler of this.logHandlers) {
235
349
  handler(line, stream);
236
350
  }
@@ -348,13 +462,27 @@ function sanitizeEnv(env) {
348
462
  }
349
463
  return result;
350
464
  }
465
+ function childEnv(extra = {}) {
466
+ const base = sanitizeEnv(process.env);
467
+ return {
468
+ ...base,
469
+ UV_THREADPOOL_SIZE: threadPoolSizeFor(base, { parallelism: os.availableParallelism() }),
470
+ ...extra,
471
+ };
472
+ }
351
473
  export class ProcessSpawner {
352
474
  constructor(logger, config = {}) {
353
475
  this.logger = logger;
354
476
  this.config = config;
477
+ this.ownSockets = new Set();
478
+ this.forkedPids = new Set();
479
+ this.staleSweep = null;
355
480
  let currentDir = SPAWNER_DIR;
356
- if (currentDir.includes('/src/') || currentDir.includes('\\src\\')) {
357
- currentDir = currentDir.replace(/[/\\]src[/\\]/, path.sep + 'dist' + path.sep);
481
+ const segments = currentDir.split(path.sep);
482
+ const srcIndex = segments.lastIndexOf('src');
483
+ if (srcIndex !== -1) {
484
+ segments[srcIndex] = 'dist';
485
+ currentDir = segments.join(path.sep);
358
486
  }
359
487
  this.workerRuntimePath = path.join(currentDir, 'worker-runtime.js');
360
488
  this.forkWorkerPath = path.join(currentDir, 'fork-worker.js');
@@ -388,12 +516,22 @@ export class ProcessSpawner {
388
516
  options: options,
389
517
  dependencies: options.dependencies,
390
518
  };
391
- const isolation = options.isolation || this.config.isolation || 'worker';
392
- const useWorkerThreads = isolation === 'worker' && (!options.execArgv || options.execArgv.length === 0);
519
+ const declaredIsolation = options.isolation;
520
+ const legacySandbox = declaredIsolation === 'vm' || declaredIsolation === 'container' ? declaredIsolation : undefined;
521
+ const strategy = (options.isolation === 'none' || options.isolation === 'worker' || options.isolation === 'child'
522
+ ? options.isolation
523
+ : undefined) ??
524
+ this.config.isolation ??
525
+ 'worker';
526
+ const sandbox = options.sandbox ?? legacySandbox;
527
+ const sandboxUnimplemented = sandbox === 'vm' || sandbox === 'container';
528
+ const useWorkerThreads = strategy === 'worker' && !sandboxUnimplemented && (!options.execArgv || options.execArgv.length === 0);
393
529
  let worker;
394
530
  let netronClient = null;
395
531
  try {
396
- if (isolation === 'vm' || isolation === 'container') {
532
+ if (sandboxUnimplemented) {
533
+ this.logger.warn({ processId, isolation: sandbox, actual: 'child' }, `Process isolation '${sandbox}' is not implemented — spawning an ordinary child process. ` +
534
+ 'It has the same filesystem and network access as its parent; do not rely on it as a security boundary.');
397
535
  worker = await this.spawnChildProcess(context, options.execArgv);
398
536
  }
399
537
  else if (useWorkerThreads) {
@@ -406,8 +544,10 @@ export class ProcessSpawner {
406
544
  const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout);
407
545
  const actualServiceName = readyInfo.serviceName || serviceName;
408
546
  const actualServiceVersion = readyInfo.serviceVersion || serviceVersion;
409
- if (isolation !== 'none') {
410
- netronClient = new NetronClient(processId, this.logger);
547
+ if (strategy !== 'none') {
548
+ netronClient = new NetronClient(processId, this.logger, {
549
+ ...(options.requestTimeout !== undefined && { requestTimeout: options.requestTimeout }),
550
+ });
411
551
  await netronClient.start();
412
552
  await netronClient.connect(transport.url);
413
553
  const qualifiedServiceName = actualServiceVersion
@@ -415,10 +555,14 @@ export class ProcessSpawner {
415
555
  : actualServiceName;
416
556
  const proxyHandler = new ServiceProxyHandler(processId, netronClient, qualifiedServiceName, this.logger);
417
557
  const proxy = proxyHandler.createProxy();
418
- return new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport);
558
+ const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
559
+ handle.seedStartupOutput(readyInfo.startupOutput);
560
+ return handle;
419
561
  }
420
562
  else {
421
- return new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport);
563
+ const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
564
+ handle.seedStartupOutput(readyInfo.startupOutput);
565
+ return handle;
422
566
  }
423
567
  }
424
568
  catch (error) {
@@ -430,13 +574,13 @@ export class ProcessSpawner {
430
574
  }
431
575
  else {
432
576
  const child = worker;
433
- if (child.pid && !child.killed) {
434
- child.kill('SIGTERM');
435
- setTimeout(() => {
436
- if (!child.killed) {
437
- child.kill('SIGKILL');
438
- }
439
- }, 3000).unref();
577
+ if (child.pid) {
578
+ const gone = await killChildAndVerify(child, this.logger, { serviceName, processId });
579
+ if (!gone) {
580
+ this.logger.error({ serviceName, processId, pid: child.pid }, 'Child survived SIGKILL after a failed spawn — it still holds whatever it held ' +
581
+ '(ports, locks). It is parented to this process, so the orphan sweep will not ' +
582
+ 'see it: it needs the abandoned-child sweep.');
583
+ }
440
584
  }
441
585
  }
442
586
  }
@@ -450,25 +594,55 @@ export class ProcessSpawner {
450
594
  }
451
595
  throw error;
452
596
  }
597
+ finally {
598
+ this.releaseForkClaim(worker?.pid);
599
+ }
453
600
  }
454
601
  async cleanup() {
455
- try {
456
- const entries = await fs.readdir(this.tempDir);
457
- for (const entry of entries) {
458
- if (entry.endsWith('.sock')) {
459
- try {
460
- await fs.unlink(path.join(this.tempDir, entry));
461
- }
462
- catch { }
463
- }
602
+ for (const socketPath of this.ownSockets) {
603
+ try {
604
+ await fs.unlink(socketPath);
464
605
  }
606
+ catch {
607
+ }
608
+ }
609
+ this.ownSockets.clear();
610
+ try {
611
+ await fs.rmdir(this.tempDir);
465
612
  }
466
613
  catch {
467
614
  }
615
+ }
616
+ async sweepStaleSockets() {
617
+ let entries;
468
618
  try {
469
- await fs.rmdir(this.tempDir);
619
+ entries = await fs.readdir(this.tempDir);
470
620
  }
471
621
  catch {
622
+ return;
623
+ }
624
+ const cutoff = Date.now() - STALE_SOCKET_MIN_AGE_MS;
625
+ for (const entry of entries) {
626
+ if (!entry.endsWith('.sock'))
627
+ continue;
628
+ const socketPath = path.join(this.tempDir, entry);
629
+ if (this.ownSockets.has(socketPath))
630
+ continue;
631
+ try {
632
+ const stat = await fs.stat(socketPath);
633
+ if (stat.mtimeMs > cutoff)
634
+ continue;
635
+ }
636
+ catch {
637
+ continue;
638
+ }
639
+ if (!(await isSocketRefused(socketPath)))
640
+ continue;
641
+ try {
642
+ await fs.unlink(socketPath);
643
+ }
644
+ catch {
645
+ }
472
646
  }
473
647
  }
474
648
  async setupTransport(processId, options) {
@@ -488,8 +662,12 @@ export class ProcessSpawner {
488
662
  }
489
663
  case 'unix':
490
664
  default: {
491
- await fs.mkdir(this.tempDir, { recursive: true });
665
+ await fs.mkdir(this.tempDir, { recursive: true, mode: 0o700 });
666
+ await fs.chmod(this.tempDir, 0o700).catch(() => {
667
+ });
492
668
  const socketPath = path.join(this.tempDir, `${processId}.sock`);
669
+ this.ownSockets.add(socketPath);
670
+ this.staleSweep ??= this.sweepStaleSockets().catch(() => undefined);
493
671
  return {
494
672
  type: 'unix',
495
673
  path: socketPath,
@@ -521,7 +699,7 @@ export class ProcessSpawner {
521
699
  const serializableContext = this.makeSerializable(context);
522
700
  const worker = new Worker(this.workerRuntimePath, {
523
701
  workerData: serializableContext,
524
- env: sanitizeEnv(process.env),
702
+ env: childEnv(),
525
703
  });
526
704
  return worker;
527
705
  }
@@ -547,25 +725,39 @@ export class ProcessSpawner {
547
725
  if (contextJson.length > 65536) {
548
726
  throw new Error(`TITAN_WORKER_CONTEXT exceeds 64KB (${contextJson.length} bytes). Reduce dependencies.`);
549
727
  }
728
+ const windowMs = shutdownLadder(context.options?.shutdownTimeout ?? DEFAULT_SHUTDOWN_BUDGET_MS).childWindowMs;
550
729
  const child = fork(this.forkWorkerPath, [], {
551
730
  env: {
552
- ...sanitizeEnv(process.env),
553
- ...(context.options?.env ?? {}),
731
+ ...childEnv(context.options?.env ?? {}),
554
732
  TITAN_WORKER_CONTEXT: contextJson,
733
+ TITAN_SHUTDOWN_TIMEOUT_MS: String(windowMs),
734
+ TITAN_DRAIN_TIMEOUT_MS: context.options?.env?.['TITAN_DRAIN_TIMEOUT_MS'] || String(childDrainMs(windowMs)),
555
735
  },
556
736
  execArgv: execArgv || [],
557
737
  silent: true,
558
738
  ...(context.options?.cwd ? { cwd: context.options.cwd } : {}),
559
739
  });
740
+ if (typeof child.pid === 'number') {
741
+ const pid = child.pid;
742
+ this.forkedPids.add(pid);
743
+ child.once('exit', () => this.forkedPids.delete(pid));
744
+ }
560
745
  return child;
561
746
  }
747
+ getForkedPids() {
748
+ return this.forkedPids;
749
+ }
750
+ releaseForkClaim(pid) {
751
+ if (typeof pid === 'number')
752
+ this.forkedPids.delete(pid);
753
+ }
562
754
  async waitForReady(worker, isWorkerThread, timeout = 5000) {
563
755
  return new Promise((resolve, reject) => {
564
756
  const STDERR_CAP_BYTES = 64 * 1024;
565
757
  const stderrBuf = [];
566
758
  const stdoutBuf = [];
567
- let stderrBytes = 0;
568
- let stdoutBytes = 0;
759
+ const stderrBytes = 0;
760
+ const stdoutBytes = 0;
569
761
  const appendBounded = (target, bytesRef, chunk) => {
570
762
  target.push(chunk);
571
763
  bytesRef.v += chunk.length;
@@ -576,6 +768,18 @@ export class ProcessSpawner {
576
768
  };
577
769
  const stderrRef = { v: stderrBytes };
578
770
  const stdoutRef = { v: stdoutBytes };
771
+ const LAG_PROBE_INTERVAL_MS = 1_000;
772
+ const LAG_SIGNIFICANT_MS = 2_000;
773
+ let maxLagMs = 0;
774
+ let expectedTickAt = Date.now() + LAG_PROBE_INTERVAL_MS;
775
+ const lagProbe = setInterval(() => {
776
+ const now = Date.now();
777
+ const lag = now - expectedTickAt;
778
+ expectedTickAt = now + LAG_PROBE_INTERVAL_MS;
779
+ if (lag > maxLagMs)
780
+ maxLagMs = lag;
781
+ }, LAG_PROBE_INTERVAL_MS);
782
+ lagProbe.unref?.();
579
783
  if (!isWorkerThread) {
580
784
  const child = worker;
581
785
  child.stderr?.on('data', (chunk) => {
@@ -596,18 +800,37 @@ export class ProcessSpawner {
596
800
  };
597
801
  };
598
802
  const timer = setTimeout(() => {
803
+ clearInterval(lagProbe);
599
804
  const pid = !isWorkerThread ? worker.pid : undefined;
600
805
  const pidHint = pid ? ` (pid: ${pid})` : '';
601
806
  const diag = buildDiagnostics();
602
807
  const stderrTail = diag.stderr
603
808
  ? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-15).join('\n')
604
809
  : '';
605
- const err = Errors.timeout(`Worker startup${pidHint}${stderrTail}`, timeout);
606
- err.details = { ...(err.details ?? {}), ...diag };
810
+ const supervisorNote = maxLagMs >= LAG_SIGNIFICANT_MS
811
+ ? `\n--- this timeout is ours, not the child's ---\n` +
812
+ `The supervisor's own 1s timer ran ${Math.round(maxLagMs / 1000)}s late ` +
813
+ `during this window, so it was not running to read the child or to ` +
814
+ `receive its ready message. Check free memory and swap on this host ` +
815
+ `before reading the application.`
816
+ : '';
817
+ const err = Errors.timeout(`Worker startup${pidHint}`, timeout);
818
+ if (supervisorNote) {
819
+ err.message += supervisorNote;
820
+ }
821
+ if (stderrTail) {
822
+ err.message += stderrTail;
823
+ }
824
+ err.details = {
825
+ ...(err.details ?? {}),
826
+ ...diag,
827
+ supervisorMaxLagMs: maxLagMs,
828
+ };
607
829
  reject(err);
608
830
  }, timeout);
609
831
  const cleanup = () => {
610
832
  clearTimeout(timer);
833
+ clearInterval(lagProbe);
611
834
  if (isWorkerThread) {
612
835
  worker.off('message', messageHandler);
613
836
  }
@@ -623,6 +846,7 @@ export class ProcessSpawner {
623
846
  resolve({
624
847
  serviceName: data.serviceName,
625
848
  serviceVersion: data.serviceVersion,
849
+ startupOutput: { stdout: stdoutBuf.join(''), stderr: stderrBuf.join('') },
626
850
  });
627
851
  }
628
852
  else if (data.type === 'error') {
@@ -636,14 +860,17 @@ export class ProcessSpawner {
636
860
  }
637
861
  }
638
862
  };
639
- const exitHandler = (code) => {
863
+ const exitHandler = (code, signal) => {
640
864
  cleanup();
641
865
  const diag = buildDiagnostics();
642
866
  const stderrTail = diag.stderr
643
867
  ? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-20).join('\n')
644
868
  : '';
645
- const err = new Error(`Worker exited during startup with code ${code}${stderrTail}`);
646
- err.details = diag;
869
+ const how = signal
870
+ ? `after being killed by ${signal} — it did not choose to exit, so its own logs will not explain this`
871
+ : `with code ${code}`;
872
+ const err = new Error(`Worker exited during startup ${how}${stderrTail}`);
873
+ err.details = { ...diag, exitCode: code, exitSignal: signal };
647
874
  reject(err);
648
875
  };
649
876
  if (isWorkerThread) {