@omnitron-dev/titan-pm 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backoff.d.ts +12 -0
- package/dist/backoff.d.ts.map +1 -0
- package/dist/backoff.js +44 -0
- package/dist/backoff.js.map +1 -0
- package/dist/child-contract.d.ts +58 -0
- package/dist/child-contract.d.ts.map +1 -0
- package/dist/child-contract.js +27 -0
- package/dist/child-contract.js.map +1 -0
- package/dist/confirm-death.d.ts +18 -0
- package/dist/confirm-death.d.ts.map +1 -0
- package/dist/confirm-death.js +49 -0
- package/dist/confirm-death.js.map +1 -0
- package/dist/decorators.d.ts +1 -10
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +43 -99
- package/dist/decorators.js.map +1 -1
- package/dist/discovery.d.ts +24 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +162 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fork-worker.js +61 -45
- package/dist/fork-worker.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/last-resort-handlers.d.ts +5 -0
- package/dist/last-resort-handlers.d.ts.map +1 -0
- package/dist/last-resort-handlers.js +40 -0
- package/dist/last-resort-handlers.js.map +1 -0
- package/dist/lifecycle-queue.d.ts +20 -0
- package/dist/lifecycle-queue.d.ts.map +1 -0
- package/dist/lifecycle-queue.js +110 -0
- package/dist/lifecycle-queue.js.map +1 -0
- package/dist/liveness.d.ts +5 -0
- package/dist/liveness.d.ts.map +1 -0
- package/dist/liveness.js +34 -0
- package/dist/liveness.js.map +1 -0
- package/dist/netron-client.d.ts +10 -8
- package/dist/netron-client.d.ts.map +1 -1
- package/dist/netron-client.js +16 -10
- package/dist/netron-client.js.map +1 -1
- package/dist/parent-death-watchdog.d.ts +13 -0
- package/dist/parent-death-watchdog.d.ts.map +1 -0
- package/dist/parent-death-watchdog.js +58 -0
- package/dist/parent-death-watchdog.js.map +1 -0
- package/dist/pm.module.d.ts.map +1 -1
- package/dist/pm.module.js.map +1 -1
- package/dist/process-health.d.ts.map +1 -1
- package/dist/process-health.js +20 -4
- package/dist/process-health.js.map +1 -1
- package/dist/process-manager.d.ts +4 -0
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +51 -4
- package/dist/process-manager.js.map +1 -1
- package/dist/process-metrics.d.ts.map +1 -1
- package/dist/process-metrics.js +10 -19
- package/dist/process-metrics.js.map +1 -1
- package/dist/process-pool.d.ts +4 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +112 -29
- package/dist/process-pool.js.map +1 -1
- package/dist/process-registry.d.ts.map +1 -1
- package/dist/process-spawner.d.ts +18 -2
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +289 -62
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +1 -0
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +59 -45
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/service-proxy.d.ts.map +1 -1
- package/dist/service-proxy.js +33 -3
- package/dist/service-proxy.js.map +1 -1
- package/dist/shutdown-windows.d.ts +19 -0
- package/dist/shutdown-windows.d.ts.map +1 -0
- package/dist/shutdown-windows.js +35 -0
- package/dist/shutdown-windows.js.map +1 -0
- package/dist/thread-pool.d.ts +7 -0
- package/dist/thread-pool.d.ts.map +1 -0
- package/dist/thread-pool.js +11 -0
- package/dist/thread-pool.js.map +1 -0
- package/dist/types.d.ts +79 -37
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +35 -39
- package/dist/types.js.map +1 -1
- package/dist/utils/deque.d.ts.map +1 -1
- package/dist/worker-health.d.ts +8 -0
- package/dist/worker-health.d.ts.map +1 -0
- package/dist/worker-health.js +63 -0
- package/dist/worker-health.js.map +1 -0
- package/dist/worker-runtime.js +17 -53
- package/dist/worker-runtime.js.map +1 -1
- package/dist/worker-traffic.d.ts +8 -0
- package/dist/worker-traffic.d.ts.map +1 -0
- package/dist/worker-traffic.js +56 -0
- package/dist/worker-traffic.js.map +1 -0
- package/package.json +9 -8
package/dist/process-spawner.js
CHANGED
|
@@ -7,16 +7,78 @@ import path from 'node:path';
|
|
|
7
7
|
import os from 'node:os';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { createInterface } from 'node:readline';
|
|
10
|
+
import net from 'node:net';
|
|
10
11
|
import { Errors } from '@omnitron-dev/titan/errors';
|
|
12
|
+
import { isAlive as isPidAlive } from './liveness.js';
|
|
11
13
|
import { ProcessStatus } from './types.js';
|
|
12
14
|
import { NetronClient } from './netron-client.js';
|
|
13
15
|
import { ServiceProxyHandler } from './service-proxy.js';
|
|
14
16
|
import { getAvailablePort } from '@omnitron-dev/titan/utils';
|
|
15
17
|
import { generateUuidV7 } from '@omnitron-dev/titan/utils';
|
|
18
|
+
import { shutdownLadder, childDrainMs, DEFAULT_SHUTDOWN_BUDGET_MS } from './shutdown-windows.js';
|
|
19
|
+
import { confirmDeath } from './confirm-death.js';
|
|
20
|
+
import { threadPoolSizeFor } from './thread-pool.js';
|
|
16
21
|
let MockProcessSpawnerClass = null;
|
|
17
22
|
const SPAWNER_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
async function isSocketRefused(socketPath, timeoutMs = 250) {
|
|
24
|
+
return new Promise((resolve) => {
|
|
25
|
+
let settled = false;
|
|
26
|
+
const done = (refused) => {
|
|
27
|
+
if (settled)
|
|
28
|
+
return;
|
|
29
|
+
settled = true;
|
|
30
|
+
clearTimeout(timer);
|
|
31
|
+
socket.destroy();
|
|
32
|
+
resolve(refused);
|
|
33
|
+
};
|
|
34
|
+
const socket = net.connect({ path: socketPath });
|
|
35
|
+
const timer = setTimeout(() => done(false), timeoutMs);
|
|
36
|
+
socket.once('connect', () => done(false));
|
|
37
|
+
socket.once('error', (error) => done(error.code === 'ECONNREFUSED'));
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const STALE_SOCKET_MIN_AGE_MS = 10 * 60 * 1000;
|
|
41
|
+
const EARLY_LOG_MAX_BYTES = 1024 * 1024;
|
|
42
|
+
export function hasExited(child) {
|
|
43
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
44
|
+
return true;
|
|
45
|
+
return child.pid == null ? true : !isPidAlive(child.pid);
|
|
46
|
+
}
|
|
47
|
+
export async function killChildAndVerify(child, logger, context, termGraceMs = 3000) {
|
|
48
|
+
if (hasExited(child))
|
|
49
|
+
return true;
|
|
50
|
+
const exited = new Promise((resolve) => {
|
|
51
|
+
const done = () => resolve(true);
|
|
52
|
+
child.once('exit', done);
|
|
53
|
+
setTimeout(() => {
|
|
54
|
+
child.off('exit', done);
|
|
55
|
+
resolve(false);
|
|
56
|
+
}, termGraceMs).unref();
|
|
57
|
+
});
|
|
58
|
+
try {
|
|
59
|
+
child.kill('SIGTERM');
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
if (await exited)
|
|
65
|
+
return true;
|
|
66
|
+
if (hasExited(child))
|
|
67
|
+
return true;
|
|
68
|
+
logger.warn({ ...context, pid: child.pid }, 'Child ignored SIGTERM during cleanup — escalating to SIGKILL');
|
|
69
|
+
try {
|
|
70
|
+
child.kill('SIGKILL');
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
await new Promise((r) => {
|
|
76
|
+
setTimeout(r, 200).unref();
|
|
77
|
+
});
|
|
78
|
+
return hasExited(child);
|
|
79
|
+
}
|
|
18
80
|
export class WorkerHandle extends EventEmitter {
|
|
19
|
-
constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig) {
|
|
81
|
+
constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true) {
|
|
20
82
|
super();
|
|
21
83
|
this.id = id;
|
|
22
84
|
this.transportUrl = transportUrl;
|
|
@@ -28,12 +90,16 @@ export class WorkerHandle extends EventEmitter {
|
|
|
28
90
|
this.transportConfig = transportConfig;
|
|
29
91
|
this.messageHandlers = new Map();
|
|
30
92
|
this.logHandlers = new Set();
|
|
93
|
+
this.earlyLogs = [];
|
|
94
|
+
this.earlyLogBytes = 0;
|
|
95
|
+
this.earlyLogsDropped = 0;
|
|
31
96
|
this.exitEmitted = false;
|
|
32
97
|
this._status = initialStatus;
|
|
33
98
|
this.worker = worker;
|
|
34
99
|
this.netronClient = netronClient;
|
|
35
100
|
this.setupMessageHandlers();
|
|
36
|
-
|
|
101
|
+
if (forwardChildLogs)
|
|
102
|
+
this.setupDefaultLogForwarding();
|
|
37
103
|
}
|
|
38
104
|
emitExit(code, signal) {
|
|
39
105
|
if (this.exitEmitted)
|
|
@@ -41,14 +107,20 @@ export class WorkerHandle extends EventEmitter {
|
|
|
41
107
|
this.exitEmitted = true;
|
|
42
108
|
const expected = this._status === ProcessStatus.STOPPING || this._status === ProcessStatus.STOPPED;
|
|
43
109
|
this._status = ProcessStatus.STOPPED;
|
|
110
|
+
const forced = expected && signal === 'SIGKILL';
|
|
44
111
|
const info = {
|
|
45
112
|
workerId: this.id,
|
|
46
113
|
serviceName: this.serviceName,
|
|
47
114
|
code,
|
|
48
115
|
signal,
|
|
49
116
|
expected,
|
|
117
|
+
forced,
|
|
50
118
|
};
|
|
51
|
-
this.logger[expected ? 'info' : 'warn'](info, expected
|
|
119
|
+
this.logger[expected && !forced ? 'info' : 'warn'](info, !expected
|
|
120
|
+
? 'Worker exited unexpectedly'
|
|
121
|
+
: forced
|
|
122
|
+
? 'Worker had to be killed — it did not exit within its shutdown window'
|
|
123
|
+
: 'Worker exited cleanly');
|
|
52
124
|
try {
|
|
53
125
|
this.emit('exit', info);
|
|
54
126
|
}
|
|
@@ -101,21 +173,10 @@ export class WorkerHandle extends EventEmitter {
|
|
|
101
173
|
}
|
|
102
174
|
}
|
|
103
175
|
async terminateChildProcess(child, totalDeadlineMs) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if (totalDeadlineMs === 0) {
|
|
109
|
-
GRACEFUL_TIMEOUT = 0;
|
|
110
|
-
SIGTERM_TIMEOUT = 0;
|
|
111
|
-
SIGKILL_TIMEOUT = 500;
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
GRACEFUL_TIMEOUT = Math.floor(totalDeadlineMs * 0.4);
|
|
115
|
-
SIGTERM_TIMEOUT = Math.floor(totalDeadlineMs * 0.4);
|
|
116
|
-
SIGKILL_TIMEOUT = totalDeadlineMs - GRACEFUL_TIMEOUT - SIGTERM_TIMEOUT;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
176
|
+
const ladder = shutdownLadder(totalDeadlineMs);
|
|
177
|
+
const GRACEFUL_TIMEOUT = ladder.gracefulMs;
|
|
178
|
+
const SIGTERM_TIMEOUT = ladder.sigtermMs;
|
|
179
|
+
const SIGKILL_TIMEOUT = ladder.sigkillMs;
|
|
119
180
|
return new Promise((resolve, reject) => {
|
|
120
181
|
let resolved = false;
|
|
121
182
|
let termTimer = null;
|
|
@@ -155,7 +216,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
155
216
|
};
|
|
156
217
|
child.once('exit', exitHandler);
|
|
157
218
|
child.once('error', errorHandler);
|
|
158
|
-
if (child
|
|
219
|
+
if (hasExited(child)) {
|
|
159
220
|
cleanup();
|
|
160
221
|
resolve();
|
|
161
222
|
return;
|
|
@@ -169,7 +230,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
169
230
|
this.logger.debug({ workerId: this.id }, 'IPC channel closed, proceeding with SIGTERM');
|
|
170
231
|
}
|
|
171
232
|
termTimer = setTimeout(() => {
|
|
172
|
-
if (resolved || child
|
|
233
|
+
if (resolved || hasExited(child))
|
|
173
234
|
return;
|
|
174
235
|
this.logger.debug({ workerId: this.id }, 'Graceful shutdown timeout, sending SIGTERM');
|
|
175
236
|
try {
|
|
@@ -179,7 +240,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
179
240
|
}
|
|
180
241
|
}, GRACEFUL_TIMEOUT);
|
|
181
242
|
killTimer = setTimeout(() => {
|
|
182
|
-
if (resolved || child
|
|
243
|
+
if (resolved || hasExited(child))
|
|
183
244
|
return;
|
|
184
245
|
this.logger.warn({ workerId: this.id }, 'SIGTERM timeout, sending SIGKILL');
|
|
185
246
|
try {
|
|
@@ -189,11 +250,27 @@ export class WorkerHandle extends EventEmitter {
|
|
|
189
250
|
}
|
|
190
251
|
}, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT);
|
|
191
252
|
finalTimer = setTimeout(() => {
|
|
192
|
-
if (
|
|
193
|
-
|
|
194
|
-
|
|
253
|
+
if (resolved)
|
|
254
|
+
return;
|
|
255
|
+
cleanup();
|
|
256
|
+
if (child.pid == null) {
|
|
195
257
|
resolve();
|
|
258
|
+
return;
|
|
196
259
|
}
|
|
260
|
+
void confirmDeath(child).then((verdict) => {
|
|
261
|
+
if (verdict.dead) {
|
|
262
|
+
resolve();
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process survived SIGKILL — escalating one more SIGKILL and logging the leak');
|
|
266
|
+
try {
|
|
267
|
+
if (child.pid != null)
|
|
268
|
+
process.kill(child.pid, 'SIGKILL');
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
}
|
|
272
|
+
resolve();
|
|
273
|
+
});
|
|
197
274
|
}, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT + SIGKILL_TIMEOUT);
|
|
198
275
|
});
|
|
199
276
|
}
|
|
@@ -204,7 +281,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
204
281
|
}
|
|
205
282
|
else {
|
|
206
283
|
const child = this.worker;
|
|
207
|
-
return !child
|
|
284
|
+
return !hasExited(child);
|
|
208
285
|
}
|
|
209
286
|
}
|
|
210
287
|
async send(message) {
|
|
@@ -227,10 +304,47 @@ export class WorkerHandle extends EventEmitter {
|
|
|
227
304
|
const id = generateUuidV7();
|
|
228
305
|
this.messageHandlers.set(id, handler);
|
|
229
306
|
}
|
|
307
|
+
seedStartupOutput(output) {
|
|
308
|
+
if (!output)
|
|
309
|
+
return;
|
|
310
|
+
for (const [text, stream] of [
|
|
311
|
+
[output.stdout, 'stdout'],
|
|
312
|
+
[output.stderr, 'stderr'],
|
|
313
|
+
]) {
|
|
314
|
+
for (const line of text.split('\n')) {
|
|
315
|
+
if (line.trim())
|
|
316
|
+
this.emitLog(line, stream);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
230
320
|
onLog(handler) {
|
|
321
|
+
const first = this.logHandlers.size === 0;
|
|
231
322
|
this.logHandlers.add(handler);
|
|
323
|
+
if (!first)
|
|
324
|
+
return;
|
|
325
|
+
const buffered = this.earlyLogs;
|
|
326
|
+
this.earlyLogs = null;
|
|
327
|
+
if (!buffered)
|
|
328
|
+
return;
|
|
329
|
+
if (this.earlyLogsDropped > 0) {
|
|
330
|
+
handler(`[titan-pm] ${this.earlyLogsDropped} earlier line(s) from this process were dropped: ` +
|
|
331
|
+
'it printed more than the pre-capture buffer holds', 'stderr');
|
|
332
|
+
}
|
|
333
|
+
for (const entry of buffered) {
|
|
334
|
+
handler(entry.line, entry.stream);
|
|
335
|
+
}
|
|
232
336
|
}
|
|
233
337
|
emitLog(line, stream) {
|
|
338
|
+
if (this.logHandlers.size === 0 && this.earlyLogs) {
|
|
339
|
+
this.earlyLogs.push({ line, stream });
|
|
340
|
+
this.earlyLogBytes += line.length;
|
|
341
|
+
while (this.earlyLogBytes > EARLY_LOG_MAX_BYTES && this.earlyLogs.length > 1) {
|
|
342
|
+
const dropped = this.earlyLogs.shift();
|
|
343
|
+
this.earlyLogBytes -= dropped.line.length;
|
|
344
|
+
this.earlyLogsDropped++;
|
|
345
|
+
}
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
234
348
|
for (const handler of this.logHandlers) {
|
|
235
349
|
handler(line, stream);
|
|
236
350
|
}
|
|
@@ -348,13 +462,27 @@ function sanitizeEnv(env) {
|
|
|
348
462
|
}
|
|
349
463
|
return result;
|
|
350
464
|
}
|
|
465
|
+
function childEnv(extra = {}) {
|
|
466
|
+
const base = sanitizeEnv(process.env);
|
|
467
|
+
return {
|
|
468
|
+
...base,
|
|
469
|
+
UV_THREADPOOL_SIZE: threadPoolSizeFor(base, { parallelism: os.availableParallelism() }),
|
|
470
|
+
...extra,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
351
473
|
export class ProcessSpawner {
|
|
352
474
|
constructor(logger, config = {}) {
|
|
353
475
|
this.logger = logger;
|
|
354
476
|
this.config = config;
|
|
477
|
+
this.ownSockets = new Set();
|
|
478
|
+
this.forkedPids = new Set();
|
|
479
|
+
this.staleSweep = null;
|
|
355
480
|
let currentDir = SPAWNER_DIR;
|
|
356
|
-
|
|
357
|
-
|
|
481
|
+
const segments = currentDir.split(path.sep);
|
|
482
|
+
const srcIndex = segments.lastIndexOf('src');
|
|
483
|
+
if (srcIndex !== -1) {
|
|
484
|
+
segments[srcIndex] = 'dist';
|
|
485
|
+
currentDir = segments.join(path.sep);
|
|
358
486
|
}
|
|
359
487
|
this.workerRuntimePath = path.join(currentDir, 'worker-runtime.js');
|
|
360
488
|
this.forkWorkerPath = path.join(currentDir, 'fork-worker.js');
|
|
@@ -388,12 +516,22 @@ export class ProcessSpawner {
|
|
|
388
516
|
options: options,
|
|
389
517
|
dependencies: options.dependencies,
|
|
390
518
|
};
|
|
391
|
-
const
|
|
392
|
-
const
|
|
519
|
+
const declaredIsolation = options.isolation;
|
|
520
|
+
const legacySandbox = declaredIsolation === 'vm' || declaredIsolation === 'container' ? declaredIsolation : undefined;
|
|
521
|
+
const strategy = (options.isolation === 'none' || options.isolation === 'worker' || options.isolation === 'child'
|
|
522
|
+
? options.isolation
|
|
523
|
+
: undefined) ??
|
|
524
|
+
this.config.isolation ??
|
|
525
|
+
'worker';
|
|
526
|
+
const sandbox = options.sandbox ?? legacySandbox;
|
|
527
|
+
const sandboxUnimplemented = sandbox === 'vm' || sandbox === 'container';
|
|
528
|
+
const useWorkerThreads = strategy === 'worker' && !sandboxUnimplemented && (!options.execArgv || options.execArgv.length === 0);
|
|
393
529
|
let worker;
|
|
394
530
|
let netronClient = null;
|
|
395
531
|
try {
|
|
396
|
-
if (
|
|
532
|
+
if (sandboxUnimplemented) {
|
|
533
|
+
this.logger.warn({ processId, isolation: sandbox, actual: 'child' }, `Process isolation '${sandbox}' is not implemented — spawning an ordinary child process. ` +
|
|
534
|
+
'It has the same filesystem and network access as its parent; do not rely on it as a security boundary.');
|
|
397
535
|
worker = await this.spawnChildProcess(context, options.execArgv);
|
|
398
536
|
}
|
|
399
537
|
else if (useWorkerThreads) {
|
|
@@ -406,8 +544,10 @@ export class ProcessSpawner {
|
|
|
406
544
|
const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout);
|
|
407
545
|
const actualServiceName = readyInfo.serviceName || serviceName;
|
|
408
546
|
const actualServiceVersion = readyInfo.serviceVersion || serviceVersion;
|
|
409
|
-
if (
|
|
410
|
-
netronClient = new NetronClient(processId, this.logger
|
|
547
|
+
if (strategy !== 'none') {
|
|
548
|
+
netronClient = new NetronClient(processId, this.logger, {
|
|
549
|
+
...(options.requestTimeout !== undefined && { requestTimeout: options.requestTimeout }),
|
|
550
|
+
});
|
|
411
551
|
await netronClient.start();
|
|
412
552
|
await netronClient.connect(transport.url);
|
|
413
553
|
const qualifiedServiceName = actualServiceVersion
|
|
@@ -415,10 +555,14 @@ export class ProcessSpawner {
|
|
|
415
555
|
: actualServiceName;
|
|
416
556
|
const proxyHandler = new ServiceProxyHandler(processId, netronClient, qualifiedServiceName, this.logger);
|
|
417
557
|
const proxy = proxyHandler.createProxy();
|
|
418
|
-
|
|
558
|
+
const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
559
|
+
handle.seedStartupOutput(readyInfo.startupOutput);
|
|
560
|
+
return handle;
|
|
419
561
|
}
|
|
420
562
|
else {
|
|
421
|
-
|
|
563
|
+
const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
564
|
+
handle.seedStartupOutput(readyInfo.startupOutput);
|
|
565
|
+
return handle;
|
|
422
566
|
}
|
|
423
567
|
}
|
|
424
568
|
catch (error) {
|
|
@@ -430,13 +574,13 @@ export class ProcessSpawner {
|
|
|
430
574
|
}
|
|
431
575
|
else {
|
|
432
576
|
const child = worker;
|
|
433
|
-
if (child.pid
|
|
434
|
-
child.
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
577
|
+
if (child.pid) {
|
|
578
|
+
const gone = await killChildAndVerify(child, this.logger, { serviceName, processId });
|
|
579
|
+
if (!gone) {
|
|
580
|
+
this.logger.error({ serviceName, processId, pid: child.pid }, 'Child survived SIGKILL after a failed spawn — it still holds whatever it held ' +
|
|
581
|
+
'(ports, locks). It is parented to this process, so the orphan sweep will not ' +
|
|
582
|
+
'see it: it needs the abandoned-child sweep.');
|
|
583
|
+
}
|
|
440
584
|
}
|
|
441
585
|
}
|
|
442
586
|
}
|
|
@@ -450,25 +594,55 @@ export class ProcessSpawner {
|
|
|
450
594
|
}
|
|
451
595
|
throw error;
|
|
452
596
|
}
|
|
597
|
+
finally {
|
|
598
|
+
this.releaseForkClaim(worker?.pid);
|
|
599
|
+
}
|
|
453
600
|
}
|
|
454
601
|
async cleanup() {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
if (entry.endsWith('.sock')) {
|
|
459
|
-
try {
|
|
460
|
-
await fs.unlink(path.join(this.tempDir, entry));
|
|
461
|
-
}
|
|
462
|
-
catch { }
|
|
463
|
-
}
|
|
602
|
+
for (const socketPath of this.ownSockets) {
|
|
603
|
+
try {
|
|
604
|
+
await fs.unlink(socketPath);
|
|
464
605
|
}
|
|
606
|
+
catch {
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
this.ownSockets.clear();
|
|
610
|
+
try {
|
|
611
|
+
await fs.rmdir(this.tempDir);
|
|
465
612
|
}
|
|
466
613
|
catch {
|
|
467
614
|
}
|
|
615
|
+
}
|
|
616
|
+
async sweepStaleSockets() {
|
|
617
|
+
let entries;
|
|
468
618
|
try {
|
|
469
|
-
await fs.
|
|
619
|
+
entries = await fs.readdir(this.tempDir);
|
|
470
620
|
}
|
|
471
621
|
catch {
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
const cutoff = Date.now() - STALE_SOCKET_MIN_AGE_MS;
|
|
625
|
+
for (const entry of entries) {
|
|
626
|
+
if (!entry.endsWith('.sock'))
|
|
627
|
+
continue;
|
|
628
|
+
const socketPath = path.join(this.tempDir, entry);
|
|
629
|
+
if (this.ownSockets.has(socketPath))
|
|
630
|
+
continue;
|
|
631
|
+
try {
|
|
632
|
+
const stat = await fs.stat(socketPath);
|
|
633
|
+
if (stat.mtimeMs > cutoff)
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
catch {
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
639
|
+
if (!(await isSocketRefused(socketPath)))
|
|
640
|
+
continue;
|
|
641
|
+
try {
|
|
642
|
+
await fs.unlink(socketPath);
|
|
643
|
+
}
|
|
644
|
+
catch {
|
|
645
|
+
}
|
|
472
646
|
}
|
|
473
647
|
}
|
|
474
648
|
async setupTransport(processId, options) {
|
|
@@ -488,8 +662,12 @@ export class ProcessSpawner {
|
|
|
488
662
|
}
|
|
489
663
|
case 'unix':
|
|
490
664
|
default: {
|
|
491
|
-
await fs.mkdir(this.tempDir, { recursive: true });
|
|
665
|
+
await fs.mkdir(this.tempDir, { recursive: true, mode: 0o700 });
|
|
666
|
+
await fs.chmod(this.tempDir, 0o700).catch(() => {
|
|
667
|
+
});
|
|
492
668
|
const socketPath = path.join(this.tempDir, `${processId}.sock`);
|
|
669
|
+
this.ownSockets.add(socketPath);
|
|
670
|
+
this.staleSweep ??= this.sweepStaleSockets().catch(() => undefined);
|
|
493
671
|
return {
|
|
494
672
|
type: 'unix',
|
|
495
673
|
path: socketPath,
|
|
@@ -521,7 +699,7 @@ export class ProcessSpawner {
|
|
|
521
699
|
const serializableContext = this.makeSerializable(context);
|
|
522
700
|
const worker = new Worker(this.workerRuntimePath, {
|
|
523
701
|
workerData: serializableContext,
|
|
524
|
-
env:
|
|
702
|
+
env: childEnv(),
|
|
525
703
|
});
|
|
526
704
|
return worker;
|
|
527
705
|
}
|
|
@@ -547,25 +725,39 @@ export class ProcessSpawner {
|
|
|
547
725
|
if (contextJson.length > 65536) {
|
|
548
726
|
throw new Error(`TITAN_WORKER_CONTEXT exceeds 64KB (${contextJson.length} bytes). Reduce dependencies.`);
|
|
549
727
|
}
|
|
728
|
+
const windowMs = shutdownLadder(context.options?.shutdownTimeout ?? DEFAULT_SHUTDOWN_BUDGET_MS).childWindowMs;
|
|
550
729
|
const child = fork(this.forkWorkerPath, [], {
|
|
551
730
|
env: {
|
|
552
|
-
...
|
|
553
|
-
...(context.options?.env ?? {}),
|
|
731
|
+
...childEnv(context.options?.env ?? {}),
|
|
554
732
|
TITAN_WORKER_CONTEXT: contextJson,
|
|
733
|
+
TITAN_SHUTDOWN_TIMEOUT_MS: String(windowMs),
|
|
734
|
+
TITAN_DRAIN_TIMEOUT_MS: context.options?.env?.['TITAN_DRAIN_TIMEOUT_MS'] || String(childDrainMs(windowMs)),
|
|
555
735
|
},
|
|
556
736
|
execArgv: execArgv || [],
|
|
557
737
|
silent: true,
|
|
558
738
|
...(context.options?.cwd ? { cwd: context.options.cwd } : {}),
|
|
559
739
|
});
|
|
740
|
+
if (typeof child.pid === 'number') {
|
|
741
|
+
const pid = child.pid;
|
|
742
|
+
this.forkedPids.add(pid);
|
|
743
|
+
child.once('exit', () => this.forkedPids.delete(pid));
|
|
744
|
+
}
|
|
560
745
|
return child;
|
|
561
746
|
}
|
|
747
|
+
getForkedPids() {
|
|
748
|
+
return this.forkedPids;
|
|
749
|
+
}
|
|
750
|
+
releaseForkClaim(pid) {
|
|
751
|
+
if (typeof pid === 'number')
|
|
752
|
+
this.forkedPids.delete(pid);
|
|
753
|
+
}
|
|
562
754
|
async waitForReady(worker, isWorkerThread, timeout = 5000) {
|
|
563
755
|
return new Promise((resolve, reject) => {
|
|
564
756
|
const STDERR_CAP_BYTES = 64 * 1024;
|
|
565
757
|
const stderrBuf = [];
|
|
566
758
|
const stdoutBuf = [];
|
|
567
|
-
|
|
568
|
-
|
|
759
|
+
const stderrBytes = 0;
|
|
760
|
+
const stdoutBytes = 0;
|
|
569
761
|
const appendBounded = (target, bytesRef, chunk) => {
|
|
570
762
|
target.push(chunk);
|
|
571
763
|
bytesRef.v += chunk.length;
|
|
@@ -576,6 +768,18 @@ export class ProcessSpawner {
|
|
|
576
768
|
};
|
|
577
769
|
const stderrRef = { v: stderrBytes };
|
|
578
770
|
const stdoutRef = { v: stdoutBytes };
|
|
771
|
+
const LAG_PROBE_INTERVAL_MS = 1_000;
|
|
772
|
+
const LAG_SIGNIFICANT_MS = 2_000;
|
|
773
|
+
let maxLagMs = 0;
|
|
774
|
+
let expectedTickAt = Date.now() + LAG_PROBE_INTERVAL_MS;
|
|
775
|
+
const lagProbe = setInterval(() => {
|
|
776
|
+
const now = Date.now();
|
|
777
|
+
const lag = now - expectedTickAt;
|
|
778
|
+
expectedTickAt = now + LAG_PROBE_INTERVAL_MS;
|
|
779
|
+
if (lag > maxLagMs)
|
|
780
|
+
maxLagMs = lag;
|
|
781
|
+
}, LAG_PROBE_INTERVAL_MS);
|
|
782
|
+
lagProbe.unref?.();
|
|
579
783
|
if (!isWorkerThread) {
|
|
580
784
|
const child = worker;
|
|
581
785
|
child.stderr?.on('data', (chunk) => {
|
|
@@ -596,18 +800,37 @@ export class ProcessSpawner {
|
|
|
596
800
|
};
|
|
597
801
|
};
|
|
598
802
|
const timer = setTimeout(() => {
|
|
803
|
+
clearInterval(lagProbe);
|
|
599
804
|
const pid = !isWorkerThread ? worker.pid : undefined;
|
|
600
805
|
const pidHint = pid ? ` (pid: ${pid})` : '';
|
|
601
806
|
const diag = buildDiagnostics();
|
|
602
807
|
const stderrTail = diag.stderr
|
|
603
808
|
? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-15).join('\n')
|
|
604
809
|
: '';
|
|
605
|
-
const
|
|
606
|
-
|
|
810
|
+
const supervisorNote = maxLagMs >= LAG_SIGNIFICANT_MS
|
|
811
|
+
? `\n--- this timeout is ours, not the child's ---\n` +
|
|
812
|
+
`The supervisor's own 1s timer ran ${Math.round(maxLagMs / 1000)}s late ` +
|
|
813
|
+
`during this window, so it was not running to read the child or to ` +
|
|
814
|
+
`receive its ready message. Check free memory and swap on this host ` +
|
|
815
|
+
`before reading the application.`
|
|
816
|
+
: '';
|
|
817
|
+
const err = Errors.timeout(`Worker startup${pidHint}`, timeout);
|
|
818
|
+
if (supervisorNote) {
|
|
819
|
+
err.message += supervisorNote;
|
|
820
|
+
}
|
|
821
|
+
if (stderrTail) {
|
|
822
|
+
err.message += stderrTail;
|
|
823
|
+
}
|
|
824
|
+
err.details = {
|
|
825
|
+
...(err.details ?? {}),
|
|
826
|
+
...diag,
|
|
827
|
+
supervisorMaxLagMs: maxLagMs,
|
|
828
|
+
};
|
|
607
829
|
reject(err);
|
|
608
830
|
}, timeout);
|
|
609
831
|
const cleanup = () => {
|
|
610
832
|
clearTimeout(timer);
|
|
833
|
+
clearInterval(lagProbe);
|
|
611
834
|
if (isWorkerThread) {
|
|
612
835
|
worker.off('message', messageHandler);
|
|
613
836
|
}
|
|
@@ -623,6 +846,7 @@ export class ProcessSpawner {
|
|
|
623
846
|
resolve({
|
|
624
847
|
serviceName: data.serviceName,
|
|
625
848
|
serviceVersion: data.serviceVersion,
|
|
849
|
+
startupOutput: { stdout: stdoutBuf.join(''), stderr: stderrBuf.join('') },
|
|
626
850
|
});
|
|
627
851
|
}
|
|
628
852
|
else if (data.type === 'error') {
|
|
@@ -636,14 +860,17 @@ export class ProcessSpawner {
|
|
|
636
860
|
}
|
|
637
861
|
}
|
|
638
862
|
};
|
|
639
|
-
const exitHandler = (code) => {
|
|
863
|
+
const exitHandler = (code, signal) => {
|
|
640
864
|
cleanup();
|
|
641
865
|
const diag = buildDiagnostics();
|
|
642
866
|
const stderrTail = diag.stderr
|
|
643
867
|
? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-20).join('\n')
|
|
644
868
|
: '';
|
|
645
|
-
const
|
|
646
|
-
|
|
869
|
+
const how = signal
|
|
870
|
+
? `after being killed by ${signal} — it did not choose to exit, so its own logs will not explain this`
|
|
871
|
+
: `with code ${code}`;
|
|
872
|
+
const err = new Error(`Worker exited during startup ${how}${stderrTail}`);
|
|
873
|
+
err.details = { ...diag, exitCode: code, exitSignal: signal };
|
|
647
874
|
reject(err);
|
|
648
875
|
};
|
|
649
876
|
if (isWorkerThread) {
|