@omnitron-dev/titan-pm 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backoff.d.ts +12 -0
- package/dist/backoff.d.ts.map +1 -0
- package/dist/backoff.js +44 -0
- package/dist/backoff.js.map +1 -0
- package/dist/child-contract.d.ts +58 -0
- package/dist/child-contract.d.ts.map +1 -0
- package/dist/child-contract.js +27 -0
- package/dist/child-contract.js.map +1 -0
- package/dist/confirm-death.d.ts +18 -0
- package/dist/confirm-death.d.ts.map +1 -0
- package/dist/confirm-death.js +49 -0
- package/dist/confirm-death.js.map +1 -0
- package/dist/decorators.d.ts +1 -10
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +43 -99
- package/dist/decorators.js.map +1 -1
- package/dist/discovery.d.ts +24 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +162 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fork-worker.js +64 -27
- package/dist/fork-worker.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/last-resort-handlers.d.ts +5 -0
- package/dist/last-resort-handlers.d.ts.map +1 -0
- package/dist/last-resort-handlers.js +40 -0
- package/dist/last-resort-handlers.js.map +1 -0
- package/dist/lifecycle-queue.d.ts +20 -0
- package/dist/lifecycle-queue.d.ts.map +1 -0
- package/dist/lifecycle-queue.js +110 -0
- package/dist/lifecycle-queue.js.map +1 -0
- package/dist/liveness.d.ts +5 -0
- package/dist/liveness.d.ts.map +1 -0
- package/dist/liveness.js +34 -0
- package/dist/liveness.js.map +1 -0
- package/dist/netron-client.d.ts +10 -9
- package/dist/netron-client.d.ts.map +1 -1
- package/dist/netron-client.js +32 -21
- package/dist/netron-client.js.map +1 -1
- package/dist/parent-death-watchdog.d.ts +13 -0
- package/dist/parent-death-watchdog.d.ts.map +1 -0
- package/dist/parent-death-watchdog.js +58 -0
- package/dist/parent-death-watchdog.js.map +1 -0
- package/dist/pm.module.d.ts.map +1 -1
- package/dist/pm.module.js.map +1 -1
- package/dist/process-health.d.ts +2 -0
- package/dist/process-health.d.ts.map +1 -1
- package/dist/process-health.js +49 -13
- package/dist/process-health.js.map +1 -1
- package/dist/process-manager.d.ts +6 -0
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +111 -4
- package/dist/process-manager.js.map +1 -1
- package/dist/process-metrics.d.ts +1 -0
- package/dist/process-metrics.d.ts.map +1 -1
- package/dist/process-metrics.js +18 -19
- package/dist/process-metrics.js.map +1 -1
- package/dist/process-pool.d.ts +12 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +340 -147
- package/dist/process-pool.js.map +1 -1
- package/dist/process-registry.d.ts.map +1 -1
- package/dist/process-spawner.d.ts +27 -5
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +397 -70
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +7 -0
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +141 -17
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/process-workflow.js +11 -4
- package/dist/process-workflow.js.map +1 -1
- package/dist/service-proxy.d.ts.map +1 -1
- package/dist/service-proxy.js +33 -3
- package/dist/service-proxy.js.map +1 -1
- package/dist/shutdown-windows.d.ts +19 -0
- package/dist/shutdown-windows.d.ts.map +1 -0
- package/dist/shutdown-windows.js +35 -0
- package/dist/shutdown-windows.js.map +1 -0
- package/dist/thread-pool.d.ts +7 -0
- package/dist/thread-pool.d.ts.map +1 -0
- package/dist/thread-pool.js +11 -0
- package/dist/thread-pool.js.map +1 -0
- package/dist/types.d.ts +91 -37
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +35 -39
- package/dist/types.js.map +1 -1
- package/dist/utils/deque.d.ts.map +1 -1
- package/dist/worker-health.d.ts +8 -0
- package/dist/worker-health.d.ts.map +1 -0
- package/dist/worker-health.js +63 -0
- package/dist/worker-health.js.map +1 -0
- package/dist/worker-runtime.js +25 -52
- package/dist/worker-runtime.js.map +1 -1
- package/dist/worker-traffic.d.ts +8 -0
- package/dist/worker-traffic.d.ts.map +1 -0
- package/dist/worker-traffic.js +56 -0
- package/dist/worker-traffic.js.map +1 -0
- package/package.json +11 -10
package/dist/process-spawner.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
1
2
|
import { Worker } from 'worker_threads';
|
|
2
3
|
import { fork } from 'node:child_process';
|
|
3
4
|
import { promises as fs } from 'node:fs';
|
|
@@ -6,16 +7,79 @@ import path from 'node:path';
|
|
|
6
7
|
import os from 'node:os';
|
|
7
8
|
import { fileURLToPath } from 'node:url';
|
|
8
9
|
import { createInterface } from 'node:readline';
|
|
10
|
+
import net from 'node:net';
|
|
9
11
|
import { Errors } from '@omnitron-dev/titan/errors';
|
|
12
|
+
import { isAlive as isPidAlive } from './liveness.js';
|
|
10
13
|
import { ProcessStatus } from './types.js';
|
|
11
14
|
import { NetronClient } from './netron-client.js';
|
|
12
15
|
import { ServiceProxyHandler } from './service-proxy.js';
|
|
13
16
|
import { getAvailablePort } from '@omnitron-dev/titan/utils';
|
|
14
17
|
import { generateUuidV7 } from '@omnitron-dev/titan/utils';
|
|
18
|
+
import { shutdownLadder, childDrainMs, DEFAULT_SHUTDOWN_BUDGET_MS } from './shutdown-windows.js';
|
|
19
|
+
import { confirmDeath } from './confirm-death.js';
|
|
20
|
+
import { threadPoolSizeFor } from './thread-pool.js';
|
|
15
21
|
let MockProcessSpawnerClass = null;
|
|
16
22
|
const SPAWNER_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
17
|
-
|
|
18
|
-
|
|
23
|
+
async function isSocketRefused(socketPath, timeoutMs = 250) {
|
|
24
|
+
return new Promise((resolve) => {
|
|
25
|
+
let settled = false;
|
|
26
|
+
const done = (refused) => {
|
|
27
|
+
if (settled)
|
|
28
|
+
return;
|
|
29
|
+
settled = true;
|
|
30
|
+
clearTimeout(timer);
|
|
31
|
+
socket.destroy();
|
|
32
|
+
resolve(refused);
|
|
33
|
+
};
|
|
34
|
+
const socket = net.connect({ path: socketPath });
|
|
35
|
+
const timer = setTimeout(() => done(false), timeoutMs);
|
|
36
|
+
socket.once('connect', () => done(false));
|
|
37
|
+
socket.once('error', (error) => done(error.code === 'ECONNREFUSED'));
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const STALE_SOCKET_MIN_AGE_MS = 10 * 60 * 1000;
|
|
41
|
+
const EARLY_LOG_MAX_BYTES = 1024 * 1024;
|
|
42
|
+
export function hasExited(child) {
|
|
43
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
44
|
+
return true;
|
|
45
|
+
return child.pid == null ? true : !isPidAlive(child.pid);
|
|
46
|
+
}
|
|
47
|
+
export async function killChildAndVerify(child, logger, context, termGraceMs = 3000) {
|
|
48
|
+
if (hasExited(child))
|
|
49
|
+
return true;
|
|
50
|
+
const exited = new Promise((resolve) => {
|
|
51
|
+
const done = () => resolve(true);
|
|
52
|
+
child.once('exit', done);
|
|
53
|
+
setTimeout(() => {
|
|
54
|
+
child.off('exit', done);
|
|
55
|
+
resolve(false);
|
|
56
|
+
}, termGraceMs).unref();
|
|
57
|
+
});
|
|
58
|
+
try {
|
|
59
|
+
child.kill('SIGTERM');
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
if (await exited)
|
|
65
|
+
return true;
|
|
66
|
+
if (hasExited(child))
|
|
67
|
+
return true;
|
|
68
|
+
logger.warn({ ...context, pid: child.pid }, 'Child ignored SIGTERM during cleanup — escalating to SIGKILL');
|
|
69
|
+
try {
|
|
70
|
+
child.kill('SIGKILL');
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
await new Promise((r) => {
|
|
76
|
+
setTimeout(r, 200).unref();
|
|
77
|
+
});
|
|
78
|
+
return hasExited(child);
|
|
79
|
+
}
|
|
80
|
+
export class WorkerHandle extends EventEmitter {
|
|
81
|
+
constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true) {
|
|
82
|
+
super();
|
|
19
83
|
this.id = id;
|
|
20
84
|
this.transportUrl = transportUrl;
|
|
21
85
|
this.serviceName = serviceName;
|
|
@@ -26,11 +90,43 @@ export class WorkerHandle {
|
|
|
26
90
|
this.transportConfig = transportConfig;
|
|
27
91
|
this.messageHandlers = new Map();
|
|
28
92
|
this.logHandlers = new Set();
|
|
93
|
+
this.earlyLogs = [];
|
|
94
|
+
this.earlyLogBytes = 0;
|
|
95
|
+
this.earlyLogsDropped = 0;
|
|
96
|
+
this.exitEmitted = false;
|
|
29
97
|
this._status = initialStatus;
|
|
30
98
|
this.worker = worker;
|
|
31
99
|
this.netronClient = netronClient;
|
|
32
100
|
this.setupMessageHandlers();
|
|
33
|
-
|
|
101
|
+
if (forwardChildLogs)
|
|
102
|
+
this.setupDefaultLogForwarding();
|
|
103
|
+
}
|
|
104
|
+
emitExit(code, signal) {
|
|
105
|
+
if (this.exitEmitted)
|
|
106
|
+
return;
|
|
107
|
+
this.exitEmitted = true;
|
|
108
|
+
const expected = this._status === ProcessStatus.STOPPING || this._status === ProcessStatus.STOPPED;
|
|
109
|
+
this._status = ProcessStatus.STOPPED;
|
|
110
|
+
const forced = expected && signal === 'SIGKILL';
|
|
111
|
+
const info = {
|
|
112
|
+
workerId: this.id,
|
|
113
|
+
serviceName: this.serviceName,
|
|
114
|
+
code,
|
|
115
|
+
signal,
|
|
116
|
+
expected,
|
|
117
|
+
forced,
|
|
118
|
+
};
|
|
119
|
+
this.logger[expected && !forced ? 'info' : 'warn'](info, !expected
|
|
120
|
+
? 'Worker exited unexpectedly'
|
|
121
|
+
: forced
|
|
122
|
+
? 'Worker had to be killed — it did not exit within its shutdown window'
|
|
123
|
+
: 'Worker exited cleanly');
|
|
124
|
+
try {
|
|
125
|
+
this.emit('exit', info);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
this.logger.error({ err, workerId: this.id }, 'exit listener threw');
|
|
129
|
+
}
|
|
34
130
|
}
|
|
35
131
|
get status() {
|
|
36
132
|
return this._status;
|
|
@@ -40,9 +136,15 @@ export class WorkerHandle {
|
|
|
40
136
|
return undefined;
|
|
41
137
|
return this.worker.pid;
|
|
42
138
|
}
|
|
43
|
-
|
|
44
|
-
|
|
139
|
+
onExit(handler) {
|
|
140
|
+
this.on('exit', handler);
|
|
141
|
+
return () => this.off('exit', handler);
|
|
142
|
+
}
|
|
143
|
+
async terminate(opts) {
|
|
144
|
+
if (this._status !== ProcessStatus.STOPPED) {
|
|
45
145
|
this._status = ProcessStatus.STOPPING;
|
|
146
|
+
}
|
|
147
|
+
try {
|
|
46
148
|
if (this.netronClient?.isConnected()) {
|
|
47
149
|
await this.netronClient.disconnect();
|
|
48
150
|
}
|
|
@@ -52,7 +154,7 @@ export class WorkerHandle {
|
|
|
52
154
|
}
|
|
53
155
|
else {
|
|
54
156
|
const child = this.worker;
|
|
55
|
-
await this.terminateChildProcess(child);
|
|
157
|
+
await this.terminateChildProcess(child, opts?.shutdownTimeout);
|
|
56
158
|
}
|
|
57
159
|
if (this.transportConfig?.type === 'unix' && this.transportConfig.path) {
|
|
58
160
|
try {
|
|
@@ -70,16 +172,32 @@ export class WorkerHandle {
|
|
|
70
172
|
throw error;
|
|
71
173
|
}
|
|
72
174
|
}
|
|
73
|
-
async terminateChildProcess(child) {
|
|
74
|
-
const
|
|
75
|
-
const
|
|
76
|
-
const
|
|
175
|
+
async terminateChildProcess(child, totalDeadlineMs) {
|
|
176
|
+
const ladder = shutdownLadder(totalDeadlineMs);
|
|
177
|
+
const GRACEFUL_TIMEOUT = ladder.gracefulMs;
|
|
178
|
+
const SIGTERM_TIMEOUT = ladder.sigtermMs;
|
|
179
|
+
const SIGKILL_TIMEOUT = ladder.sigkillMs;
|
|
77
180
|
return new Promise((resolve, reject) => {
|
|
78
181
|
let resolved = false;
|
|
182
|
+
let termTimer = null;
|
|
183
|
+
let killTimer = null;
|
|
184
|
+
let finalTimer = null;
|
|
79
185
|
const cleanup = () => {
|
|
80
186
|
resolved = true;
|
|
81
187
|
child.off('exit', exitHandler);
|
|
82
188
|
child.off('error', errorHandler);
|
|
189
|
+
if (termTimer) {
|
|
190
|
+
clearTimeout(termTimer);
|
|
191
|
+
termTimer = null;
|
|
192
|
+
}
|
|
193
|
+
if (killTimer) {
|
|
194
|
+
clearTimeout(killTimer);
|
|
195
|
+
killTimer = null;
|
|
196
|
+
}
|
|
197
|
+
if (finalTimer) {
|
|
198
|
+
clearTimeout(finalTimer);
|
|
199
|
+
finalTimer = null;
|
|
200
|
+
}
|
|
83
201
|
};
|
|
84
202
|
const exitHandler = () => {
|
|
85
203
|
if (!resolved) {
|
|
@@ -98,7 +216,7 @@ export class WorkerHandle {
|
|
|
98
216
|
};
|
|
99
217
|
child.once('exit', exitHandler);
|
|
100
218
|
child.once('error', errorHandler);
|
|
101
|
-
if (child
|
|
219
|
+
if (hasExited(child)) {
|
|
102
220
|
cleanup();
|
|
103
221
|
resolve();
|
|
104
222
|
return;
|
|
@@ -111,8 +229,8 @@ export class WorkerHandle {
|
|
|
111
229
|
catch (_e) {
|
|
112
230
|
this.logger.debug({ workerId: this.id }, 'IPC channel closed, proceeding with SIGTERM');
|
|
113
231
|
}
|
|
114
|
-
setTimeout(() => {
|
|
115
|
-
if (resolved || child
|
|
232
|
+
termTimer = setTimeout(() => {
|
|
233
|
+
if (resolved || hasExited(child))
|
|
116
234
|
return;
|
|
117
235
|
this.logger.debug({ workerId: this.id }, 'Graceful shutdown timeout, sending SIGTERM');
|
|
118
236
|
try {
|
|
@@ -121,8 +239,8 @@ export class WorkerHandle {
|
|
|
121
239
|
catch (_e) {
|
|
122
240
|
}
|
|
123
241
|
}, GRACEFUL_TIMEOUT);
|
|
124
|
-
setTimeout(() => {
|
|
125
|
-
if (resolved || child
|
|
242
|
+
killTimer = setTimeout(() => {
|
|
243
|
+
if (resolved || hasExited(child))
|
|
126
244
|
return;
|
|
127
245
|
this.logger.warn({ workerId: this.id }, 'SIGTERM timeout, sending SIGKILL');
|
|
128
246
|
try {
|
|
@@ -131,12 +249,28 @@ export class WorkerHandle {
|
|
|
131
249
|
catch (_e) {
|
|
132
250
|
}
|
|
133
251
|
}, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT);
|
|
134
|
-
setTimeout(() => {
|
|
135
|
-
if (
|
|
136
|
-
|
|
137
|
-
|
|
252
|
+
finalTimer = setTimeout(() => {
|
|
253
|
+
if (resolved)
|
|
254
|
+
return;
|
|
255
|
+
cleanup();
|
|
256
|
+
if (child.pid == null) {
|
|
138
257
|
resolve();
|
|
258
|
+
return;
|
|
139
259
|
}
|
|
260
|
+
void confirmDeath(child).then((verdict) => {
|
|
261
|
+
if (verdict.dead) {
|
|
262
|
+
resolve();
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process survived SIGKILL — escalating one more SIGKILL and logging the leak');
|
|
266
|
+
try {
|
|
267
|
+
if (child.pid != null)
|
|
268
|
+
process.kill(child.pid, 'SIGKILL');
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
}
|
|
272
|
+
resolve();
|
|
273
|
+
});
|
|
140
274
|
}, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT + SIGKILL_TIMEOUT);
|
|
141
275
|
});
|
|
142
276
|
}
|
|
@@ -147,29 +281,70 @@ export class WorkerHandle {
|
|
|
147
281
|
}
|
|
148
282
|
else {
|
|
149
283
|
const child = this.worker;
|
|
150
|
-
return !child
|
|
284
|
+
return !hasExited(child);
|
|
151
285
|
}
|
|
152
286
|
}
|
|
153
287
|
async send(message) {
|
|
154
288
|
if (this.isWorkerThread) {
|
|
155
289
|
const worker = this.worker;
|
|
290
|
+
if (this._status !== ProcessStatus.RUNNING && this._status !== ProcessStatus.STARTING) {
|
|
291
|
+
throw Errors.unavailable(`Worker thread ${this.id} is ${this._status}; cannot post message`);
|
|
292
|
+
}
|
|
156
293
|
worker.postMessage(message);
|
|
157
294
|
}
|
|
158
295
|
else {
|
|
159
296
|
const child = this.worker;
|
|
160
|
-
if (child.send) {
|
|
161
|
-
child.send
|
|
297
|
+
if (!child.send || !child.connected) {
|
|
298
|
+
throw Errors.unavailable(`IPC channel to child ${this.id} is closed; cannot send`);
|
|
162
299
|
}
|
|
300
|
+
child.send(message);
|
|
163
301
|
}
|
|
164
302
|
}
|
|
165
303
|
onMessage(handler) {
|
|
166
304
|
const id = generateUuidV7();
|
|
167
305
|
this.messageHandlers.set(id, handler);
|
|
168
306
|
}
|
|
307
|
+
seedStartupOutput(output) {
|
|
308
|
+
if (!output)
|
|
309
|
+
return;
|
|
310
|
+
for (const [text, stream] of [
|
|
311
|
+
[output.stdout, 'stdout'],
|
|
312
|
+
[output.stderr, 'stderr'],
|
|
313
|
+
]) {
|
|
314
|
+
for (const line of text.split('\n')) {
|
|
315
|
+
if (line.trim())
|
|
316
|
+
this.emitLog(line, stream);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
169
320
|
onLog(handler) {
|
|
321
|
+
const first = this.logHandlers.size === 0;
|
|
170
322
|
this.logHandlers.add(handler);
|
|
323
|
+
if (!first)
|
|
324
|
+
return;
|
|
325
|
+
const buffered = this.earlyLogs;
|
|
326
|
+
this.earlyLogs = null;
|
|
327
|
+
if (!buffered)
|
|
328
|
+
return;
|
|
329
|
+
if (this.earlyLogsDropped > 0) {
|
|
330
|
+
handler(`[titan-pm] ${this.earlyLogsDropped} earlier line(s) from this process were dropped: ` +
|
|
331
|
+
'it printed more than the pre-capture buffer holds', 'stderr');
|
|
332
|
+
}
|
|
333
|
+
for (const entry of buffered) {
|
|
334
|
+
handler(entry.line, entry.stream);
|
|
335
|
+
}
|
|
171
336
|
}
|
|
172
337
|
emitLog(line, stream) {
|
|
338
|
+
if (this.logHandlers.size === 0 && this.earlyLogs) {
|
|
339
|
+
this.earlyLogs.push({ line, stream });
|
|
340
|
+
this.earlyLogBytes += line.length;
|
|
341
|
+
while (this.earlyLogBytes > EARLY_LOG_MAX_BYTES && this.earlyLogs.length > 1) {
|
|
342
|
+
const dropped = this.earlyLogs.shift();
|
|
343
|
+
this.earlyLogBytes -= dropped.line.length;
|
|
344
|
+
this.earlyLogsDropped++;
|
|
345
|
+
}
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
173
348
|
for (const handler of this.logHandlers) {
|
|
174
349
|
handler(line, stream);
|
|
175
350
|
}
|
|
@@ -185,11 +360,12 @@ export class WorkerHandle {
|
|
|
185
360
|
});
|
|
186
361
|
worker.on('error', (error) => {
|
|
187
362
|
this.logger.error({ error, workerId: this.id }, 'Worker error');
|
|
188
|
-
this._status
|
|
363
|
+
if (this._status !== ProcessStatus.STOPPING && this._status !== ProcessStatus.STOPPED) {
|
|
364
|
+
this._status = ProcessStatus.FAILED;
|
|
365
|
+
}
|
|
189
366
|
});
|
|
190
367
|
worker.on('exit', (code) => {
|
|
191
|
-
this.
|
|
192
|
-
this._status = ProcessStatus.STOPPED;
|
|
368
|
+
this.emitExit(code, null);
|
|
193
369
|
});
|
|
194
370
|
}
|
|
195
371
|
else {
|
|
@@ -202,11 +378,12 @@ export class WorkerHandle {
|
|
|
202
378
|
});
|
|
203
379
|
child.on('error', (error) => {
|
|
204
380
|
this.logger.error({ error, workerId: this.id }, 'Child process error');
|
|
205
|
-
this._status
|
|
381
|
+
if (this._status !== ProcessStatus.STOPPING && this._status !== ProcessStatus.STOPPED) {
|
|
382
|
+
this._status = ProcessStatus.FAILED;
|
|
383
|
+
}
|
|
206
384
|
});
|
|
207
385
|
child.on('exit', (code, signal) => {
|
|
208
|
-
this.
|
|
209
|
-
this._status = ProcessStatus.STOPPED;
|
|
386
|
+
this.emitExit(code, signal);
|
|
210
387
|
});
|
|
211
388
|
this.setupLogCapture(child);
|
|
212
389
|
}
|
|
@@ -285,13 +462,27 @@ function sanitizeEnv(env) {
|
|
|
285
462
|
}
|
|
286
463
|
return result;
|
|
287
464
|
}
|
|
465
|
+
function childEnv(extra = {}) {
|
|
466
|
+
const base = sanitizeEnv(process.env);
|
|
467
|
+
return {
|
|
468
|
+
...base,
|
|
469
|
+
UV_THREADPOOL_SIZE: threadPoolSizeFor(base, { parallelism: os.availableParallelism() }),
|
|
470
|
+
...extra,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
288
473
|
export class ProcessSpawner {
|
|
289
474
|
constructor(logger, config = {}) {
|
|
290
475
|
this.logger = logger;
|
|
291
476
|
this.config = config;
|
|
477
|
+
this.ownSockets = new Set();
|
|
478
|
+
this.forkedPids = new Set();
|
|
479
|
+
this.staleSweep = null;
|
|
292
480
|
let currentDir = SPAWNER_DIR;
|
|
293
|
-
|
|
294
|
-
|
|
481
|
+
const segments = currentDir.split(path.sep);
|
|
482
|
+
const srcIndex = segments.lastIndexOf('src');
|
|
483
|
+
if (srcIndex !== -1) {
|
|
484
|
+
segments[srcIndex] = 'dist';
|
|
485
|
+
currentDir = segments.join(path.sep);
|
|
295
486
|
}
|
|
296
487
|
this.workerRuntimePath = path.join(currentDir, 'worker-runtime.js');
|
|
297
488
|
this.forkWorkerPath = path.join(currentDir, 'fork-worker.js');
|
|
@@ -325,12 +516,22 @@ export class ProcessSpawner {
|
|
|
325
516
|
options: options,
|
|
326
517
|
dependencies: options.dependencies,
|
|
327
518
|
};
|
|
328
|
-
const
|
|
329
|
-
const
|
|
519
|
+
const declaredIsolation = options.isolation;
|
|
520
|
+
const legacySandbox = declaredIsolation === 'vm' || declaredIsolation === 'container' ? declaredIsolation : undefined;
|
|
521
|
+
const strategy = (options.isolation === 'none' || options.isolation === 'worker' || options.isolation === 'child'
|
|
522
|
+
? options.isolation
|
|
523
|
+
: undefined) ??
|
|
524
|
+
this.config.isolation ??
|
|
525
|
+
'worker';
|
|
526
|
+
const sandbox = options.sandbox ?? legacySandbox;
|
|
527
|
+
const sandboxUnimplemented = sandbox === 'vm' || sandbox === 'container';
|
|
528
|
+
const useWorkerThreads = strategy === 'worker' && !sandboxUnimplemented && (!options.execArgv || options.execArgv.length === 0);
|
|
330
529
|
let worker;
|
|
331
530
|
let netronClient = null;
|
|
332
531
|
try {
|
|
333
|
-
if (
|
|
532
|
+
if (sandboxUnimplemented) {
|
|
533
|
+
this.logger.warn({ processId, isolation: sandbox, actual: 'child' }, `Process isolation '${sandbox}' is not implemented — spawning an ordinary child process. ` +
|
|
534
|
+
'It has the same filesystem and network access as its parent; do not rely on it as a security boundary.');
|
|
334
535
|
worker = await this.spawnChildProcess(context, options.execArgv);
|
|
335
536
|
}
|
|
336
537
|
else if (useWorkerThreads) {
|
|
@@ -343,8 +544,10 @@ export class ProcessSpawner {
|
|
|
343
544
|
const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout);
|
|
344
545
|
const actualServiceName = readyInfo.serviceName || serviceName;
|
|
345
546
|
const actualServiceVersion = readyInfo.serviceVersion || serviceVersion;
|
|
346
|
-
if (
|
|
347
|
-
netronClient = new NetronClient(processId, this.logger
|
|
547
|
+
if (strategy !== 'none') {
|
|
548
|
+
netronClient = new NetronClient(processId, this.logger, {
|
|
549
|
+
...(options.requestTimeout !== undefined && { requestTimeout: options.requestTimeout }),
|
|
550
|
+
});
|
|
348
551
|
await netronClient.start();
|
|
349
552
|
await netronClient.connect(transport.url);
|
|
350
553
|
const qualifiedServiceName = actualServiceVersion
|
|
@@ -352,10 +555,14 @@ export class ProcessSpawner {
|
|
|
352
555
|
: actualServiceName;
|
|
353
556
|
const proxyHandler = new ServiceProxyHandler(processId, netronClient, qualifiedServiceName, this.logger);
|
|
354
557
|
const proxy = proxyHandler.createProxy();
|
|
355
|
-
|
|
558
|
+
const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
559
|
+
handle.seedStartupOutput(readyInfo.startupOutput);
|
|
560
|
+
return handle;
|
|
356
561
|
}
|
|
357
562
|
else {
|
|
358
|
-
|
|
563
|
+
const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
564
|
+
handle.seedStartupOutput(readyInfo.startupOutput);
|
|
565
|
+
return handle;
|
|
359
566
|
}
|
|
360
567
|
}
|
|
361
568
|
catch (error) {
|
|
@@ -367,13 +574,13 @@ export class ProcessSpawner {
|
|
|
367
574
|
}
|
|
368
575
|
else {
|
|
369
576
|
const child = worker;
|
|
370
|
-
if (child.pid
|
|
371
|
-
child.
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
577
|
+
if (child.pid) {
|
|
578
|
+
const gone = await killChildAndVerify(child, this.logger, { serviceName, processId });
|
|
579
|
+
if (!gone) {
|
|
580
|
+
this.logger.error({ serviceName, processId, pid: child.pid }, 'Child survived SIGKILL after a failed spawn — it still holds whatever it held ' +
|
|
581
|
+
'(ports, locks). It is parented to this process, so the orphan sweep will not ' +
|
|
582
|
+
'see it: it needs the abandoned-child sweep.');
|
|
583
|
+
}
|
|
377
584
|
}
|
|
378
585
|
}
|
|
379
586
|
}
|
|
@@ -387,25 +594,55 @@ export class ProcessSpawner {
|
|
|
387
594
|
}
|
|
388
595
|
throw error;
|
|
389
596
|
}
|
|
597
|
+
finally {
|
|
598
|
+
this.releaseForkClaim(worker?.pid);
|
|
599
|
+
}
|
|
390
600
|
}
|
|
391
601
|
async cleanup() {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
await fs.unlink(path.join(this.tempDir, entry));
|
|
398
|
-
}
|
|
399
|
-
catch { }
|
|
400
|
-
}
|
|
602
|
+
for (const socketPath of this.ownSockets) {
|
|
603
|
+
try {
|
|
604
|
+
await fs.unlink(socketPath);
|
|
605
|
+
}
|
|
606
|
+
catch {
|
|
401
607
|
}
|
|
402
608
|
}
|
|
609
|
+
this.ownSockets.clear();
|
|
610
|
+
try {
|
|
611
|
+
await fs.rmdir(this.tempDir);
|
|
612
|
+
}
|
|
403
613
|
catch {
|
|
404
614
|
}
|
|
615
|
+
}
|
|
616
|
+
async sweepStaleSockets() {
|
|
617
|
+
let entries;
|
|
405
618
|
try {
|
|
406
|
-
await fs.
|
|
619
|
+
entries = await fs.readdir(this.tempDir);
|
|
407
620
|
}
|
|
408
621
|
catch {
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
const cutoff = Date.now() - STALE_SOCKET_MIN_AGE_MS;
|
|
625
|
+
for (const entry of entries) {
|
|
626
|
+
if (!entry.endsWith('.sock'))
|
|
627
|
+
continue;
|
|
628
|
+
const socketPath = path.join(this.tempDir, entry);
|
|
629
|
+
if (this.ownSockets.has(socketPath))
|
|
630
|
+
continue;
|
|
631
|
+
try {
|
|
632
|
+
const stat = await fs.stat(socketPath);
|
|
633
|
+
if (stat.mtimeMs > cutoff)
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
catch {
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
639
|
+
if (!(await isSocketRefused(socketPath)))
|
|
640
|
+
continue;
|
|
641
|
+
try {
|
|
642
|
+
await fs.unlink(socketPath);
|
|
643
|
+
}
|
|
644
|
+
catch {
|
|
645
|
+
}
|
|
409
646
|
}
|
|
410
647
|
}
|
|
411
648
|
async setupTransport(processId, options) {
|
|
@@ -425,8 +662,12 @@ export class ProcessSpawner {
|
|
|
425
662
|
}
|
|
426
663
|
case 'unix':
|
|
427
664
|
default: {
|
|
428
|
-
await fs.mkdir(this.tempDir, { recursive: true });
|
|
665
|
+
await fs.mkdir(this.tempDir, { recursive: true, mode: 0o700 });
|
|
666
|
+
await fs.chmod(this.tempDir, 0o700).catch(() => {
|
|
667
|
+
});
|
|
429
668
|
const socketPath = path.join(this.tempDir, `${processId}.sock`);
|
|
669
|
+
this.ownSockets.add(socketPath);
|
|
670
|
+
this.staleSweep ??= this.sweepStaleSockets().catch(() => undefined);
|
|
430
671
|
return {
|
|
431
672
|
type: 'unix',
|
|
432
673
|
path: socketPath,
|
|
@@ -458,7 +699,7 @@ export class ProcessSpawner {
|
|
|
458
699
|
const serializableContext = this.makeSerializable(context);
|
|
459
700
|
const worker = new Worker(this.workerRuntimePath, {
|
|
460
701
|
workerData: serializableContext,
|
|
461
|
-
env:
|
|
702
|
+
env: childEnv(),
|
|
462
703
|
});
|
|
463
704
|
return worker;
|
|
464
705
|
}
|
|
@@ -484,39 +725,112 @@ export class ProcessSpawner {
|
|
|
484
725
|
if (contextJson.length > 65536) {
|
|
485
726
|
throw new Error(`TITAN_WORKER_CONTEXT exceeds 64KB (${contextJson.length} bytes). Reduce dependencies.`);
|
|
486
727
|
}
|
|
728
|
+
const windowMs = shutdownLadder(context.options?.shutdownTimeout ?? DEFAULT_SHUTDOWN_BUDGET_MS).childWindowMs;
|
|
487
729
|
const child = fork(this.forkWorkerPath, [], {
|
|
488
730
|
env: {
|
|
489
|
-
...
|
|
490
|
-
...(context.options?.env ?? {}),
|
|
731
|
+
...childEnv(context.options?.env ?? {}),
|
|
491
732
|
TITAN_WORKER_CONTEXT: contextJson,
|
|
733
|
+
TITAN_SHUTDOWN_TIMEOUT_MS: String(windowMs),
|
|
734
|
+
TITAN_DRAIN_TIMEOUT_MS: context.options?.env?.['TITAN_DRAIN_TIMEOUT_MS'] || String(childDrainMs(windowMs)),
|
|
492
735
|
},
|
|
493
736
|
execArgv: execArgv || [],
|
|
494
737
|
silent: true,
|
|
495
738
|
...(context.options?.cwd ? { cwd: context.options.cwd } : {}),
|
|
496
739
|
});
|
|
740
|
+
if (typeof child.pid === 'number') {
|
|
741
|
+
const pid = child.pid;
|
|
742
|
+
this.forkedPids.add(pid);
|
|
743
|
+
child.once('exit', () => this.forkedPids.delete(pid));
|
|
744
|
+
}
|
|
497
745
|
return child;
|
|
498
746
|
}
|
|
747
|
+
getForkedPids() {
|
|
748
|
+
return this.forkedPids;
|
|
749
|
+
}
|
|
750
|
+
releaseForkClaim(pid) {
|
|
751
|
+
if (typeof pid === 'number')
|
|
752
|
+
this.forkedPids.delete(pid);
|
|
753
|
+
}
|
|
499
754
|
async waitForReady(worker, isWorkerThread, timeout = 5000) {
|
|
500
755
|
return new Promise((resolve, reject) => {
|
|
501
|
-
const
|
|
756
|
+
const STDERR_CAP_BYTES = 64 * 1024;
|
|
757
|
+
const stderrBuf = [];
|
|
758
|
+
const stdoutBuf = [];
|
|
759
|
+
const stderrBytes = 0;
|
|
760
|
+
const stdoutBytes = 0;
|
|
761
|
+
const appendBounded = (target, bytesRef, chunk) => {
|
|
762
|
+
target.push(chunk);
|
|
763
|
+
bytesRef.v += chunk.length;
|
|
764
|
+
while (bytesRef.v > STDERR_CAP_BYTES && target.length > 1) {
|
|
765
|
+
const dropped = target.shift();
|
|
766
|
+
bytesRef.v -= dropped.length;
|
|
767
|
+
}
|
|
768
|
+
};
|
|
769
|
+
const stderrRef = { v: stderrBytes };
|
|
770
|
+
const stdoutRef = { v: stdoutBytes };
|
|
771
|
+
const LAG_PROBE_INTERVAL_MS = 1_000;
|
|
772
|
+
const LAG_SIGNIFICANT_MS = 2_000;
|
|
773
|
+
let maxLagMs = 0;
|
|
774
|
+
let expectedTickAt = Date.now() + LAG_PROBE_INTERVAL_MS;
|
|
775
|
+
const lagProbe = setInterval(() => {
|
|
776
|
+
const now = Date.now();
|
|
777
|
+
const lag = now - expectedTickAt;
|
|
778
|
+
expectedTickAt = now + LAG_PROBE_INTERVAL_MS;
|
|
779
|
+
if (lag > maxLagMs)
|
|
780
|
+
maxLagMs = lag;
|
|
781
|
+
}, LAG_PROBE_INTERVAL_MS);
|
|
782
|
+
lagProbe.unref?.();
|
|
502
783
|
if (!isWorkerThread) {
|
|
503
784
|
const child = worker;
|
|
504
785
|
child.stderr?.on('data', (chunk) => {
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
stderrChunks.shift();
|
|
786
|
+
appendBounded(stderrBuf, stderrRef, chunk.toString());
|
|
787
|
+
});
|
|
788
|
+
child.stdout?.on('data', (chunk) => {
|
|
789
|
+
appendBounded(stdoutBuf, stdoutRef, chunk.toString());
|
|
510
790
|
});
|
|
511
791
|
}
|
|
792
|
+
const buildDiagnostics = () => {
|
|
793
|
+
const stderr = stderrBuf.join('').trimEnd();
|
|
794
|
+
const stdout = stdoutBuf.join('').trimEnd();
|
|
795
|
+
return {
|
|
796
|
+
stderr: stderr || undefined,
|
|
797
|
+
stdout: stdout || undefined,
|
|
798
|
+
stderrBytes: stderrRef.v,
|
|
799
|
+
stdoutBytes: stdoutRef.v,
|
|
800
|
+
};
|
|
801
|
+
};
|
|
512
802
|
const timer = setTimeout(() => {
|
|
803
|
+
clearInterval(lagProbe);
|
|
513
804
|
const pid = !isWorkerThread ? worker.pid : undefined;
|
|
514
805
|
const pidHint = pid ? ` (pid: ${pid})` : '';
|
|
515
|
-
const
|
|
516
|
-
|
|
806
|
+
const diag = buildDiagnostics();
|
|
807
|
+
const stderrTail = diag.stderr
|
|
808
|
+
? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-15).join('\n')
|
|
809
|
+
: '';
|
|
810
|
+
const supervisorNote = maxLagMs >= LAG_SIGNIFICANT_MS
|
|
811
|
+
? `\n--- this timeout is ours, not the child's ---\n` +
|
|
812
|
+
`The supervisor's own 1s timer ran ${Math.round(maxLagMs / 1000)}s late ` +
|
|
813
|
+
`during this window, so it was not running to read the child or to ` +
|
|
814
|
+
`receive its ready message. Check free memory and swap on this host ` +
|
|
815
|
+
`before reading the application.`
|
|
816
|
+
: '';
|
|
817
|
+
const err = Errors.timeout(`Worker startup${pidHint}`, timeout);
|
|
818
|
+
if (supervisorNote) {
|
|
819
|
+
err.message += supervisorNote;
|
|
820
|
+
}
|
|
821
|
+
if (stderrTail) {
|
|
822
|
+
err.message += stderrTail;
|
|
823
|
+
}
|
|
824
|
+
err.details = {
|
|
825
|
+
...(err.details ?? {}),
|
|
826
|
+
...diag,
|
|
827
|
+
supervisorMaxLagMs: maxLagMs,
|
|
828
|
+
};
|
|
829
|
+
reject(err);
|
|
517
830
|
}, timeout);
|
|
518
831
|
const cleanup = () => {
|
|
519
832
|
clearTimeout(timer);
|
|
833
|
+
clearInterval(lagProbe);
|
|
520
834
|
if (isWorkerThread) {
|
|
521
835
|
worker.off('message', messageHandler);
|
|
522
836
|
}
|
|
@@ -532,19 +846,32 @@ export class ProcessSpawner {
|
|
|
532
846
|
resolve({
|
|
533
847
|
serviceName: data.serviceName,
|
|
534
848
|
serviceVersion: data.serviceVersion,
|
|
849
|
+
startupOutput: { stdout: stdoutBuf.join(''), stderr: stderrBuf.join('') },
|
|
535
850
|
});
|
|
536
851
|
}
|
|
537
852
|
else if (data.type === 'error') {
|
|
538
853
|
cleanup();
|
|
539
854
|
const errMsg = data.error?.message || data.message || 'Worker initialization failed';
|
|
540
|
-
|
|
855
|
+
const err = new Error(errMsg);
|
|
856
|
+
err.details = buildDiagnostics();
|
|
857
|
+
if (data.error?.stack)
|
|
858
|
+
err.childStack = data.error.stack;
|
|
859
|
+
reject(err);
|
|
541
860
|
}
|
|
542
861
|
}
|
|
543
862
|
};
|
|
544
|
-
const exitHandler = (code) => {
|
|
863
|
+
const exitHandler = (code, signal) => {
|
|
545
864
|
cleanup();
|
|
546
|
-
const
|
|
547
|
-
|
|
865
|
+
const diag = buildDiagnostics();
|
|
866
|
+
const stderrTail = diag.stderr
|
|
867
|
+
? '\n--- last child stderr ---\n' + diag.stderr.split('\n').slice(-20).join('\n')
|
|
868
|
+
: '';
|
|
869
|
+
const how = signal
|
|
870
|
+
? `after being killed by ${signal} — it did not choose to exit, so its own logs will not explain this`
|
|
871
|
+
: `with code ${code}`;
|
|
872
|
+
const err = new Error(`Worker exited during startup ${how}${stderrTail}`);
|
|
873
|
+
err.details = { ...diag, exitCode: code, exitSignal: signal };
|
|
874
|
+
reject(err);
|
|
548
875
|
};
|
|
549
876
|
if (isWorkerThread) {
|
|
550
877
|
worker.on('message', messageHandler);
|