@omnitron-dev/titan-pm 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/controller-handoff.d.ts +61 -0
- package/dist/controller-handoff.d.ts.map +1 -0
- package/dist/controller-handoff.js +163 -0
- package/dist/controller-handoff.js.map +1 -0
- package/dist/decorators.d.ts.map +1 -1
- package/dist/decorators.js +26 -14
- package/dist/decorators.js.map +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/output-capture.d.ts +24 -0
- package/dist/output-capture.d.ts.map +1 -0
- package/dist/output-capture.js +112 -0
- package/dist/output-capture.js.map +1 -0
- package/dist/process-manager.d.ts +8 -1
- package/dist/process-manager.d.ts.map +1 -1
- package/dist/process-manager.js +84 -58
- package/dist/process-manager.js.map +1 -1
- package/dist/process-pool.d.ts +11 -0
- package/dist/process-pool.d.ts.map +1 -1
- package/dist/process-pool.js +126 -44
- package/dist/process-pool.js.map +1 -1
- package/dist/process-spawner.d.ts +11 -4
- package/dist/process-spawner.d.ts.map +1 -1
- package/dist/process-spawner.js +140 -108
- package/dist/process-spawner.js.map +1 -1
- package/dist/process-supervisor.d.ts +12 -1
- package/dist/process-supervisor.d.ts.map +1 -1
- package/dist/process-supervisor.js +122 -61
- package/dist/process-supervisor.js.map +1 -1
- package/dist/process-workflow.d.ts +10 -3
- package/dist/process-workflow.d.ts.map +1 -1
- package/dist/process-workflow.js +72 -100
- package/dist/process-workflow.js.map +1 -1
- package/dist/restart-window.d.ts +9 -0
- package/dist/restart-window.d.ts.map +1 -0
- package/dist/restart-window.js +28 -0
- package/dist/restart-window.js.map +1 -0
- package/dist/types.d.ts +17 -13
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +17 -10
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
2
2
|
import { Worker } from 'worker_threads';
|
|
3
3
|
import { ChildProcess } from 'node:child_process';
|
|
4
|
+
import { type ProcessOutputContext } from './output-capture.js';
|
|
4
5
|
import type { ILogger } from '@omnitron-dev/titan/module/logger';
|
|
5
6
|
import type { IProcessSpawner, ISpawnOptions, IWorkerHandle, IWorkerExitInfo, IProcessManagerConfig } from './types.js';
|
|
6
7
|
import { ProcessStatus } from './types.js';
|
|
@@ -17,7 +18,11 @@ export interface IStartupOutput {
|
|
|
17
18
|
stderr: string;
|
|
18
19
|
}
|
|
19
20
|
export declare function hasExited(child: ChildProcess): boolean;
|
|
20
|
-
export declare function killChildAndVerify(child: ChildProcess, logger: ILogger, context: Record<string, unknown>, termGraceMs?: number
|
|
21
|
+
export declare function killChildAndVerify(child: ChildProcess, logger: ILogger, context: Record<string, unknown>, termGraceMs?: number, options?: {
|
|
22
|
+
force?: boolean;
|
|
23
|
+
processGroup?: boolean;
|
|
24
|
+
killGraceMs?: number;
|
|
25
|
+
}): Promise<boolean>;
|
|
21
26
|
export declare class WorkerHandle extends EventEmitter implements IWorkerHandle {
|
|
22
27
|
readonly id: string;
|
|
23
28
|
readonly transportUrl: string;
|
|
@@ -27,6 +32,7 @@ export declare class WorkerHandle extends EventEmitter implements IWorkerHandle
|
|
|
27
32
|
private readonly isWorkerThread;
|
|
28
33
|
readonly proxy?: any;
|
|
29
34
|
readonly transportConfig?: ITransportConfig | undefined;
|
|
35
|
+
private readonly captureInstalled;
|
|
30
36
|
private _status;
|
|
31
37
|
private messageHandlers;
|
|
32
38
|
private logHandlers;
|
|
@@ -36,7 +42,7 @@ export declare class WorkerHandle extends EventEmitter implements IWorkerHandle
|
|
|
36
42
|
private exitEmitted;
|
|
37
43
|
readonly worker: Worker | ChildProcess;
|
|
38
44
|
readonly netronClient: NetronClient | null;
|
|
39
|
-
constructor(id: string, worker: Worker | ChildProcess, netronClient: NetronClient | null, transportUrl: string, serviceName: string, serviceVersion: string, logger: ILogger, isWorkerThread: boolean, proxy?: any, initialStatus?: ProcessStatus, transportConfig?: ITransportConfig | undefined, forwardChildLogs?: boolean);
|
|
45
|
+
constructor(id: string, worker: Worker | ChildProcess, netronClient: NetronClient | null, transportUrl: string, serviceName: string, serviceVersion: string, logger: ILogger, isWorkerThread: boolean, proxy?: any, initialStatus?: ProcessStatus, transportConfig?: ITransportConfig | undefined, forwardChildLogs?: boolean, captureInstalled?: boolean);
|
|
40
46
|
private emitExit;
|
|
41
47
|
get status(): ProcessStatus;
|
|
42
48
|
get pid(): number | undefined;
|
|
@@ -49,8 +55,9 @@ export declare class WorkerHandle extends EventEmitter implements IWorkerHandle
|
|
|
49
55
|
send(message: any): Promise<void>;
|
|
50
56
|
onMessage(handler: (data: any) => void): void;
|
|
51
57
|
seedStartupOutput(output?: IStartupOutput): void;
|
|
52
|
-
onLog(handler: (line: string, stream: 'stdout' | 'stderr') => void): void;
|
|
53
|
-
|
|
58
|
+
onLog(handler: (line: string, stream: 'stdout' | 'stderr', context?: ProcessOutputContext) => void): void;
|
|
59
|
+
hasLogConsumers(): boolean;
|
|
60
|
+
emitLog(line: string, stream: 'stdout' | 'stderr', context?: ProcessOutputContext): void;
|
|
54
61
|
private setupMessageHandlers;
|
|
55
62
|
private setupLogCapture;
|
|
56
63
|
private setupDefaultLogForwarding;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-spawner.d.ts","sourceRoot":"","sources":["../src/process-spawner.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAQ,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"process-spawner.d.ts","sourceRoot":"","sources":["../src/process-spawner.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAQ,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAMxD,OAAO,EAAiB,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAI/E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,aAAa,EACb,eAAe,EAEf,qBAAqB,EACtB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAgBlD,UAAU,gBAAgB;IACxB,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA2ED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AA0BD,wBAAgB,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAGtD;AAsBD,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,WAAW,SAAO,EAClB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9E,OAAO,CAAC,OAAO,CAAC,CAqClB;AAED,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;aAiCnD,EAAE,EAAE,MAAM;aAGV,YAAY,EAAE,MAAM;aACpB,WAAW,EAAE,MAAM;aACnB,cAAc,EAAE,MAAM;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;aACf,KAAK,CAAC,EAAE,GAAG;aAEX,eAAe,CAAC,EAAE,gBAAgB;IAElD,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IA5CnC,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,eAAe,CAA0C;IACjE,OAAO,CAAC,WAAW,CAAkG;IAiBrH,OAAO,CAAC,SAAS,CAAmG;IACpH,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,gBAAgB,CAAK;IAM7B,OAAO,CAAC,WAAW,CAAS;IAC5B,SAAgB,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAC9C,SAAgB,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD,YACkB,EAAE,EAAE,MAAM,EAC1B,MAAM,EAAE,MAAM,GAAG,YAAY,EAC7B,YAAY,EAAE,YAAY,GAAG,IAAI,EACjB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,EACrB,MAAM,EAAE,OAAO,EACf,cAAc,EAAE,OAAO,EACxB,KAAK,CAAC,EAAE,GAAG,EAC3B,aAAa,GAAE,aAAsC,EACrC,eAAe,CAAC,EAAE,gBAAgB,YAAA,EAClD,gBAAgB,GAAE,OAAc,EACf,gBAAgB,UAAQ,EAU1C;IAED,OAAO,CAAC,QAAQ;IAyChB,IAAI,MAAM,IAAI,aAAa,CAE1B;IAGD,IAAI,GAAG,IAAI,MAAM,GAAG,SAAS,CAG5B;IAQD,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,GAAG,MAAM,IAAI,CAG3D;IAEK,SAAS,CAAC,IAAI,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA2ClE;YAqBa,qBAAqB;IAwInC,OAAO,IAAI,OAAO,CAajB;IAEK,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBtC;IAED,SAAS,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI,CAG5C;IAcD,iBAAiB,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAa/C;IAED,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,EAAE,OAAO,CAAC,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI,CAqBxG;IAGD,eAAe,IAAI,OAAO,CAAsC;IAGhE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAcvF;IAED,OAAO,CAAC,oBAAoB;IA0D5B,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,yBAAyB;CAuBlC;AAqDD,qBAAa,cAAe,YAAW,eAAe;IAwClD,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAxCzB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IAGjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IA4BhD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAGhD,OAAO,CAAC,UAAU,CAA8B;IAEhD,YACmB,MAAM,EAAE,OAAO,EACf,MAAM,GAAE,qBAA0B,EAgCpD;IAQK,KAAK,CAAC,CAAC,EACX,kBAAkB,EAAE,MAAM,GAAG,CAAC,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,EACxD,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAgRxB;IAKK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB7B;YAiBa,iBAAiB;YA6CjB,cAAc;YAsDd,kBAAkB;YAmClB,iBAAiB;IAmB/B,OAAO,CAAC,gBAAgB;YA4BV,iBAAiB;IA6C/B,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAEnC;IAOD,OAAO,CAAC,gBAAgB;YASV,YAAY;CA0M3B;AAKD,qBAAa,qBAAqB;IAMhC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAE,qBAA0B,GAAG,eAAe,CA+BlF;IAcD,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,GAAG,GAAG,IAAI,CAIjD;IAMD,OAAa,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC,CAYtG;CACF"}
|
package/dist/process-spawner.js
CHANGED
|
@@ -6,7 +6,7 @@ import fsSync from 'node:fs';
|
|
|
6
6
|
import path from 'node:path';
|
|
7
7
|
import os from 'node:os';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
|
-
import {
|
|
9
|
+
import { OutputCapture } from './output-capture.js';
|
|
10
10
|
import net from 'node:net';
|
|
11
11
|
import { Errors } from '@omnitron-dev/titan/errors';
|
|
12
12
|
import { isAlive as isPidAlive } from './liveness.js';
|
|
@@ -33,7 +33,9 @@ async function isSocketRefused(socketPath, timeoutMs = 250) {
|
|
|
33
33
|
};
|
|
34
34
|
const socket = net.connect({ path: socketPath });
|
|
35
35
|
const timer = setTimeout(() => done(false), timeoutMs);
|
|
36
|
-
socket.once('
|
|
36
|
+
socket.once('data', () => done(false));
|
|
37
|
+
socket.once('end', () => done(false));
|
|
38
|
+
socket.once('close', () => done(false));
|
|
37
39
|
socket.once('error', (error) => done(error.code === 'ECONNREFUSED'));
|
|
38
40
|
});
|
|
39
41
|
}
|
|
@@ -44,41 +46,55 @@ export function hasExited(child) {
|
|
|
44
46
|
return true;
|
|
45
47
|
return child.pid == null ? true : !isPidAlive(child.pid);
|
|
46
48
|
}
|
|
47
|
-
export async function killChildAndVerify(child, logger, context, termGraceMs = 3000) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
49
|
+
export async function killChildAndVerify(child, logger, context, termGraceMs = 3000, options = {}) {
|
|
50
|
+
const pid = child.pid;
|
|
51
|
+
const groupAlive = () => {
|
|
52
|
+
if (!options.processGroup || pid == null)
|
|
53
|
+
return false;
|
|
54
|
+
try {
|
|
55
|
+
process.kill(-pid, 0);
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (error.code === 'ESRCH')
|
|
60
|
+
return false;
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const gone = () => hasExited(child) && !groupAlive();
|
|
65
|
+
const signal = (value) => {
|
|
66
|
+
if (options.processGroup && pid != null && groupAlive())
|
|
67
|
+
process.kill(-pid, value);
|
|
68
|
+
else if (!hasExited(child))
|
|
69
|
+
child.kill(value);
|
|
70
|
+
};
|
|
71
|
+
const waitForExit = async (milliseconds) => {
|
|
72
|
+
const deadline = performance.now() + milliseconds;
|
|
73
|
+
while (!gone()) {
|
|
74
|
+
const remaining = deadline - performance.now();
|
|
75
|
+
if (remaining <= 0)
|
|
76
|
+
return false;
|
|
77
|
+
await new Promise(resolve => {
|
|
78
|
+
const finish = () => { clearTimeout(timer); child.off('exit', finish); resolve(); };
|
|
79
|
+
const timer = setTimeout(finish, Math.min(50, remaining));
|
|
80
|
+
child.once('exit', finish);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
67
83
|
return true;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
child.kill('SIGKILL');
|
|
71
|
-
}
|
|
72
|
-
catch {
|
|
84
|
+
};
|
|
85
|
+
if (gone())
|
|
73
86
|
return true;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
87
|
+
if (!options.force) {
|
|
88
|
+
signal('SIGTERM');
|
|
89
|
+
if (await waitForExit(termGraceMs))
|
|
90
|
+
return true;
|
|
91
|
+
logger.warn({ ...context, pid }, 'Child ignored SIGTERM — escalating to SIGKILL');
|
|
92
|
+
}
|
|
93
|
+
signal('SIGKILL');
|
|
94
|
+
return waitForExit(options.killGraceMs ?? 1000);
|
|
79
95
|
}
|
|
80
96
|
export class WorkerHandle extends EventEmitter {
|
|
81
|
-
constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true) {
|
|
97
|
+
constructor(id, worker, netronClient, transportUrl, serviceName, serviceVersion, logger, isWorkerThread, proxy, initialStatus = ProcessStatus.STARTING, transportConfig, forwardChildLogs = true, captureInstalled = false) {
|
|
82
98
|
super();
|
|
83
99
|
this.id = id;
|
|
84
100
|
this.transportUrl = transportUrl;
|
|
@@ -88,6 +104,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
88
104
|
this.isWorkerThread = isWorkerThread;
|
|
89
105
|
this.proxy = proxy;
|
|
90
106
|
this.transportConfig = transportConfig;
|
|
107
|
+
this.captureInstalled = captureInstalled;
|
|
91
108
|
this.messageHandlers = new Map();
|
|
92
109
|
this.logHandlers = new Set();
|
|
93
110
|
this.earlyLogs = [];
|
|
@@ -207,11 +224,11 @@ export class WorkerHandle extends EventEmitter {
|
|
|
207
224
|
};
|
|
208
225
|
const errorHandler = (err) => {
|
|
209
226
|
if (!resolved) {
|
|
210
|
-
cleanup();
|
|
211
227
|
if (err.code !== 'ERR_IPC_CHANNEL_CLOSED') {
|
|
212
228
|
this.logger.warn({ error: err, workerId: this.id }, 'Error during child termination');
|
|
213
229
|
}
|
|
214
|
-
|
|
230
|
+
if (hasExited(child))
|
|
231
|
+
exitHandler();
|
|
215
232
|
}
|
|
216
233
|
};
|
|
217
234
|
child.once('exit', exitHandler);
|
|
@@ -262,15 +279,9 @@ export class WorkerHandle extends EventEmitter {
|
|
|
262
279
|
resolve();
|
|
263
280
|
return;
|
|
264
281
|
}
|
|
265
|
-
this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
process.kill(child.pid, 'SIGKILL');
|
|
269
|
-
}
|
|
270
|
-
catch {
|
|
271
|
-
}
|
|
272
|
-
resolve();
|
|
273
|
-
});
|
|
282
|
+
this.logger.error({ workerId: this.id, pid: child.pid, state: verdict.state, waitedMs: verdict.waitedMs }, 'Process termination is unconfirmed; retaining ownership for recovery');
|
|
283
|
+
reject(new Error(`Worker ${this.id} (pid ${child.pid}) did not confirm exit after termination: ${verdict.state}`));
|
|
284
|
+
}, reject);
|
|
274
285
|
}, GRACEFUL_TIMEOUT + SIGTERM_TIMEOUT + SIGKILL_TIMEOUT);
|
|
275
286
|
});
|
|
276
287
|
}
|
|
@@ -331,12 +342,13 @@ export class WorkerHandle extends EventEmitter {
|
|
|
331
342
|
'it printed more than the pre-capture buffer holds', 'stderr');
|
|
332
343
|
}
|
|
333
344
|
for (const entry of buffered) {
|
|
334
|
-
handler(entry.line, entry.stream);
|
|
345
|
+
handler(entry.line, entry.stream, entry.context);
|
|
335
346
|
}
|
|
336
347
|
}
|
|
337
|
-
|
|
348
|
+
hasLogConsumers() { return this.logHandlers.size > 0; }
|
|
349
|
+
emitLog(line, stream, context) {
|
|
338
350
|
if (this.logHandlers.size === 0 && this.earlyLogs) {
|
|
339
|
-
this.earlyLogs.push({ line, stream });
|
|
351
|
+
this.earlyLogs.push({ line, stream, ...(context ? { context } : {}) });
|
|
340
352
|
this.earlyLogBytes += line.length;
|
|
341
353
|
while (this.earlyLogBytes > EARLY_LOG_MAX_BYTES && this.earlyLogs.length > 1) {
|
|
342
354
|
const dropped = this.earlyLogs.shift();
|
|
@@ -346,7 +358,7 @@ export class WorkerHandle extends EventEmitter {
|
|
|
346
358
|
return;
|
|
347
359
|
}
|
|
348
360
|
for (const handler of this.logHandlers) {
|
|
349
|
-
handler(line, stream);
|
|
361
|
+
handler(line, stream, context);
|
|
350
362
|
}
|
|
351
363
|
}
|
|
352
364
|
setupMessageHandlers() {
|
|
@@ -385,24 +397,14 @@ export class WorkerHandle extends EventEmitter {
|
|
|
385
397
|
child.on('exit', (code, signal) => {
|
|
386
398
|
this.emitExit(code, signal);
|
|
387
399
|
});
|
|
388
|
-
this.
|
|
400
|
+
if (!this.captureInstalled)
|
|
401
|
+
this.setupLogCapture(child);
|
|
389
402
|
}
|
|
390
403
|
}
|
|
391
404
|
setupLogCapture(child) {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
if (line.trim())
|
|
396
|
-
this.emitLog(line, 'stdout');
|
|
397
|
-
});
|
|
398
|
-
}
|
|
399
|
-
if (child.stderr) {
|
|
400
|
-
const rl = createInterface({ input: child.stderr, crlfDelay: Infinity });
|
|
401
|
-
rl.on('line', (line) => {
|
|
402
|
-
if (line.trim())
|
|
403
|
-
this.emitLog(line, 'stderr');
|
|
404
|
-
});
|
|
405
|
-
}
|
|
405
|
+
const capture = new OutputCapture(this.id, (line, stream, context) => this.emitLog(line, stream, context));
|
|
406
|
+
capture.attach(child.stdout, 'stdout');
|
|
407
|
+
capture.attach(child.stderr, 'stderr');
|
|
406
408
|
}
|
|
407
409
|
setupDefaultLogForwarding() {
|
|
408
410
|
const childLogger = this.logger.child({
|
|
@@ -509,11 +511,14 @@ export class ProcessSpawner {
|
|
|
509
511
|
throw Errors.notFound('Process file', processPath);
|
|
510
512
|
}
|
|
511
513
|
const transport = await this.setupTransport(processId, options);
|
|
514
|
+
const workerOptions = { ...options };
|
|
515
|
+
delete workerOptions.config;
|
|
516
|
+
delete workerOptions.onOutput;
|
|
512
517
|
const context = {
|
|
513
518
|
processId,
|
|
514
519
|
processPath,
|
|
515
520
|
transport,
|
|
516
|
-
options:
|
|
521
|
+
options: workerOptions,
|
|
517
522
|
dependencies: options.dependencies,
|
|
518
523
|
};
|
|
519
524
|
const declaredIsolation = options.isolation;
|
|
@@ -528,7 +533,34 @@ export class ProcessSpawner {
|
|
|
528
533
|
const useWorkerThreads = strategy === 'worker' && !sandboxUnimplemented && (!options.execArgv || options.execArgv.length === 0);
|
|
529
534
|
let worker;
|
|
530
535
|
let netronClient = null;
|
|
536
|
+
let outputHandle;
|
|
537
|
+
const early = [];
|
|
538
|
+
let earlyBytes = 0;
|
|
539
|
+
let earlyDropped = 0;
|
|
540
|
+
const tails = { stdout: '', stderr: '' };
|
|
541
|
+
const capture = new OutputCapture(processId, (line, stream, outputContext) => {
|
|
542
|
+
options.onOutput?.(line, stream, outputContext);
|
|
543
|
+
tails[stream] = (tails[stream] + line + '\n').slice(-64 * 1024);
|
|
544
|
+
if (outputHandle) {
|
|
545
|
+
if (!options.onOutput || this.config.forwardChildLogs !== false || outputHandle.hasLogConsumers())
|
|
546
|
+
outputHandle.emitLog(line, stream, outputContext);
|
|
547
|
+
}
|
|
548
|
+
else if (!options.onOutput || this.config.forwardChildLogs !== false) {
|
|
549
|
+
early.push({ line, stream, context: outputContext });
|
|
550
|
+
earlyBytes += line.length;
|
|
551
|
+
while (earlyBytes > EARLY_LOG_MAX_BYTES && early.length > 1) {
|
|
552
|
+
earlyBytes -= early.shift().line.length;
|
|
553
|
+
earlyDropped++;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
});
|
|
531
557
|
try {
|
|
558
|
+
if (useWorkerThreads && options.cwd) {
|
|
559
|
+
const [requested, current] = await Promise.all([fs.realpath(options.cwd), fs.realpath(process.cwd())]);
|
|
560
|
+
if (requested !== current) {
|
|
561
|
+
throw new Error('A separate working directory requires child isolation; worker threads share the parent cwd');
|
|
562
|
+
}
|
|
563
|
+
}
|
|
532
564
|
if (sandboxUnimplemented) {
|
|
533
565
|
this.logger.warn({ processId, isolation: sandbox, actual: 'child' }, `Process isolation '${sandbox}' is not implemented — spawning an ordinary child process. ` +
|
|
534
566
|
'It has the same filesystem and network access as its parent; do not rely on it as a security boundary.');
|
|
@@ -540,8 +572,11 @@ export class ProcessSpawner {
|
|
|
540
572
|
else {
|
|
541
573
|
worker = await this.spawnChildProcess(context, options.execArgv);
|
|
542
574
|
}
|
|
575
|
+
worker.on('error', () => { });
|
|
576
|
+
capture.attach(worker.stdout, 'stdout');
|
|
577
|
+
capture.attach(worker.stderr, 'stderr');
|
|
543
578
|
const startupTimeout = options.startupTimeout ?? this.config.resources?.timeout ?? 30_000;
|
|
544
|
-
const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout);
|
|
579
|
+
const readyInfo = await this.waitForReady(worker, useWorkerThreads, startupTimeout, tails, !!options.onOutput);
|
|
545
580
|
const actualServiceName = readyInfo.serviceName || serviceName;
|
|
546
581
|
const actualServiceVersion = readyInfo.serviceVersion || serviceVersion;
|
|
547
582
|
if (strategy !== 'none') {
|
|
@@ -555,13 +590,23 @@ export class ProcessSpawner {
|
|
|
555
590
|
: actualServiceName;
|
|
556
591
|
const proxyHandler = new ServiceProxyHandler(processId, netronClient, qualifiedServiceName, this.logger);
|
|
557
592
|
const proxy = proxyHandler.createProxy();
|
|
558
|
-
const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
559
|
-
handle
|
|
593
|
+
const handle = new WorkerHandle(processId, worker, netronClient, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, proxy, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true, true);
|
|
594
|
+
outputHandle = handle;
|
|
595
|
+
if (earlyDropped)
|
|
596
|
+
handle.emitLog(`[titan-pm] ${earlyDropped} lines exceeded the optional readiness replay buffer`, 'stderr');
|
|
597
|
+
for (const item of early)
|
|
598
|
+
handle.emitLog(item.line, item.stream, item.context);
|
|
599
|
+
early.length = 0;
|
|
560
600
|
return handle;
|
|
561
601
|
}
|
|
562
602
|
else {
|
|
563
|
-
const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true);
|
|
564
|
-
handle
|
|
603
|
+
const handle = new WorkerHandle(processId, worker, null, transport.url, actualServiceName, actualServiceVersion, this.logger, useWorkerThreads, undefined, ProcessStatus.RUNNING, transport, this.config.forwardChildLogs ?? true, true);
|
|
604
|
+
outputHandle = handle;
|
|
605
|
+
if (earlyDropped)
|
|
606
|
+
handle.emitLog(`[titan-pm] ${earlyDropped} lines exceeded the optional readiness replay buffer`, 'stderr');
|
|
607
|
+
for (const item of early)
|
|
608
|
+
handle.emitLog(item.line, item.stream, item.context);
|
|
609
|
+
early.length = 0;
|
|
565
610
|
return handle;
|
|
566
611
|
}
|
|
567
612
|
}
|
|
@@ -592,6 +637,12 @@ export class ProcessSpawner {
|
|
|
592
637
|
this.logger.error({ error: disconnectError, serviceName, processId }, 'Failed to disconnect Netron client during error cleanup');
|
|
593
638
|
});
|
|
594
639
|
}
|
|
640
|
+
await new Promise(resolve => setImmediate(resolve));
|
|
641
|
+
if (error instanceof Error)
|
|
642
|
+
error.details = {
|
|
643
|
+
...(error.details ?? {}), ...tails,
|
|
644
|
+
outputCaptured: !!options.onOutput,
|
|
645
|
+
};
|
|
595
646
|
throw error;
|
|
596
647
|
}
|
|
597
648
|
finally {
|
|
@@ -699,6 +750,8 @@ export class ProcessSpawner {
|
|
|
699
750
|
const serializableContext = this.makeSerializable(context);
|
|
700
751
|
const worker = new Worker(this.workerRuntimePath, {
|
|
701
752
|
workerData: serializableContext,
|
|
753
|
+
stdout: true,
|
|
754
|
+
stderr: true,
|
|
702
755
|
env: childEnv(),
|
|
703
756
|
});
|
|
704
757
|
return worker;
|
|
@@ -751,23 +804,8 @@ export class ProcessSpawner {
|
|
|
751
804
|
if (typeof pid === 'number')
|
|
752
805
|
this.forkedPids.delete(pid);
|
|
753
806
|
}
|
|
754
|
-
async waitForReady(worker, isWorkerThread, timeout = 5000) {
|
|
807
|
+
async waitForReady(worker, isWorkerThread, timeout = 5000, tails = { stdout: '', stderr: '' }, outputCaptured = false) {
|
|
755
808
|
return new Promise((resolve, reject) => {
|
|
756
|
-
const STDERR_CAP_BYTES = 64 * 1024;
|
|
757
|
-
const stderrBuf = [];
|
|
758
|
-
const stdoutBuf = [];
|
|
759
|
-
const stderrBytes = 0;
|
|
760
|
-
const stdoutBytes = 0;
|
|
761
|
-
const appendBounded = (target, bytesRef, chunk) => {
|
|
762
|
-
target.push(chunk);
|
|
763
|
-
bytesRef.v += chunk.length;
|
|
764
|
-
while (bytesRef.v > STDERR_CAP_BYTES && target.length > 1) {
|
|
765
|
-
const dropped = target.shift();
|
|
766
|
-
bytesRef.v -= dropped.length;
|
|
767
|
-
}
|
|
768
|
-
};
|
|
769
|
-
const stderrRef = { v: stderrBytes };
|
|
770
|
-
const stdoutRef = { v: stdoutBytes };
|
|
771
809
|
const LAG_PROBE_INTERVAL_MS = 1_000;
|
|
772
810
|
const LAG_SIGNIFICANT_MS = 2_000;
|
|
773
811
|
let maxLagMs = 0;
|
|
@@ -780,27 +818,12 @@ export class ProcessSpawner {
|
|
|
780
818
|
maxLagMs = lag;
|
|
781
819
|
}, LAG_PROBE_INTERVAL_MS);
|
|
782
820
|
lagProbe.unref?.();
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
});
|
|
788
|
-
child.stdout?.on('data', (chunk) => {
|
|
789
|
-
appendBounded(stdoutBuf, stdoutRef, chunk.toString());
|
|
790
|
-
});
|
|
791
|
-
}
|
|
792
|
-
const buildDiagnostics = () => {
|
|
793
|
-
const stderr = stderrBuf.join('').trimEnd();
|
|
794
|
-
const stdout = stdoutBuf.join('').trimEnd();
|
|
795
|
-
return {
|
|
796
|
-
stderr: stderr || undefined,
|
|
797
|
-
stdout: stdout || undefined,
|
|
798
|
-
stderrBytes: stderrRef.v,
|
|
799
|
-
stdoutBytes: stdoutRef.v,
|
|
800
|
-
};
|
|
801
|
-
};
|
|
821
|
+
const buildDiagnostics = () => ({
|
|
822
|
+
stdout: tails.stdout || undefined, stderr: tails.stderr || undefined,
|
|
823
|
+
stdoutBytes: Buffer.byteLength(tails.stdout), stderrBytes: Buffer.byteLength(tails.stderr), outputCaptured,
|
|
824
|
+
});
|
|
802
825
|
const timer = setTimeout(() => {
|
|
803
|
-
|
|
826
|
+
cleanup();
|
|
804
827
|
const pid = !isWorkerThread ? worker.pid : undefined;
|
|
805
828
|
const pidHint = pid ? ` (pid: ${pid})` : '';
|
|
806
829
|
const diag = buildDiagnostics();
|
|
@@ -829,16 +852,24 @@ export class ProcessSpawner {
|
|
|
829
852
|
reject(err);
|
|
830
853
|
}, timeout);
|
|
831
854
|
const cleanup = () => {
|
|
855
|
+
worker.off('error', errorHandler);
|
|
832
856
|
clearTimeout(timer);
|
|
833
857
|
clearInterval(lagProbe);
|
|
834
858
|
if (isWorkerThread) {
|
|
835
859
|
worker.off('message', messageHandler);
|
|
860
|
+
worker.off('exit', exitHandler);
|
|
836
861
|
}
|
|
837
862
|
else {
|
|
838
863
|
worker.off('message', messageHandler);
|
|
839
|
-
worker.off('
|
|
864
|
+
worker.off('close', exitHandler);
|
|
840
865
|
}
|
|
841
866
|
};
|
|
867
|
+
const errorHandler = (error) => {
|
|
868
|
+
cleanup();
|
|
869
|
+
error.details = buildDiagnostics();
|
|
870
|
+
reject(error);
|
|
871
|
+
};
|
|
872
|
+
worker.on('error', errorHandler);
|
|
842
873
|
const messageHandler = (data) => {
|
|
843
874
|
if (data && typeof data === 'object' && 'type' in data) {
|
|
844
875
|
if (data.type === 'ready') {
|
|
@@ -846,7 +877,7 @@ export class ProcessSpawner {
|
|
|
846
877
|
resolve({
|
|
847
878
|
serviceName: data.serviceName,
|
|
848
879
|
serviceVersion: data.serviceVersion,
|
|
849
|
-
startupOutput: {
|
|
880
|
+
startupOutput: { ...tails },
|
|
850
881
|
});
|
|
851
882
|
}
|
|
852
883
|
else if (data.type === 'error') {
|
|
@@ -875,10 +906,11 @@ export class ProcessSpawner {
|
|
|
875
906
|
};
|
|
876
907
|
if (isWorkerThread) {
|
|
877
908
|
worker.on('message', messageHandler);
|
|
909
|
+
worker.on('exit', exitHandler);
|
|
878
910
|
}
|
|
879
911
|
else {
|
|
880
912
|
worker.on('message', messageHandler);
|
|
881
|
-
worker.on('
|
|
913
|
+
worker.on('close', exitHandler);
|
|
882
914
|
}
|
|
883
915
|
});
|
|
884
916
|
}
|