nestworker 2.1.0 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -2
- package/dist/core/worker.interfaces.d.ts +30 -15
- package/dist/core/worker.pool.d.ts +34 -5
- package/dist/core/worker.pool.js +388 -196
- package/dist/core/worker.pool.js.map +1 -1
- package/dist/core/worker.service.d.ts +2 -0
- package/dist/core/worker.service.js +71 -31
- package/dist/core/worker.service.js.map +1 -1
- package/dist/example/bench.d.ts +13 -0
- package/dist/example/bench.js +86 -0
- package/dist/example/bench.js.map +1 -0
- package/dist/example/image.service.d.ts +2 -0
- package/dist/example/image.service.js +10 -0
- package/dist/example/image.service.js.map +1 -1
- package/dist/example/main.js +1 -1
- package/dist/example/main.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/worker/worker-runtime.js +151 -73
- package/dist/worker/worker-runtime.js.map +1 -1
- package/package.json +10 -1
package/dist/core/worker.pool.js
CHANGED
|
@@ -13,24 +13,50 @@ const PRIORITY_WEIGHT = {
|
|
|
13
13
|
NORMAL: 2,
|
|
14
14
|
LOW: 1,
|
|
15
15
|
};
|
|
16
|
+
// Per-worker state attached via a symbol-keyed slot — avoids a Map lookup
|
|
17
|
+
// per message and per dispatch on the hot path.
|
|
18
|
+
const STATE = Symbol('nestworker:state');
|
|
19
|
+
function getState(worker) {
|
|
20
|
+
return worker[STATE];
|
|
21
|
+
}
|
|
16
22
|
class WorkerPool extends node_events_1.EventEmitter {
|
|
17
23
|
services;
|
|
18
24
|
size;
|
|
19
25
|
shutdownTimeout;
|
|
20
26
|
workers = [];
|
|
27
|
+
/**
|
|
28
|
+
* Available "slots" — each worker is pushed `concurrency` times when it
|
|
29
|
+
* becomes ready, then popped/pushed as jobs are dispatched/completed.
|
|
30
|
+
* This naturally supports per-worker pipelining without any per-job
|
|
31
|
+
* counter bookkeeping.
|
|
32
|
+
*/
|
|
21
33
|
idle = [];
|
|
22
34
|
warmingUp = new Set();
|
|
35
|
+
// Head-index FIFO: queue.shift() is O(n); a head pointer makes pop O(1)
|
|
36
|
+
// and the array is compacted lazily when it grows wasteful.
|
|
23
37
|
queue = [];
|
|
38
|
+
queueHead = 0;
|
|
24
39
|
destroyed = false;
|
|
25
|
-
|
|
40
|
+
activeCount = 0;
|
|
41
|
+
/**
|
|
42
|
+
* `schedule()` is invoked many times in a single synchronous burst (e.g.
|
|
43
|
+
* `for (...) ws.run(...)` floods 20k enqueues). Running the dispatch loop
|
|
44
|
+
* after every enqueue limits us to batches of size 1 per worker — the
|
|
45
|
+
* whole point of batching is then defeated. Defer to the next microtask
|
|
46
|
+
* so all synchronously-enqueued jobs land in one schedule pass and get
|
|
47
|
+
* coalesced into a single postMessage per worker.
|
|
48
|
+
*/
|
|
49
|
+
scheduleQueued = false;
|
|
26
50
|
/** Maps abortSignalId → worker currently running that job */
|
|
27
51
|
signalWorkerMap = new Map();
|
|
52
|
+
concurrency;
|
|
28
53
|
proxyMap = new Map();
|
|
29
|
-
constructor(services, proxyInstances, size = node_os_1.default.cpus().length, shutdownTimeout = 30_000) {
|
|
54
|
+
constructor(services, proxyInstances, size = node_os_1.default.cpus().length, shutdownTimeout = 30_000, concurrency = 1) {
|
|
30
55
|
super();
|
|
31
56
|
this.services = services;
|
|
32
57
|
this.size = size;
|
|
33
58
|
this.shutdownTimeout = shutdownTimeout;
|
|
59
|
+
this.concurrency = concurrency > 0 ? concurrency : 1;
|
|
34
60
|
for (const { propertyKey, instance } of proxyInstances) {
|
|
35
61
|
this.proxyMap.set(propertyKey, instance);
|
|
36
62
|
}
|
|
@@ -38,7 +64,7 @@ class WorkerPool extends node_events_1.EventEmitter {
|
|
|
38
64
|
this.spawnWorker();
|
|
39
65
|
}
|
|
40
66
|
}
|
|
41
|
-
execute(job, signal) {
|
|
67
|
+
execute(job, meta, signal) {
|
|
42
68
|
if (this.destroyed)
|
|
43
69
|
return Promise.reject(new Error('WorkerPool destroyed'));
|
|
44
70
|
return new Promise((resolve, reject) => {
|
|
@@ -49,6 +75,10 @@ class WorkerPool extends node_events_1.EventEmitter {
|
|
|
49
75
|
}
|
|
50
76
|
const task = {
|
|
51
77
|
job,
|
|
78
|
+
priority: meta.priority,
|
|
79
|
+
timeout: meta.timeout,
|
|
80
|
+
retry: meta.retry,
|
|
81
|
+
retryDelay: meta.retryDelay,
|
|
52
82
|
resolve: resolve,
|
|
53
83
|
reject,
|
|
54
84
|
attempts: 0,
|
|
@@ -56,21 +86,29 @@ class WorkerPool extends node_events_1.EventEmitter {
|
|
|
56
86
|
};
|
|
57
87
|
if (signal) {
|
|
58
88
|
signal.addEventListener('abort', () => {
|
|
59
|
-
// Remove from queue if not yet dispatched
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
89
|
+
// Remove from queue if not yet dispatched. Use head-index aware
|
|
90
|
+
// search and tombstone with `undefined` to avoid an O(n) splice.
|
|
91
|
+
const q = this.queue;
|
|
92
|
+
for (let i = this.queueHead; i < q.length; i++) {
|
|
93
|
+
if (q[i] === task) {
|
|
94
|
+
q[i] = undefined;
|
|
95
|
+
// If it's at the head, advance past it.
|
|
96
|
+
while (this.queueHead < q.length && q[this.queueHead] === undefined) {
|
|
97
|
+
this.queueHead++;
|
|
98
|
+
}
|
|
99
|
+
reject(new DOMException('Task aborted', 'AbortError'));
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
65
102
|
}
|
|
66
103
|
// Already running — send abort signal to worker
|
|
67
|
-
if (job.abortSignalId) {
|
|
104
|
+
if (job.abortSignalId !== undefined) {
|
|
68
105
|
const worker = this.signalWorkerMap.get(job.abortSignalId);
|
|
69
106
|
if (worker) {
|
|
70
107
|
try {
|
|
71
108
|
worker.postMessage({ type: 'abort', abortSignalId: job.abortSignalId });
|
|
72
109
|
}
|
|
73
|
-
catch { /* worker gone */
|
|
110
|
+
catch { /* worker gone */
|
|
111
|
+
}
|
|
74
112
|
}
|
|
75
113
|
}
|
|
76
114
|
}, { once: true });
|
|
@@ -83,230 +121,369 @@ class WorkerPool extends node_events_1.EventEmitter {
|
|
|
83
121
|
return {
|
|
84
122
|
poolSize: this.size,
|
|
85
123
|
idle: this.idle.length,
|
|
86
|
-
busy: this.
|
|
87
|
-
queued: this.queue.length,
|
|
124
|
+
busy: this.activeCount,
|
|
125
|
+
queued: this.queue.length - this.queueHead,
|
|
88
126
|
warmingUp: this.warmingUp.size,
|
|
89
127
|
};
|
|
90
128
|
}
|
|
91
129
|
spawnWorker() {
|
|
92
130
|
const worker = new node_worker_threads_1.Worker(node_path_1.default.resolve(__dirname, '../worker/worker-runtime.js'), { workerData: { services: this.services } });
|
|
131
|
+
worker[STATE] = { active: new Map() };
|
|
93
132
|
this.workers.push(worker);
|
|
94
133
|
this.warmingUp.add(worker);
|
|
95
|
-
|
|
134
|
+
// ── Single persistent message listener ───────────────────────────────
|
|
135
|
+
// We used to add/remove a listener per dispatch — that allocated several
|
|
136
|
+
// closures and mutated the EventEmitter's listener array on every task.
|
|
137
|
+
// Instead we install one listener whose behaviour switches based on
|
|
138
|
+
// whether this worker is currently warming up or running a task.
|
|
139
|
+
const onMessage = (msg) => {
|
|
96
140
|
const message = msg;
|
|
97
|
-
if (
|
|
141
|
+
if (this.warmingUp.has(worker)) {
|
|
142
|
+
if (message?.type !== 'worker:ready')
|
|
143
|
+
return;
|
|
144
|
+
this.warmingUp.delete(worker);
|
|
145
|
+
if (!this.destroyed) {
|
|
146
|
+
// Push the worker into the idle queue once per concurrency slot
|
|
147
|
+
// so the scheduler will pipeline up to N jobs into it.
|
|
148
|
+
for (let i = 0; i < this.concurrency; i++)
|
|
149
|
+
this.idle.push(worker);
|
|
150
|
+
this.schedule();
|
|
151
|
+
}
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
// Spurious `worker:ready` after warmup → ignore.
|
|
155
|
+
if (message?.type === 'worker:ready')
|
|
156
|
+
return;
|
|
157
|
+
// IPC invoke from worker is not job-scoped — handle inline.
|
|
158
|
+
if (message?.type === 'ipc:invoke') {
|
|
159
|
+
this.handleIpcInvoke(worker, message);
|
|
98
160
|
return;
|
|
99
|
-
worker.removeListener('message', onReady);
|
|
100
|
-
this.warmingUp.delete(worker);
|
|
101
|
-
if (!this.destroyed) {
|
|
102
|
-
this.idle.push(worker);
|
|
103
|
-
this.schedule();
|
|
104
161
|
}
|
|
162
|
+
// Batched job results — route each by jobId.
|
|
163
|
+
if (message?.type === 'results') {
|
|
164
|
+
const batch = message;
|
|
165
|
+
const state = getState(worker);
|
|
166
|
+
const results = batch.results;
|
|
167
|
+
for (let i = 0; i < results.length; i++) {
|
|
168
|
+
const r = results[i];
|
|
169
|
+
const slot = r.jobId !== undefined ? state.active.get(r.jobId) : undefined;
|
|
170
|
+
if (!slot)
|
|
171
|
+
continue;
|
|
172
|
+
state.active.delete(r.jobId);
|
|
173
|
+
this.completeJob(worker, slot, r);
|
|
174
|
+
}
|
|
175
|
+
if (!this.destroyed)
|
|
176
|
+
this.schedule();
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
// Job result — route by jobId to the right active slot.
|
|
180
|
+
const result = message;
|
|
181
|
+
const state = getState(worker);
|
|
182
|
+
const jobId = result.jobId;
|
|
183
|
+
let slot;
|
|
184
|
+
if (jobId !== undefined) {
|
|
185
|
+
slot = state.active.get(jobId);
|
|
186
|
+
if (slot)
|
|
187
|
+
state.active.delete(jobId);
|
|
188
|
+
}
|
|
189
|
+
else if (state.active.size === 1) {
|
|
190
|
+
// Concurrency == 1 back-compat: single in-flight job, no need for ID.
|
|
191
|
+
const it = state.active.values().next();
|
|
192
|
+
slot = it.value;
|
|
193
|
+
state.active.clear();
|
|
194
|
+
}
|
|
195
|
+
if (!slot)
|
|
196
|
+
return; // late message for an aborted/timed-out task
|
|
197
|
+
this.completeJob(worker, slot, result);
|
|
198
|
+
if (!this.destroyed)
|
|
199
|
+
this.schedule();
|
|
105
200
|
};
|
|
106
|
-
worker.on('message',
|
|
201
|
+
worker.on('message', onMessage);
|
|
107
202
|
worker.once('error', (err) => this.handleWorkerError(worker, err));
|
|
108
203
|
worker.once('exit', (code) => this.handleWorkerExit(worker, code));
|
|
109
204
|
return worker;
|
|
110
205
|
}
|
|
206
|
+
handleIpcInvoke(worker, req) {
|
|
207
|
+
const svcInstance = this.proxyMap.get(req.propertyKey);
|
|
208
|
+
const reply = (res) => {
|
|
209
|
+
try {
|
|
210
|
+
worker.postMessage(res);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
try {
|
|
214
|
+
worker.postMessage({
|
|
215
|
+
type: 'ipc:result', callId: res.callId, ok: false,
|
|
216
|
+
error: `IPC result for "${req.propertyKey}.${req.methodName}" ` +
|
|
217
|
+
`is not structuredClone-compatible.`,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
catch { /* worker gone */ }
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
if (!svcInstance) {
|
|
224
|
+
reply({
|
|
225
|
+
type: 'ipc:result', callId: req.callId, ok: false,
|
|
226
|
+
error: `No proxy registered for "${req.propertyKey}"`,
|
|
227
|
+
});
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
const method = svcInstance[req.methodName];
|
|
231
|
+
if (typeof method !== 'function') {
|
|
232
|
+
reply({
|
|
233
|
+
type: 'ipc:result', callId: req.callId, ok: false,
|
|
234
|
+
error: `Method "${req.methodName}" not found on "${req.propertyKey}"`,
|
|
235
|
+
});
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
let p;
|
|
239
|
+
try {
|
|
240
|
+
p = svcInstance[req.methodName](...req.args);
|
|
241
|
+
}
|
|
242
|
+
catch (err) {
|
|
243
|
+
reply({
|
|
244
|
+
type: 'ipc:result', callId: req.callId, ok: false,
|
|
245
|
+
error: err.message ?? String(err),
|
|
246
|
+
});
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
// Sync fast path for proxies that return plain values.
|
|
250
|
+
if (p === null || typeof p !== 'object' || typeof p.then !== 'function') {
|
|
251
|
+
reply({ type: 'ipc:result', callId: req.callId, ok: true, data: p });
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
Promise.resolve(p).then((data) => reply({ type: 'ipc:result', callId: req.callId, ok: true, data }), (err) => reply({
|
|
255
|
+
type: 'ipc:result', callId: req.callId, ok: false,
|
|
256
|
+
error: err.message ?? String(err),
|
|
257
|
+
}));
|
|
258
|
+
}
|
|
111
259
|
enqueue(task) {
|
|
112
|
-
const weight = PRIORITY_WEIGHT[task.
|
|
113
|
-
|
|
260
|
+
const weight = PRIORITY_WEIGHT[task.priority];
|
|
261
|
+
const q = this.queue;
|
|
262
|
+
const head = this.queueHead;
|
|
263
|
+
const n = q.length;
|
|
264
|
+
// Fast path: empty queue, or tail has >= priority → just push (O(1)).
|
|
265
|
+
// This is by far the most common case under steady-state load.
|
|
266
|
+
if (n === head || PRIORITY_WEIGHT[q[n - 1].priority] >= weight) {
|
|
267
|
+
q.push(task);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
// Binary search over the live region [head, n) for the insertion point.
|
|
271
|
+
let lo = head, hi = n;
|
|
114
272
|
while (lo < hi) {
|
|
115
273
|
const mid = (lo + hi) >>> 1;
|
|
116
|
-
if (PRIORITY_WEIGHT[
|
|
274
|
+
if (PRIORITY_WEIGHT[q[mid].priority] < weight)
|
|
117
275
|
hi = mid;
|
|
118
276
|
else
|
|
119
277
|
lo = mid + 1;
|
|
120
278
|
}
|
|
121
|
-
|
|
279
|
+
q.splice(lo, 0, task);
|
|
122
280
|
}
|
|
123
281
|
schedule() {
|
|
282
|
+
if (this.destroyed || this.scheduleQueued)
|
|
283
|
+
return;
|
|
284
|
+
if (this.idle.length === 0 || this.queueHead >= this.queue.length)
|
|
285
|
+
return;
|
|
286
|
+
this.scheduleQueued = true;
|
|
287
|
+
queueMicrotask(this.drain);
|
|
288
|
+
}
|
|
289
|
+
/** Pre-bound for queueMicrotask — avoids closure allocation per schedule. */
|
|
290
|
+
drain = () => {
|
|
291
|
+
this.scheduleQueued = false;
|
|
124
292
|
if (this.destroyed)
|
|
125
293
|
return;
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
294
|
+
const idle = this.idle;
|
|
295
|
+
const q = this.queue;
|
|
296
|
+
// Collect per-worker dispatches built during this schedule pass so we
|
|
297
|
+
// can ship them in ONE postMessage envelope per worker. Each postMessage
|
|
298
|
+
// pays a fixed structuredClone setup cost — batching amortises it.
|
|
299
|
+
let batches;
|
|
300
|
+
while (idle.length > 0 && this.queueHead < q.length) {
|
|
301
|
+
const worker = idle.pop();
|
|
302
|
+
const task = q[this.queueHead];
|
|
303
|
+
q[this.queueHead] = undefined;
|
|
304
|
+
this.queueHead++;
|
|
305
|
+
if (this.queueHead > 1024 && this.queueHead * 2 > q.length) {
|
|
306
|
+
q.splice(0, this.queueHead);
|
|
307
|
+
this.queueHead = 0;
|
|
308
|
+
}
|
|
309
|
+
this.prepareDispatch(worker, task);
|
|
310
|
+
if (!batches)
|
|
311
|
+
batches = new Map();
|
|
312
|
+
let arr = batches.get(worker);
|
|
313
|
+
if (!arr) {
|
|
314
|
+
arr = [];
|
|
315
|
+
batches.set(worker, arr);
|
|
316
|
+
}
|
|
317
|
+
arr.push(task.job);
|
|
130
318
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
319
|
+
if (!batches)
|
|
320
|
+
return;
|
|
321
|
+
for (const [worker, jobs] of batches) {
|
|
322
|
+
if (jobs.length === 1) {
|
|
323
|
+
worker.postMessage(jobs[0]);
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
worker.postMessage({ type: 'batch', jobs });
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
};
|
|
330
|
+
prepareDispatch(worker, task) {
|
|
331
|
+
// Only pay the syscall cost when someone is actually listening.
|
|
332
|
+
const wantTiming = this.listenerCount('taskEnd') > 0 || this.listenerCount('taskStart') > 0;
|
|
333
|
+
const startedAt = wantTiming ? Date.now() : 0;
|
|
136
334
|
task.attempts++;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
this.signalWorkerMap.set(task.job.abortSignalId, worker);
|
|
335
|
+
const sigId = task.job.abortSignalId;
|
|
336
|
+
if (sigId !== undefined) {
|
|
337
|
+
this.signalWorkerMap.set(sigId, worker);
|
|
141
338
|
}
|
|
142
|
-
|
|
143
|
-
const
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
339
|
+
const slot = { task, startedAt, settled: false };
|
|
340
|
+
const state = getState(worker);
|
|
341
|
+
state.active.set(task.job.jobId, slot);
|
|
342
|
+
this.activeCount++;
|
|
343
|
+
if (this.listenerCount('taskStart') > 0)
|
|
344
|
+
this.emit('taskStart', task.job);
|
|
345
|
+
if (task.timeout && task.timeout > 0) {
|
|
346
|
+
slot.timeoutHandle = setTimeout(() => this.handleTimeout(worker, slot), task.timeout);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
/** Called from the persistent message listener when a job result arrives. */
|
|
350
|
+
completeJob(worker, slot, result) {
|
|
351
|
+
if (slot.settled)
|
|
352
|
+
return;
|
|
353
|
+
slot.settled = true;
|
|
354
|
+
if (slot.timeoutHandle)
|
|
355
|
+
clearTimeout(slot.timeoutHandle);
|
|
356
|
+
const sigId = slot.task.job.abortSignalId;
|
|
357
|
+
if (sigId !== undefined)
|
|
358
|
+
this.signalWorkerMap.delete(sigId);
|
|
359
|
+
this.activeCount--;
|
|
360
|
+
if (result.ok) {
|
|
361
|
+
if (this.listenerCount('taskEnd') > 0 && slot.startedAt > 0) {
|
|
362
|
+
this.emit('taskEnd', slot.task.job, Date.now() - slot.startedAt);
|
|
157
363
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
|
|
364
|
+
slot.task.resolve(result.data);
|
|
365
|
+
}
|
|
366
|
+
else {
|
|
367
|
+
this.handleFailure(slot.task, result.error ?? {
|
|
368
|
+
name: 'Error',
|
|
369
|
+
message: 'Unknown worker error',
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
// Give the slot back to the pool. Scheduling is the *caller*'s job so
|
|
373
|
+
// batched-result paths can release N slots before re-scheduling — that
|
|
374
|
+
// way schedule() can batch N new dispatches into a single postMessage
|
|
375
|
+
// back to the worker.
|
|
376
|
+
if (!this.destroyed) {
|
|
377
|
+
this.idle.push(worker);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
handleFailure(task, serializedError) {
|
|
381
|
+
const retry = task.retry ?? 0;
|
|
382
|
+
const retryDelay = task.retryDelay ?? 0;
|
|
383
|
+
if (this.listenerCount('taskError') > 0) {
|
|
168
384
|
this.emit('taskError', task.job, serializedError);
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
this.schedule();
|
|
175
|
-
};
|
|
176
|
-
if (delay > 0)
|
|
177
|
-
setTimeout(scheduleRetry, delay);
|
|
178
|
-
else
|
|
179
|
-
scheduleRetry();
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
// All attempts exhausted → dead letter
|
|
183
|
-
const dlEvent = {
|
|
184
|
-
jobId: task.job.jobId,
|
|
185
|
-
serviceName: task.job.serviceName,
|
|
186
|
-
methodName: task.job.methodName,
|
|
187
|
-
args: task.job.args,
|
|
188
|
-
attempts: task.attempts,
|
|
189
|
-
error: serializedError,
|
|
190
|
-
failedAt: new Date(),
|
|
385
|
+
}
|
|
386
|
+
if (task.attempts < retry + 1) {
|
|
387
|
+
const scheduleRetry = () => {
|
|
388
|
+
this.enqueue(task);
|
|
389
|
+
this.schedule();
|
|
191
390
|
};
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
try {
|
|
207
|
-
worker.postMessage({
|
|
208
|
-
type: 'ipc:result', callId: res.callId, ok: false,
|
|
209
|
-
error: `IPC result for "${req.propertyKey}.${req.methodName}" ` +
|
|
210
|
-
`is not structuredClone-compatible.`,
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
catch { /* worker gone */ }
|
|
214
|
-
}
|
|
215
|
-
};
|
|
216
|
-
if (!svcInstance) {
|
|
217
|
-
reply({ type: 'ipc:result', callId: req.callId, ok: false,
|
|
218
|
-
error: `No proxy registered for "${req.propertyKey}"` });
|
|
219
|
-
return;
|
|
220
|
-
}
|
|
221
|
-
const method = svcInstance[req.methodName];
|
|
222
|
-
if (typeof method !== 'function') {
|
|
223
|
-
reply({ type: 'ipc:result', callId: req.callId, ok: false,
|
|
224
|
-
error: `Method "${req.methodName}" not found on "${req.propertyKey}"` });
|
|
225
|
-
return;
|
|
226
|
-
}
|
|
227
|
-
try {
|
|
228
|
-
// Call via svcInstance[method]() to preserve `this` binding.
|
|
229
|
-
// Detached call (const fn = svcInstance[m]; fn()) loses `this`
|
|
230
|
-
// in strict mode, breaking any method that reads instance properties.
|
|
231
|
-
const data = await svcInstance[req.methodName](...req.args);
|
|
232
|
-
reply({ type: 'ipc:result', callId: req.callId, ok: true, data });
|
|
233
|
-
}
|
|
234
|
-
catch (err) {
|
|
235
|
-
reply({ type: 'ipc:result', callId: req.callId, ok: false,
|
|
236
|
-
error: err.message ?? String(err) });
|
|
237
|
-
}
|
|
238
|
-
return;
|
|
239
|
-
}
|
|
240
|
-
if (message.type === 'worker:ready')
|
|
241
|
-
return;
|
|
242
|
-
// ── Job result ────────────────────────────────────────────────────
|
|
243
|
-
const result = message;
|
|
244
|
-
const durationMs = Date.now() - startedAt;
|
|
245
|
-
settle(() => {
|
|
246
|
-
if (result.ok) {
|
|
247
|
-
this.emit('taskEnd', task.job, durationMs);
|
|
248
|
-
task.resolve(result.data);
|
|
249
|
-
}
|
|
250
|
-
else {
|
|
251
|
-
handleFailure(result.error ?? {
|
|
252
|
-
name: 'Error',
|
|
253
|
-
message: 'Unknown worker error',
|
|
254
|
-
});
|
|
255
|
-
}
|
|
256
|
-
});
|
|
391
|
+
if (retryDelay > 0)
|
|
392
|
+
setTimeout(scheduleRetry, retryDelay);
|
|
393
|
+
else
|
|
394
|
+
scheduleRetry();
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
const dlEvent = {
|
|
398
|
+
jobId: task.job.jobId,
|
|
399
|
+
serviceName: task.job.serviceName,
|
|
400
|
+
methodName: task.job.methodName,
|
|
401
|
+
args: task.job.args,
|
|
402
|
+
attempts: task.attempts,
|
|
403
|
+
error: serializedError,
|
|
404
|
+
failedAt: new Date(),
|
|
257
405
|
};
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
406
|
+
this.emit('dead', dlEvent);
|
|
407
|
+
task.reject(deserializeError(serializedError));
|
|
408
|
+
}
|
|
409
|
+
async handleTimeout(worker, slot) {
|
|
410
|
+
if (slot.settled)
|
|
411
|
+
return;
|
|
412
|
+
slot.settled = true;
|
|
413
|
+
const sigId = slot.task.job.abortSignalId;
|
|
414
|
+
if (sigId !== undefined)
|
|
415
|
+
this.signalWorkerMap.delete(sigId);
|
|
416
|
+
const state = getState(worker);
|
|
417
|
+
state.active.delete(slot.task.job.jobId);
|
|
418
|
+
this.activeCount--;
|
|
419
|
+
this.handleFailure(slot.task, {
|
|
420
|
+
name: 'TimeoutError',
|
|
421
|
+
message: `Task "${slot.task.job.serviceName}.${slot.task.job.methodName}" ` +
|
|
422
|
+
`timed out after ${slot.task.timeout}ms`,
|
|
423
|
+
});
|
|
424
|
+
// Timeouts terminate the worker (its event loop may be wedged) and
|
|
425
|
+
// replace it. All other in-flight jobs on this worker fail too.
|
|
426
|
+
try {
|
|
427
|
+
await worker.terminate();
|
|
279
428
|
}
|
|
280
|
-
|
|
429
|
+
catch { /* ignore */ }
|
|
430
|
+
this.replaceWorker(worker);
|
|
431
|
+
this.schedule();
|
|
281
432
|
}
|
|
282
433
|
handleWorkerError(worker, err) {
|
|
283
|
-
const
|
|
284
|
-
if (
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
434
|
+
const state = worker[STATE];
|
|
435
|
+
if (state) {
|
|
436
|
+
for (const slot of state.active.values()) {
|
|
437
|
+
if (slot.settled)
|
|
438
|
+
continue;
|
|
439
|
+
slot.settled = true;
|
|
440
|
+
if (slot.timeoutHandle)
|
|
441
|
+
clearTimeout(slot.timeoutHandle);
|
|
442
|
+
const sigId = slot.task.job.abortSignalId;
|
|
443
|
+
if (sigId !== undefined)
|
|
444
|
+
this.signalWorkerMap.delete(sigId);
|
|
445
|
+
const { serviceName, methodName } = slot.task.job;
|
|
446
|
+
const wrapped = new Error(`Worker crashed in "${serviceName}.${methodName}": ${err.message}`);
|
|
447
|
+
wrapped.stack = err.stack;
|
|
448
|
+
this.emit('error', wrapped, slot.task.job);
|
|
449
|
+
slot.task.reject(wrapped);
|
|
450
|
+
this.activeCount--;
|
|
451
|
+
}
|
|
452
|
+
state.active.clear();
|
|
291
453
|
}
|
|
292
454
|
this.replaceWorker(worker);
|
|
293
455
|
}
|
|
294
456
|
handleWorkerExit(worker, code) {
|
|
295
457
|
if (this.destroyed)
|
|
296
458
|
return;
|
|
297
|
-
const
|
|
298
|
-
if (
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
459
|
+
const state = worker[STATE];
|
|
460
|
+
if (state) {
|
|
461
|
+
for (const slot of state.active.values()) {
|
|
462
|
+
if (slot.settled)
|
|
463
|
+
continue;
|
|
464
|
+
slot.settled = true;
|
|
465
|
+
if (slot.timeoutHandle)
|
|
466
|
+
clearTimeout(slot.timeoutHandle);
|
|
467
|
+
const sigId = slot.task.job.abortSignalId;
|
|
468
|
+
if (sigId !== undefined)
|
|
469
|
+
this.signalWorkerMap.delete(sigId);
|
|
470
|
+
slot.task.reject(new Error(`Worker exited with code ${code} while running ` +
|
|
471
|
+
`"${slot.task.job.serviceName}.${slot.task.job.methodName}"`));
|
|
472
|
+
this.activeCount--;
|
|
473
|
+
}
|
|
474
|
+
state.active.clear();
|
|
302
475
|
}
|
|
303
476
|
this.replaceWorker(worker);
|
|
304
477
|
}
|
|
305
478
|
replaceWorker(oldWorker) {
|
|
306
479
|
const remove = (arr) => {
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
480
|
+
// Remove ALL occurrences (idle holds up to `concurrency` slots per worker).
|
|
481
|
+
let w = 0;
|
|
482
|
+
for (let r = 0; r < arr.length; r++) {
|
|
483
|
+
if (arr[r] !== oldWorker)
|
|
484
|
+
arr[w++] = arr[r];
|
|
485
|
+
}
|
|
486
|
+
arr.length = w;
|
|
310
487
|
};
|
|
311
488
|
remove(this.workers);
|
|
312
489
|
remove(this.idle);
|
|
@@ -319,24 +496,39 @@ class WorkerPool extends node_events_1.EventEmitter {
|
|
|
319
496
|
async destroy() {
|
|
320
497
|
this.destroyed = true;
|
|
321
498
|
// Drain: wait for all active jobs to finish, up to shutdownTimeout
|
|
322
|
-
if (this.
|
|
499
|
+
if (this.activeCount > 0) {
|
|
500
|
+
const activeTasks = [];
|
|
501
|
+
for (const worker of this.workers) {
|
|
502
|
+
const state = worker[STATE];
|
|
503
|
+
if (!state)
|
|
504
|
+
continue;
|
|
505
|
+
for (const slot of state.active.values())
|
|
506
|
+
activeTasks.push(slot.task);
|
|
507
|
+
}
|
|
323
508
|
await Promise.race([
|
|
324
|
-
|
|
325
|
-
Promise.allSettled(Array.from(this.active.values()).map(({ task }) => new Promise((res) => {
|
|
509
|
+
Promise.allSettled(activeTasks.map((task) => new Promise((res) => {
|
|
326
510
|
const orig = task.resolve;
|
|
327
511
|
const origRej = task.reject;
|
|
328
|
-
task.resolve = (v) => {
|
|
329
|
-
|
|
512
|
+
task.resolve = (v) => {
|
|
513
|
+
orig(v);
|
|
514
|
+
res();
|
|
515
|
+
};
|
|
516
|
+
task.reject = (e) => {
|
|
517
|
+
origRej(e);
|
|
518
|
+
res();
|
|
519
|
+
};
|
|
330
520
|
}))),
|
|
331
|
-
// Force after timeout
|
|
332
521
|
new Promise((res) => setTimeout(res, this.shutdownTimeout)),
|
|
333
522
|
]);
|
|
334
523
|
}
|
|
335
524
|
// Reject anything still queued
|
|
336
|
-
for (
|
|
337
|
-
queued
|
|
525
|
+
for (let i = this.queueHead; i < this.queue.length; i++) {
|
|
526
|
+
const queued = this.queue[i];
|
|
527
|
+
if (queued)
|
|
528
|
+
queued.reject(new Error('WorkerPool destroyed'));
|
|
338
529
|
}
|
|
339
530
|
this.queue.length = 0;
|
|
531
|
+
this.queueHead = 0;
|
|
340
532
|
await Promise.allSettled(this.workers.map((w) => w.terminate()));
|
|
341
533
|
this.workers.length = 0;
|
|
342
534
|
this.idle.length = 0;
|