@offmain/workerkit 0.14.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -46
- package/dist/define-worker.cjs +1 -0
- package/dist/define-worker.js +4 -0
- package/dist/index-2AONniOz.js +59 -0
- package/dist/index-CXKVsLvY.cjs +1 -0
- package/dist/index.cjs +195 -43
- package/dist/index.js +945 -487
- package/dist/types/tools/collect-results/collect-results.d.ts +26 -0
- package/dist/types/tools/collect-results/index.d.ts +2 -0
- package/dist/types/tools/collect-results/types.d.ts +10 -0
- package/dist/types/tools/define-worker/define-worker.d.ts +31 -0
- package/dist/types/tools/define-worker/define-worker.test.d.ts +1 -0
- package/dist/types/tools/define-worker/index.d.ts +1 -0
- package/dist/types/tools/define-worker-config/define-worker-config.d.ts +23 -0
- package/dist/types/tools/define-worker-config/index.d.ts +1 -0
- package/dist/types/tools/extract-transferable/extract-transferable.d.ts +15 -0
- package/dist/types/tools/extract-transferable/extract-transferable.test.d.ts +1 -0
- package/dist/types/tools/extract-transferable/index.d.ts +1 -0
- package/dist/types/tools/index.d.ts +2 -0
- package/dist/types/tools/logger/index.d.ts +2 -0
- package/dist/types/tools/logger/logger.d.ts +21 -0
- package/dist/types/tools/logger/logger.test.d.ts +1 -0
- package/dist/types/tools/logger/types.d.ts +6 -0
- package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +94 -228
- package/dist/types/tools/main-worker-factory/types.d.ts +71 -4
- package/dist/types/tools/memory-store/index.d.ts +3 -0
- package/dist/types/tools/memory-store/memory-store.d.ts +50 -0
- package/dist/types/tools/memory-store/memory-store.test.d.ts +1 -0
- package/dist/types/tools/memory-store/memory-worker-proxy.d.ts +75 -0
- package/dist/types/tools/memory-store/memory-worker.d.ts +11 -0
- package/dist/types/tools/orchestrator/index.d.ts +2 -0
- package/dist/types/tools/orchestrator/orchestrator.d.ts +21 -0
- package/dist/types/tools/orchestrator/orchestrator.test.d.ts +1 -0
- package/dist/types/tools/orchestrator/types.d.ts +10 -0
- package/dist/types/tools/partition-array/index.d.ts +1 -0
- package/dist/types/tools/partition-array/partition-array.d.ts +14 -0
- package/dist/types/tools/partition-array/partition-array.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/index.d.ts +2 -0
- package/dist/types/tools/persistent-manager/persistent-manager.d.ts +25 -0
- package/dist/types/tools/persistent-manager/persistent-manager.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/types.d.ts +9 -0
- package/dist/types/tools/pipeline/index.d.ts +2 -0
- package/dist/types/tools/pipeline/pipeline.d.ts +17 -0
- package/dist/types/tools/pipeline/pipeline.test.d.ts +1 -0
- package/dist/types/tools/pipeline/types.d.ts +13 -0
- package/dist/types/tools/run-worker/index.d.ts +2 -0
- package/dist/types/tools/run-worker/run-worker.d.ts +26 -0
- package/dist/types/tools/run-worker/run-worker.test.d.ts +1 -0
- package/dist/types/tools/run-worker/types.d.ts +14 -0
- package/dist/types/tools/worker-factory/index.d.ts +1 -1
- package/dist/types/tools/worker-factory/worker-factory.d.ts +7 -1
- package/dist/types/workers/initiator.d.ts +1 -1
- package/dist/types/workers/initiator.test.d.ts +1 -0
- package/package.json +11 -5
- package/dist/types/tools/define-worker.d.ts +0 -21
- /package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts} +0 -0
package/dist/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var y = (
|
|
4
|
-
|
|
1
|
+
var C = Object.defineProperty;
|
|
2
|
+
var O = (o, e, r) => e in o ? C(o, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : o[e] = r;
|
|
3
|
+
var y = (o, e, r) => O(o, typeof e != "symbol" ? e + "" : e, r);
|
|
4
|
+
import { e as E } from "./index-2AONniOz.js";
|
|
5
|
+
import { d as ie } from "./index-2AONniOz.js";
|
|
6
|
+
const R = `
|
|
5
7
|
const extractTransferables = (value, seen = new Set()) => {
|
|
6
8
|
if (value === null || typeof value !== 'object') return [];
|
|
7
9
|
if (seen.has(value)) return [];
|
|
@@ -15,31 +17,73 @@ const extractTransferables = (value, seen = new Set()) => {
|
|
|
15
17
|
if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
|
|
16
18
|
return Object.values(value).flatMap(v => extractTransferables(v, seen));
|
|
17
19
|
};
|
|
18
|
-
|
|
20
|
+
`, U = (o) => `
|
|
21
|
+
${R}
|
|
19
22
|
self.addEventListener('message', async (event) => {
|
|
20
23
|
try {
|
|
21
|
-
const output = await ${
|
|
24
|
+
const output = await ${o}(event.data);
|
|
22
25
|
self.postMessage({ ok: true, data: output }, extractTransferables(output));
|
|
23
26
|
} catch (err) {
|
|
24
27
|
self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
25
28
|
}
|
|
26
29
|
})
|
|
27
|
-
`,
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
30
|
+
`, N = (o) => `
|
|
31
|
+
${R}
|
|
32
|
+
const workerFn = ${o};
|
|
33
|
+
let memPort = null;
|
|
34
|
+
let factoryToken = null;
|
|
35
|
+
let pendingPayload = null;
|
|
36
|
+
|
|
37
|
+
async function runAndStore(payload) {
|
|
38
|
+
try {
|
|
39
|
+
const output = await workerFn(payload);
|
|
40
|
+
if (!memPort) {
|
|
41
|
+
// Fallback: no port provided, post data directly (legacy / opt-out path)
|
|
42
|
+
self.postMessage({ ok: true, data: output }, extractTransferables(output));
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
// Store directly into MemoryWorker
|
|
46
|
+
const ref = 'mem_' + crypto.randomUUID();
|
|
47
|
+
await new Promise((resolve, reject) => {
|
|
48
|
+
memPort.onmessage = (e) => {
|
|
49
|
+
if (e.data && e.data.ref === ref) {
|
|
50
|
+
if (e.data.ok) resolve(e.data.ref);
|
|
51
|
+
else reject(new Error(e.data.error || 'MemoryWorker SET failed'));
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
memPort.postMessage({ action: 'SET', factoryToken, ref, data: output });
|
|
55
|
+
});
|
|
56
|
+
// Only the token reaches the main thread
|
|
57
|
+
self.postMessage({ ok: true, __memory_ref__: ref });
|
|
58
|
+
} catch (err) {
|
|
59
|
+
self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
36
60
|
}
|
|
37
|
-
|
|
38
|
-
if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
|
|
39
|
-
return Object.values(value).flatMap(v => extractTransferables(v, seen));
|
|
40
|
-
};
|
|
61
|
+
}
|
|
41
62
|
|
|
42
|
-
|
|
63
|
+
self.addEventListener('message', async (event) => {
|
|
64
|
+
// Init message: receive the MemoryWorker port and factory token
|
|
65
|
+
if (event.data && event.data.__init_memory_port__) {
|
|
66
|
+
memPort = event.ports[0] ?? event.data.memPort;
|
|
67
|
+
factoryToken = event.data.factoryToken;
|
|
68
|
+
if (memPort) memPort.start();
|
|
69
|
+
// If a payload arrived before the port, process it now
|
|
70
|
+
if (pendingPayload !== null) {
|
|
71
|
+
const p = pendingPayload;
|
|
72
|
+
pendingPayload = null;
|
|
73
|
+
await runAndStore(p);
|
|
74
|
+
}
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
// If port not yet received, queue the payload
|
|
78
|
+
if (!memPort) {
|
|
79
|
+
pendingPayload = event.data;
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
await runAndStore(event.data);
|
|
83
|
+
});
|
|
84
|
+
`, j = (o) => `
|
|
85
|
+
${R}
|
|
86
|
+
const workerFn = ${o};
|
|
43
87
|
let outputPort = null;
|
|
44
88
|
let inputPort = null;
|
|
45
89
|
let pendingData = null;
|
|
@@ -104,22 +148,9 @@ self.addEventListener('message', (event) => {
|
|
|
104
148
|
pendingData = event.data;
|
|
105
149
|
}
|
|
106
150
|
});
|
|
107
|
-
`,
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (seen.has(value)) return [];
|
|
111
|
-
seen.add(value);
|
|
112
|
-
if (value instanceof ArrayBuffer || value instanceof MessagePort ||
|
|
113
|
-
(typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
|
|
114
|
-
(typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
|
|
115
|
-
return [value];
|
|
116
|
-
}
|
|
117
|
-
if (ArrayBuffer.isView(value)) return [value.buffer];
|
|
118
|
-
if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
|
|
119
|
-
return Object.values(value).flatMap(v => extractTransferables(v, seen));
|
|
120
|
-
};
|
|
121
|
-
|
|
122
|
-
const workerFn = ${n};
|
|
151
|
+
`, G = (o) => `
|
|
152
|
+
${R}
|
|
153
|
+
const workerFn = ${o};
|
|
123
154
|
let cachedDataset = null;
|
|
124
155
|
|
|
125
156
|
self.addEventListener('message', async (event) => {
|
|
@@ -161,13 +192,14 @@ self.addEventListener('message', async (event) => {
|
|
|
161
192
|
}
|
|
162
193
|
});
|
|
163
194
|
`;
|
|
164
|
-
var
|
|
165
|
-
const
|
|
166
|
-
persistent:
|
|
167
|
-
pipeline:
|
|
168
|
-
default:
|
|
195
|
+
var D = /* @__PURE__ */ ((o) => (o.Default = "default", o.Pipeline = "pipeline", o.Persistent = "persistent", o.Memory = "memory", o))(D || {});
|
|
196
|
+
const q = Object.freeze({
|
|
197
|
+
persistent: G,
|
|
198
|
+
pipeline: j,
|
|
199
|
+
default: U,
|
|
200
|
+
memory: N
|
|
169
201
|
});
|
|
170
|
-
class
|
|
202
|
+
class b {
|
|
171
203
|
/**
|
|
172
204
|
* Creates a new `Worker` from the given function or factory option.
|
|
173
205
|
*
|
|
@@ -181,7 +213,7 @@ class W {
|
|
|
181
213
|
if (r != null && r.createWorker)
|
|
182
214
|
this._worker = r.createWorker();
|
|
183
215
|
else if (e) {
|
|
184
|
-
const
|
|
216
|
+
const a = (r == null ? void 0 : r.mode) ?? "default", t = q[a](e.toString()), l = new Blob([t], {
|
|
185
217
|
type: "application/javascript"
|
|
186
218
|
});
|
|
187
219
|
this._worker = new Worker(URL.createObjectURL(l));
|
|
@@ -200,592 +232,1018 @@ class W {
|
|
|
200
232
|
return this._worker;
|
|
201
233
|
}
|
|
202
234
|
}
|
|
203
|
-
class
|
|
204
|
-
constructor(
|
|
205
|
-
this
|
|
235
|
+
class z {
|
|
236
|
+
constructor() {
|
|
237
|
+
y(this, "store", /* @__PURE__ */ new Map());
|
|
206
238
|
}
|
|
207
|
-
}
|
|
208
|
-
function m(n, e = /* @__PURE__ */ new Set()) {
|
|
209
|
-
return n === null || typeof n != "object" ? [] : e.has(n) ? [] : (e.add(n), n instanceof ArrayBuffer || n instanceof MessagePort || typeof ImageBitmap < "u" && n instanceof ImageBitmap || typeof OffscreenCanvas < "u" && n instanceof OffscreenCanvas ? [n] : ArrayBuffer.isView(n) ? [n.buffer] : Array.isArray(n) ? n.flatMap((r) => m(r, e)) : Object.values(n).flatMap(
|
|
210
|
-
(r) => m(r, e)
|
|
211
|
-
));
|
|
212
|
-
}
|
|
213
|
-
class U {
|
|
214
239
|
/**
|
|
215
|
-
*
|
|
240
|
+
* Registers a ref in the registry.
|
|
216
241
|
*
|
|
217
|
-
* @param
|
|
242
|
+
* @param ref - The memory reference ID (generated by MemoryWorkerProxy).
|
|
243
|
+
* @param metadata - Optional metadata (size hint, type tag, etc.).
|
|
218
244
|
*/
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
y(this, "_threads");
|
|
222
|
-
y(this, "_persistentWorkers", /* @__PURE__ */ new Map());
|
|
223
|
-
y(this, "_activeWorkers", /* @__PURE__ */ new Set());
|
|
224
|
-
y(this, "_isTerminated", !1);
|
|
225
|
-
this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
|
|
245
|
+
register(e, r = {}) {
|
|
246
|
+
this.store.set(e, r);
|
|
226
247
|
}
|
|
227
248
|
/**
|
|
228
|
-
*
|
|
249
|
+
* Checks if a reference ID is registered.
|
|
250
|
+
* @param refId - The reference ID to check.
|
|
229
251
|
*/
|
|
230
|
-
|
|
231
|
-
return this.
|
|
252
|
+
has(e) {
|
|
253
|
+
return this.store.has(e);
|
|
232
254
|
}
|
|
233
255
|
/**
|
|
234
|
-
*
|
|
256
|
+
* Unregisters a reference ID.
|
|
257
|
+
*
|
|
258
|
+
* @param refId - The reference ID to unregister.
|
|
259
|
+
* @returns `true` if the key existed and was removed, `false` otherwise.
|
|
235
260
|
*/
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
throw e.terminate(), new Error("MainWorkerFactory has been terminated");
|
|
239
|
-
return this._activeWorkers.add(e), e;
|
|
261
|
+
delete(e) {
|
|
262
|
+
return this.store.delete(e);
|
|
240
263
|
}
|
|
241
264
|
/**
|
|
242
|
-
*
|
|
265
|
+
* Clears all registered references.
|
|
243
266
|
*/
|
|
244
|
-
|
|
245
|
-
this.
|
|
246
|
-
try {
|
|
247
|
-
e.terminate();
|
|
248
|
-
} catch {
|
|
249
|
-
}
|
|
267
|
+
clear() {
|
|
268
|
+
this.store.clear();
|
|
250
269
|
}
|
|
251
270
|
/**
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
* @param config - The worker configuration containing `func` or `createWorker`.
|
|
255
|
-
* @returns A new `WorkerFactory` wrapping the worker.
|
|
271
|
+
* Returns statistics about current registered handles.
|
|
256
272
|
*/
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
273
|
+
stats() {
|
|
274
|
+
return {
|
|
275
|
+
count: this.store.size,
|
|
276
|
+
refs: Array.from(this.store.keys())
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const K = `
|
|
281
|
+
const store = new Map();
|
|
282
|
+
|
|
283
|
+
function handleMessage(msg, replyTarget) {
|
|
284
|
+
if (!msg || typeof msg !== 'object') return;
|
|
285
|
+
|
|
286
|
+
const { action, factoryToken, expectedToken, ref, data, id } = msg;
|
|
287
|
+
|
|
288
|
+
// Initial handshake to set expected token if needed
|
|
289
|
+
if (action === 'INIT_TOKEN') {
|
|
290
|
+
self.__expectedToken = expectedToken;
|
|
291
|
+
replyTarget.postMessage({ ok: true, action: 'INIT_TOKEN_ACK' });
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// Validate factory token
|
|
296
|
+
if (self.__expectedToken && factoryToken !== self.__expectedToken) {
|
|
297
|
+
replyTarget.postMessage({ ok: false, error: 'Unauthorized: invalid factory token', id });
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
try {
|
|
302
|
+
switch (action) {
|
|
303
|
+
case 'SET': {
|
|
304
|
+
const refId = ref || ('mem_' + crypto.randomUUID());
|
|
305
|
+
store.set(refId, data);
|
|
306
|
+
replyTarget.postMessage({ ok: true, ref: refId, id });
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
case 'GET': {
|
|
310
|
+
const resultData = store.get(ref);
|
|
311
|
+
const exists = store.has(ref);
|
|
312
|
+
replyTarget.postMessage({ ok: true, exists, data: resultData, ref, id });
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
case 'DELETE': {
|
|
316
|
+
const deleted = store.delete(ref);
|
|
317
|
+
replyTarget.postMessage({ ok: true, deleted, ref, id });
|
|
318
|
+
break;
|
|
319
|
+
}
|
|
320
|
+
case 'CLEAR': {
|
|
321
|
+
store.clear();
|
|
322
|
+
replyTarget.postMessage({ ok: true, action: 'CLEAR_ACK', id });
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
case 'STATS': {
|
|
326
|
+
replyTarget.postMessage({
|
|
327
|
+
ok: true,
|
|
328
|
+
stats: {
|
|
329
|
+
count: store.size,
|
|
330
|
+
refs: Array.from(store.keys()),
|
|
331
|
+
},
|
|
332
|
+
id,
|
|
333
|
+
});
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
case 'REGISTER_PORT': {
|
|
337
|
+
// Register a MessagePort from a computing or reducer worker.
|
|
338
|
+
// All messages arriving on this port are handled with the same
|
|
339
|
+
// store operations, enabling direct worker-to-MemoryWorker data flow.
|
|
340
|
+
const port = msg.port;
|
|
341
|
+
if (!port) {
|
|
342
|
+
replyTarget.postMessage({ ok: false, error: 'REGISTER_PORT requires a port', id });
|
|
343
|
+
break;
|
|
344
|
+
}
|
|
345
|
+
port.onmessage = (event) => handleMessage(event.data, port);
|
|
346
|
+
port.start();
|
|
347
|
+
replyTarget.postMessage({ ok: true, action: 'PORT_REGISTERED', id });
|
|
348
|
+
break;
|
|
349
|
+
}
|
|
350
|
+
default:
|
|
351
|
+
replyTarget.postMessage({ ok: false, error: 'Unknown action: ' + action, id });
|
|
352
|
+
}
|
|
353
|
+
} catch (err) {
|
|
354
|
+
replyTarget.postMessage({
|
|
355
|
+
ok: false,
|
|
356
|
+
error: err instanceof Error ? err.message : String(err),
|
|
357
|
+
id,
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
self.addEventListener('message', (event) => {
|
|
363
|
+
// Handle REGISTER_PORT specially — the port itself is a Transferable in event.ports
|
|
364
|
+
if (event.data && event.data.action === 'REGISTER_PORT') {
|
|
365
|
+
const port = event.ports[0] ?? event.data.port;
|
|
366
|
+
const msg = { ...event.data, port };
|
|
367
|
+
handleMessage(msg, self);
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
handleMessage(event.data, self);
|
|
371
|
+
});
|
|
372
|
+
`;
|
|
373
|
+
class H {
|
|
374
|
+
constructor(e) {
|
|
375
|
+
y(this, "worker");
|
|
376
|
+
y(this, "factoryToken");
|
|
377
|
+
y(this, "pending", /* @__PURE__ */ new Map());
|
|
378
|
+
this.factoryToken = e;
|
|
379
|
+
const r = new Blob([K], {
|
|
380
|
+
type: "application/javascript"
|
|
381
|
+
});
|
|
382
|
+
this.worker = new Worker(URL.createObjectURL(r)), this.worker.onmessage = (a) => {
|
|
383
|
+
const { id: t } = a.data ?? {};
|
|
384
|
+
if (t && this.pending.has(t)) {
|
|
385
|
+
const l = this.pending.get(t);
|
|
386
|
+
this.pending.delete(t), l(a.data);
|
|
387
|
+
}
|
|
388
|
+
}, this.worker.postMessage({
|
|
389
|
+
action: "INIT_TOKEN",
|
|
390
|
+
expectedToken: e
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
send(e, r = {}) {
|
|
394
|
+
return new Promise((a) => {
|
|
395
|
+
const t = `req_${crypto.randomUUID()}`;
|
|
396
|
+
this.pending.set(t, a), this.worker.postMessage({
|
|
397
|
+
action: e,
|
|
398
|
+
factoryToken: this.factoryToken,
|
|
399
|
+
id: t,
|
|
400
|
+
...r
|
|
401
|
+
});
|
|
260
402
|
});
|
|
261
|
-
return this.trackWorker(r.getWorker), r;
|
|
262
403
|
}
|
|
263
404
|
/**
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
* When the array length is not evenly divisible, the first `remainder`
|
|
267
|
-
* chunks receive one extra element so no data is lost.
|
|
405
|
+
* Stores a dataset in the MemoryWorker.
|
|
268
406
|
*
|
|
269
|
-
* @param
|
|
270
|
-
* @param
|
|
271
|
-
*
|
|
272
|
-
* @returns An array of sub-arrays. Returns `[]` when `array` is empty.
|
|
273
|
-
* @throws {Error} When `numChunks` is not a positive integer.
|
|
274
|
-
*
|
|
275
|
-
* @example
|
|
276
|
-
* partitionArray([1, 2, 3, 4, 5], 3);
|
|
277
|
-
* // → [[1, 2], [3, 4], [5]]
|
|
407
|
+
* @param data - The dataset to store.
|
|
408
|
+
* @param ref - Optional ref ID; if omitted, MemoryWorker generates one.
|
|
409
|
+
* @returns The ref ID under which the data is stored.
|
|
278
410
|
*/
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
if (
|
|
282
|
-
|
|
283
|
-
let f = 0;
|
|
284
|
-
for (let a = 0; a < s; a++) {
|
|
285
|
-
const o = t + (a < l ? 1 : 0);
|
|
286
|
-
c.push(e.slice(f, f + o)), f += o;
|
|
287
|
-
}
|
|
288
|
-
return c;
|
|
411
|
+
async set(e, r) {
|
|
412
|
+
const a = await this.send("SET", { data: e, ref: r });
|
|
413
|
+
if (!a.ok) throw new Error(a.error);
|
|
414
|
+
return a.ref;
|
|
289
415
|
}
|
|
290
416
|
/**
|
|
291
|
-
*
|
|
417
|
+
* Retrieves a dataset from the MemoryWorker by ref ID.
|
|
292
418
|
*
|
|
293
|
-
* @param
|
|
294
|
-
* @returns The
|
|
419
|
+
* @param ref - The ref ID to retrieve.
|
|
420
|
+
* @returns The stored dataset, or `undefined` if not found.
|
|
295
421
|
*/
|
|
296
|
-
|
|
297
|
-
|
|
422
|
+
async get(e) {
|
|
423
|
+
const r = await this.send("GET", { ref: e });
|
|
424
|
+
if (!r.ok) throw new Error(r.error);
|
|
425
|
+
return r.data;
|
|
298
426
|
}
|
|
299
427
|
/**
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
*
|
|
310
|
-
* the settled output.
|
|
311
|
-
*
|
|
312
|
-
* @typeParam TName - The literal name of the worker to run (inferred from
|
|
313
|
-
* the registered `workers` tuple).
|
|
314
|
-
*
|
|
315
|
-
* @param workerName - Name of the worker as declared in the `workers` config.
|
|
316
|
-
* @param params - Object containing `srcData` (the payload) plus any
|
|
317
|
-
* additional key/value pairs forwarded to the worker verbatim.
|
|
318
|
-
*
|
|
319
|
-
* @returns A {@link TypedSettledResults} wrapping the settled promises from
|
|
320
|
-
* all spawned worker threads.
|
|
428
|
+
* Checks if a ref ID exists in the MemoryWorker.
|
|
429
|
+
* @param ref - The ref ID to check.
|
|
430
|
+
*/
|
|
431
|
+
async has(e) {
|
|
432
|
+
const r = await this.send("GET", { ref: e });
|
|
433
|
+
if (!r.ok) throw new Error(r.error);
|
|
434
|
+
return !!r.exists;
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Deletes a ref from the MemoryWorker.
|
|
321
438
|
*
|
|
322
|
-
* @
|
|
323
|
-
*
|
|
439
|
+
* @param ref - The ref ID to delete.
|
|
440
|
+
* @returns `true` if the ref existed and was deleted.
|
|
324
441
|
*/
|
|
325
|
-
async
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
const
|
|
335
|
-
|
|
336
|
-
e,
|
|
337
|
-
{ data: f, ...s },
|
|
338
|
-
l,
|
|
339
|
-
c
|
|
340
|
-
), o = await Promise.allSettled(a);
|
|
341
|
-
return new L(o);
|
|
442
|
+
async delete(e) {
|
|
443
|
+
const r = await this.send("DELETE", { ref: e });
|
|
444
|
+
if (!r.ok) throw new Error(r.error);
|
|
445
|
+
return !!r.deleted;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Clears all data from the MemoryWorker.
|
|
449
|
+
*/
|
|
450
|
+
async clear() {
|
|
451
|
+
const e = await this.send("CLEAR");
|
|
452
|
+
if (!e.ok) throw new Error(e.error);
|
|
342
453
|
}
|
|
343
454
|
/**
|
|
344
|
-
*
|
|
345
|
-
|
|
455
|
+
* Returns statistics about data held in the MemoryWorker.
|
|
456
|
+
*/
|
|
457
|
+
async stats() {
|
|
458
|
+
const e = await this.send("STATS");
|
|
459
|
+
if (!e.ok) throw new Error(e.error);
|
|
460
|
+
return e.stats;
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Allocates a direct `MessagePort` to the MemoryWorker for a computing or
|
|
464
|
+
* reducer worker.
|
|
346
465
|
*
|
|
347
|
-
*
|
|
348
|
-
* `
|
|
466
|
+
* Creates a `MessageChannel` and registers one port with MemoryWorker via
|
|
467
|
+
* `REGISTER_PORT`. The other port (the "worker-side" port) is returned as a
|
|
468
|
+
* `Transferable` — it should be passed to the target worker via
|
|
469
|
+
* `postMessage(..., [workerPort])` so the worker can communicate with
|
|
470
|
+
* MemoryWorker directly without routing through the main thread.
|
|
349
471
|
*
|
|
350
|
-
* @
|
|
351
|
-
* @param workerName - Name used in error/retry logging.
|
|
352
|
-
* @param srcWorkerData - Combined `{ data, ...otherParams }` payload.
|
|
353
|
-
* @param threadCount - Number of parallel worker threads to spawn.
|
|
354
|
-
* @param isPartitioned - Whether `data` is a pre-split array of shards.
|
|
355
|
-
* @returns An array of promises, one per thread.
|
|
472
|
+
* @returns The worker-side `MessagePort` ready to be transferred.
|
|
356
473
|
*/
|
|
357
|
-
|
|
358
|
-
const {
|
|
359
|
-
return
|
|
360
|
-
|
|
474
|
+
async allocateWorkerPort() {
|
|
475
|
+
const e = new MessageChannel(), { port1: r, port2: a } = e, t = `req_${crypto.randomUUID()}`;
|
|
476
|
+
return this.pending.set(t, () => {
|
|
477
|
+
}), this.worker.postMessage(
|
|
478
|
+
{
|
|
479
|
+
action: "REGISTER_PORT",
|
|
480
|
+
factoryToken: this.factoryToken,
|
|
481
|
+
id: t
|
|
482
|
+
},
|
|
483
|
+
[r]
|
|
484
|
+
), await new Promise((l) => {
|
|
485
|
+
const f = this.pending.get(t);
|
|
486
|
+
this.pending.set(t, (m) => {
|
|
487
|
+
f(m), l();
|
|
488
|
+
});
|
|
489
|
+
}), a;
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* Terminates the MemoryWorker thread and clears all pending requests.
|
|
493
|
+
*/
|
|
494
|
+
terminate() {
|
|
495
|
+
this.pending.clear(), this.worker.terminate();
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
class J {
|
|
499
|
+
constructor(e = "error") {
|
|
500
|
+
y(this, "level");
|
|
501
|
+
this.level = e;
|
|
502
|
+
}
|
|
503
|
+
setLevel(e) {
|
|
504
|
+
this.level = e;
|
|
505
|
+
}
|
|
506
|
+
verbose(...e) {
|
|
507
|
+
this.level === "verbose" && console.debug("[WorkerManager]", ...e);
|
|
508
|
+
}
|
|
509
|
+
info(...e) {
|
|
510
|
+
(this.level === "verbose" || this.level === "info") && console.info("[WorkerManager]", ...e);
|
|
511
|
+
}
|
|
512
|
+
error(...e) {
|
|
513
|
+
this.level !== "silent" && console.error("[WorkerManager]", ...e);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
class V {
|
|
517
|
+
constructor(e) {
|
|
518
|
+
this.results = e;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
function B(o, e) {
|
|
522
|
+
if (!o.length) return [];
|
|
523
|
+
if (e <= 0) throw new Error("numChunks must be positive");
|
|
524
|
+
const r = Math.min(e, o.length), a = Math.floor(o.length / r), t = o.length % r, l = [];
|
|
525
|
+
let f = 0;
|
|
526
|
+
for (let m = 0; m < r; m++) {
|
|
527
|
+
const i = a + (m < t ? 1 : 0);
|
|
528
|
+
l.push(o.slice(f, f + i)), f += i;
|
|
529
|
+
}
|
|
530
|
+
return l;
|
|
531
|
+
}
|
|
532
|
+
async function x(o, e = {}, r) {
|
|
533
|
+
if (r.isTerminated())
|
|
534
|
+
throw r.logger.error(
|
|
535
|
+
"Attempted to collect results after MainWorkerFactory was terminated"
|
|
536
|
+
), new Error("MainWorkerFactory has been terminated");
|
|
537
|
+
const a = o.results.filter(
|
|
538
|
+
(n) => n.status === "fulfilled"
|
|
539
|
+
), t = o.results.filter(
|
|
540
|
+
(n) => n.status === "rejected"
|
|
541
|
+
);
|
|
542
|
+
if (a.length === 0)
|
|
543
|
+
return {
|
|
544
|
+
data: [],
|
|
545
|
+
succeeded: 0,
|
|
546
|
+
failed: t.length,
|
|
547
|
+
errors: t
|
|
548
|
+
};
|
|
549
|
+
const l = a.map((n) => {
|
|
550
|
+
var k;
|
|
551
|
+
const c = (k = n.value.successResult) == null ? void 0 : k.data;
|
|
552
|
+
return c == null ? void 0 : c.__memory_ref__;
|
|
553
|
+
}), f = l.every((n) => typeof n == "string"), m = e.reducer ? e.reducer.toString() : "(shards) => shards.flat()";
|
|
554
|
+
let i;
|
|
555
|
+
if (f && typeof Worker < "u" && typeof Blob < "u" && typeof MessageChannel < "u") {
|
|
556
|
+
const n = await r.memoryWorkerProxy.allocateWorkerPort(), c = `
|
|
557
|
+
const refs = ${JSON.stringify(l)};
|
|
558
|
+
const factoryToken = ${JSON.stringify(r.factoryToken)};
|
|
559
|
+
const reducer = ${m};
|
|
560
|
+
|
|
561
|
+
// collect-results.ts
|
|
562
|
+
|
|
563
|
+
self.addEventListener('message', async (event) => {
|
|
564
|
+
// event.ports[0] is the direct line to MemoryWorker
|
|
565
|
+
const memPort = event.ports[0];
|
|
566
|
+
if (!memPort) {
|
|
567
|
+
self.postMessage({ ok: false, error: 'No MemoryWorker port provided' });
|
|
568
|
+
self.close();
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
memPort.start();
|
|
572
|
+
|
|
573
|
+
try {
|
|
574
|
+
// Fetch each shard directly from MemoryWorker
|
|
575
|
+
const shards = await Promise.all(refs.map((ref) =>
|
|
576
|
+
new Promise((resolve, reject) => {
|
|
577
|
+
const reqId = 'fetch_' + ref;
|
|
578
|
+
memPort.addEventListener('message', function handler(e) {
|
|
579
|
+
if (e.data && e.data.ref === ref) {
|
|
580
|
+
memPort.removeEventListener('message', handler);
|
|
581
|
+
if (e.data.ok) resolve(e.data.data);
|
|
582
|
+
else reject(new Error(e.data.error));
|
|
583
|
+
}
|
|
584
|
+
});
|
|
585
|
+
memPort.postMessage({ action: 'GET', factoryToken, ref, id: reqId });
|
|
586
|
+
})
|
|
587
|
+
));
|
|
588
|
+
|
|
589
|
+
const result = reducer(shards);
|
|
590
|
+
self.postMessage({ ok: true, data: result });
|
|
591
|
+
} catch (error) {
|
|
592
|
+
self.postMessage({ ok: false, error: String(error) });
|
|
593
|
+
} finally {
|
|
594
|
+
self.close();
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
`;
|
|
598
|
+
try {
|
|
599
|
+
i = await new Promise((k, d) => {
|
|
600
|
+
const p = new Blob([c], { type: "application/javascript" }), s = r.trackWorker(
|
|
601
|
+
new Worker(URL.createObjectURL(p))
|
|
602
|
+
);
|
|
603
|
+
s.onmessage = (u) => {
|
|
604
|
+
r.terminateWorker(s), u.data.ok ? k(u.data.data) : (r.logger.error(
|
|
605
|
+
"Collect results reducer worker failed",
|
|
606
|
+
u.data.error
|
|
607
|
+
), d(new Error(u.data.error)));
|
|
608
|
+
}, s.onerror = (u) => {
|
|
609
|
+
r.terminateWorker(s), r.logger.error(
|
|
610
|
+
"Collect results reducer worker encountered an error event",
|
|
611
|
+
u
|
|
612
|
+
), d(u);
|
|
613
|
+
}, s.postMessage({ __start__: !0 }, [n]);
|
|
614
|
+
});
|
|
615
|
+
} catch {
|
|
616
|
+
const k = await Promise.all(
|
|
617
|
+
l.map((s) => r.memoryWorkerProxy.get(s))
|
|
618
|
+
), d = (s) => s.flat();
|
|
619
|
+
i = (e.reducer ?? d)(k);
|
|
620
|
+
}
|
|
621
|
+
} else if (f) {
|
|
622
|
+
const n = await Promise.all(
|
|
623
|
+
l.map((d) => r.memoryWorkerProxy.get(d))
|
|
624
|
+
), c = (d) => d.flat();
|
|
625
|
+
i = (e.reducer ?? c)(n);
|
|
626
|
+
} else {
|
|
627
|
+
const n = a.map((d) => d.value.successResult.data), c = (d) => d.flat();
|
|
628
|
+
i = (e.reducer ?? c)(n);
|
|
629
|
+
}
|
|
630
|
+
return {
|
|
631
|
+
data: i,
|
|
632
|
+
succeeded: a.length,
|
|
633
|
+
failed: t.length,
|
|
634
|
+
errors: t
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
async function X(o, e, r) {
|
|
638
|
+
var v, S;
|
|
639
|
+
if (r.isTerminated())
|
|
640
|
+
return r.logger.error(
|
|
641
|
+
"Attempted to execute worker after MainWorkerFactory was terminated"
|
|
642
|
+
), Promise.reject(new Error("MainWorkerFactory has been terminated"));
|
|
643
|
+
const a = r.findWorkerByName(o);
|
|
644
|
+
if (!a)
|
|
645
|
+
return r.logger.error(`Worker config not found for worker: "${o}"`), Promise.reject(new Error(`Worker "${o}" not found`));
|
|
646
|
+
const {
|
|
647
|
+
srcData: t,
|
|
648
|
+
reducer: l,
|
|
649
|
+
autoCollect: f = !0,
|
|
650
|
+
...m
|
|
651
|
+
} = e || {};
|
|
652
|
+
let i = t;
|
|
653
|
+
const n = m.__memory_ref__, c = !!m.deleteMemory;
|
|
654
|
+
if (i === void 0 && n) {
|
|
655
|
+
if (!r.memoryStore.has(n))
|
|
656
|
+
return r.logger.error(
|
|
657
|
+
`Attempted to use invalid memory reference: "${n}"`
|
|
658
|
+
), Promise.reject(
|
|
659
|
+
new Error(`Memory reference "${n}" not found in MemoryStore`)
|
|
660
|
+
);
|
|
661
|
+
i = await r.memoryWorkerProxy.get(n), delete m.__memory_ref__, delete m.deleteMemory;
|
|
662
|
+
} else
|
|
663
|
+
delete m.deleteMemory;
|
|
664
|
+
const k = a.maxConcurrency ?? r.threads, d = !!(Array.isArray(i) && a.partition), p = d ? B(i, k) : i, s = d && Array.isArray(p) ? p.length : k, u = r.orchestrator.createWorkerPromises(
|
|
665
|
+
a,
|
|
666
|
+
o,
|
|
667
|
+
{ data: p, ...m },
|
|
668
|
+
s,
|
|
669
|
+
d
|
|
670
|
+
), h = await Promise.allSettled(u);
|
|
671
|
+
c && n && (await r.memoryWorkerProxy.delete(n), r.memoryStore.delete(n));
|
|
672
|
+
for (const g of h)
|
|
673
|
+
if (g.status === "fulfilled") {
|
|
674
|
+
const W = (v = g.value.successResult) == null ? void 0 : v.data, w = W == null ? void 0 : W.__memory_ref__;
|
|
675
|
+
w && r.memoryStore.register(w);
|
|
676
|
+
}
|
|
677
|
+
const _ = new V(h);
|
|
678
|
+
if (!f) {
|
|
679
|
+
const g = h.filter((w) => w.status === "fulfilled").length, W = h.filter(
|
|
680
|
+
(w) => w.status === "rejected"
|
|
681
|
+
);
|
|
682
|
+
return {
|
|
683
|
+
data: _,
|
|
684
|
+
succeeded: g,
|
|
685
|
+
failed: W.length,
|
|
686
|
+
errors: W
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
const T = await x(
|
|
690
|
+
_,
|
|
691
|
+
{ reducer: l },
|
|
692
|
+
{
|
|
693
|
+
isTerminated: () => r.isTerminated(),
|
|
694
|
+
trackWorker: r.orchestrator.context ? (g) => g : (g) => g,
|
|
695
|
+
terminateWorker: (g) => {
|
|
696
|
+
},
|
|
697
|
+
logger: r.logger,
|
|
698
|
+
memoryWorkerProxy: r.memoryWorkerProxy,
|
|
699
|
+
factoryToken: r.factoryToken
|
|
700
|
+
}
|
|
701
|
+
), M = !!(a.memory || a.memoryOnly);
|
|
702
|
+
for (const g of h)
|
|
703
|
+
if (g.status === "fulfilled") {
|
|
704
|
+
const W = (S = g.value.successResult) == null ? void 0 : S.data, w = W == null ? void 0 : W.__memory_ref__;
|
|
705
|
+
w && (r.memoryStore.delete(w), await r.memoryWorkerProxy.delete(w));
|
|
706
|
+
}
|
|
707
|
+
if (M) {
|
|
708
|
+
const g = await r.memoryWorkerProxy.set(T.data);
|
|
709
|
+
return r.memoryStore.register(g), { ...T, __memory_ref__: g };
|
|
710
|
+
}
|
|
711
|
+
return T;
|
|
712
|
+
}
|
|
713
|
+
async function Q(o, e) {
|
|
714
|
+
if (e.isTerminated())
|
|
715
|
+
throw e.logger.error(
|
|
716
|
+
"Attempted to execute pipeline after MainWorkerFactory was terminated"
|
|
717
|
+
), new Error("MainWorkerFactory has been terminated");
|
|
718
|
+
if (o.length === 0)
|
|
719
|
+
throw e.logger.error("Attempted to execute an empty pipeline"), new Error("Pipeline requires at least one step");
|
|
720
|
+
if (o.length === 1) {
|
|
721
|
+
const r = o[0], { worker: a, srcData: t, ...l } = r, f = e.findWorkerByName(r.worker);
|
|
722
|
+
if (!f)
|
|
723
|
+
throw e.logger.error(
|
|
724
|
+
`Pipeline step config not found for worker: "${r.worker}"`
|
|
725
|
+
), new Error(`Worker "${r.worker}" not found`);
|
|
726
|
+
const m = new b(f.func, {
|
|
727
|
+
createWorker: f.createWorker
|
|
728
|
+
});
|
|
729
|
+
e.trackWorker(m.getWorker);
|
|
730
|
+
const i = m.getWorker;
|
|
731
|
+
return new Promise((n, c) => {
|
|
732
|
+
i.onmessage = (p) => {
|
|
733
|
+
var s, u;
|
|
734
|
+
e.terminateWorker(i), ((s = p.data) == null ? void 0 : s.ok) === !1 ? (e.logger.error(
|
|
735
|
+
`Pipeline step ${r.worker} failed`,
|
|
736
|
+
p.data.error
|
|
737
|
+
), c(new Error(p.data.error))) : n({
|
|
738
|
+
data: (u = p.data) == null ? void 0 : u.data,
|
|
739
|
+
succeeded: 1,
|
|
740
|
+
failed: 0,
|
|
741
|
+
errors: []
|
|
742
|
+
});
|
|
743
|
+
}, i.onerror = (p) => {
|
|
744
|
+
e.terminateWorker(i), e.logger.error(
|
|
745
|
+
`Pipeline step ${r.worker} encountered an error event`,
|
|
746
|
+
p
|
|
747
|
+
), c(p);
|
|
748
|
+
};
|
|
749
|
+
const d = { data: t ?? {}, ...l, index: 0 };
|
|
750
|
+
e.logger.verbose(
|
|
751
|
+
`Starting single-step pipeline for worker ${r.worker}`
|
|
752
|
+
), i.postMessage(d, E(d));
|
|
753
|
+
});
|
|
754
|
+
}
|
|
755
|
+
return new Promise(async (r, a) => {
|
|
756
|
+
const t = [], l = [];
|
|
757
|
+
for (const s of o) {
|
|
758
|
+
const u = e.findWorkerByName(s.worker);
|
|
759
|
+
if (!u) {
|
|
760
|
+
e.logger.error(
|
|
761
|
+
`Pipeline step config not found for worker: "${s.worker}"`
|
|
762
|
+
), a(new Error(`Worker "${s.worker}" not found`));
|
|
763
|
+
return;
|
|
764
|
+
}
|
|
765
|
+
const h = new b(u.func, {
|
|
766
|
+
mode: D.Pipeline,
|
|
767
|
+
createWorker: u.createWorker
|
|
768
|
+
}), _ = e.trackWorker(h.getWorker);
|
|
769
|
+
t.push(_);
|
|
770
|
+
}
|
|
771
|
+
for (let s = 0; s < t.length - 1; s++)
|
|
772
|
+
l.push(new MessageChannel());
|
|
773
|
+
for (let s = 0; s < t.length; s++) {
|
|
774
|
+
const {
|
|
775
|
+
worker: u,
|
|
776
|
+
srcData: h,
|
|
777
|
+
..._
|
|
778
|
+
} = o[s], T = [], M = {};
|
|
779
|
+
if (s > 0 && (M.inputPort = l[s - 1].port1, T.push(M.inputPort)), s < t.length - 1 && (M.outputPort = l[s].port2, T.push(M.outputPort)), t[s].postMessage(
|
|
780
|
+
{ __pipeline_ports__: !0, stepParams: _, ...M },
|
|
781
|
+
T
|
|
782
|
+
), s < t.length - 1) {
|
|
783
|
+
const v = t[s], S = t[s + 1], {
|
|
784
|
+
worker: g,
|
|
785
|
+
srcData: W,
|
|
786
|
+
...w
|
|
787
|
+
} = o[s + 1];
|
|
788
|
+
v.onmessage = (P) => {
|
|
789
|
+
var I, F;
|
|
790
|
+
if (P.data && P.data.__pipeline_ports__) return;
|
|
791
|
+
if (((I = P.data) == null ? void 0 : I.ok) === !1) {
|
|
792
|
+
e.logger.error(
|
|
793
|
+
`Pipeline step ${o[s].worker} failed`,
|
|
794
|
+
P.data.error
|
|
795
|
+
), t.forEach((L) => e.terminateWorker(L)), a(new Error(P.data.error));
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
const $ = { data: ((F = P.data) == null ? void 0 : F.ok) !== void 0 ? P.data.data : P.data, ...w, index: 0 };
|
|
799
|
+
S.postMessage($, E($));
|
|
800
|
+
}, v.onerror = (P) => {
|
|
801
|
+
e.logger.error(
|
|
802
|
+
`Pipeline step ${o[s].worker} encountered an error event`,
|
|
803
|
+
P
|
|
804
|
+
), t.forEach((A) => e.terminateWorker(A)), a(P);
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
const f = t[t.length - 1];
|
|
809
|
+
f.onmessage = (s) => {
|
|
810
|
+
var u, h;
|
|
811
|
+
t.forEach((_) => e.terminateWorker(_)), ((u = s.data) == null ? void 0 : u.ok) === !1 ? (e.logger.error("Final pipeline step failed", s.data.error), a(new Error(s.data.error))) : r({
|
|
812
|
+
data: (h = s.data) == null ? void 0 : h.data,
|
|
813
|
+
succeeded: 1,
|
|
814
|
+
failed: 0,
|
|
815
|
+
errors: []
|
|
816
|
+
});
|
|
817
|
+
}, f.onerror = (s) => {
|
|
818
|
+
t.forEach((u) => e.terminateWorker(u)), e.logger.error(
|
|
819
|
+
"Final pipeline step encountered an error event",
|
|
820
|
+
s
|
|
821
|
+
), a(s);
|
|
822
|
+
};
|
|
823
|
+
const {
|
|
824
|
+
worker: m,
|
|
825
|
+
srcData: i,
|
|
826
|
+
...n
|
|
827
|
+
} = o[0];
|
|
828
|
+
let c = i;
|
|
829
|
+
const k = n.__memory_ref__, d = !!n.deleteMemory;
|
|
830
|
+
if (c === void 0 && k) {
|
|
831
|
+
if (!e.memoryStore.has(k)) {
|
|
832
|
+
t.forEach((s) => e.terminateWorker(s)), e.logger.error(
|
|
833
|
+
`Memory reference "${k}" not found in MemoryStore`
|
|
834
|
+
), a(
|
|
835
|
+
new Error(`Memory reference "${k}" not found in MemoryStore`)
|
|
836
|
+
);
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
c = await e.memoryWorkerProxy.get(k), d && (e.memoryStore.delete(k), await e.memoryWorkerProxy.delete(k)), delete n.__memory_ref__, delete n.deleteMemory;
|
|
840
|
+
}
|
|
841
|
+
c === void 0 && (c = {});
|
|
842
|
+
const p = {
|
|
843
|
+
data: c,
|
|
844
|
+
...n,
|
|
845
|
+
index: 0
|
|
846
|
+
};
|
|
847
|
+
e.logger.verbose(`Starting full pipeline with ${o.length} steps`), t[0].postMessage(p, E(p));
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
class Y {
|
|
851
|
+
constructor(e) {
|
|
852
|
+
y(this, "_persistentWorkers", /* @__PURE__ */ new Map());
|
|
853
|
+
this.context = e;
|
|
854
|
+
}
|
|
855
|
+
async runPersistent(e, r) {
|
|
856
|
+
if (this.context.isTerminated())
|
|
857
|
+
throw this.context.logger.error(
|
|
858
|
+
"Attempted to run persistent worker after MainWorkerFactory was terminated"
|
|
859
|
+
), new Error("MainWorkerFactory has been terminated");
|
|
860
|
+
const a = this.context.findWorkerByName(e);
|
|
861
|
+
if (!a)
|
|
862
|
+
throw this.context.logger.error(
|
|
863
|
+
`Persistent worker config not found for worker: "${e}"`
|
|
864
|
+
), new Error(`Worker "${e}" not found`);
|
|
865
|
+
let t = this._persistentWorkers.get(e);
|
|
866
|
+
if (!t) {
|
|
867
|
+
const l = new b(a.func, {
|
|
868
|
+
mode: D.Persistent,
|
|
869
|
+
createWorker: a.createWorker
|
|
870
|
+
});
|
|
871
|
+
t = this.context.trackWorker(l.getWorker), this.context.logger.verbose(
|
|
872
|
+
`Created persistent worker for ${e}`
|
|
873
|
+
), this._persistentWorkers.set(e, t);
|
|
874
|
+
}
|
|
875
|
+
return new Promise((l, f) => {
|
|
876
|
+
t.onmessage = (i) => {
|
|
877
|
+
var n, c;
|
|
878
|
+
((n = i.data) == null ? void 0 : n.ok) === !1 ? (this.context.logger.error(
|
|
879
|
+
`Persistent worker ${e} failed`,
|
|
880
|
+
i.data.error
|
|
881
|
+
), f(new Error(i.data.error))) : l((c = i.data) == null ? void 0 : c.data);
|
|
882
|
+
}, t.onerror = (i) => {
|
|
883
|
+
this.context.logger.error(
|
|
884
|
+
`Persistent worker ${e} encountered an error event`,
|
|
885
|
+
i
|
|
886
|
+
), f(i);
|
|
887
|
+
};
|
|
888
|
+
const m = {
|
|
889
|
+
type: "run",
|
|
890
|
+
config: r.config
|
|
891
|
+
};
|
|
892
|
+
r.dataset !== void 0 && (m.dataset = r.dataset), t.postMessage(m, E(m));
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
release(e) {
|
|
896
|
+
const r = this._persistentWorkers.get(e);
|
|
897
|
+
if (r) {
|
|
898
|
+
try {
|
|
899
|
+
r.postMessage({ type: "release" });
|
|
900
|
+
} catch {
|
|
901
|
+
}
|
|
902
|
+
this.context.terminateWorker(r), this._persistentWorkers.delete(e);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
terminateAll() {
|
|
906
|
+
for (const e of this._persistentWorkers.values()) {
|
|
907
|
+
try {
|
|
908
|
+
e.postMessage({ type: "release" });
|
|
909
|
+
} catch {
|
|
910
|
+
}
|
|
911
|
+
this.context.terminateWorker(e);
|
|
912
|
+
}
|
|
913
|
+
this._persistentWorkers.clear();
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
class Z {
|
|
917
|
+
constructor(e) {
|
|
918
|
+
this.context = e;
|
|
919
|
+
}
|
|
920
|
+
createWorkerPromises(e, r, a, t, l) {
|
|
921
|
+
const { data: f, ...m } = a;
|
|
922
|
+
return Array.from({ length: t }, (i, n) => {
|
|
923
|
+
const c = l && Array.isArray(f) ? f[n] : f;
|
|
361
924
|
return this.runWorkerWithRetry(
|
|
362
925
|
{
|
|
363
926
|
workerFunc: e.func,
|
|
364
927
|
createWorker: e.createWorker,
|
|
365
928
|
workerName: r,
|
|
366
|
-
index:
|
|
367
|
-
data: { data:
|
|
929
|
+
index: n,
|
|
930
|
+
data: { data: c, ...m }
|
|
368
931
|
},
|
|
369
932
|
e.retries
|
|
370
933
|
);
|
|
371
934
|
});
|
|
372
935
|
}
|
|
373
|
-
/**
|
|
374
|
-
* Runs a single worker instance, retrying on failure up to `retryCount`
|
|
375
|
-
* times before re-throwing the last error.
|
|
376
|
-
*
|
|
377
|
-
* Each retry is logged to `console.error` with the remaining attempt count
|
|
378
|
-
* so failures are visible during development.
|
|
379
|
-
*
|
|
380
|
-
* @param instanceConfig - Full configuration for the worker instance.
|
|
381
|
-
* @param retryCount - Remaining retry attempts (default `2`).
|
|
382
|
-
* @returns The successful {@link WorkerResult} once the worker resolves.
|
|
383
|
-
* @throws The last caught error when all retries are exhausted.
|
|
384
|
-
*/
|
|
385
936
|
async runWorkerWithRetry(e, r = 2) {
|
|
386
937
|
try {
|
|
387
938
|
return await this.initiateWorker(e);
|
|
388
|
-
} catch (
|
|
939
|
+
} catch (a) {
|
|
389
940
|
if (r > 0)
|
|
390
|
-
return
|
|
941
|
+
return this.context.logger.info(
|
|
391
942
|
`Worker ${e.index} failed, retrying (${r} left):`,
|
|
392
|
-
|
|
943
|
+
a
|
|
393
944
|
), this.runWorkerWithRetry(e, r - 1);
|
|
394
|
-
throw
|
|
945
|
+
throw this.context.logger.error("Worker failed after all retries:", a), a;
|
|
395
946
|
}
|
|
396
947
|
}
|
|
397
|
-
|
|
398
|
-
* Spawns a single worker thread, posts the payload, and resolves or rejects
|
|
399
|
-
* based on the message the worker sends back.
|
|
400
|
-
*
|
|
401
|
-
* The worker is expected to respond with either:
|
|
402
|
-
* - `{ ok: true, data: T }` — success; resolves with a {@link WorkerResult}.
|
|
403
|
-
* - `{ ok: false, error: string }` — logical failure; rejects with a
|
|
404
|
-
* structured error object.
|
|
405
|
-
*
|
|
406
|
-
* Any transferable objects found in the payload are moved (not copied) to
|
|
407
|
-
* the worker via the `transfer` list of `postMessage`.
|
|
408
|
-
*
|
|
409
|
-
* The underlying `Worker` is always terminated after the message completes.
|
|
410
|
-
*
|
|
411
|
-
* @param instanceConfig - Worker function, factory, name, shard index, and data.
|
|
412
|
-
* @returns A promise that resolves with the worker's result.
|
|
413
|
-
*/
|
|
414
|
-
initiateWorker({
|
|
948
|
+
async initiateWorker({
|
|
415
949
|
workerFunc: e,
|
|
416
950
|
createWorker: r,
|
|
417
|
-
workerName:
|
|
951
|
+
workerName: a,
|
|
418
952
|
index: t,
|
|
419
953
|
data: l
|
|
420
954
|
}) {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
955
|
+
const f = await this.context.memoryWorkerProxy.allocateWorkerPort();
|
|
956
|
+
return new Promise((m, i) => {
|
|
957
|
+
if (this.context.isTerminated()) {
|
|
958
|
+
i(new Error("MainWorkerFactory has been terminated"));
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
961
|
+
const n = new b(e, {
|
|
962
|
+
createWorker: r,
|
|
963
|
+
mode: D.Memory
|
|
964
|
+
}), c = this.context.trackWorker(n.getWorker);
|
|
965
|
+
c.onerror = (d) => {
|
|
966
|
+
this.context.terminateWorker(c), i({
|
|
430
967
|
index: t,
|
|
431
968
|
workerConfigs: {
|
|
432
969
|
workerFunc: e,
|
|
433
970
|
createWorker: r,
|
|
434
|
-
workerName:
|
|
971
|
+
workerName: a,
|
|
435
972
|
index: t,
|
|
436
973
|
data: l
|
|
437
974
|
},
|
|
438
|
-
failedResult:
|
|
975
|
+
failedResult: d
|
|
439
976
|
});
|
|
440
|
-
},
|
|
441
|
-
var
|
|
442
|
-
if (((
|
|
443
|
-
this.terminateWorker(
|
|
977
|
+
}, c.onmessage = (d) => {
|
|
978
|
+
var u, h, _;
|
|
979
|
+
if (((u = d.data) == null ? void 0 : u.ok) === !1) {
|
|
980
|
+
this.context.terminateWorker(c), i({
|
|
444
981
|
index: t,
|
|
445
982
|
workerConfigs: {
|
|
446
983
|
workerFunc: e,
|
|
447
984
|
createWorker: r,
|
|
448
|
-
workerName:
|
|
985
|
+
workerName: a,
|
|
449
986
|
index: t,
|
|
450
987
|
data: l
|
|
451
988
|
},
|
|
452
989
|
failedResult: new ErrorEvent("error", {
|
|
453
|
-
message:
|
|
990
|
+
message: d.data.error
|
|
454
991
|
})
|
|
455
992
|
});
|
|
456
993
|
return;
|
|
457
994
|
}
|
|
458
|
-
|
|
995
|
+
const p = (h = d.data) == null ? void 0 : h.__memory_ref__, s = p !== void 0 ? { __memory_ref__: p } : ((_ = d.data) == null ? void 0 : _.ok) !== void 0 ? d.data.data : d.data;
|
|
996
|
+
m({
|
|
459
997
|
index: t,
|
|
460
998
|
workerConfigs: {
|
|
461
999
|
workerFunc: e,
|
|
462
1000
|
createWorker: r,
|
|
463
|
-
workerName:
|
|
1001
|
+
workerName: a,
|
|
464
1002
|
index: t,
|
|
465
1003
|
data: l
|
|
466
1004
|
},
|
|
467
1005
|
successResult: new MessageEvent("message", {
|
|
468
|
-
data:
|
|
1006
|
+
data: s
|
|
469
1007
|
})
|
|
470
|
-
}), this.terminateWorker(
|
|
1008
|
+
}), this.context.terminateWorker(c);
|
|
471
1009
|
};
|
|
472
|
-
const
|
|
1010
|
+
const k = {
|
|
473
1011
|
index: t,
|
|
474
1012
|
...Array.isArray(l) ? { data: l } : l
|
|
475
1013
|
};
|
|
476
|
-
|
|
1014
|
+
c.postMessage(
|
|
1015
|
+
{
|
|
1016
|
+
__init_memory_port__: !0,
|
|
1017
|
+
factoryToken: this.context.factoryToken
|
|
1018
|
+
},
|
|
1019
|
+
[f]
|
|
1020
|
+
), c.postMessage(k, E(k));
|
|
477
1021
|
});
|
|
478
1022
|
}
|
|
1023
|
+
}
|
|
1024
|
+
class te {
|
|
479
1025
|
/**
|
|
480
|
-
*
|
|
481
|
-
* main thread.
|
|
482
|
-
*
|
|
483
|
-
* Fulfilled shards are extracted and passed to the `reducer` function, which
|
|
484
|
-
* runs inside a dedicated inline worker so the merge itself never blocks the
|
|
485
|
-
* main thread. Failed shards are counted and their raw rejection reasons are
|
|
486
|
-
* preserved in `errors`.
|
|
487
|
-
*
|
|
488
|
-
* @typeParam T - The per-shard data type (inferred from `settled`).
|
|
489
|
-
* @typeParam R - The final merged output type (defaults to a flat array of
|
|
490
|
-
* `T` items when no custom reducer is provided).
|
|
491
|
-
*
|
|
492
|
-
* @param settled - The {@link TypedSettledResults} returned by `runWorker`.
|
|
493
|
-
* @param options - Optional {@link CollectOptions}. Supply a `reducer` to
|
|
494
|
-
* control how shards are merged. The reducer **must be self-contained**
|
|
495
|
-
* (no closures over external variables) because it is serialised and run
|
|
496
|
-
* inside a worker.
|
|
497
|
-
*
|
|
498
|
-
* @returns A {@link CollectedResult} with the merged `data`, counts of
|
|
499
|
-
* `succeeded`/`failed` shards, and the raw `errors` array.
|
|
500
|
-
*
|
|
501
|
-
* @example
|
|
502
|
-
* // default: flat array of all shard data
|
|
503
|
-
* const { data, succeeded, failed } = await foreman.collectResults(res);
|
|
1026
|
+
* Initializes a new MainWorkerFactory.
|
|
504
1027
|
*
|
|
505
|
-
* @
|
|
506
|
-
*
|
|
507
|
-
*
|
|
508
|
-
* reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
|
|
509
|
-
* });
|
|
1028
|
+
* @param options - Configuration options.
|
|
1029
|
+
* @param options.workers - The list of worker configurations.
|
|
1030
|
+
* @param options.logLevel - Optional logging level (defaults to 'error').
|
|
510
1031
|
*/
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}, i.onerror = (u) => {
|
|
535
|
-
this.terminateWorker(i), o(u);
|
|
536
|
-
}, i.postMessage(l);
|
|
537
|
-
}),
|
|
538
|
-
succeeded: s.length,
|
|
539
|
-
failed: t.length,
|
|
540
|
-
errors: t
|
|
1032
|
+
constructor(e) {
|
|
1033
|
+
y(this, "_workers");
|
|
1034
|
+
y(this, "_threads");
|
|
1035
|
+
y(this, "_activeWorkers", /* @__PURE__ */ new Set());
|
|
1036
|
+
y(this, "_memoryStore", new z());
|
|
1037
|
+
y(this, "_memoryWorkerProxy");
|
|
1038
|
+
y(this, "_factoryToken");
|
|
1039
|
+
y(this, "_isTerminated", !1);
|
|
1040
|
+
y(this, "_persistentManager");
|
|
1041
|
+
y(this, "_orchestrator");
|
|
1042
|
+
/**
|
|
1043
|
+
* The logger instance used by this factory.
|
|
1044
|
+
*/
|
|
1045
|
+
y(this, "logger");
|
|
1046
|
+
this._workers = e.workers, this.logger = new J(e.logLevel ?? "error"), this._threads = typeof navigator < "u" && navigator.hardwareConcurrency ? navigator.hardwareConcurrency : 4, this._factoryToken = crypto.randomUUID(), this._memoryWorkerProxy = new H(this._factoryToken);
|
|
1047
|
+
const r = {
|
|
1048
|
+
isTerminated: () => this.isTerminated,
|
|
1049
|
+
findWorkerByName: this.findWorkerByName.bind(this),
|
|
1050
|
+
trackWorker: this.trackWorker.bind(this),
|
|
1051
|
+
terminateWorker: this.terminateWorker.bind(this),
|
|
1052
|
+
logger: this.logger,
|
|
1053
|
+
memoryWorkerProxy: this._memoryWorkerProxy,
|
|
1054
|
+
factoryToken: this._factoryToken
|
|
541
1055
|
};
|
|
1056
|
+
this._persistentManager = new Y(r), this._orchestrator = new Z(r);
|
|
542
1057
|
}
|
|
543
1058
|
/**
|
|
544
|
-
*
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
1059
|
+
* Indicates whether the factory has been terminated.
|
|
1060
|
+
*/
|
|
1061
|
+
get isTerminated() {
|
|
1062
|
+
return this._isTerminated;
|
|
1063
|
+
}
|
|
1064
|
+
trackWorker(e) {
|
|
1065
|
+
if (this._isTerminated)
|
|
1066
|
+
throw e.terminate(), new Error("MainWorkerFactory has been terminated");
|
|
1067
|
+
return this._activeWorkers.add(e), e;
|
|
1068
|
+
}
|
|
1069
|
+
terminateWorker(e) {
|
|
1070
|
+
this._activeWorkers.delete(e);
|
|
1071
|
+
try {
|
|
1072
|
+
e.terminate();
|
|
1073
|
+
} catch {
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
/**
|
|
1077
|
+
* Helper utility to partition an array into a specified number of chunks.
|
|
550
1078
|
*
|
|
551
|
-
* @
|
|
552
|
-
*
|
|
1079
|
+
* @param array - The array to partition.
|
|
1080
|
+
* @param numChunks - The desired number of chunks.
|
|
1081
|
+
* @returns An array of array chunks.
|
|
1082
|
+
* @template T - The type of elements in the array.
|
|
1083
|
+
*/
|
|
1084
|
+
partitionArray(e, r) {
|
|
1085
|
+
return B(e, r);
|
|
1086
|
+
}
|
|
1087
|
+
findWorkerByName(e) {
|
|
1088
|
+
return this._workers.find((r) => r.name === e);
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Dispatches a worker task with the given parameters.
|
|
553
1092
|
*
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
1093
|
+
* Workers store their results directly in the `MemoryWorker` thread — large data
|
|
1094
|
+
* never touches the main thread heap. Results are auto-collected and merged in a
|
|
1095
|
+
* dedicated reducer worker. Returns a `CollectedResult<R>` directly.
|
|
557
1096
|
*
|
|
558
|
-
*
|
|
559
|
-
*
|
|
1097
|
+
* Pass `autoCollect: false` in `rawParams` to skip auto-collection and receive
|
|
1098
|
+
* the raw settled state (escape hatch for advanced custom reducers).
|
|
560
1099
|
*
|
|
561
|
-
* @
|
|
562
|
-
*
|
|
563
|
-
*
|
|
564
|
-
*
|
|
565
|
-
*
|
|
566
|
-
*
|
|
567
|
-
* console.log(result); // final transformed + filtered data
|
|
1100
|
+
* @param workerName - The name of the registered worker to execute.
|
|
1101
|
+
* @param rawParams - The payload, options, and reducer for the worker.
|
|
1102
|
+
* @returns A promise resolving to a CollectedResult with merged data and shard stats.
|
|
1103
|
+
* @template TName - The name of the registered worker.
|
|
1104
|
+
* @template T - The return type of the worker function.
|
|
1105
|
+
* @template R - The merged return type after reducing shards.
|
|
568
1106
|
*/
|
|
569
|
-
async
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
a.onmessage = (u) => {
|
|
580
|
-
var k, g;
|
|
581
|
-
this.terminateWorker(a), ((k = u.data) == null ? void 0 : k.ok) === !1 ? d(new Error(u.data.error)) : o((g = u.data) == null ? void 0 : g.data);
|
|
582
|
-
}, a.onerror = (u) => {
|
|
583
|
-
this.terminateWorker(a), d(u);
|
|
584
|
-
};
|
|
585
|
-
const i = { data: t ?? {}, ...l, index: 0 };
|
|
586
|
-
a.postMessage(i, m(i));
|
|
587
|
-
});
|
|
588
|
-
}
|
|
589
|
-
return new Promise((r, s) => {
|
|
590
|
-
const t = [], l = [];
|
|
591
|
-
for (const i of e) {
|
|
592
|
-
const u = this.findWorkerByName(i.worker);
|
|
593
|
-
if (!u) {
|
|
594
|
-
s(new Error(`Worker "${i.worker}" not found`));
|
|
595
|
-
return;
|
|
596
|
-
}
|
|
597
|
-
const k = new W(u.func, {
|
|
598
|
-
mode: P.Pipeline,
|
|
599
|
-
createWorker: u.createWorker
|
|
600
|
-
}), g = this.trackWorker(k.getWorker);
|
|
601
|
-
t.push(g);
|
|
602
|
-
}
|
|
603
|
-
for (let i = 0; i < t.length - 1; i++)
|
|
604
|
-
l.push(new MessageChannel());
|
|
605
|
-
for (let i = 0; i < t.length; i++) {
|
|
606
|
-
const { worker: u, srcData: k, ...g } = e[i], v = [], w = {};
|
|
607
|
-
if (i > 0 && (w.inputPort = l[i - 1].port1, v.push(w.inputPort)), i < t.length - 1 && (w.outputPort = l[i].port2, v.push(w.outputPort)), t[i].postMessage(
|
|
608
|
-
{ __pipeline_ports__: !0, stepParams: g, ...w },
|
|
609
|
-
v
|
|
610
|
-
), i < t.length - 1) {
|
|
611
|
-
const _ = t[i], A = t[i + 1], { worker: j, srcData: $, ...D } = e[i + 1];
|
|
612
|
-
_.onmessage = (h) => {
|
|
613
|
-
var E, T;
|
|
614
|
-
if (h.data && h.data.__pipeline_ports__) return;
|
|
615
|
-
if (((E = h.data) == null ? void 0 : E.ok) === !1) {
|
|
616
|
-
t.forEach((B) => this.terminateWorker(B)), s(new Error(h.data.error));
|
|
617
|
-
return;
|
|
618
|
-
}
|
|
619
|
-
const b = { data: ((T = h.data) == null ? void 0 : T.ok) !== void 0 ? h.data.data : h.data, ...D, index: 0 };
|
|
620
|
-
A.postMessage(b, m(b));
|
|
621
|
-
}, _.onerror = (h) => {
|
|
622
|
-
t.forEach((M) => this.terminateWorker(M)), s(h);
|
|
623
|
-
};
|
|
624
|
-
}
|
|
625
|
-
}
|
|
626
|
-
const c = t[t.length - 1];
|
|
627
|
-
c.onmessage = (i) => {
|
|
628
|
-
var u, k;
|
|
629
|
-
t.forEach((g) => this.terminateWorker(g)), ((u = i.data) == null ? void 0 : u.ok) === !1 ? s(new Error(i.data.error)) : r((k = i.data) == null ? void 0 : k.data);
|
|
630
|
-
}, c.onerror = (i) => {
|
|
631
|
-
t.forEach((u) => this.terminateWorker(u)), s(i);
|
|
632
|
-
};
|
|
633
|
-
const { worker: f, srcData: a, ...o } = e[0], p = { data: a ?? {}, ...o, index: 0 };
|
|
634
|
-
t[0].postMessage(p, m(p));
|
|
1107
|
+
async runWorker(e, r) {
|
|
1108
|
+
return X(e, r, {
|
|
1109
|
+
isTerminated: () => this.isTerminated,
|
|
1110
|
+
findWorkerByName: this.findWorkerByName.bind(this),
|
|
1111
|
+
memoryStore: this._memoryStore,
|
|
1112
|
+
memoryWorkerProxy: this._memoryWorkerProxy,
|
|
1113
|
+
factoryToken: this._factoryToken,
|
|
1114
|
+
threads: this._threads,
|
|
1115
|
+
orchestrator: this._orchestrator,
|
|
1116
|
+
logger: this.logger
|
|
635
1117
|
});
|
|
636
1118
|
}
|
|
637
1119
|
/**
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
* On the first call, provide both `dataset` and `config`. The worker stores
|
|
641
|
-
* the dataset in memory. On subsequent calls, only `config` is needed — the
|
|
642
|
-
* worker reuses the cached dataset and reprocesses it with the new config.
|
|
643
|
-
*
|
|
644
|
-
* The worker stays alive until {@link release} is called.
|
|
1120
|
+
* Deletes a specific reference from the memory store and MemoryWorker.
|
|
645
1121
|
*
|
|
646
|
-
* @param
|
|
647
|
-
* @
|
|
648
|
-
|
|
1122
|
+
* @param ref - The memory reference ID to delete.
|
|
1123
|
+
* @returns True if the reference existed and was deleted, false otherwise.
|
|
1124
|
+
*/
|
|
1125
|
+
async deleteMemory(e) {
|
|
1126
|
+
return this._memoryStore.delete(e), this._memoryWorkerProxy.delete(e);
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* Clears all references from the memory store and MemoryWorker.
|
|
1130
|
+
*/
|
|
1131
|
+
async clearMemory() {
|
|
1132
|
+
this._memoryStore.clear(), await this._memoryWorkerProxy.clear();
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* Retrieves statistics about the currently stored memory handles.
|
|
649
1136
|
*
|
|
650
|
-
* @
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
1137
|
+
* @returns The MemoryStats containing count and active reference IDs.
|
|
1138
|
+
*/
|
|
1139
|
+
async getMemoryStats() {
|
|
1140
|
+
return this._memoryWorkerProxy.stats();
|
|
1141
|
+
}
|
|
1142
|
+
/**
|
|
1143
|
+
* Collects and reduces the results from a `runWorker` execution.
|
|
656
1144
|
*
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
*
|
|
660
|
-
*
|
|
1145
|
+
* @deprecated `runWorker` now auto-collects results. This method is kept as
|
|
1146
|
+
* an escape hatch for advanced cases where `autoCollect: false` was passed to
|
|
1147
|
+
* `runWorker`. In the common case, the return value of `runWorker` already
|
|
1148
|
+
* contains the merged `CollectedResult`.
|
|
661
1149
|
*
|
|
662
|
-
*
|
|
663
|
-
*
|
|
664
|
-
*
|
|
665
|
-
*
|
|
666
|
-
*
|
|
1150
|
+
* @param settled - The settled results from `runWorker` (when `autoCollect: false`).
|
|
1151
|
+
* @param options - Options containing the reducer function.
|
|
1152
|
+
* @returns A structured CollectedResult object.
|
|
1153
|
+
* @template T - The type of the worker result data.
|
|
1154
|
+
* @template R - The type of the merged result data.
|
|
1155
|
+
*/
|
|
1156
|
+
async collectResults(e, r = {}) {
|
|
1157
|
+
return x(e, r, {
|
|
1158
|
+
isTerminated: () => this.isTerminated,
|
|
1159
|
+
trackWorker: this.trackWorker.bind(this),
|
|
1160
|
+
terminateWorker: this.terminateWorker.bind(this),
|
|
1161
|
+
logger: this.logger,
|
|
1162
|
+
memoryWorkerProxy: this._memoryWorkerProxy,
|
|
1163
|
+
factoryToken: this._factoryToken
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* Executes a sequential pipeline of worker steps.
|
|
1168
|
+
* Passes the output of one step as the input to the next step.
|
|
667
1169
|
*
|
|
668
|
-
*
|
|
669
|
-
*
|
|
1170
|
+
* @param steps - An array of PipelineStep configurations.
|
|
1171
|
+
* @returns A CollectedResult containing the final pipeline output and shard stats,
|
|
1172
|
+
* consistent with the shape returned by `runWorker()`.
|
|
1173
|
+
* @template TResult - The type of the pipeline result.
|
|
670
1174
|
*/
|
|
671
|
-
async
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
createWorker: s.createWorker
|
|
681
|
-
});
|
|
682
|
-
t = this.trackWorker(l.getWorker), this._persistentWorkers.set(e, t);
|
|
683
|
-
}
|
|
684
|
-
return new Promise((l, c) => {
|
|
685
|
-
t.onmessage = (a) => {
|
|
686
|
-
var o, d;
|
|
687
|
-
((o = a.data) == null ? void 0 : o.ok) === !1 ? c(new Error(a.data.error)) : l((d = a.data) == null ? void 0 : d.data);
|
|
688
|
-
}, t.onerror = (a) => {
|
|
689
|
-
c(a);
|
|
690
|
-
};
|
|
691
|
-
const f = {
|
|
692
|
-
type: "run",
|
|
693
|
-
config: r.config
|
|
694
|
-
};
|
|
695
|
-
r.dataset !== void 0 && (f.dataset = r.dataset), t.postMessage(f, m(f));
|
|
1175
|
+
async pipeline(e) {
|
|
1176
|
+
return Q(e, {
|
|
1177
|
+
memoryStore: this._memoryStore,
|
|
1178
|
+
memoryWorkerProxy: this._memoryWorkerProxy,
|
|
1179
|
+
isTerminated: () => this.isTerminated,
|
|
1180
|
+
findWorkerByName: this.findWorkerByName.bind(this),
|
|
1181
|
+
trackWorker: this.trackWorker.bind(this),
|
|
1182
|
+
terminateWorker: this.terminateWorker.bind(this),
|
|
1183
|
+
logger: this.logger
|
|
696
1184
|
});
|
|
697
1185
|
}
|
|
698
1186
|
/**
|
|
699
|
-
*
|
|
700
|
-
*
|
|
1187
|
+
* Executes a task on a persistent (long-lived) worker thread.
|
|
1188
|
+
* Useful for workers that maintain local state (like WebAssembly modules or databases)
|
|
1189
|
+
* across multiple invocations.
|
|
701
1190
|
*
|
|
702
|
-
*
|
|
703
|
-
*
|
|
1191
|
+
* @param workerName - The name of the persistent worker to run.
|
|
1192
|
+
* @param params - The input data and configuration for the task.
|
|
1193
|
+
* @returns The result from the persistent worker thread.
|
|
1194
|
+
* @template TResult - The expected type of the result from the worker thread.
|
|
1195
|
+
*/
|
|
1196
|
+
async runPersistent(e, r) {
|
|
1197
|
+
return this._persistentManager.runPersistent(e, r);
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Releases and terminates a persistent worker by name.
|
|
704
1201
|
*
|
|
705
|
-
* @param workerName -
|
|
1202
|
+
* @param workerName - The name of the persistent worker to release.
|
|
706
1203
|
*/
|
|
707
1204
|
release(e) {
|
|
708
|
-
|
|
709
|
-
if (r) {
|
|
710
|
-
try {
|
|
711
|
-
r.postMessage({ type: "release" });
|
|
712
|
-
} catch {
|
|
713
|
-
}
|
|
714
|
-
this.terminateWorker(r), this._persistentWorkers.delete(e);
|
|
715
|
-
}
|
|
1205
|
+
this._persistentManager.release(e);
|
|
716
1206
|
}
|
|
717
1207
|
/**
|
|
718
|
-
* Terminates
|
|
719
|
-
*
|
|
720
|
-
* Calling `terminate()` immediately stops all running worker threads, releases
|
|
721
|
-
* cached persistent workers, and clears all internal worker state.
|
|
1208
|
+
* Terminates all active worker threads, persistent workers, MemoryWorker, and clears the memory store.
|
|
1209
|
+
* Marks this factory instance as terminated.
|
|
722
1210
|
*/
|
|
723
1211
|
terminate() {
|
|
724
|
-
this._isTerminated = !0;
|
|
725
|
-
for (const e of this._persistentWorkers.values()) {
|
|
726
|
-
try {
|
|
727
|
-
e.postMessage({ type: "release" });
|
|
728
|
-
} catch {
|
|
729
|
-
}
|
|
730
|
-
this.terminateWorker(e);
|
|
731
|
-
}
|
|
732
|
-
this._persistentWorkers.clear();
|
|
1212
|
+
this._isTerminated = !0, this._persistentManager.terminateAll();
|
|
733
1213
|
for (const e of Array.from(this._activeWorkers))
|
|
734
1214
|
this.terminateWorker(e);
|
|
735
|
-
this._activeWorkers.clear();
|
|
1215
|
+
this._activeWorkers.clear(), this._memoryStore.clear(), this._memoryWorkerProxy.terminate();
|
|
736
1216
|
}
|
|
737
1217
|
/**
|
|
738
|
-
* Alias for {@link terminate}. Terminates
|
|
1218
|
+
* Alias for {@link terminate}. Terminates all active resources.
|
|
739
1219
|
*/
|
|
740
1220
|
destroy() {
|
|
741
1221
|
this.terminate();
|
|
742
1222
|
}
|
|
743
1223
|
/**
|
|
744
|
-
*
|
|
745
|
-
* and resetting the factory state, allowing new worker instances to be initiated.
|
|
1224
|
+
* Terminates all resources and resets the factory to an active state.
|
|
746
1225
|
*/
|
|
747
1226
|
reset() {
|
|
748
1227
|
this.terminate(), this._isTerminated = !1;
|
|
749
1228
|
}
|
|
750
1229
|
/**
|
|
751
|
-
* Alias for {@link reset}.
|
|
1230
|
+
* Alias for {@link reset}. Terminates and reactivates the factory.
|
|
752
1231
|
*/
|
|
753
1232
|
restart() {
|
|
754
1233
|
this.reset();
|
|
755
1234
|
}
|
|
756
1235
|
}
|
|
757
|
-
function
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
};
|
|
763
|
-
async function c(f) {
|
|
764
|
-
try {
|
|
765
|
-
const a = typeof f == "object" && f !== null && "data" in f ? { ...t, ...f } : { data: f, ...t, index: 0 }, o = await n(a), d = { ok: !0, data: o }, p = m(o);
|
|
766
|
-
e ? e.postMessage(d, p) : l(d, p);
|
|
767
|
-
} catch (a) {
|
|
768
|
-
const o = {
|
|
769
|
-
ok: !1,
|
|
770
|
-
error: a instanceof Error ? a.message : String(a)
|
|
771
|
-
};
|
|
772
|
-
e ? e.postMessage(o) : l(o);
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
self.addEventListener("message", (f) => {
|
|
776
|
-
const a = f.data;
|
|
777
|
-
if (a && a.__pipeline_ports__) {
|
|
778
|
-
a.stepParams && (t = a.stepParams), a.outputPort && (e = a.outputPort), a.inputPort && (r = a.inputPort, r.onmessage = (o) => {
|
|
779
|
-
var d;
|
|
780
|
-
o.data && o.data.ok === !1 ? e ? e.postMessage(o.data) : l(o.data) : c({ data: (d = o.data) == null ? void 0 : d.data, ...t, index: 0 });
|
|
781
|
-
}), s !== null && (c(s), s = null);
|
|
782
|
-
return;
|
|
783
|
-
}
|
|
784
|
-
r ? s = a : c(a);
|
|
785
|
-
});
|
|
1236
|
+
function oe(o) {
|
|
1237
|
+
return o === void 0 ? (e) => e : o;
|
|
1238
|
+
}
|
|
1239
|
+
function ae(...o) {
|
|
1240
|
+
return o;
|
|
786
1241
|
}
|
|
787
1242
|
export {
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
1243
|
+
te as MainWorkerFactory,
|
|
1244
|
+
b as WorkerFactory,
|
|
1245
|
+
D as WorkerMode,
|
|
1246
|
+
ie as defineWorker,
|
|
1247
|
+
oe as defineWorkerConfig,
|
|
1248
|
+
ae as defineWorkerConfigs
|
|
791
1249
|
};
|