@offmain/workerkit 0.12.3 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -40
- package/dist/index.cjs +16 -8
- package/dist/index.js +317 -175
- package/dist/types/tools/define-worker.d.ts +21 -0
- package/dist/types/tools/define-worker.test.d.ts +1 -0
- package/dist/types/tools/index.d.ts +1 -0
- package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +39 -6
- package/dist/types/tools/main-worker-factory/types.d.ts +10 -7
- package/dist/types/tools/worker-factory/worker-factory.d.ts +4 -10
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -57,48 +57,55 @@ console.log(data); // [15]
|
|
|
57
57
|
|
|
58
58
|
## WorkerConfig Options
|
|
59
59
|
|
|
60
|
-
| Option | Type
|
|
61
|
-
| ---------------- |
|
|
62
|
-
| `name` | `string`
|
|
63
|
-
| `role` | `string`
|
|
64
|
-
| `func` | `Function`
|
|
65
|
-
| `
|
|
66
|
-
| `maxConcurrency` | `number`
|
|
67
|
-
| `retries` | `number`
|
|
68
|
-
| `partition` | `boolean`
|
|
60
|
+
| Option | Type | Default | Description |
|
|
61
|
+
| ---------------- | -------------- | ------------------------------- | --------------------------------------------------------------------------------------------------- |
|
|
62
|
+
| `name` | `string` | — | Unique identifier used to call the worker |
|
|
63
|
+
| `role` | `string` | — | Logical grouping label |
|
|
64
|
+
| `func` | `Function` | — | The exported worker function to run (optional if `createWorker` is provided) |
|
|
65
|
+
| `createWorker` | `() => Worker` | — | Worker factory function `() => new Worker(new URL(...))` for Webpack 5 / Vite static analysis |
|
|
66
|
+
| `maxConcurrency` | `number` | `navigator.hardwareConcurrency` | Max parallel worker instances — defaults to the number of logical CPU cores reported by the browser |
|
|
67
|
+
| `retries` | `number` | `0` | How many times to retry a failed shard before marking it as rejected |
|
|
68
|
+
| `partition` | `boolean` | `false` | Split array input across multiple workers automatically |
|
|
69
69
|
|
|
70
70
|
---
|
|
71
71
|
|
|
72
|
-
## Module Bundler Integration (`
|
|
72
|
+
## Module Bundler Integration (`createWorker`)
|
|
73
73
|
|
|
74
74
|
When worker logic relies on external npm packages (such as Luxon, `date-fns`, `i18next`, or custom data transformation modules), dynamic inline stringification (`.toString()`) cannot access those closed-over imports.
|
|
75
75
|
|
|
76
|
-
Passing `
|
|
76
|
+
Passing `createWorker` enables modern module bundlers (Webpack 5, Vite, Rollup, Parcel) to analyze and bundle the worker file along with all of its dependencies into a dedicated ES module worker chunk.
|
|
77
77
|
|
|
78
|
-
### Dedicated Worker Script
|
|
78
|
+
### Dedicated Worker Script with `defineWorker`
|
|
79
|
+
|
|
80
|
+
When writing standalone worker files, wrap your export in `defineWorker`. This automatically handles standard runs, worker-to-worker pipelines (`pipeline()`), and persistent dataset caching without manual `postMessage` / `onmessage` boilerplate:
|
|
79
81
|
|
|
80
82
|
```ts
|
|
81
83
|
// transform-data.worker.ts
|
|
84
|
+
import { defineWorker } from '@offmain/workerkit';
|
|
82
85
|
import { format } from 'date-fns';
|
|
83
86
|
import { t } from './i18n.ts';
|
|
84
87
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
88
|
+
export default defineWorker(
|
|
89
|
+
async ({
|
|
90
|
+
data,
|
|
91
|
+
options,
|
|
92
|
+
}: {
|
|
93
|
+
data: { items: any[]; locale: string };
|
|
94
|
+
options?: { prefix?: string };
|
|
95
|
+
}) => {
|
|
96
|
+
const { items, locale } = data;
|
|
97
|
+
return items.map((item: any) => ({
|
|
89
98
|
...item,
|
|
90
99
|
formattedDate: format(new Date(item.timestamp), 'yyyy-MM-dd'),
|
|
91
|
-
label: t('transaction', locale),
|
|
100
|
+
label: (options?.prefix ?? '') + t('transaction', locale),
|
|
92
101
|
}));
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
} catch (err) {
|
|
96
|
-
self.postMessage({ ok: false, error: (err as Error).message });
|
|
97
|
-
}
|
|
98
|
-
});
|
|
102
|
+
},
|
|
103
|
+
);
|
|
99
104
|
```
|
|
100
105
|
|
|
101
|
-
###
|
|
106
|
+
### Webpack 5 & Vite Static Analysis (`createWorker`)
|
|
107
|
+
|
|
108
|
+
Webpack 5 and Vite look for literal `new Worker(new URL(..., import.meta.url))` calls inside consumer source files. By providing a `createWorker` factory function, bundlers statically detect and bundle the worker into a separate JS file, while allowing `MainWorkerFactory` to scale `maxConcurrency` across multiple threads:
|
|
102
109
|
|
|
103
110
|
```ts
|
|
104
111
|
import { MainWorkerFactory } from '@offmain/workerkit';
|
|
@@ -108,9 +115,13 @@ const factory = new MainWorkerFactory({
|
|
|
108
115
|
{
|
|
109
116
|
name: 'transformData',
|
|
110
117
|
role: 'compute',
|
|
111
|
-
// Webpack 5
|
|
112
|
-
|
|
113
|
-
|
|
118
|
+
// Webpack 5 and Vite statically analyze new Worker(new URL(..., import.meta.url))
|
|
119
|
+
// written inside this factory function and emit an individual bundled JS chunk.
|
|
120
|
+
createWorker: () =>
|
|
121
|
+
new Worker(new URL('./transform-data.worker.ts', import.meta.url), {
|
|
122
|
+
type: 'module',
|
|
123
|
+
}),
|
|
124
|
+
maxConcurrency: 4, // Spawns up to 4 parallel worker instances
|
|
114
125
|
},
|
|
115
126
|
] as const,
|
|
116
127
|
});
|
|
@@ -170,7 +181,7 @@ Only the final result crosses back to the main thread. If your pipeline generate
|
|
|
170
181
|
|
|
171
182
|
### Usage
|
|
172
183
|
|
|
173
|
-
|
|
184
|
+
````ts
|
|
174
185
|
import { MainWorkerFactory } from '@offmain/workerkit';
|
|
175
186
|
import { fetchData, transform, aggregate } from './workers.ts';
|
|
176
187
|
|
|
@@ -182,20 +193,33 @@ const factory = new MainWorkerFactory({
|
|
|
182
193
|
] as const,
|
|
183
194
|
});
|
|
184
195
|
|
|
196
|
+
### Step-Specific Options and Configs in Pipeline
|
|
197
|
+
|
|
198
|
+
You can pass step-specific parameters (such as `options`, `configs`, etc.) directly to each pipeline step:
|
|
199
|
+
|
|
200
|
+
```ts
|
|
185
201
|
const result = await factory.pipeline<AggregateResult>([
|
|
186
|
-
{
|
|
187
|
-
|
|
188
|
-
|
|
202
|
+
{
|
|
203
|
+
worker: 'fetchData',
|
|
204
|
+
srcData: { url: '/api/records' },
|
|
205
|
+
options: { timeout: 5000 },
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
worker: 'transform',
|
|
209
|
+
configs: { multiplier: 2 },
|
|
210
|
+
},
|
|
211
|
+
{
|
|
212
|
+
worker: 'aggregate',
|
|
213
|
+
options: { threshold: 10 },
|
|
214
|
+
},
|
|
189
215
|
]);
|
|
216
|
+
````
|
|
190
217
|
|
|
191
|
-
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
### How each step receives data
|
|
218
|
+
### How each step receives data and parameters
|
|
195
219
|
|
|
196
|
-
- The first step receives `srcData` as `{ data: srcData,
|
|
197
|
-
- Each subsequent step receives the previous step's output as `{ data: previousOutput, index: 0 }`.
|
|
198
|
-
- Worker functions
|
|
220
|
+
- The first step receives `srcData` merged with its step parameters as `{ data: srcData, options: { timeout: 5000 }, index: 0 }`.
|
|
221
|
+
- Each subsequent step receives the previous step's output merged with its step parameters as `{ data: previousOutput, configs: { multiplier: 2 }, index: 0 }`.
|
|
222
|
+
- Worker functions (both inline functions and native scripts using `defineWorker`) receive all step parameters in their first argument.
|
|
199
223
|
|
|
200
224
|
### When to use pipeline vs runWorker
|
|
201
225
|
|
|
@@ -304,11 +328,42 @@ The framework handles the caching transparently — your function always receive
|
|
|
304
328
|
|
|
305
329
|
The cached dataset lives in worker memory until `release()` is called. For large datasets, always call `release()` when you're done to free the memory:
|
|
306
330
|
|
|
331
|
+
After releasing, the next `runPersistent` call will create a fresh worker instance (requiring a new dataset).
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
335
|
+
## Lifecycle Management (`terminate`, `destroy`, `reset`, `restart`)
|
|
336
|
+
|
|
337
|
+
`MainWorkerFactory` provides built-in lifecycle management to terminate running workers and clean up browser resources:
|
|
338
|
+
|
|
339
|
+
### `terminate()` / `destroy()`
|
|
340
|
+
|
|
341
|
+
Immediately stops all active workers (one-shot tasks, pipelines, reducers) and releases all cached persistent workers:
|
|
342
|
+
|
|
307
343
|
```ts
|
|
308
|
-
|
|
344
|
+
// Stop all active threads and release persistent workers
|
|
345
|
+
factory.terminate();
|
|
346
|
+
// or
|
|
347
|
+
factory.destroy(); // Alias for terminate()
|
|
348
|
+
|
|
349
|
+
console.log(factory.isTerminated); // true
|
|
309
350
|
```
|
|
310
351
|
|
|
311
|
-
After
|
|
352
|
+
After calling `terminate()`, any attempt to run workers on the factory instance will immediately reject.
|
|
353
|
+
|
|
354
|
+
### `reset()` / `restart()`
|
|
355
|
+
|
|
356
|
+
Terminates all active/persistent worker instances and restores the factory to an active state (`isTerminated = false`), allowing new worker instances to be initiated cleanly:
|
|
357
|
+
|
|
358
|
+
```ts
|
|
359
|
+
// Stop existing workers and reset factory state
|
|
360
|
+
factory.reset(); // or factory.restart()
|
|
361
|
+
|
|
362
|
+
console.log(factory.isTerminated); // false
|
|
363
|
+
|
|
364
|
+
// Factory is ready to initiate fresh worker instances again
|
|
365
|
+
const settled = await factory.runWorker('sum', { srcData: [1, 2, 3] });
|
|
366
|
+
```
|
|
312
367
|
|
|
313
368
|
---
|
|
314
369
|
|
package/dist/index.cjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"use strict";var
|
|
1
|
+
"use strict";var S=Object.defineProperty;var x=(n,e,r)=>e in n?S(n,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):n[e]=r;var y=(n,e,r)=>x(n,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const O=n=>`
|
|
2
2
|
const extractTransferables = (value, seen = new Set()) => {
|
|
3
3
|
if (value === null || typeof value !== 'object') return [];
|
|
4
4
|
if (seen.has(value)) return [];
|
|
@@ -21,7 +21,7 @@ self.addEventListener('message', async (event) => {
|
|
|
21
21
|
self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
22
22
|
}
|
|
23
23
|
})
|
|
24
|
-
`,
|
|
24
|
+
`,F=n=>`
|
|
25
25
|
const extractTransferables = (value, seen = new Set()) => {
|
|
26
26
|
if (value === null || typeof value !== 'object') return [];
|
|
27
27
|
if (seen.has(value)) return [];
|
|
@@ -40,10 +40,15 @@ const workerFn = ${n};
|
|
|
40
40
|
let outputPort = null;
|
|
41
41
|
let inputPort = null;
|
|
42
42
|
let pendingData = null;
|
|
43
|
+
let stepParams = {};
|
|
43
44
|
|
|
44
45
|
async function processData(data) {
|
|
45
46
|
try {
|
|
46
|
-
const
|
|
47
|
+
const payload =
|
|
48
|
+
typeof data === 'object' && data !== null && 'data' in data
|
|
49
|
+
? { ...stepParams, ...data }
|
|
50
|
+
: { data, ...stepParams, index: 0 };
|
|
51
|
+
const output = await workerFn(payload);
|
|
47
52
|
const result = { ok: true, data: output };
|
|
48
53
|
const transfers = extractTransferables(output);
|
|
49
54
|
if (outputPort) {
|
|
@@ -63,6 +68,9 @@ async function processData(data) {
|
|
|
63
68
|
|
|
64
69
|
self.addEventListener('message', (event) => {
|
|
65
70
|
if (event.data && event.data.__pipeline_ports__) {
|
|
71
|
+
if (event.data.stepParams) {
|
|
72
|
+
stepParams = event.data.stepParams;
|
|
73
|
+
}
|
|
66
74
|
if (event.data.outputPort) {
|
|
67
75
|
outputPort = event.data.outputPort;
|
|
68
76
|
}
|
|
@@ -74,7 +82,7 @@ self.addEventListener('message', (event) => {
|
|
|
74
82
|
if (outputPort) outputPort.postMessage(e.data);
|
|
75
83
|
else self.postMessage(e.data);
|
|
76
84
|
} else {
|
|
77
|
-
processData({ data: e.data.data, index: 0 });
|
|
85
|
+
processData({ data: e.data.data, ...stepParams, index: 0 });
|
|
78
86
|
}
|
|
79
87
|
};
|
|
80
88
|
}
|
|
@@ -93,7 +101,7 @@ self.addEventListener('message', (event) => {
|
|
|
93
101
|
pendingData = event.data;
|
|
94
102
|
}
|
|
95
103
|
});
|
|
96
|
-
`,
|
|
104
|
+
`,R=n=>`
|
|
97
105
|
const extractTransferables = (value, seen = new Set()) => {
|
|
98
106
|
if (value === null || typeof value !== 'object') return [];
|
|
99
107
|
if (seen.has(value)) return [];
|
|
@@ -149,8 +157,8 @@ self.addEventListener('message', async (event) => {
|
|
|
149
157
|
self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
150
158
|
}
|
|
151
159
|
});
|
|
152
|
-
`;var
|
|
153
|
-
const reducer = ${
|
|
160
|
+
`;var P=(n=>(n.Default="default",n.Pipeline="pipeline",n.Persistent="persistent",n))(P||{});const C=Object.freeze({persistent:R,pipeline:F,default:O});class v{constructor(e,r){y(this,"_worker");if(r!=null&&r.createWorker)this._worker=r.createWorker();else if(e){const s=(r==null?void 0:r.mode)??"default",t=C[s](e.toString()),l=new Blob([t],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(l))}else throw new Error("Either workerFunction or options.createWorker must be provided to WorkerFactory.")}get getWorker(){return this._worker}}class j{constructor(e){this.results=e}}function m(n,e=new Set){return n===null||typeof n!="object"?[]:e.has(n)?[]:(e.add(n),n instanceof ArrayBuffer||n instanceof MessagePort||typeof ImageBitmap<"u"&&n instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&n instanceof OffscreenCanvas?[n]:ArrayBuffer.isView(n)?[n.buffer]:Array.isArray(n)?n.flatMap(r=>m(r,e)):Object.values(n).flatMap(r=>m(r,e)))}class L{constructor(e){y(this,"_workers");y(this,"_threads");y(this,"_persistentWorkers",new Map);y(this,"_activeWorkers",new Set);y(this,"_isTerminated",!1);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}get isTerminated(){return this._isTerminated}trackWorker(e){if(this._isTerminated)throw e.terminate(),new Error("MainWorkerFactory has been terminated");return this._activeWorkers.add(e),e}terminateWorker(e){this._activeWorkers.delete(e);try{e.terminate()}catch{}}initWorker(e){const r=new v(e.func,{createWorker:e.createWorker});return this.trackWorker(r.getWorker),r}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const s=Math.min(r,e.length),t=Math.floor(e.length/s),l=e.length%s,c=[];let f=0;for(let a=0;a<s;a++){const o=t+(a<l?1:0);c.push(e.slice(f,f+o)),f+=o}return c}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,{srcData:r,...s}){if(this._isTerminated)return Promise.reject(new Error("MainWorkerFactory has been terminated"));const t=this.findWorkerByName(e);if(!t)return Promise.reject(new Error(`Worker "${e}" not found`));const l=t.maxConcurrency??this._threads,c=!!(Array.isArray(r)&&r.length>1&&t.partition),f=c?this.partitionArray(r,l):r,a=this.createWorkerPromises(t,e,{data:f,...s},l,c),o=await Promise.allSettled(a);return new j(o)}createWorkerPromises(e,r,s,t,l){const{data:c,...f}=s;return Array.from({length:t},(a,o)=>{const d=l&&Array.isArray(c)?c[o]:c;return this.runWorkerWithRetry({workerFunc:e.func,createWorker:e.createWorker,workerName:r,index:o,data:{data:d,...f}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(s){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,s),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",s),s}}initiateWorker({workerFunc:e,createWorker:r,workerName:s,index:t,data:l}){return new Promise((c,f)=>{const o=this.initWorker({name:s,role:"",func:e,createWorker:r}).getWorker;o.onerror=p=>{this.terminateWorker(o),f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},failedResult:p})},o.onmessage=p=>{var i,u;if(((i=p.data)==null?void 0:i.ok)===!1){this.terminateWorker(o),f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},failedResult:new ErrorEvent("error",{message:p.data.error})});return}c({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},successResult:new MessageEvent("message",{data:(u=p.data)==null?void 0:u.data})}),this.terminateWorker(o)};const d={index:t,...Array.isArray(l)?{data:l}:l};o.postMessage(d,m(d))})}async collectResults(e,r={}){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const s=e.results.filter(a=>a.status==="fulfilled"),t=e.results.filter(a=>a.status==="rejected"),l=s.map(a=>a.value.successResult.data),c=r.reducer?r.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((a,o)=>{const d=`
|
|
161
|
+
const reducer = ${c};
|
|
154
162
|
self.addEventListener('message', (event) => {
|
|
155
163
|
try {
|
|
156
164
|
const result = reducer(event.data);
|
|
@@ -159,4 +167,4 @@ self.addEventListener('message', async (event) => {
|
|
|
159
167
|
self.postMessage({ ok: false, error: String(err) });
|
|
160
168
|
}
|
|
161
169
|
});
|
|
162
|
-
`,
|
|
170
|
+
`,p=new Blob([d],{type:"application/javascript"}),i=this.trackWorker(new Worker(URL.createObjectURL(p)));i.onmessage=u=>{this.terminateWorker(i),u.data.ok?a(u.data.data):o(new Error(u.data.error))},i.onerror=u=>{this.terminateWorker(i),o(u)},i.postMessage(l)}),succeeded:s.length,failed:t.length,errors:t}}async pipeline(e){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const r=e[0],{worker:s,srcData:t,...l}=r,c=this.findWorkerByName(r.worker);if(!c)throw new Error(`Worker "${r.worker}" not found`);const a=this.initWorker(c).getWorker;return new Promise((o,d)=>{a.onmessage=u=>{var k,g;this.terminateWorker(a),((k=u.data)==null?void 0:k.ok)===!1?d(new Error(u.data.error)):o((g=u.data)==null?void 0:g.data)},a.onerror=u=>{this.terminateWorker(a),d(u)};const i={data:t??{},...l,index:0};a.postMessage(i,m(i))})}return new Promise((r,s)=>{const t=[],l=[];for(const i of e){const u=this.findWorkerByName(i.worker);if(!u){s(new Error(`Worker "${i.worker}" not found`));return}const k=new v(u.func,{mode:P.Pipeline,createWorker:u.createWorker}),g=this.trackWorker(k.getWorker);t.push(g)}for(let i=0;i<t.length-1;i++)l.push(new MessageChannel);for(let i=0;i<t.length;i++){const{worker:u,srcData:k,...g}=e[i],W=[],w={};if(i>0&&(w.inputPort=l[i-1].port1,W.push(w.inputPort)),i<t.length-1&&(w.outputPort=l[i].port2,W.push(w.outputPort)),t[i].postMessage({__pipeline_ports__:!0,stepParams:g,...w},W),i<t.length-1){const _=t[i],A=t[i+1],{worker:I,srcData:U,...D}=e[i+1];_.onmessage=h=>{var E,T;if(h.data&&h.data.__pipeline_ports__)return;if(((E=h.data)==null?void 0:E.ok)===!1){t.forEach(B=>this.terminateWorker(B)),s(new Error(h.data.error));return}const b={data:((T=h.data)==null?void 0:T.ok)!==void 0?h.data.data:h.data,...D,index:0};A.postMessage(b,m(b))},_.onerror=h=>{t.forEach(M=>this.terminateWorker(M)),s(h)}}}const c=t[t.length-1];c.onmessage=i=>{var u,k;t.forEach(g=>this.terminateWorker(g)),((u=i.data)==null?void 0:u.ok)===!1?s(new Error(i.data.error)):r((k=i.data)==null?void 0:k.data)},c.onerror=i=>{t.forEach(u=>this.terminateWorker(u)),s(i)};const{worker:f,srcData:a,...o}=e[0],p={data:a??{},...o,index:0};t[0].postMessage(p,m(p))})}async runPersistent(e,r){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const s=this.findWorkerByName(e);if(!s)throw new Error(`Worker "${e}" not found`);let t=this._persistentWorkers.get(e);if(!t){const l=new v(s.func,{mode:P.Persistent,createWorker:s.createWorker});t=this.trackWorker(l.getWorker),this._persistentWorkers.set(e,t)}return new Promise((l,c)=>{t.onmessage=a=>{var o,d;((o=a.data)==null?void 0:o.ok)===!1?c(new Error(a.data.error)):l((d=a.data)==null?void 0:d.data)},t.onerror=a=>{c(a)};const f={type:"run",config:r.config};r.dataset!==void 0&&(f.dataset=r.dataset),t.postMessage(f,m(f))})}release(e){const r=this._persistentWorkers.get(e);if(r){try{r.postMessage({type:"release"})}catch{}this.terminateWorker(r),this._persistentWorkers.delete(e)}}terminate(){this._isTerminated=!0;for(const e of this._persistentWorkers.values()){try{e.postMessage({type:"release"})}catch{}this.terminateWorker(e)}this._persistentWorkers.clear();for(const e of Array.from(this._activeWorkers))this.terminateWorker(e);this._activeWorkers.clear()}destroy(){this.terminate()}reset(){this.terminate(),this._isTerminated=!1}restart(){this.reset()}}function $(n){if(typeof self>"u")return;let e=null,r=null,s=null,t={};const l=(f,a)=>{self.postMessage(f,a)};async function c(f){try{const a=typeof f=="object"&&f!==null&&"data"in f?{...t,...f}:{data:f,...t,index:0},o=await n(a),d={ok:!0,data:o},p=m(o);e?e.postMessage(d,p):l(d,p)}catch(a){const o={ok:!1,error:a instanceof Error?a.message:String(a)};e?e.postMessage(o):l(o)}}self.addEventListener("message",f=>{const a=f.data;if(a&&a.__pipeline_ports__){a.stepParams&&(t=a.stepParams),a.outputPort&&(e=a.outputPort),a.inputPort&&(r=a.inputPort,r.onmessage=o=>{var d;o.data&&o.data.ok===!1?e?e.postMessage(o.data):l(o.data):c({data:(d=o.data)==null?void 0:d.data,...t,index:0})}),s!==null&&(c(s),s=null);return}r?s=a:c(a)})}exports.MainWorkerFactory=L;exports.WorkerFactory=v;exports.defineWorker=$;
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
const
|
|
1
|
+
var S = Object.defineProperty;
|
|
2
|
+
var x = (n, e, r) => e in n ? S(n, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : n[e] = r;
|
|
3
|
+
var y = (n, e, r) => x(n, typeof e != "symbol" ? e + "" : e, r);
|
|
4
|
+
const O = (n) => `
|
|
5
5
|
const extractTransferables = (value, seen = new Set()) => {
|
|
6
6
|
if (value === null || typeof value !== 'object') return [];
|
|
7
7
|
if (seen.has(value)) return [];
|
|
@@ -24,7 +24,7 @@ self.addEventListener('message', async (event) => {
|
|
|
24
24
|
self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
25
25
|
}
|
|
26
26
|
})
|
|
27
|
-
`,
|
|
27
|
+
`, R = (n) => `
|
|
28
28
|
const extractTransferables = (value, seen = new Set()) => {
|
|
29
29
|
if (value === null || typeof value !== 'object') return [];
|
|
30
30
|
if (seen.has(value)) return [];
|
|
@@ -43,10 +43,15 @@ const workerFn = ${n};
|
|
|
43
43
|
let outputPort = null;
|
|
44
44
|
let inputPort = null;
|
|
45
45
|
let pendingData = null;
|
|
46
|
+
let stepParams = {};
|
|
46
47
|
|
|
47
48
|
async function processData(data) {
|
|
48
49
|
try {
|
|
49
|
-
const
|
|
50
|
+
const payload =
|
|
51
|
+
typeof data === 'object' && data !== null && 'data' in data
|
|
52
|
+
? { ...stepParams, ...data }
|
|
53
|
+
: { data, ...stepParams, index: 0 };
|
|
54
|
+
const output = await workerFn(payload);
|
|
50
55
|
const result = { ok: true, data: output };
|
|
51
56
|
const transfers = extractTransferables(output);
|
|
52
57
|
if (outputPort) {
|
|
@@ -66,6 +71,9 @@ async function processData(data) {
|
|
|
66
71
|
|
|
67
72
|
self.addEventListener('message', (event) => {
|
|
68
73
|
if (event.data && event.data.__pipeline_ports__) {
|
|
74
|
+
if (event.data.stepParams) {
|
|
75
|
+
stepParams = event.data.stepParams;
|
|
76
|
+
}
|
|
69
77
|
if (event.data.outputPort) {
|
|
70
78
|
outputPort = event.data.outputPort;
|
|
71
79
|
}
|
|
@@ -77,7 +85,7 @@ self.addEventListener('message', (event) => {
|
|
|
77
85
|
if (outputPort) outputPort.postMessage(e.data);
|
|
78
86
|
else self.postMessage(e.data);
|
|
79
87
|
} else {
|
|
80
|
-
processData({ data: e.data.data, index: 0 });
|
|
88
|
+
processData({ data: e.data.data, ...stepParams, index: 0 });
|
|
81
89
|
}
|
|
82
90
|
};
|
|
83
91
|
}
|
|
@@ -96,7 +104,7 @@ self.addEventListener('message', (event) => {
|
|
|
96
104
|
pendingData = event.data;
|
|
97
105
|
}
|
|
98
106
|
});
|
|
99
|
-
`,
|
|
107
|
+
`, F = (n) => `
|
|
100
108
|
const extractTransferables = (value, seen = new Set()) => {
|
|
101
109
|
if (value === null || typeof value !== 'object') return [];
|
|
102
110
|
if (seen.has(value)) return [];
|
|
@@ -153,39 +161,33 @@ self.addEventListener('message', async (event) => {
|
|
|
153
161
|
}
|
|
154
162
|
});
|
|
155
163
|
`;
|
|
156
|
-
var
|
|
157
|
-
const
|
|
158
|
-
persistent:
|
|
159
|
-
pipeline:
|
|
160
|
-
default:
|
|
164
|
+
var P = /* @__PURE__ */ ((n) => (n.Default = "default", n.Pipeline = "pipeline", n.Persistent = "persistent", n))(P || {});
|
|
165
|
+
const C = Object.freeze({
|
|
166
|
+
persistent: F,
|
|
167
|
+
pipeline: R,
|
|
168
|
+
default: O
|
|
161
169
|
});
|
|
162
|
-
class
|
|
170
|
+
class W {
|
|
163
171
|
/**
|
|
164
|
-
* Creates a new `Worker` from the given function.
|
|
165
|
-
*
|
|
166
|
-
* The function is stringified, embedded into a self-contained worker script,
|
|
167
|
-
* converted to a `Blob` URL, and passed to the `Worker` constructor.
|
|
172
|
+
* Creates a new `Worker` from the given function or factory option.
|
|
168
173
|
*
|
|
169
174
|
* @param workerFunction - The function to run inside the worker thread.
|
|
170
175
|
* Must be self-contained — it cannot reference variables from the outer
|
|
171
176
|
* scope because it is serialized via `.toString()`.
|
|
172
|
-
* @param options - Optional configuration
|
|
173
|
-
* worker execution mode (default, pipeline, or persistent).
|
|
177
|
+
* @param options - Optional configuration containing `createWorker` or `mode`.
|
|
174
178
|
*/
|
|
175
179
|
constructor(e, r) {
|
|
176
|
-
|
|
177
|
-
if (r != null && r.
|
|
178
|
-
this._worker = r.
|
|
179
|
-
else if (r != null && r.workerURL)
|
|
180
|
-
this._worker = new Worker(r.workerURL, { type: "module" });
|
|
180
|
+
y(this, "_worker");
|
|
181
|
+
if (r != null && r.createWorker)
|
|
182
|
+
this._worker = r.createWorker();
|
|
181
183
|
else if (e) {
|
|
182
|
-
const
|
|
184
|
+
const s = (r == null ? void 0 : r.mode) ?? "default", t = C[s](e.toString()), l = new Blob([t], {
|
|
183
185
|
type: "application/javascript"
|
|
184
186
|
});
|
|
185
|
-
this._worker = new Worker(URL.createObjectURL(
|
|
187
|
+
this._worker = new Worker(URL.createObjectURL(l));
|
|
186
188
|
} else
|
|
187
189
|
throw new Error(
|
|
188
|
-
"Either workerFunction or options.
|
|
190
|
+
"Either workerFunction or options.createWorker must be provided to WorkerFactory."
|
|
189
191
|
);
|
|
190
192
|
}
|
|
191
193
|
/**
|
|
@@ -198,38 +200,65 @@ class h {
|
|
|
198
200
|
return this._worker;
|
|
199
201
|
}
|
|
200
202
|
}
|
|
201
|
-
class
|
|
203
|
+
class L {
|
|
202
204
|
constructor(e) {
|
|
203
205
|
this.results = e;
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
|
-
function
|
|
207
|
-
return n === null || typeof n != "object" ? [] : e.has(n) ? [] : (e.add(n), n instanceof ArrayBuffer || n instanceof MessagePort || typeof ImageBitmap < "u" && n instanceof ImageBitmap || typeof OffscreenCanvas < "u" && n instanceof OffscreenCanvas ? [n] : ArrayBuffer.isView(n) ? [n.buffer] : Array.isArray(n) ? n.flatMap((r) =>
|
|
208
|
-
(r) =>
|
|
208
|
+
function m(n, e = /* @__PURE__ */ new Set()) {
|
|
209
|
+
return n === null || typeof n != "object" ? [] : e.has(n) ? [] : (e.add(n), n instanceof ArrayBuffer || n instanceof MessagePort || typeof ImageBitmap < "u" && n instanceof ImageBitmap || typeof OffscreenCanvas < "u" && n instanceof OffscreenCanvas ? [n] : ArrayBuffer.isView(n) ? [n.buffer] : Array.isArray(n) ? n.flatMap((r) => m(r, e)) : Object.values(n).flatMap(
|
|
210
|
+
(r) => m(r, e)
|
|
209
211
|
));
|
|
210
212
|
}
|
|
211
|
-
class
|
|
213
|
+
class U {
|
|
212
214
|
/**
|
|
213
215
|
* Creates a new `MainWorkerFactory`.
|
|
214
216
|
*
|
|
215
217
|
* @param options - Configuration object containing the `workers` registry.
|
|
216
218
|
*/
|
|
217
219
|
constructor(e) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
220
|
+
y(this, "_workers");
|
|
221
|
+
y(this, "_threads");
|
|
222
|
+
y(this, "_persistentWorkers", /* @__PURE__ */ new Map());
|
|
223
|
+
y(this, "_activeWorkers", /* @__PURE__ */ new Set());
|
|
224
|
+
y(this, "_isTerminated", !1);
|
|
221
225
|
this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
|
|
222
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* Returns `true` if the factory has been terminated.
|
|
229
|
+
*/
|
|
230
|
+
get isTerminated() {
|
|
231
|
+
return this._isTerminated;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Registers an active worker instance for lifecycle tracking.
|
|
235
|
+
*/
|
|
236
|
+
trackWorker(e) {
|
|
237
|
+
if (this._isTerminated)
|
|
238
|
+
throw e.terminate(), new Error("MainWorkerFactory has been terminated");
|
|
239
|
+
return this._activeWorkers.add(e), e;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Terminates a worker instance and removes it from tracking.
|
|
243
|
+
*/
|
|
244
|
+
terminateWorker(e) {
|
|
245
|
+
this._activeWorkers.delete(e);
|
|
246
|
+
try {
|
|
247
|
+
e.terminate();
|
|
248
|
+
} catch {
|
|
249
|
+
}
|
|
250
|
+
}
|
|
223
251
|
/**
|
|
224
252
|
* Instantiates a {@link WorkerFactory} for the given worker configuration.
|
|
225
253
|
*
|
|
226
|
-
* @param config - The worker configuration containing `func` or `
|
|
254
|
+
* @param config - The worker configuration containing `func` or `createWorker`.
|
|
227
255
|
* @returns A new `WorkerFactory` wrapping the worker.
|
|
228
256
|
*/
|
|
229
257
|
initWorker(e) {
|
|
230
|
-
|
|
231
|
-
|
|
258
|
+
const r = new W(e.func, {
|
|
259
|
+
createWorker: e.createWorker
|
|
232
260
|
});
|
|
261
|
+
return this.trackWorker(r.getWorker), r;
|
|
233
262
|
}
|
|
234
263
|
/**
|
|
235
264
|
* Splits an array into up to `numChunks` evenly-sized sub-arrays.
|
|
@@ -250,13 +279,13 @@ class _ {
|
|
|
250
279
|
partitionArray(e, r) {
|
|
251
280
|
if (!e.length) return [];
|
|
252
281
|
if (r <= 0) throw new Error("numChunks must be positive");
|
|
253
|
-
const
|
|
254
|
-
let
|
|
255
|
-
for (let
|
|
256
|
-
const
|
|
257
|
-
|
|
282
|
+
const s = Math.min(r, e.length), t = Math.floor(e.length / s), l = e.length % s, c = [];
|
|
283
|
+
let f = 0;
|
|
284
|
+
for (let a = 0; a < s; a++) {
|
|
285
|
+
const o = t + (a < l ? 1 : 0);
|
|
286
|
+
c.push(e.slice(f, f + o)), f += o;
|
|
258
287
|
}
|
|
259
|
-
return
|
|
288
|
+
return c;
|
|
260
289
|
}
|
|
261
290
|
/**
|
|
262
291
|
* Looks up a registered worker configuration by name.
|
|
@@ -295,19 +324,21 @@ class _ {
|
|
|
295
324
|
*/
|
|
296
325
|
async runWorker(e, {
|
|
297
326
|
srcData: r,
|
|
298
|
-
...
|
|
327
|
+
...s
|
|
299
328
|
}) {
|
|
300
|
-
|
|
301
|
-
|
|
329
|
+
if (this._isTerminated)
|
|
330
|
+
return Promise.reject(new Error("MainWorkerFactory has been terminated"));
|
|
331
|
+
const t = this.findWorkerByName(e);
|
|
332
|
+
if (!t)
|
|
302
333
|
return Promise.reject(new Error(`Worker "${e}" not found`));
|
|
303
|
-
const
|
|
304
|
-
|
|
334
|
+
const l = t.maxConcurrency ?? this._threads, c = !!(Array.isArray(r) && r.length > 1 && t.partition), f = c ? this.partitionArray(r, l) : r, a = this.createWorkerPromises(
|
|
335
|
+
t,
|
|
305
336
|
e,
|
|
306
|
-
{ data:
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
),
|
|
310
|
-
return new
|
|
337
|
+
{ data: f, ...s },
|
|
338
|
+
l,
|
|
339
|
+
c
|
|
340
|
+
), o = await Promise.allSettled(a);
|
|
341
|
+
return new L(o);
|
|
311
342
|
}
|
|
312
343
|
/**
|
|
313
344
|
* Builds the array of per-thread worker promises for a single `runWorker`
|
|
@@ -323,17 +354,17 @@ class _ {
|
|
|
323
354
|
* @param isPartitioned - Whether `data` is a pre-split array of shards.
|
|
324
355
|
* @returns An array of promises, one per thread.
|
|
325
356
|
*/
|
|
326
|
-
createWorkerPromises(e, r,
|
|
327
|
-
const { data:
|
|
328
|
-
return Array.from({ length:
|
|
329
|
-
const
|
|
357
|
+
createWorkerPromises(e, r, s, t, l) {
|
|
358
|
+
const { data: c, ...f } = s;
|
|
359
|
+
return Array.from({ length: t }, (a, o) => {
|
|
360
|
+
const d = l && Array.isArray(c) ? c[o] : c;
|
|
330
361
|
return this.runWorkerWithRetry(
|
|
331
362
|
{
|
|
332
363
|
workerFunc: e.func,
|
|
333
|
-
|
|
364
|
+
createWorker: e.createWorker,
|
|
334
365
|
workerName: r,
|
|
335
|
-
index:
|
|
336
|
-
data: { data:
|
|
366
|
+
index: o,
|
|
367
|
+
data: { data: d, ...f }
|
|
337
368
|
},
|
|
338
369
|
e.retries
|
|
339
370
|
);
|
|
@@ -354,13 +385,13 @@ class _ {
|
|
|
354
385
|
async runWorkerWithRetry(e, r = 2) {
|
|
355
386
|
try {
|
|
356
387
|
return await this.initiateWorker(e);
|
|
357
|
-
} catch (
|
|
388
|
+
} catch (s) {
|
|
358
389
|
if (r > 0)
|
|
359
390
|
return console.error(
|
|
360
391
|
`Worker ${e.index} failed, retrying (${r} left):`,
|
|
361
|
-
|
|
392
|
+
s
|
|
362
393
|
), this.runWorkerWithRetry(e, r - 1);
|
|
363
|
-
throw console.error("Worker failed after all retries:",
|
|
394
|
+
throw console.error("Worker failed after all retries:", s), s;
|
|
364
395
|
}
|
|
365
396
|
}
|
|
366
397
|
/**
|
|
@@ -375,57 +406,74 @@ class _ {
|
|
|
375
406
|
* Any transferable objects found in the payload are moved (not copied) to
|
|
376
407
|
* the worker via the `transfer` list of `postMessage`.
|
|
377
408
|
*
|
|
378
|
-
* The underlying `Worker` is always terminated after the
|
|
379
|
-
* whether it succeeded or failed.
|
|
409
|
+
* The underlying `Worker` is always terminated after the message completes.
|
|
380
410
|
*
|
|
381
|
-
* @param instanceConfig - Worker function,
|
|
411
|
+
* @param instanceConfig - Worker function, factory, name, shard index, and data.
|
|
382
412
|
* @returns A promise that resolves with the worker's result.
|
|
383
413
|
*/
|
|
384
414
|
initiateWorker({
|
|
385
415
|
workerFunc: e,
|
|
386
|
-
|
|
387
|
-
workerName:
|
|
388
|
-
index:
|
|
389
|
-
data:
|
|
416
|
+
createWorker: r,
|
|
417
|
+
workerName: s,
|
|
418
|
+
index: t,
|
|
419
|
+
data: l
|
|
390
420
|
}) {
|
|
391
|
-
return new Promise((
|
|
392
|
-
const
|
|
393
|
-
name:
|
|
421
|
+
return new Promise((c, f) => {
|
|
422
|
+
const o = this.initWorker({
|
|
423
|
+
name: s,
|
|
394
424
|
role: "",
|
|
395
425
|
func: e,
|
|
396
|
-
|
|
426
|
+
createWorker: r
|
|
397
427
|
}).getWorker;
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
index:
|
|
401
|
-
workerConfigs: {
|
|
402
|
-
|
|
428
|
+
o.onerror = (p) => {
|
|
429
|
+
this.terminateWorker(o), f({
|
|
430
|
+
index: t,
|
|
431
|
+
workerConfigs: {
|
|
432
|
+
workerFunc: e,
|
|
433
|
+
createWorker: r,
|
|
434
|
+
workerName: s,
|
|
435
|
+
index: t,
|
|
436
|
+
data: l
|
|
437
|
+
},
|
|
438
|
+
failedResult: p
|
|
403
439
|
});
|
|
404
|
-
},
|
|
405
|
-
var
|
|
406
|
-
if (((
|
|
407
|
-
|
|
408
|
-
index:
|
|
409
|
-
workerConfigs: {
|
|
440
|
+
}, o.onmessage = (p) => {
|
|
441
|
+
var i, u;
|
|
442
|
+
if (((i = p.data) == null ? void 0 : i.ok) === !1) {
|
|
443
|
+
this.terminateWorker(o), f({
|
|
444
|
+
index: t,
|
|
445
|
+
workerConfigs: {
|
|
446
|
+
workerFunc: e,
|
|
447
|
+
createWorker: r,
|
|
448
|
+
workerName: s,
|
|
449
|
+
index: t,
|
|
450
|
+
data: l
|
|
451
|
+
},
|
|
410
452
|
failedResult: new ErrorEvent("error", {
|
|
411
|
-
message:
|
|
453
|
+
message: p.data.error
|
|
412
454
|
})
|
|
413
455
|
});
|
|
414
456
|
return;
|
|
415
457
|
}
|
|
416
|
-
|
|
417
|
-
index:
|
|
418
|
-
workerConfigs: {
|
|
458
|
+
c({
|
|
459
|
+
index: t,
|
|
460
|
+
workerConfigs: {
|
|
461
|
+
workerFunc: e,
|
|
462
|
+
createWorker: r,
|
|
463
|
+
workerName: s,
|
|
464
|
+
index: t,
|
|
465
|
+
data: l
|
|
466
|
+
},
|
|
419
467
|
successResult: new MessageEvent("message", {
|
|
420
|
-
data: (
|
|
468
|
+
data: (u = p.data) == null ? void 0 : u.data
|
|
421
469
|
})
|
|
422
|
-
}),
|
|
470
|
+
}), this.terminateWorker(o);
|
|
423
471
|
};
|
|
424
|
-
const
|
|
425
|
-
index:
|
|
426
|
-
...Array.isArray(
|
|
472
|
+
const d = {
|
|
473
|
+
index: t,
|
|
474
|
+
...Array.isArray(l) ? { data: l } : l
|
|
427
475
|
};
|
|
428
|
-
|
|
476
|
+
o.postMessage(d, m(d));
|
|
429
477
|
});
|
|
430
478
|
}
|
|
431
479
|
/**
|
|
@@ -461,15 +509,17 @@ class _ {
|
|
|
461
509
|
* });
|
|
462
510
|
*/
|
|
463
511
|
async collectResults(e, r = {}) {
|
|
464
|
-
|
|
465
|
-
(
|
|
466
|
-
|
|
467
|
-
(
|
|
468
|
-
),
|
|
512
|
+
if (this._isTerminated)
|
|
513
|
+
throw new Error("MainWorkerFactory has been terminated");
|
|
514
|
+
const s = e.results.filter(
|
|
515
|
+
(a) => a.status === "fulfilled"
|
|
516
|
+
), t = e.results.filter(
|
|
517
|
+
(a) => a.status === "rejected"
|
|
518
|
+
), l = s.map((a) => a.value.successResult.data), c = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
|
|
469
519
|
return {
|
|
470
|
-
data: await new Promise((
|
|
471
|
-
const
|
|
472
|
-
const reducer = ${
|
|
520
|
+
data: await new Promise((a, o) => {
|
|
521
|
+
const d = `
|
|
522
|
+
const reducer = ${c};
|
|
473
523
|
self.addEventListener('message', (event) => {
|
|
474
524
|
try {
|
|
475
525
|
const result = reducer(event.data);
|
|
@@ -478,16 +528,16 @@ class _ {
|
|
|
478
528
|
self.postMessage({ ok: false, error: String(err) });
|
|
479
529
|
}
|
|
480
530
|
});
|
|
481
|
-
`,
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
},
|
|
485
|
-
|
|
486
|
-
},
|
|
531
|
+
`, p = new Blob([d], { type: "application/javascript" }), i = this.trackWorker(new Worker(URL.createObjectURL(p)));
|
|
532
|
+
i.onmessage = (u) => {
|
|
533
|
+
this.terminateWorker(i), u.data.ok ? a(u.data.data) : o(new Error(u.data.error));
|
|
534
|
+
}, i.onerror = (u) => {
|
|
535
|
+
this.terminateWorker(i), o(u);
|
|
536
|
+
}, i.postMessage(l);
|
|
487
537
|
}),
|
|
488
|
-
succeeded:
|
|
489
|
-
failed:
|
|
490
|
-
errors:
|
|
538
|
+
succeeded: s.length,
|
|
539
|
+
failed: t.length,
|
|
540
|
+
errors: t
|
|
491
541
|
};
|
|
492
542
|
}
|
|
493
543
|
/**
|
|
@@ -517,61 +567,71 @@ class _ {
|
|
|
517
567
|
* console.log(result); // final transformed + filtered data
|
|
518
568
|
*/
|
|
519
569
|
async pipeline(e) {
|
|
570
|
+
if (this._isTerminated)
|
|
571
|
+
throw new Error("MainWorkerFactory has been terminated");
|
|
520
572
|
if (e.length === 0)
|
|
521
573
|
throw new Error("Pipeline requires at least one step");
|
|
522
574
|
if (e.length === 1) {
|
|
523
|
-
const r = e[0],
|
|
524
|
-
if (!
|
|
525
|
-
const
|
|
526
|
-
return new Promise((
|
|
527
|
-
|
|
528
|
-
var
|
|
529
|
-
|
|
530
|
-
},
|
|
531
|
-
|
|
575
|
+
const r = e[0], { worker: s, srcData: t, ...l } = r, c = this.findWorkerByName(r.worker);
|
|
576
|
+
if (!c) throw new Error(`Worker "${r.worker}" not found`);
|
|
577
|
+
const a = this.initWorker(c).getWorker;
|
|
578
|
+
return new Promise((o, d) => {
|
|
579
|
+
a.onmessage = (u) => {
|
|
580
|
+
var k, g;
|
|
581
|
+
this.terminateWorker(a), ((k = u.data) == null ? void 0 : k.ok) === !1 ? d(new Error(u.data.error)) : o((g = u.data) == null ? void 0 : g.data);
|
|
582
|
+
}, a.onerror = (u) => {
|
|
583
|
+
this.terminateWorker(a), d(u);
|
|
532
584
|
};
|
|
533
|
-
const
|
|
534
|
-
|
|
535
|
-
{ data: t, index: 0 },
|
|
536
|
-
g(t)
|
|
537
|
-
);
|
|
585
|
+
const i = { data: t ?? {}, ...l, index: 0 };
|
|
586
|
+
a.postMessage(i, m(i));
|
|
538
587
|
});
|
|
539
588
|
}
|
|
540
|
-
return new Promise((r,
|
|
541
|
-
const
|
|
542
|
-
for (const
|
|
543
|
-
const
|
|
544
|
-
if (!
|
|
545
|
-
|
|
589
|
+
return new Promise((r, s) => {
|
|
590
|
+
const t = [], l = [];
|
|
591
|
+
for (const i of e) {
|
|
592
|
+
const u = this.findWorkerByName(i.worker);
|
|
593
|
+
if (!u) {
|
|
594
|
+
s(new Error(`Worker "${i.worker}" not found`));
|
|
546
595
|
return;
|
|
547
596
|
}
|
|
548
|
-
const
|
|
549
|
-
mode:
|
|
550
|
-
|
|
551
|
-
});
|
|
552
|
-
|
|
597
|
+
const k = new W(u.func, {
|
|
598
|
+
mode: P.Pipeline,
|
|
599
|
+
createWorker: u.createWorker
|
|
600
|
+
}), g = this.trackWorker(k.getWorker);
|
|
601
|
+
t.push(g);
|
|
553
602
|
}
|
|
554
|
-
for (let
|
|
555
|
-
|
|
556
|
-
for (let
|
|
557
|
-
const
|
|
558
|
-
|
|
559
|
-
{ __pipeline_ports__: !0, ...
|
|
560
|
-
|
|
561
|
-
)
|
|
603
|
+
for (let i = 0; i < t.length - 1; i++)
|
|
604
|
+
l.push(new MessageChannel());
|
|
605
|
+
for (let i = 0; i < t.length; i++) {
|
|
606
|
+
const { worker: u, srcData: k, ...g } = e[i], v = [], w = {};
|
|
607
|
+
if (i > 0 && (w.inputPort = l[i - 1].port1, v.push(w.inputPort)), i < t.length - 1 && (w.outputPort = l[i].port2, v.push(w.outputPort)), t[i].postMessage(
|
|
608
|
+
{ __pipeline_ports__: !0, stepParams: g, ...w },
|
|
609
|
+
v
|
|
610
|
+
), i < t.length - 1) {
|
|
611
|
+
const _ = t[i], A = t[i + 1], { worker: j, srcData: $, ...D } = e[i + 1];
|
|
612
|
+
_.onmessage = (h) => {
|
|
613
|
+
var E, T;
|
|
614
|
+
if (h.data && h.data.__pipeline_ports__) return;
|
|
615
|
+
if (((E = h.data) == null ? void 0 : E.ok) === !1) {
|
|
616
|
+
t.forEach((B) => this.terminateWorker(B)), s(new Error(h.data.error));
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
const b = { data: ((T = h.data) == null ? void 0 : T.ok) !== void 0 ? h.data.data : h.data, ...D, index: 0 };
|
|
620
|
+
A.postMessage(b, m(b));
|
|
621
|
+
}, _.onerror = (h) => {
|
|
622
|
+
t.forEach((M) => this.terminateWorker(M)), s(h);
|
|
623
|
+
};
|
|
624
|
+
}
|
|
562
625
|
}
|
|
563
|
-
const
|
|
564
|
-
|
|
565
|
-
var
|
|
566
|
-
|
|
567
|
-
},
|
|
568
|
-
|
|
626
|
+
const c = t[t.length - 1];
|
|
627
|
+
c.onmessage = (i) => {
|
|
628
|
+
var u, k;
|
|
629
|
+
t.forEach((g) => this.terminateWorker(g)), ((u = i.data) == null ? void 0 : u.ok) === !1 ? s(new Error(i.data.error)) : r((k = i.data) == null ? void 0 : k.data);
|
|
630
|
+
}, c.onerror = (i) => {
|
|
631
|
+
t.forEach((u) => this.terminateWorker(u)), s(i);
|
|
569
632
|
};
|
|
570
|
-
const
|
|
571
|
-
|
|
572
|
-
{ data: l, index: 0 },
|
|
573
|
-
g(l)
|
|
574
|
-
);
|
|
633
|
+
const { worker: f, srcData: a, ...o } = e[0], p = { data: a ?? {}, ...o, index: 0 };
|
|
634
|
+
t[0].postMessage(p, m(p));
|
|
575
635
|
});
|
|
576
636
|
}
|
|
577
637
|
/**
|
|
@@ -609,24 +669,30 @@ class _ {
|
|
|
609
669
|
* factory.release('transform');
|
|
610
670
|
*/
|
|
611
671
|
async runPersistent(e, r) {
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
672
|
+
if (this._isTerminated)
|
|
673
|
+
throw new Error("MainWorkerFactory has been terminated");
|
|
674
|
+
const s = this.findWorkerByName(e);
|
|
675
|
+
if (!s) throw new Error(`Worker "${e}" not found`);
|
|
676
|
+
let t = this._persistentWorkers.get(e);
|
|
677
|
+
if (!t) {
|
|
678
|
+
const l = new W(s.func, {
|
|
679
|
+
mode: P.Persistent,
|
|
680
|
+
createWorker: s.createWorker
|
|
681
|
+
});
|
|
682
|
+
t = this.trackWorker(l.getWorker), this._persistentWorkers.set(e, t);
|
|
683
|
+
}
|
|
684
|
+
return new Promise((l, c) => {
|
|
685
|
+
t.onmessage = (a) => {
|
|
686
|
+
var o, d;
|
|
687
|
+
((o = a.data) == null ? void 0 : o.ok) === !1 ? c(new Error(a.data.error)) : l((d = a.data) == null ? void 0 : d.data);
|
|
688
|
+
}, t.onerror = (a) => {
|
|
689
|
+
c(a);
|
|
624
690
|
};
|
|
625
|
-
const
|
|
691
|
+
const f = {
|
|
626
692
|
type: "run",
|
|
627
693
|
config: r.config
|
|
628
694
|
};
|
|
629
|
-
r.dataset !== void 0 && (
|
|
695
|
+
r.dataset !== void 0 && (f.dataset = r.dataset), t.postMessage(f, m(f));
|
|
630
696
|
});
|
|
631
697
|
}
|
|
632
698
|
/**
|
|
@@ -640,10 +706,86 @@ class _ {
|
|
|
640
706
|
*/
|
|
641
707
|
release(e) {
|
|
642
708
|
const r = this._persistentWorkers.get(e);
|
|
643
|
-
|
|
709
|
+
if (r) {
|
|
710
|
+
try {
|
|
711
|
+
r.postMessage({ type: "release" });
|
|
712
|
+
} catch {
|
|
713
|
+
}
|
|
714
|
+
this.terminateWorker(r), this._persistentWorkers.delete(e);
|
|
715
|
+
}
|
|
644
716
|
}
|
|
717
|
+
/**
|
|
718
|
+
* Terminates the factory and all active and persistent worker instances.
|
|
719
|
+
*
|
|
720
|
+
* Calling `terminate()` immediately stops all running worker threads, releases
|
|
721
|
+
* cached persistent workers, and clears all internal worker state.
|
|
722
|
+
*/
|
|
723
|
+
terminate() {
|
|
724
|
+
this._isTerminated = !0;
|
|
725
|
+
for (const e of this._persistentWorkers.values()) {
|
|
726
|
+
try {
|
|
727
|
+
e.postMessage({ type: "release" });
|
|
728
|
+
} catch {
|
|
729
|
+
}
|
|
730
|
+
this.terminateWorker(e);
|
|
731
|
+
}
|
|
732
|
+
this._persistentWorkers.clear();
|
|
733
|
+
for (const e of Array.from(this._activeWorkers))
|
|
734
|
+
this.terminateWorker(e);
|
|
735
|
+
this._activeWorkers.clear();
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Alias for {@link terminate}. Terminates the factory and all worker instances.
|
|
739
|
+
*/
|
|
740
|
+
destroy() {
|
|
741
|
+
this.terminate();
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Resets the factory by terminating all active and persistent workers
|
|
745
|
+
* and resetting the factory state, allowing new worker instances to be initiated.
|
|
746
|
+
*/
|
|
747
|
+
reset() {
|
|
748
|
+
this.terminate(), this._isTerminated = !1;
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Alias for {@link reset}. Resets the factory state to initiate new worker instances.
|
|
752
|
+
*/
|
|
753
|
+
restart() {
|
|
754
|
+
this.reset();
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
function V(n) {
|
|
758
|
+
if (typeof self > "u") return;
|
|
759
|
+
let e = null, r = null, s = null, t = {};
|
|
760
|
+
const l = (f, a) => {
|
|
761
|
+
self.postMessage(f, a);
|
|
762
|
+
};
|
|
763
|
+
async function c(f) {
|
|
764
|
+
try {
|
|
765
|
+
const a = typeof f == "object" && f !== null && "data" in f ? { ...t, ...f } : { data: f, ...t, index: 0 }, o = await n(a), d = { ok: !0, data: o }, p = m(o);
|
|
766
|
+
e ? e.postMessage(d, p) : l(d, p);
|
|
767
|
+
} catch (a) {
|
|
768
|
+
const o = {
|
|
769
|
+
ok: !1,
|
|
770
|
+
error: a instanceof Error ? a.message : String(a)
|
|
771
|
+
};
|
|
772
|
+
e ? e.postMessage(o) : l(o);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
self.addEventListener("message", (f) => {
|
|
776
|
+
const a = f.data;
|
|
777
|
+
if (a && a.__pipeline_ports__) {
|
|
778
|
+
a.stepParams && (t = a.stepParams), a.outputPort && (e = a.outputPort), a.inputPort && (r = a.inputPort, r.onmessage = (o) => {
|
|
779
|
+
var d;
|
|
780
|
+
o.data && o.data.ok === !1 ? e ? e.postMessage(o.data) : l(o.data) : c({ data: (d = o.data) == null ? void 0 : d.data, ...t, index: 0 });
|
|
781
|
+
}), s !== null && (c(s), s = null);
|
|
782
|
+
return;
|
|
783
|
+
}
|
|
784
|
+
r ? s = a : c(a);
|
|
785
|
+
});
|
|
645
786
|
}
|
|
646
787
|
export {
|
|
647
|
-
|
|
648
|
-
|
|
788
|
+
U as MainWorkerFactory,
|
|
789
|
+
W as WorkerFactory,
|
|
790
|
+
V as defineWorker
|
|
649
791
|
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { WorkerFunction } from './main-worker-factory/types';
|
|
2
|
+
/**
|
|
3
|
+
* Defines and exports a function to run inside a native Web Worker script,
|
|
4
|
+
* providing full compatibility with `MainWorkerFactory` features including
|
|
5
|
+
* standard runs, worker-to-worker pipelines (`foreman.pipeline()`), and
|
|
6
|
+
* dataset caching (`foreman.runPersistent()`).
|
|
7
|
+
*
|
|
8
|
+
* @typeParam TParams - The type of the payload sent to the worker.
|
|
9
|
+
* @typeParam TResult - The return type of the worker function.
|
|
10
|
+
*
|
|
11
|
+
* @param workerFn - The worker execution function.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* // my-native-worker.ts
|
|
15
|
+
* import { defineWorker } from '@offmain/workerkit';
|
|
16
|
+
*
|
|
17
|
+
* export default defineWorker(async ({ data }: { data: number[] }) => {
|
|
18
|
+
* return data.map((x) => x * 2);
|
|
19
|
+
* });
|
|
20
|
+
*/
|
|
21
|
+
export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
|
|
2
2
|
/**
|
|
3
3
|
* Recursively collects all Transferable objects from a value.
|
|
4
|
-
*
|
|
4
|
+
* Transferable (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
|
|
5
5
|
* are zero-copy — they are moved to the worker instead of cloned.
|
|
6
6
|
*/
|
|
7
|
-
export declare function
|
|
7
|
+
export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
|
|
8
8
|
/**
|
|
9
9
|
* Central orchestrator for running typed Web Workers in parallel.
|
|
10
10
|
*
|
|
@@ -36,6 +36,8 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
|
|
|
36
36
|
private readonly _workers;
|
|
37
37
|
private readonly _threads;
|
|
38
38
|
private readonly _persistentWorkers;
|
|
39
|
+
private readonly _activeWorkers;
|
|
40
|
+
private _isTerminated;
|
|
39
41
|
/**
|
|
40
42
|
* Creates a new `MainWorkerFactory`.
|
|
41
43
|
*
|
|
@@ -44,10 +46,22 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
|
|
|
44
46
|
constructor(options: {
|
|
45
47
|
workers: TConfigs;
|
|
46
48
|
});
|
|
49
|
+
/**
|
|
50
|
+
* Returns `true` if the factory has been terminated.
|
|
51
|
+
*/
|
|
52
|
+
get isTerminated(): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Registers an active worker instance for lifecycle tracking.
|
|
55
|
+
*/
|
|
56
|
+
private trackWorker;
|
|
57
|
+
/**
|
|
58
|
+
* Terminates a worker instance and removes it from tracking.
|
|
59
|
+
*/
|
|
60
|
+
private terminateWorker;
|
|
47
61
|
/**
|
|
48
62
|
* Instantiates a {@link WorkerFactory} for the given worker configuration.
|
|
49
63
|
*
|
|
50
|
-
* @param config - The worker configuration containing `func` or `
|
|
64
|
+
* @param config - The worker configuration containing `func` or `createWorker`.
|
|
51
65
|
* @returns A new `WorkerFactory` wrapping the worker.
|
|
52
66
|
*/
|
|
53
67
|
private initWorker;
|
|
@@ -144,10 +158,9 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
|
|
|
144
158
|
* Any transferable objects found in the payload are moved (not copied) to
|
|
145
159
|
* the worker via the `transfer` list of `postMessage`.
|
|
146
160
|
*
|
|
147
|
-
* The underlying `Worker` is always terminated after the
|
|
148
|
-
* whether it succeeded or failed.
|
|
161
|
+
* The underlying `Worker` is always terminated after the message completes.
|
|
149
162
|
*
|
|
150
|
-
* @param instanceConfig - Worker function,
|
|
163
|
+
* @param instanceConfig - Worker function, factory, name, shard index, and data.
|
|
151
164
|
* @returns A promise that resolves with the worker's result.
|
|
152
165
|
*/
|
|
153
166
|
private initiateWorker;
|
|
@@ -259,5 +272,25 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
|
|
|
259
272
|
* @param workerName - Name of the persistent worker to release.
|
|
260
273
|
*/
|
|
261
274
|
release(workerName: string): void;
|
|
275
|
+
/**
|
|
276
|
+
* Terminates the factory and all active and persistent worker instances.
|
|
277
|
+
*
|
|
278
|
+
* Calling `terminate()` immediately stops all running worker threads, releases
|
|
279
|
+
* cached persistent workers, and clears all internal worker state.
|
|
280
|
+
*/
|
|
281
|
+
terminate(): void;
|
|
282
|
+
/**
|
|
283
|
+
* Alias for {@link terminate}. Terminates the factory and all worker instances.
|
|
284
|
+
*/
|
|
285
|
+
destroy(): void;
|
|
286
|
+
/**
|
|
287
|
+
* Resets the factory by terminating all active and persistent workers
|
|
288
|
+
* and resetting the factory state, allowing new worker instances to be initiated.
|
|
289
|
+
*/
|
|
290
|
+
reset(): void;
|
|
291
|
+
/**
|
|
292
|
+
* Alias for {@link reset}. Resets the factory state to initiate new worker instances.
|
|
293
|
+
*/
|
|
294
|
+
restart(): void;
|
|
262
295
|
}
|
|
263
296
|
export default MainWorkerFactory;
|
|
@@ -23,13 +23,14 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
|
|
|
23
23
|
name: WorkerName;
|
|
24
24
|
/** Human-readable role label (e.g. `'compute'`, `'transform'`). */
|
|
25
25
|
role: WorkerRole;
|
|
26
|
-
/** The worker function that will be serialised and run in a thread. Optional if `
|
|
26
|
+
/** The worker function that will be serialised and run in a thread. Optional if `createWorker` is provided. */
|
|
27
27
|
func?: TFunc;
|
|
28
28
|
/**
|
|
29
|
-
*
|
|
30
|
-
* Enables
|
|
29
|
+
* Factory function returning a native `Worker` instance (e.g. `() => new Worker(new URL('./worker.ts', import.meta.url))`).
|
|
30
|
+
* Enables bundlers like Webpack 5, Vite, Rollup, and Parcel to statically analyze and bundle worker code into individual chunks,
|
|
31
|
+
* while allowing `MainWorkerFactory` to scale concurrency and manage worker thread lifecycles.
|
|
31
32
|
*/
|
|
32
|
-
|
|
33
|
+
createWorker?: () => Worker;
|
|
33
34
|
/**
|
|
34
35
|
* Maximum number of parallel threads to spawn for this worker.
|
|
35
36
|
* Defaults to `navigator.hardwareConcurrency` when omitted.
|
|
@@ -103,10 +104,10 @@ export interface MainWorkerFactoryWorker extends WorkerConfig {
|
|
|
103
104
|
export interface WorkerInstanceConfig<TFunc extends WorkerFunction = WorkerFunction> {
|
|
104
105
|
/** Name of the parent worker config, used in logs and error objects. */
|
|
105
106
|
workerName: WorkerName;
|
|
106
|
-
/** The function serialised and executed inside the thread (optional if `
|
|
107
|
+
/** The function serialised and executed inside the thread (optional if `createWorker` is set). */
|
|
107
108
|
workerFunc?: TFunc;
|
|
108
|
-
/**
|
|
109
|
-
|
|
109
|
+
/** Factory function returning a Worker instance. */
|
|
110
|
+
createWorker?: () => Worker;
|
|
110
111
|
/** Zero-based shard index assigned to this thread. */
|
|
111
112
|
index: number;
|
|
112
113
|
/** The data payload (full or partitioned shard) sent to the thread. */
|
|
@@ -167,4 +168,6 @@ export interface PipelineStep {
|
|
|
167
168
|
worker: string;
|
|
168
169
|
/** Input data for the first step (subsequent steps receive previous output) */
|
|
169
170
|
srcData?: unknown;
|
|
171
|
+
/** Any additional step parameters, configs, or options forwarded to the worker payload. */
|
|
172
|
+
[key: string]: unknown;
|
|
170
173
|
}
|
|
@@ -7,10 +7,8 @@ export declare enum WorkerMode {
|
|
|
7
7
|
export interface WorkerFactoryOptions {
|
|
8
8
|
/** The worker execution mode. Defaults to `WorkerMode.Default`. */
|
|
9
9
|
mode?: WorkerMode;
|
|
10
|
-
/**
|
|
11
|
-
|
|
12
|
-
/** The worker URL to use instead of creating a worker from a function. */
|
|
13
|
-
workerURL?: string | URL;
|
|
10
|
+
/** Factory function returning a native `Worker` instance. */
|
|
11
|
+
createWorker?: () => Worker;
|
|
14
12
|
}
|
|
15
13
|
/**
|
|
16
14
|
* Low-level factory that serializes a {@link WorkerFunction} into a Blob URL
|
|
@@ -36,16 +34,12 @@ export interface WorkerFactoryOptions {
|
|
|
36
34
|
declare class WorkerFactory {
|
|
37
35
|
readonly _worker: Worker;
|
|
38
36
|
/**
|
|
39
|
-
* Creates a new `Worker` from the given function.
|
|
40
|
-
*
|
|
41
|
-
* The function is stringified, embedded into a self-contained worker script,
|
|
42
|
-
* converted to a `Blob` URL, and passed to the `Worker` constructor.
|
|
37
|
+
* Creates a new `Worker` from the given function or factory option.
|
|
43
38
|
*
|
|
44
39
|
* @param workerFunction - The function to run inside the worker thread.
|
|
45
40
|
* Must be self-contained — it cannot reference variables from the outer
|
|
46
41
|
* scope because it is serialized via `.toString()`.
|
|
47
|
-
* @param options - Optional configuration
|
|
48
|
-
* worker execution mode (default, pipeline, or persistent).
|
|
42
|
+
* @param options - Optional configuration containing `createWorker` or `mode`.
|
|
49
43
|
*/
|
|
50
44
|
constructor(workerFunction?: WorkerFunction, options?: WorkerFactoryOptions);
|
|
51
45
|
/**
|
package/package.json
CHANGED