@offmain/workerkit 0.14.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -46
- package/dist/define-worker.cjs +1 -0
- package/dist/define-worker.js +4 -0
- package/dist/index-2AONniOz.js +59 -0
- package/dist/index-CXKVsLvY.cjs +1 -0
- package/dist/index.cjs +195 -43
- package/dist/index.js +945 -487
- package/dist/types/tools/collect-results/collect-results.d.ts +26 -0
- package/dist/types/tools/collect-results/index.d.ts +2 -0
- package/dist/types/tools/collect-results/types.d.ts +10 -0
- package/dist/types/tools/define-worker/define-worker.d.ts +31 -0
- package/dist/types/tools/define-worker/define-worker.test.d.ts +1 -0
- package/dist/types/tools/define-worker/index.d.ts +1 -0
- package/dist/types/tools/define-worker-config/define-worker-config.d.ts +23 -0
- package/dist/types/tools/define-worker-config/index.d.ts +1 -0
- package/dist/types/tools/extract-transferable/extract-transferable.d.ts +15 -0
- package/dist/types/tools/extract-transferable/extract-transferable.test.d.ts +1 -0
- package/dist/types/tools/extract-transferable/index.d.ts +1 -0
- package/dist/types/tools/index.d.ts +2 -0
- package/dist/types/tools/logger/index.d.ts +2 -0
- package/dist/types/tools/logger/logger.d.ts +21 -0
- package/dist/types/tools/logger/logger.test.d.ts +1 -0
- package/dist/types/tools/logger/types.d.ts +6 -0
- package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +94 -228
- package/dist/types/tools/main-worker-factory/types.d.ts +71 -4
- package/dist/types/tools/memory-store/index.d.ts +3 -0
- package/dist/types/tools/memory-store/memory-store.d.ts +50 -0
- package/dist/types/tools/memory-store/memory-store.test.d.ts +1 -0
- package/dist/types/tools/memory-store/memory-worker-proxy.d.ts +75 -0
- package/dist/types/tools/memory-store/memory-worker.d.ts +11 -0
- package/dist/types/tools/orchestrator/index.d.ts +2 -0
- package/dist/types/tools/orchestrator/orchestrator.d.ts +21 -0
- package/dist/types/tools/orchestrator/orchestrator.test.d.ts +1 -0
- package/dist/types/tools/orchestrator/types.d.ts +10 -0
- package/dist/types/tools/partition-array/index.d.ts +1 -0
- package/dist/types/tools/partition-array/partition-array.d.ts +14 -0
- package/dist/types/tools/partition-array/partition-array.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/index.d.ts +2 -0
- package/dist/types/tools/persistent-manager/persistent-manager.d.ts +25 -0
- package/dist/types/tools/persistent-manager/persistent-manager.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/types.d.ts +9 -0
- package/dist/types/tools/pipeline/index.d.ts +2 -0
- package/dist/types/tools/pipeline/pipeline.d.ts +17 -0
- package/dist/types/tools/pipeline/pipeline.test.d.ts +1 -0
- package/dist/types/tools/pipeline/types.d.ts +13 -0
- package/dist/types/tools/run-worker/index.d.ts +2 -0
- package/dist/types/tools/run-worker/run-worker.d.ts +26 -0
- package/dist/types/tools/run-worker/run-worker.test.d.ts +1 -0
- package/dist/types/tools/run-worker/types.d.ts +14 -0
- package/dist/types/tools/worker-factory/index.d.ts +1 -1
- package/dist/types/tools/worker-factory/worker-factory.d.ts +7 -1
- package/dist/types/workers/initiator.d.ts +1 -1
- package/dist/types/workers/initiator.test.d.ts +1 -0
- package/package.json +11 -5
- package/dist/types/tools/define-worker.d.ts +0 -21
- /package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts} +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { WorkerFactory } from '../worker-factory';
|
|
2
|
+
import { LogLevel } from '../logger';
|
|
2
3
|
/** A unique string identifier for a worker, matching its `name` field. */
|
|
3
4
|
export type WorkerName = string;
|
|
4
5
|
/** A descriptive label for the worker's role (e.g. `'compute'`, `'io'`). */
|
|
@@ -12,7 +13,7 @@ export type WorkerRole = string;
|
|
|
12
13
|
* @typeParam TParams - The type of the message payload sent to the worker.
|
|
13
14
|
* @typeParam TResult - The type of the value the worker posts back.
|
|
14
15
|
*/
|
|
15
|
-
export type WorkerFunction<TParams =
|
|
16
|
+
export type WorkerFunction<TParams = any, TResult = any> = (params: TParams) => TResult;
|
|
16
17
|
/**
|
|
17
18
|
* Configuration object that registers a named worker with the factory.
|
|
18
19
|
*
|
|
@@ -31,6 +32,11 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
|
|
|
31
32
|
* while allowing `MainWorkerFactory` to scale concurrency and manage worker thread lifecycles.
|
|
32
33
|
*/
|
|
33
34
|
createWorker?: () => Worker;
|
|
35
|
+
/**
|
|
36
|
+
* Optional compile-time type hint for workers that only use `createWorker`.
|
|
37
|
+
* Will not exist or be used at runtime.
|
|
38
|
+
*/
|
|
39
|
+
_typeHint?: TFunc;
|
|
34
40
|
/**
|
|
35
41
|
* Maximum number of parallel threads to spawn for this worker.
|
|
36
42
|
* Defaults to `navigator.hardwareConcurrency` when omitted.
|
|
@@ -52,6 +58,23 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
|
|
|
52
58
|
* If any dependency fails, this worker will not run.
|
|
53
59
|
*/
|
|
54
60
|
dependencies?: Array<() => void>;
|
|
61
|
+
/**
|
|
62
|
+
* When `true`, the worker's output is saved in the isolated `MemoryWorker`
|
|
63
|
+
* and returned alongside a `__memory_ref__` token.
|
|
64
|
+
*/
|
|
65
|
+
memory?: boolean;
|
|
66
|
+
/**
|
|
67
|
+
* When `true`, the worker's output is saved in the isolated `MemoryWorker`
|
|
68
|
+
* and **only** the `__memory_ref__` token is returned (0 bytes data transferred back to main thread).
|
|
69
|
+
*/
|
|
70
|
+
memoryOnly?: boolean;
|
|
71
|
+
}
|
|
72
|
+
/** Statistics about stored memory handles in MemoryWorker. */
|
|
73
|
+
export interface MemoryStats {
|
|
74
|
+
/** Total number of active memory references. */
|
|
75
|
+
count: number;
|
|
76
|
+
/** Array of active reference IDs. */
|
|
77
|
+
refs: string[];
|
|
55
78
|
}
|
|
56
79
|
/**
|
|
57
80
|
* Derives a `name → function` map from a readonly tuple of
|
|
@@ -62,9 +85,13 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
|
|
|
62
85
|
*
|
|
63
86
|
* @typeParam T - The readonly tuple of `WorkerConfig` values.
|
|
64
87
|
*/
|
|
65
|
-
export type WorkerConfigMap<T extends readonly WorkerConfig<
|
|
88
|
+
export type WorkerConfigMap<T extends readonly WorkerConfig<any>[]> = {
|
|
66
89
|
[K in T[number]['name']]: NonNullable<Extract<T[number], {
|
|
67
90
|
name: K;
|
|
91
|
+
}>['_typeHint']> extends WorkerFunction ? NonNullable<Extract<T[number], {
|
|
92
|
+
name: K;
|
|
93
|
+
}>['_typeHint']> : NonNullable<Extract<T[number], {
|
|
94
|
+
name: K;
|
|
68
95
|
}>['func']> extends WorkerFunction ? NonNullable<Extract<T[number], {
|
|
69
96
|
name: K;
|
|
70
97
|
}>['func']> : WorkerFunction;
|
|
@@ -91,6 +118,7 @@ export type WorkerReturnType<TFunc extends WorkerFunction> = TFunc extends Worke
|
|
|
91
118
|
/** Options passed to the `MainWorkerFactory` constructor. */
|
|
92
119
|
export interface MainWorkerFactoryOptions {
|
|
93
120
|
workers: WorkerConfig[];
|
|
121
|
+
logLevel?: LogLevel;
|
|
94
122
|
}
|
|
95
123
|
/** Internal representation of a worker config that has been instantiated. */
|
|
96
124
|
export interface MainWorkerFactoryWorker extends WorkerConfig {
|
|
@@ -131,6 +159,10 @@ export interface WorkerResult {
|
|
|
131
159
|
/**
|
|
132
160
|
* Typed wrapper around the settled results from `runWorker`.
|
|
133
161
|
* Carries `T` (the worker's return type) so `collectResults` can infer it.
|
|
162
|
+
*
|
|
163
|
+
* @internal Not part of the public API. `runWorker` now returns `CollectedResult<R>` directly.
|
|
164
|
+
* @deprecated Use the return value of `runWorker` directly.
|
|
165
|
+
* @template T - The worker's return type.
|
|
134
166
|
*/
|
|
135
167
|
export declare class TypedSettledResults<T> {
|
|
136
168
|
readonly results: PromiseSettledResult<WorkerResult>[];
|
|
@@ -138,7 +170,11 @@ export declare class TypedSettledResults<T> {
|
|
|
138
170
|
/** Never actually exists at runtime — used only for type inference. */
|
|
139
171
|
readonly __type: T;
|
|
140
172
|
}
|
|
141
|
-
/**
|
|
173
|
+
/**
|
|
174
|
+
* Options passed to `collectResults` (kept for the opt-out escape hatch).
|
|
175
|
+
* @template T - The type of a single shard result.
|
|
176
|
+
* @template R - The type of the merged result.
|
|
177
|
+
*/
|
|
142
178
|
export interface CollectOptions<T, R = T[]> {
|
|
143
179
|
/**
|
|
144
180
|
* Custom reducer applied to the array of fulfilled shard values.
|
|
@@ -151,7 +187,32 @@ export interface CollectOptions<T, R = T[]> {
|
|
|
151
187
|
*/
|
|
152
188
|
reducer?: (shards: T[]) => R;
|
|
153
189
|
}
|
|
154
|
-
/**
|
|
190
|
+
/**
|
|
191
|
+
* Options that can be passed alongside `srcData` in `runWorker` to control
|
|
192
|
+
* auto-collection behaviour.
|
|
193
|
+
* @template T - The type of a single shard result.
|
|
194
|
+
* @template R - The type of the merged result.
|
|
195
|
+
*/
|
|
196
|
+
export interface RunWorkerOptions<T = unknown, R = T[]> {
|
|
197
|
+
/**
|
|
198
|
+
* Custom reducer to merge shard results.
|
|
199
|
+
* Must be a self-contained function (no closures over external variables)
|
|
200
|
+
* since it is serialised via `.toString()` and run inside a worker.
|
|
201
|
+
* Defaults to `(shards) => shards.flat()`.
|
|
202
|
+
*/
|
|
203
|
+
reducer?: (shards: T[]) => R;
|
|
204
|
+
/**
|
|
205
|
+
* Set to `false` to skip auto-collection and receive a `TypedSettledResults`
|
|
206
|
+
* object instead of `CollectedResult`. Useful for advanced scenarios where
|
|
207
|
+
* you need to call `collectResults` with custom options manually.
|
|
208
|
+
* @default true
|
|
209
|
+
*/
|
|
210
|
+
autoCollect?: boolean;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Result returned by collectResults
|
|
214
|
+
* @template R - The type of the merged data.
|
|
215
|
+
*/
|
|
155
216
|
export interface CollectedResult<R> {
|
|
156
217
|
/** The merged output produced by the reducer */
|
|
157
218
|
data: R;
|
|
@@ -161,6 +222,12 @@ export interface CollectedResult<R> {
|
|
|
161
222
|
failed: number;
|
|
162
223
|
/** Raw rejected results, if any */
|
|
163
224
|
errors: PromiseRejectedResult[];
|
|
225
|
+
/**
|
|
226
|
+
* Present when the worker config has `memory: true`.
|
|
227
|
+
* The ref under which the merged result is stored in MemoryWorker,
|
|
228
|
+
* allowing it to be passed to subsequent workers without re-serializing.
|
|
229
|
+
*/
|
|
230
|
+
__memory_ref__?: string;
|
|
164
231
|
}
|
|
165
232
|
/** A single step in a worker pipeline */
|
|
166
233
|
export interface PipelineStep {
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryStore is a lightweight ref registry that lives on the main thread.
|
|
3
|
+
*
|
|
4
|
+
* It tracks **which memory refs exist** (and optional metadata) but does NOT
|
|
5
|
+
* store actual data. All large datasets are held inside the dedicated
|
|
6
|
+
* `MemoryWorker` thread, keeping the main thread heap free.
|
|
7
|
+
*
|
|
8
|
+
* Each ref is indexed by a cryptographically generated UUID to ensure
|
|
9
|
+
* collision-free, secure handle management.
|
|
10
|
+
*/
|
|
11
|
+
export declare class MemoryStore {
|
|
12
|
+
private readonly store;
|
|
13
|
+
/**
|
|
14
|
+
* Registers a ref in the registry.
|
|
15
|
+
*
|
|
16
|
+
* @param ref - The memory reference ID (generated by MemoryWorkerProxy).
|
|
17
|
+
* @param metadata - Optional metadata (size hint, type tag, etc.).
|
|
18
|
+
*/
|
|
19
|
+
register(ref: string, metadata?: RefMetadata): void;
|
|
20
|
+
/**
|
|
21
|
+
* Checks if a reference ID is registered.
|
|
22
|
+
* @param refId - The reference ID to check.
|
|
23
|
+
*/
|
|
24
|
+
has(refId: string): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Unregisters a reference ID.
|
|
27
|
+
*
|
|
28
|
+
* @param refId - The reference ID to unregister.
|
|
29
|
+
* @returns `true` if the key existed and was removed, `false` otherwise.
|
|
30
|
+
*/
|
|
31
|
+
delete(refId: string): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Clears all registered references.
|
|
34
|
+
*/
|
|
35
|
+
clear(): void;
|
|
36
|
+
/**
|
|
37
|
+
* Returns statistics about current registered handles.
|
|
38
|
+
*/
|
|
39
|
+
stats(): {
|
|
40
|
+
count: number;
|
|
41
|
+
refs: string[];
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
/** Lightweight metadata stored alongside a ref in the main-thread registry. */
|
|
45
|
+
export interface RefMetadata {
|
|
46
|
+
/** Optional size hint in bytes for diagnostics. */
|
|
47
|
+
size?: number;
|
|
48
|
+
/** Optional type tag (e.g. 'array', 'object'). */
|
|
49
|
+
type?: string;
|
|
50
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryWorkerProxy provides a Promise-based API for communicating with the
|
|
3
|
+
* dedicated `MemoryWorker` thread.
|
|
4
|
+
*
|
|
5
|
+
* It manages the worker's lifecycle, authenticates all requests with the
|
|
6
|
+
* factory token, and exposes high-level methods for storing, retrieving,
|
|
7
|
+
* and deleting datasets held off the main thread.
|
|
8
|
+
*
|
|
9
|
+
* The key method is `allocateWorkerPort()`, which creates a `MessageChannel`
|
|
10
|
+
* and registers one port with the MemoryWorker so that a computing worker
|
|
11
|
+
* or reducer worker can read/write data **directly** — without routing
|
|
12
|
+
* through the main thread.
|
|
13
|
+
*/
|
|
14
|
+
export declare class MemoryWorkerProxy {
|
|
15
|
+
private readonly worker;
|
|
16
|
+
private readonly factoryToken;
|
|
17
|
+
private readonly pending;
|
|
18
|
+
constructor(factoryToken: string);
|
|
19
|
+
private send;
|
|
20
|
+
/**
|
|
21
|
+
* Stores a dataset in the MemoryWorker.
|
|
22
|
+
*
|
|
23
|
+
* @param data - The dataset to store.
|
|
24
|
+
* @param ref - Optional ref ID; if omitted, MemoryWorker generates one.
|
|
25
|
+
* @returns The ref ID under which the data is stored.
|
|
26
|
+
*/
|
|
27
|
+
set(data: unknown, ref?: string): Promise<string>;
|
|
28
|
+
/**
|
|
29
|
+
* Retrieves a dataset from the MemoryWorker by ref ID.
|
|
30
|
+
*
|
|
31
|
+
* @param ref - The ref ID to retrieve.
|
|
32
|
+
* @returns The stored dataset, or `undefined` if not found.
|
|
33
|
+
*/
|
|
34
|
+
get(ref: string): Promise<unknown>;
|
|
35
|
+
/**
|
|
36
|
+
* Checks if a ref ID exists in the MemoryWorker.
|
|
37
|
+
* @param ref - The ref ID to check.
|
|
38
|
+
*/
|
|
39
|
+
has(ref: string): Promise<boolean>;
|
|
40
|
+
/**
|
|
41
|
+
* Deletes a ref from the MemoryWorker.
|
|
42
|
+
*
|
|
43
|
+
* @param ref - The ref ID to delete.
|
|
44
|
+
* @returns `true` if the ref existed and was deleted.
|
|
45
|
+
*/
|
|
46
|
+
delete(ref: string): Promise<boolean>;
|
|
47
|
+
/**
|
|
48
|
+
* Clears all data from the MemoryWorker.
|
|
49
|
+
*/
|
|
50
|
+
clear(): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Returns statistics about data held in the MemoryWorker.
|
|
53
|
+
*/
|
|
54
|
+
stats(): Promise<{
|
|
55
|
+
count: number;
|
|
56
|
+
refs: string[];
|
|
57
|
+
}>;
|
|
58
|
+
/**
|
|
59
|
+
* Allocates a direct `MessagePort` to the MemoryWorker for a computing or
|
|
60
|
+
* reducer worker.
|
|
61
|
+
*
|
|
62
|
+
* Creates a `MessageChannel` and registers one port with MemoryWorker via
|
|
63
|
+
* `REGISTER_PORT`. The other port (the "worker-side" port) is returned as a
|
|
64
|
+
* `Transferable` — it should be passed to the target worker via
|
|
65
|
+
* `postMessage(..., [workerPort])` so the worker can communicate with
|
|
66
|
+
* MemoryWorker directly without routing through the main thread.
|
|
67
|
+
*
|
|
68
|
+
* @returns The worker-side `MessagePort` ready to be transferred.
|
|
69
|
+
*/
|
|
70
|
+
allocateWorkerPort(): Promise<MessagePort>;
|
|
71
|
+
/**
|
|
72
|
+
* Terminates the MemoryWorker thread and clears all pending requests.
|
|
73
|
+
*/
|
|
74
|
+
terminate(): void;
|
|
75
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Script source for the dedicated MemoryWorker thread.
|
|
3
|
+
*
|
|
4
|
+
* Runs inside an isolated Web Worker thread and maintains the MemoryStore.
|
|
5
|
+
* Handshakes and authenticates all incoming MessagePort requests using a secret `factoryToken`.
|
|
6
|
+
*
|
|
7
|
+
* Supports direct worker-to-MemoryWorker communication via `REGISTER_PORT`:
|
|
8
|
+
* A `MessagePort` can be registered so that computing workers and reducer workers
|
|
9
|
+
* can read/write data directly without routing through the main thread.
|
|
10
|
+
*/
|
|
11
|
+
export declare const memoryWorkerScript = "\nconst store = new Map();\n\nfunction handleMessage(msg, replyTarget) {\n if (!msg || typeof msg !== 'object') return;\n\n const { action, factoryToken, expectedToken, ref, data, id } = msg;\n\n // Initial handshake to set expected token if needed\n if (action === 'INIT_TOKEN') {\n self.__expectedToken = expectedToken;\n replyTarget.postMessage({ ok: true, action: 'INIT_TOKEN_ACK' });\n return;\n }\n\n // Validate factory token\n if (self.__expectedToken && factoryToken !== self.__expectedToken) {\n replyTarget.postMessage({ ok: false, error: 'Unauthorized: invalid factory token', id });\n return;\n }\n\n try {\n switch (action) {\n case 'SET': {\n const refId = ref || ('mem_' + crypto.randomUUID());\n store.set(refId, data);\n replyTarget.postMessage({ ok: true, ref: refId, id });\n break;\n }\n case 'GET': {\n const resultData = store.get(ref);\n const exists = store.has(ref);\n replyTarget.postMessage({ ok: true, exists, data: resultData, ref, id });\n break;\n }\n case 'DELETE': {\n const deleted = store.delete(ref);\n replyTarget.postMessage({ ok: true, deleted, ref, id });\n break;\n }\n case 'CLEAR': {\n store.clear();\n replyTarget.postMessage({ ok: true, action: 'CLEAR_ACK', id });\n break;\n }\n case 'STATS': {\n replyTarget.postMessage({\n ok: true,\n stats: {\n count: store.size,\n refs: Array.from(store.keys()),\n },\n id,\n });\n break;\n }\n case 'REGISTER_PORT': {\n // Register a MessagePort from a computing or reducer worker.\n // All messages arriving on this port are handled with the same\n // store operations, enabling direct worker-to-MemoryWorker data flow.\n const port = msg.port;\n if (!port) {\n replyTarget.postMessage({ ok: false, error: 'REGISTER_PORT requires a port', id });\n break;\n }\n port.onmessage = (event) => handleMessage(event.data, port);\n port.start();\n replyTarget.postMessage({ ok: true, action: 'PORT_REGISTERED', id });\n break;\n }\n default:\n replyTarget.postMessage({ ok: false, error: 'Unknown action: ' + action, id });\n }\n } catch (err) {\n replyTarget.postMessage({\n ok: false,\n error: err instanceof Error ? err.message : String(err),\n id,\n });\n }\n}\n\nself.addEventListener('message', (event) => {\n // Handle REGISTER_PORT specially \u2014 the port itself is a Transferable in event.ports\n if (event.data && event.data.action === 'REGISTER_PORT') {\n const port = event.ports[0] ?? event.data.port;\n const msg = { ...event.data, port };\n handleMessage(msg, self);\n return;\n }\n handleMessage(event.data, self);\n});\n";
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { WorkerConfig, WorkerInstanceConfig, WorkerResult } from '../main-worker-factory/types';
|
|
2
|
+
import { OrchestratorContext } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* WorkerOrchestrator manages the lifecycle and execution of dynamically spawned Web Workers.
|
|
5
|
+
*
|
|
6
|
+
* Responsibilities:
|
|
7
|
+
* - Spawns requested worker instances based on a provided configuration.
|
|
8
|
+
* - Slices and partitions data payloads (if `isPartitioned` is true) to distribute work.
|
|
9
|
+
* - Handles auto-retries for failing workers without bubbling up errors prematurely.
|
|
10
|
+
* - Integrates with the context to track active workers and safely clean them up upon completion.
|
|
11
|
+
* - Standardizes the result wrapping into structured `WorkerResult` objects.
|
|
12
|
+
*/
|
|
13
|
+
export declare class WorkerOrchestrator {
|
|
14
|
+
private context;
|
|
15
|
+
constructor(context: OrchestratorContext);
|
|
16
|
+
createWorkerPromises(config: WorkerConfig, workerName: string, srcWorkerData: {
|
|
17
|
+
data: unknown;
|
|
18
|
+
} & Record<string, unknown>, threadCount: number, isPartitioned: boolean): Promise<WorkerResult>[];
|
|
19
|
+
runWorkerWithRetry(instanceConfig: WorkerInstanceConfig, retryCount?: number): Promise<WorkerResult>;
|
|
20
|
+
private initiateWorker;
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ILogger } from '../logger';
|
|
2
|
+
import { MemoryWorkerProxy } from '../memory-store';
|
|
3
|
+
export interface OrchestratorContext {
|
|
4
|
+
isTerminated: () => boolean;
|
|
5
|
+
trackWorker: (worker: Worker) => Worker;
|
|
6
|
+
terminateWorker: (worker: Worker) => void;
|
|
7
|
+
logger: ILogger;
|
|
8
|
+
memoryWorkerProxy: MemoryWorkerProxy;
|
|
9
|
+
factoryToken: string;
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './partition-array';
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Partitions a flat array into a specified number of smaller chunks.
|
|
3
|
+
*
|
|
4
|
+
* This is used to distribute a large dataset across multiple worker threads.
|
|
5
|
+
* If the array length is not perfectly divisible by `numChunks`, the remainder
|
|
6
|
+
* elements are distributed evenly across the first few chunks (1 extra element per chunk).
|
|
7
|
+
*
|
|
8
|
+
* @typeParam T - The type of elements within the array.
|
|
9
|
+
* @param array - The array to be partitioned.
|
|
10
|
+
* @param numChunks - The desired number of sub-arrays (chunks).
|
|
11
|
+
* @returns An array containing the partitioned sub-arrays. If the input array is empty, returns `[]`.
|
|
12
|
+
* @throws If `numChunks` is less than or equal to 0.
|
|
13
|
+
*/
|
|
14
|
+
export declare function partitionArray<T>(array: T[], numChunks: number): T[][];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { PersistentManagerContext } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* PersistentWorkerManager handles Web Workers that remain alive across multiple executions.
|
|
4
|
+
*
|
|
5
|
+
* Unlike transient workers which are terminated after a single run, persistent workers
|
|
6
|
+
* maintain their internal state and memory context. This makes them ideal for tasks that
|
|
7
|
+
* require heavy initialization, database connections, or large datasets.
|
|
8
|
+
*
|
|
9
|
+
* Responsibilities:
|
|
10
|
+
* - Lazily instantiates persistent workers on first use and caches their instances.
|
|
11
|
+
* - Routes execution requests (`type: 'run'`) to the correct running worker instance.
|
|
12
|
+
* - Provides memory safety by explicitly releasing specific workers or all workers
|
|
13
|
+
* (`terminateAll`) when they are no longer needed.
|
|
14
|
+
*/
|
|
15
|
+
export declare class PersistentWorkerManager {
|
|
16
|
+
private context;
|
|
17
|
+
private readonly _persistentWorkers;
|
|
18
|
+
constructor(context: PersistentManagerContext);
|
|
19
|
+
runPersistent<TResult = unknown>(workerName: string, params: {
|
|
20
|
+
dataset?: unknown;
|
|
21
|
+
config: unknown;
|
|
22
|
+
}): Promise<TResult>;
|
|
23
|
+
release(workerName: string): void;
|
|
24
|
+
terminateAll(): void;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { WorkerConfig } from '../main-worker-factory/types';
|
|
2
|
+
import { ILogger } from '../logger';
|
|
3
|
+
export interface PersistentManagerContext {
|
|
4
|
+
isTerminated: () => boolean;
|
|
5
|
+
findWorkerByName: (name: string) => WorkerConfig | undefined;
|
|
6
|
+
trackWorker: (worker: Worker) => Worker;
|
|
7
|
+
logger: ILogger;
|
|
8
|
+
terminateWorker: (worker: Worker) => void;
|
|
9
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { PipelineStep, CollectedResult } from '../main-worker-factory/types';
|
|
2
|
+
import { PipelineContext } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Executes a pipeline of workers, handling initialization, data flow, and termination.
|
|
5
|
+
*
|
|
6
|
+
* This function can execute either a single-step worker or a multi-step worker pipeline.
|
|
7
|
+
* For multi-step pipelines, workers are linked together via `MessageChannel`s.
|
|
8
|
+
* Data flows directly from one worker to the next without routing through the main thread,
|
|
9
|
+
* while errors and pipeline status communicate back to the main thread.
|
|
10
|
+
*
|
|
11
|
+
* @param steps - An array of pipeline steps, where each step defines the worker name and input parameters.
|
|
12
|
+
* @param context - The execution context (provides orchestrator, logger, memory store).
|
|
13
|
+
* @returns A promise that resolves to a CollectedResult containing the final output of the pipeline, or rejects on error.
|
|
14
|
+
* @throws {Error} If the pipeline is empty, the worker manager is terminated, or a worker is not found.
|
|
15
|
+
* @template TResult - The type of the pipeline result.
|
|
16
|
+
*/
|
|
17
|
+
export declare function executePipeline<TResult = unknown>(steps: PipelineStep[], context: PipelineContext): Promise<CollectedResult<TResult>>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { WorkerConfig } from '../main-worker-factory/types';
|
|
2
|
+
import { MemoryStore } from '../memory-store';
|
|
3
|
+
import { MemoryWorkerProxy } from '../memory-store/memory-worker-proxy';
|
|
4
|
+
import { ILogger } from '../logger';
|
|
5
|
+
export interface PipelineContext {
|
|
6
|
+
memoryStore: MemoryStore;
|
|
7
|
+
memoryWorkerProxy: MemoryWorkerProxy;
|
|
8
|
+
isTerminated: () => boolean;
|
|
9
|
+
findWorkerByName: (name: string) => WorkerConfig | undefined;
|
|
10
|
+
trackWorker: (worker: Worker) => Worker;
|
|
11
|
+
terminateWorker: (worker: Worker) => void;
|
|
12
|
+
logger: ILogger;
|
|
13
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { WorkerConfig, WorkerDataParam, WorkerReturnType, WorkerConfigMap, CollectedResult, RunWorkerOptions } from '../main-worker-factory/types';
|
|
2
|
+
import { RunWorkerContext } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Executes a specific worker with the provided parameters, optionally handling partitioning
|
|
5
|
+
* and data resolution from the memory store.
|
|
6
|
+
*
|
|
7
|
+
* Workers now run in Memory mode: they store their results directly into MemoryWorker
|
|
8
|
+
* via a pre-allocated MessagePort. Only `__memory_ref__` tokens return to the main thread.
|
|
9
|
+
* Auto-collect is then triggered to merge shards in a reducer worker (also without
|
|
10
|
+
* routing data through the main thread), and a `CollectedResult<R>` is returned.
|
|
11
|
+
*
|
|
12
|
+
* @param workerName - The name of the worker to execute (from configured workers).
|
|
13
|
+
* @param rawParams - The parameters to pass to the worker, including optional `srcData` and memory options.
|
|
14
|
+
* @param context - The execution context (provides orchestrator, logger, memory store, proxy).
|
|
15
|
+
* @returns A CollectedResult containing the merged data and shard stats.
|
|
16
|
+
* @throws {Error} If the worker factory is terminated or the worker is not found.
|
|
17
|
+
* @template TConfigs - A tuple of WorkerConfig configurations.
|
|
18
|
+
* @template TName - The name of the registered worker.
|
|
19
|
+
* @template T - The return type of the worker function.
|
|
20
|
+
* @template R - The merged return type after reducing shards.
|
|
21
|
+
*/
|
|
22
|
+
export declare function executeWorker<TConfigs extends readonly WorkerConfig[], TName extends keyof WorkerConfigMap<TConfigs> & string, T = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>, R = T extends (infer Item)[] ? Item[] : T[]>(workerName: TName, rawParams: {
|
|
23
|
+
srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
|
|
24
|
+
__memory_ref__?: string;
|
|
25
|
+
deleteMemory?: boolean;
|
|
26
|
+
} & RunWorkerOptions<T, R> & Record<string, unknown>, context: RunWorkerContext): Promise<CollectedResult<R>>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { WorkerConfig } from '../main-worker-factory/types';
|
|
2
|
+
import { WorkerOrchestrator } from '../orchestrator';
|
|
3
|
+
import { MemoryStore, MemoryWorkerProxy } from '../memory-store';
|
|
4
|
+
import { ILogger } from '../logger/types';
|
|
5
|
+
export interface RunWorkerContext {
|
|
6
|
+
isTerminated: () => boolean;
|
|
7
|
+
findWorkerByName: (name: string) => WorkerConfig | undefined;
|
|
8
|
+
memoryStore: MemoryStore;
|
|
9
|
+
memoryWorkerProxy: MemoryWorkerProxy;
|
|
10
|
+
factoryToken: string;
|
|
11
|
+
threads: number;
|
|
12
|
+
orchestrator: WorkerOrchestrator;
|
|
13
|
+
logger: ILogger;
|
|
14
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { default as WorkerFactory } from './worker-factory';
|
|
1
|
+
export { default as WorkerFactory, WorkerMode } from './worker-factory';
|
|
@@ -2,7 +2,13 @@ import { WorkerFunction } from '../main-worker-factory/types';
|
|
|
2
2
|
export declare enum WorkerMode {
|
|
3
3
|
Default = "default",
|
|
4
4
|
Pipeline = "pipeline",
|
|
5
|
-
Persistent = "persistent"
|
|
5
|
+
Persistent = "persistent",
|
|
6
|
+
/**
|
|
7
|
+
* Memory mode: the worker stores its result directly into the MemoryWorker
|
|
8
|
+
* thread via a pre-allocated `MessagePort`. Only a `__memory_ref__` token
|
|
9
|
+
* is posted back to the main thread — large data never touches main thread heap.
|
|
10
|
+
*/
|
|
11
|
+
Memory = "memory"
|
|
6
12
|
}
|
|
7
13
|
export interface WorkerFactoryOptions {
|
|
8
14
|
/** The worker execution mode. Defaults to `WorkerMode.Default`. */
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@offmain/workerkit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"description": "A lightweight manager for running functions in Web Workers with partitioning, retries, and concurrency control",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|
|
@@ -12,6 +12,11 @@
|
|
|
12
12
|
"import": "./dist/index.js",
|
|
13
13
|
"require": "./dist/index.cjs"
|
|
14
14
|
},
|
|
15
|
+
"./define-worker": {
|
|
16
|
+
"types": "./dist/types/tools/define-worker/index.d.ts",
|
|
17
|
+
"import": "./dist/define-worker.js",
|
|
18
|
+
"require": "./dist/define-worker.cjs"
|
|
19
|
+
},
|
|
15
20
|
"./eslint-plugin": "./eslint-rules/index.cjs",
|
|
16
21
|
"./eslint-rules/no-dom-in-worker": "./eslint-rules/no-dom-in-worker.cjs",
|
|
17
22
|
"./eslint-rules/worker-exportable": "./eslint-rules/worker-exportable.cjs"
|
|
@@ -60,6 +65,8 @@
|
|
|
60
65
|
},
|
|
61
66
|
"devDependencies": {
|
|
62
67
|
"@release-it/conventional-changelog": "11.0.0",
|
|
68
|
+
"@types/luxon": "^3.7.2",
|
|
69
|
+
"@types/node": "^26.4.0",
|
|
63
70
|
"@typescript-eslint/eslint-plugin": "^8.58.2",
|
|
64
71
|
"@typescript-eslint/parser": "^8.58.2",
|
|
65
72
|
"date-fns": "^4.1.0",
|
|
@@ -69,16 +76,15 @@
|
|
|
69
76
|
"eslint-plugin-local-rules": "^3.0.2",
|
|
70
77
|
"eslint-plugin-prettier": "^5.5.5",
|
|
71
78
|
"husky": "^9.1.7",
|
|
79
|
+
"i18next": "^25.5.1",
|
|
72
80
|
"jsdom": "^24.0.0",
|
|
73
81
|
"lint-staged": "^16.4.0",
|
|
82
|
+
"luxon": "^3.7.2",
|
|
74
83
|
"prettier": "^3.4.2",
|
|
75
84
|
"release-it": "19.0.3",
|
|
76
85
|
"typescript": "^5.7.2",
|
|
77
86
|
"vite": "^6.3.3",
|
|
78
87
|
"vite-plugin-dts": "^4.5.4",
|
|
79
|
-
"vitest": "^1.0.0"
|
|
80
|
-
"luxon": "^3.7.2",
|
|
81
|
-
"i18next": "^25.5.1",
|
|
82
|
-
"@types/luxon": "^3.7.2"
|
|
88
|
+
"vitest": "^1.0.0"
|
|
83
89
|
}
|
|
84
90
|
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { WorkerFunction } from './main-worker-factory/types';
|
|
2
|
-
/**
|
|
3
|
-
* Defines and exports a function to run inside a native Web Worker script,
|
|
4
|
-
* providing full compatibility with `MainWorkerFactory` features including
|
|
5
|
-
* standard runs, worker-to-worker pipelines (`foreman.pipeline()`), and
|
|
6
|
-
* dataset caching (`foreman.runPersistent()`).
|
|
7
|
-
*
|
|
8
|
-
* @typeParam TParams - The type of the payload sent to the worker.
|
|
9
|
-
* @typeParam TResult - The return type of the worker function.
|
|
10
|
-
*
|
|
11
|
-
* @param workerFn - The worker execution function.
|
|
12
|
-
*
|
|
13
|
-
* @example
|
|
14
|
-
* // my-native-worker.ts
|
|
15
|
-
* import { defineWorker } from '@offmain/workerkit';
|
|
16
|
-
*
|
|
17
|
-
* export default defineWorker(async ({ data }: { data: number[] }) => {
|
|
18
|
-
* return data.map((x) => x * 2);
|
|
19
|
-
* });
|
|
20
|
-
*/
|
|
21
|
-
export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;
|
/package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts}
RENAMED
|
File without changes
|