@offmain/workerkit 0.14.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -46
- package/dist/define-worker.cjs +1 -0
- package/dist/define-worker.js +4 -0
- package/dist/index-2AONniOz.js +59 -0
- package/dist/index-CXKVsLvY.cjs +1 -0
- package/dist/index.cjs +195 -43
- package/dist/index.js +945 -487
- package/dist/types/tools/collect-results/collect-results.d.ts +26 -0
- package/dist/types/tools/collect-results/index.d.ts +2 -0
- package/dist/types/tools/collect-results/types.d.ts +10 -0
- package/dist/types/tools/define-worker/define-worker.d.ts +31 -0
- package/dist/types/tools/define-worker/define-worker.test.d.ts +1 -0
- package/dist/types/tools/define-worker/index.d.ts +1 -0
- package/dist/types/tools/define-worker-config/define-worker-config.d.ts +23 -0
- package/dist/types/tools/define-worker-config/index.d.ts +1 -0
- package/dist/types/tools/extract-transferable/extract-transferable.d.ts +15 -0
- package/dist/types/tools/extract-transferable/extract-transferable.test.d.ts +1 -0
- package/dist/types/tools/extract-transferable/index.d.ts +1 -0
- package/dist/types/tools/index.d.ts +2 -0
- package/dist/types/tools/logger/index.d.ts +2 -0
- package/dist/types/tools/logger/logger.d.ts +21 -0
- package/dist/types/tools/logger/logger.test.d.ts +1 -0
- package/dist/types/tools/logger/types.d.ts +6 -0
- package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +94 -228
- package/dist/types/tools/main-worker-factory/types.d.ts +71 -4
- package/dist/types/tools/memory-store/index.d.ts +3 -0
- package/dist/types/tools/memory-store/memory-store.d.ts +50 -0
- package/dist/types/tools/memory-store/memory-store.test.d.ts +1 -0
- package/dist/types/tools/memory-store/memory-worker-proxy.d.ts +75 -0
- package/dist/types/tools/memory-store/memory-worker.d.ts +11 -0
- package/dist/types/tools/orchestrator/index.d.ts +2 -0
- package/dist/types/tools/orchestrator/orchestrator.d.ts +21 -0
- package/dist/types/tools/orchestrator/orchestrator.test.d.ts +1 -0
- package/dist/types/tools/orchestrator/types.d.ts +10 -0
- package/dist/types/tools/partition-array/index.d.ts +1 -0
- package/dist/types/tools/partition-array/partition-array.d.ts +14 -0
- package/dist/types/tools/partition-array/partition-array.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/index.d.ts +2 -0
- package/dist/types/tools/persistent-manager/persistent-manager.d.ts +25 -0
- package/dist/types/tools/persistent-manager/persistent-manager.test.d.ts +1 -0
- package/dist/types/tools/persistent-manager/types.d.ts +9 -0
- package/dist/types/tools/pipeline/index.d.ts +2 -0
- package/dist/types/tools/pipeline/pipeline.d.ts +17 -0
- package/dist/types/tools/pipeline/pipeline.test.d.ts +1 -0
- package/dist/types/tools/pipeline/types.d.ts +13 -0
- package/dist/types/tools/run-worker/index.d.ts +2 -0
- package/dist/types/tools/run-worker/run-worker.d.ts +26 -0
- package/dist/types/tools/run-worker/run-worker.test.d.ts +1 -0
- package/dist/types/tools/run-worker/types.d.ts +14 -0
- package/dist/types/tools/worker-factory/index.d.ts +1 -1
- package/dist/types/tools/worker-factory/worker-factory.d.ts +7 -1
- package/dist/types/workers/initiator.d.ts +1 -1
- package/dist/types/workers/initiator.test.d.ts +1 -0
- package/package.json +11 -5
- package/dist/types/tools/define-worker.d.ts +0 -21
- /package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts} +0 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { CollectOptions, CollectedResult, TypedSettledResults } from '../main-worker-factory/types';
|
|
2
|
+
import { CollectResultsContext } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Collects and merges the results of multiple worker execution promises.
|
|
5
|
+
*
|
|
6
|
+
* This function processes an array of settled promises from worker executions,
|
|
7
|
+
* segregates them into fulfilled and rejected results, and then reduces (merges)
|
|
8
|
+
* the fulfilled data into a single combined output.
|
|
9
|
+
*
|
|
10
|
+
* If a `reducer` is provided in `options`, it will be used to merge the shards.
|
|
11
|
+
* Otherwise, a default reducer that flats the array will be used.
|
|
12
|
+
*
|
|
13
|
+
* The merge operation is offloaded to an ephemeral Web Worker. The reducer worker
|
|
14
|
+
* receives a direct `MessagePort` to the MemoryWorker, so it fetches shard data
|
|
15
|
+
* directly without routing through the main thread.
|
|
16
|
+
*
|
|
17
|
+
* @template T - The expected type of the data returned from an individual worker.
|
|
18
|
+
* @template R - The expected type of the merged data.
|
|
19
|
+
*
|
|
20
|
+
* @param settled - The settled results (`Promise.allSettled` output) from the workers.
|
|
21
|
+
* @param options - Options for collection, such as a custom `reducer` function.
|
|
22
|
+
* @param context - Context object providing logger, worker tracking, termination checks, and MemoryWorkerProxy.
|
|
23
|
+
* @returns A promise resolving to the collected results containing the merged data, success count, failure count, and errors.
|
|
24
|
+
* @throws Will throw an error if the `MainWorkerFactory` context has been terminated prior to collection.
|
|
25
|
+
*/
|
|
26
|
+
export declare function collectWorkerResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options: CollectOptions<T, R> | undefined, context: CollectResultsContext): Promise<CollectedResult<R>>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ILogger } from '../logger';
|
|
2
|
+
import { MemoryWorkerProxy } from '../memory-store';
|
|
3
|
+
export interface CollectResultsContext {
|
|
4
|
+
isTerminated: () => boolean;
|
|
5
|
+
trackWorker: (worker: Worker) => Worker;
|
|
6
|
+
terminateWorker: (worker: Worker) => void;
|
|
7
|
+
logger: ILogger;
|
|
8
|
+
memoryWorkerProxy: MemoryWorkerProxy;
|
|
9
|
+
factoryToken: string;
|
|
10
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { WorkerFunction } from '../main-worker-factory/types';
|
|
2
|
+
/**
|
|
3
|
+
* Defines a function to run inside a native Web Worker script, establishing
|
|
4
|
+
* a standard messaging interface that provides full compatibility with `MainWorkerFactory`.
|
|
5
|
+
*
|
|
6
|
+
* This helper wraps your worker logic and automatically handles:
|
|
7
|
+
* - Standard single-execution runs (`foreman.run()`).
|
|
8
|
+
* - Worker-to-worker message passing in pipelines (`foreman.pipeline()`).
|
|
9
|
+
* - Dataset caching and persistent state (`foreman.runPersistent()`).
|
|
10
|
+
* - Extracting and passing transferable objects (like `ArrayBuffer` or `MessagePort`)
|
|
11
|
+
* automatically to optimize memory usage without structured cloning overhead.
|
|
12
|
+
* - Catching synchronous and asynchronous errors and formatting them for the main thread.
|
|
13
|
+
*
|
|
14
|
+
* @typeParam TParams - The type of the payload/parameters sent to the worker.
|
|
15
|
+
* It typically includes a `data` field alongside optional configurations.
|
|
16
|
+
* @typeParam TResult - The return type of the worker function. Can be a promise or a direct value.
|
|
17
|
+
*
|
|
18
|
+
* @param workerFn - The function containing the worker's execution logic. It takes the parsed
|
|
19
|
+
* payload and returns the computed result (or a Promise resolving to it).
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* // my-native-worker.ts
|
|
23
|
+
* import { defineWorker } from '@offmain/workerkit';
|
|
24
|
+
*
|
|
25
|
+
* // This worker function is compatible with standard runs and pipelines
|
|
26
|
+
* export default defineWorker(async ({ data, options }: { data: number[], options?: { mult?: number } }) => {
|
|
27
|
+
* const mult = options?.mult ?? 2;
|
|
28
|
+
* return data.map((x) => x * mult);
|
|
29
|
+
* });
|
|
30
|
+
*/
|
|
31
|
+
export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './define-worker';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { WorkerConfig, WorkerFunction } from '../main-worker-factory/types';
|
|
2
|
+
/**
|
|
3
|
+
* Helper to strictly type a single WorkerConfig.
|
|
4
|
+
* Especially useful for `createWorker` configs where the function type is not natively inferrable.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* defineWorkerConfig<typeof myFunc>({ name: 'myWorker', createWorker: () => new Worker(...) })
|
|
8
|
+
*/
|
|
9
|
+
export declare function defineWorkerConfig<TFunc extends WorkerFunction<any, any>>(): <const TConfig extends WorkerConfig<TFunc>>(config: TConfig) => TConfig & {
|
|
10
|
+
_typeHint: TFunc;
|
|
11
|
+
};
|
|
12
|
+
export declare function defineWorkerConfig<const TConfig extends WorkerConfig<any>>(config: TConfig): TConfig;
|
|
13
|
+
/**
|
|
14
|
+
* Helper to strictly type an array of WorkerConfigs.
|
|
15
|
+
* Eliminates the need for `as const` while preserving the exact generics of each worker.
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* const workers = defineWorkerConfigs(
|
|
19
|
+
* defineWorkerConfig({ name: 'a', func: funcA }),
|
|
20
|
+
* defineWorkerConfig<typeof b>()({ name: 'b', createWorker: () => new Worker(...) })
|
|
21
|
+
* );
|
|
22
|
+
*/
|
|
23
|
+
export declare function defineWorkerConfigs<T extends readonly WorkerConfig<any>[]>(...workers: T): T;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './define-worker-config';
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recursively extracts transferable objects from a given payload to optimize Web Worker messages.
|
|
3
|
+
*
|
|
4
|
+
* Transferable objects (like `ArrayBuffer`, `MessagePort`, `ImageBitmap`, `OffscreenCanvas`)
|
|
5
|
+
* are transferred by reference rather than being copied via Structured Clone, which significantly
|
|
6
|
+
* reduces memory usage and improves performance when sending large datasets.
|
|
7
|
+
*
|
|
8
|
+
* This utility traverses arrays, objects, and nested structures to collect all valid transferables.
|
|
9
|
+
* It also intelligently handles circular references by maintaining a `seen` Set.
|
|
10
|
+
*
|
|
11
|
+
* @param value - The input value to scan for transferable objects.
|
|
12
|
+
* @param seen - An optional Set used internally to track visited objects and prevent infinite recursion in cyclic structures.
|
|
13
|
+
* @returns An array containing all unique transferable objects found within the value.
|
|
14
|
+
*/
|
|
15
|
+
export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './extract-transferable';
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { LogLevel, ILogger } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* A lightweight logging utility designed for the Web Worker Manager.
|
|
4
|
+
*
|
|
5
|
+
* Supports various log levels to control the verbosity of the output:
|
|
6
|
+
* - `verbose`: Outputs debug, info, and error messages.
|
|
7
|
+
* - `info`: Outputs info and error messages (suppresses debug).
|
|
8
|
+
* - `error`: Outputs only error messages (suppresses debug and info).
|
|
9
|
+
* - `silent`: Suppresses all messages.
|
|
10
|
+
*
|
|
11
|
+
* All log messages are automatically prefixed with `[WorkerManager]` to
|
|
12
|
+
* distinguish them easily in the console.
|
|
13
|
+
*/
|
|
14
|
+
export declare class Logger implements ILogger {
|
|
15
|
+
private level;
|
|
16
|
+
constructor(level?: LogLevel);
|
|
17
|
+
setLevel(level: LogLevel): void;
|
|
18
|
+
verbose(...args: unknown[]): void;
|
|
19
|
+
info(...args: unknown[]): void;
|
|
20
|
+
error(...args: unknown[]): void;
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,295 +1,161 @@
|
|
|
1
|
-
import { CollectOptions, CollectedResult,
|
|
1
|
+
import { CollectOptions, CollectedResult, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerReturnType, TypedSettledResults, MemoryStats, RunWorkerOptions } from './types';
|
|
2
|
+
import { Logger, LogLevel } from '../logger';
|
|
2
3
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*/
|
|
7
|
-
export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
|
|
8
|
-
/**
|
|
9
|
-
* Central orchestrator for running typed Web Workers in parallel.
|
|
10
|
-
*
|
|
11
|
-
* `MainWorkerFactory` manages a registry of named worker configurations and
|
|
12
|
-
* handles the full lifecycle of each worker: spawning, partitioning input
|
|
13
|
-
* data across threads, retrying on failure, and collecting results.
|
|
4
|
+
* Main orchestration class for Web Worker management.
|
|
5
|
+
* Provides high-level APIs to configure, dispatch, and monitor workers,
|
|
6
|
+
* handling memory isolation, sharding, and concurrency.
|
|
14
7
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* re-running it with different configs without re-sending the data
|
|
20
|
-
* ({@link runPersistent}, {@link release}).
|
|
8
|
+
* Workers store their results directly into the dedicated `MemoryWorker` thread
|
|
9
|
+
* via a pre-allocated `MessagePort`. Only lightweight `__memory_ref__` tokens
|
|
10
|
+
* cross the main thread boundary. `runWorker` auto-collects shards and returns
|
|
11
|
+
* a `CollectedResult<R>` directly — no manual `collectResults` call needed.
|
|
21
12
|
*
|
|
22
|
-
* @typeParam TConfigs - A
|
|
23
|
-
* defines the set of available workers and their typed signatures.
|
|
24
|
-
*
|
|
25
|
-
* @example
|
|
26
|
-
* const foreman = new MainWorkerFactory({
|
|
27
|
-
* workers: [
|
|
28
|
-
* { name: 'sum', role: 'compute', func: sumWorker, partition: true },
|
|
29
|
-
* ] as const,
|
|
30
|
-
* });
|
|
31
|
-
*
|
|
32
|
-
* const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3, 4] });
|
|
33
|
-
* const { data } = await foreman.collectResults(settled);
|
|
13
|
+
* @typeParam TConfigs - A tuple of WorkerConfig configurations.
|
|
34
14
|
*/
|
|
35
|
-
declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<
|
|
15
|
+
declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<any>[]> {
|
|
36
16
|
private readonly _workers;
|
|
37
17
|
private readonly _threads;
|
|
38
|
-
private readonly _persistentWorkers;
|
|
39
18
|
private readonly _activeWorkers;
|
|
19
|
+
private readonly _memoryStore;
|
|
20
|
+
private readonly _memoryWorkerProxy;
|
|
21
|
+
private readonly _factoryToken;
|
|
40
22
|
private _isTerminated;
|
|
23
|
+
private readonly _persistentManager;
|
|
24
|
+
private readonly _orchestrator;
|
|
41
25
|
/**
|
|
42
|
-
*
|
|
26
|
+
* The logger instance used by this factory.
|
|
27
|
+
*/
|
|
28
|
+
readonly logger: Logger;
|
|
29
|
+
/**
|
|
30
|
+
* Initializes a new MainWorkerFactory.
|
|
43
31
|
*
|
|
44
|
-
* @param options - Configuration
|
|
32
|
+
* @param options - Configuration options.
|
|
33
|
+
* @param options.workers - The list of worker configurations.
|
|
34
|
+
* @param options.logLevel - Optional logging level (defaults to 'error').
|
|
45
35
|
*/
|
|
46
36
|
constructor(options: {
|
|
47
37
|
workers: TConfigs;
|
|
38
|
+
logLevel?: LogLevel;
|
|
48
39
|
});
|
|
49
40
|
/**
|
|
50
|
-
*
|
|
41
|
+
* Indicates whether the factory has been terminated.
|
|
51
42
|
*/
|
|
52
43
|
get isTerminated(): boolean;
|
|
53
|
-
/**
|
|
54
|
-
* Registers an active worker instance for lifecycle tracking.
|
|
55
|
-
*/
|
|
56
44
|
private trackWorker;
|
|
57
|
-
/**
|
|
58
|
-
* Terminates a worker instance and removes it from tracking.
|
|
59
|
-
*/
|
|
60
45
|
private terminateWorker;
|
|
61
46
|
/**
|
|
62
|
-
*
|
|
47
|
+
* Helper utility to partition an array into a specified number of chunks.
|
|
63
48
|
*
|
|
64
|
-
* @param
|
|
65
|
-
* @
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* Splits an array into up to `numChunks` evenly-sized sub-arrays.
|
|
70
|
-
*
|
|
71
|
-
* When the array length is not evenly divisible, the first `remainder`
|
|
72
|
-
* chunks receive one extra element so no data is lost.
|
|
73
|
-
*
|
|
74
|
-
* @param array - The source array to partition.
|
|
75
|
-
* @param numChunks - Maximum number of chunks to produce.
|
|
76
|
-
* Clamped to `array.length` so you never get empty chunks.
|
|
77
|
-
* @returns An array of sub-arrays. Returns `[]` when `array` is empty.
|
|
78
|
-
* @throws {Error} When `numChunks` is not a positive integer.
|
|
79
|
-
*
|
|
80
|
-
* @example
|
|
81
|
-
* partitionArray([1, 2, 3, 4, 5], 3);
|
|
82
|
-
* // → [[1, 2], [3, 4], [5]]
|
|
49
|
+
* @param array - The array to partition.
|
|
50
|
+
* @param numChunks - The desired number of chunks.
|
|
51
|
+
* @returns An array of array chunks.
|
|
52
|
+
* @template T - The type of elements in the array.
|
|
83
53
|
*/
|
|
84
54
|
partitionArray<T>(array: T[], numChunks: number): T[][];
|
|
85
|
-
/**
|
|
86
|
-
* Looks up a registered worker configuration by name.
|
|
87
|
-
*
|
|
88
|
-
* @param name - The `name` field of the target {@link WorkerConfig}.
|
|
89
|
-
* @returns The matching config, or `undefined` if not found.
|
|
90
|
-
*/
|
|
91
55
|
private findWorkerByName;
|
|
92
56
|
/**
|
|
93
|
-
*
|
|
94
|
-
* threads when the worker is configured for partitioning.
|
|
95
|
-
*
|
|
96
|
-
* When `config.partition` is `true` and `srcData` is an array with more
|
|
97
|
-
* than one element, the array is split into up to `maxConcurrency` (or
|
|
98
|
-
* `navigator.hardwareConcurrency`) shards and each shard is processed by
|
|
99
|
-
* a separate worker thread in parallel.
|
|
100
|
-
*
|
|
101
|
-
* All threads are awaited with `Promise.allSettled`, so a failure in one
|
|
102
|
-
* shard does not cancel the others. Use {@link collectResults} to merge
|
|
103
|
-
* the settled output.
|
|
104
|
-
*
|
|
105
|
-
* @typeParam TName - The literal name of the worker to run (inferred from
|
|
106
|
-
* the registered `workers` tuple).
|
|
57
|
+
* Dispatches a worker task with the given parameters.
|
|
107
58
|
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
59
|
+
* Workers store their results directly in the `MemoryWorker` thread — large data
|
|
60
|
+
* never touches the main thread heap. Results are auto-collected and merged in a
|
|
61
|
+
* dedicated reducer worker. Returns a `CollectedResult<R>` directly.
|
|
111
62
|
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
63
|
+
* Pass `autoCollect: false` in `rawParams` to skip auto-collection and receive
|
|
64
|
+
* the raw settled state (escape hatch for advanced custom reducers).
|
|
114
65
|
*
|
|
115
|
-
* @
|
|
116
|
-
*
|
|
66
|
+
* @param workerName - The name of the registered worker to execute.
|
|
67
|
+
* @param rawParams - The payload, options, and reducer for the worker.
|
|
68
|
+
* @returns A promise resolving to a CollectedResult with merged data and shard stats.
|
|
69
|
+
* @template TName - The name of the registered worker.
|
|
70
|
+
* @template T - The return type of the worker function.
|
|
71
|
+
* @template R - The merged return type after reducing shards.
|
|
117
72
|
*/
|
|
118
|
-
runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName,
|
|
119
|
-
srcData
|
|
120
|
-
|
|
73
|
+
runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string, T = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>, R = T extends (infer Item)[] ? Item[] : T[]>(workerName: TName, rawParams: {
|
|
74
|
+
srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
|
|
75
|
+
__memory_ref__?: string;
|
|
76
|
+
deleteMemory?: boolean;
|
|
77
|
+
} & RunWorkerOptions<T, R> & Record<string, unknown>): Promise<CollectedResult<R>>;
|
|
121
78
|
/**
|
|
122
|
-
*
|
|
123
|
-
* call.
|
|
79
|
+
* Deletes a specific reference from the memory store and MemoryWorker.
|
|
124
80
|
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
* @param config - The resolved {@link WorkerConfig} for this run.
|
|
129
|
-
* @param workerName - Name used in error/retry logging.
|
|
130
|
-
* @param srcWorkerData - Combined `{ data, ...otherParams }` payload.
|
|
131
|
-
* @param threadCount - Number of parallel worker threads to spawn.
|
|
132
|
-
* @param isPartitioned - Whether `data` is a pre-split array of shards.
|
|
133
|
-
* @returns An array of promises, one per thread.
|
|
81
|
+
* @param ref - The memory reference ID to delete.
|
|
82
|
+
* @returns True if the reference existed and was deleted, false otherwise.
|
|
134
83
|
*/
|
|
135
|
-
|
|
84
|
+
deleteMemory(ref: string): Promise<boolean>;
|
|
136
85
|
/**
|
|
137
|
-
*
|
|
138
|
-
* times before re-throwing the last error.
|
|
139
|
-
*
|
|
140
|
-
* Each retry is logged to `console.error` with the remaining attempt count
|
|
141
|
-
* so failures are visible during development.
|
|
142
|
-
*
|
|
143
|
-
* @param instanceConfig - Full configuration for the worker instance.
|
|
144
|
-
* @param retryCount - Remaining retry attempts (default `2`).
|
|
145
|
-
* @returns The successful {@link WorkerResult} once the worker resolves.
|
|
146
|
-
* @throws The last caught error when all retries are exhausted.
|
|
86
|
+
* Clears all references from the memory store and MemoryWorker.
|
|
147
87
|
*/
|
|
148
|
-
|
|
88
|
+
clearMemory(): Promise<void>;
|
|
149
89
|
/**
|
|
150
|
-
*
|
|
151
|
-
* based on the message the worker sends back.
|
|
152
|
-
*
|
|
153
|
-
* The worker is expected to respond with either:
|
|
154
|
-
* - `{ ok: true, data: T }` — success; resolves with a {@link WorkerResult}.
|
|
155
|
-
* - `{ ok: false, error: string }` — logical failure; rejects with a
|
|
156
|
-
* structured error object.
|
|
157
|
-
*
|
|
158
|
-
* Any transferable objects found in the payload are moved (not copied) to
|
|
159
|
-
* the worker via the `transfer` list of `postMessage`.
|
|
160
|
-
*
|
|
161
|
-
* The underlying `Worker` is always terminated after the message completes.
|
|
90
|
+
* Retrieves statistics about the currently stored memory handles.
|
|
162
91
|
*
|
|
163
|
-
* @
|
|
164
|
-
* @returns A promise that resolves with the worker's result.
|
|
92
|
+
* @returns The MemoryStats containing count and active reference IDs.
|
|
165
93
|
*/
|
|
166
|
-
|
|
94
|
+
getMemoryStats(): Promise<MemoryStats>;
|
|
167
95
|
/**
|
|
168
|
-
* Collects and
|
|
169
|
-
* main thread.
|
|
96
|
+
* Collects and reduces the results from a `runWorker` execution.
|
|
170
97
|
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
98
|
+
* @deprecated `runWorker` now auto-collects results. This method is kept as
|
|
99
|
+
* an escape hatch for advanced cases where `autoCollect: false` was passed to
|
|
100
|
+
* `runWorker`. In the common case, the return value of `runWorker` already
|
|
101
|
+
* contains the merged `CollectedResult`.
|
|
175
102
|
*
|
|
176
|
-
* @
|
|
177
|
-
* @
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
* @
|
|
181
|
-
* @param options - Optional {@link CollectOptions}. Supply a `reducer` to
|
|
182
|
-
* control how shards are merged. The reducer **must be self-contained**
|
|
183
|
-
* (no closures over external variables) because it is serialised and run
|
|
184
|
-
* inside a worker.
|
|
185
|
-
*
|
|
186
|
-
* @returns A {@link CollectedResult} with the merged `data`, counts of
|
|
187
|
-
* `succeeded`/`failed` shards, and the raw `errors` array.
|
|
188
|
-
*
|
|
189
|
-
* @example
|
|
190
|
-
* // default: flat array of all shard data
|
|
191
|
-
* const { data, succeeded, failed } = await foreman.collectResults(res);
|
|
192
|
-
*
|
|
193
|
-
* @example
|
|
194
|
-
* // custom reducer: sum numbers across shards
|
|
195
|
-
* const { data } = await foreman.collectResults<number[], number>(res, {
|
|
196
|
-
* reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
|
|
197
|
-
* });
|
|
103
|
+
* @param settled - The settled results from `runWorker` (when `autoCollect: false`).
|
|
104
|
+
* @param options - Options containing the reducer function.
|
|
105
|
+
* @returns A structured CollectedResult object.
|
|
106
|
+
* @template T - The type of the worker result data.
|
|
107
|
+
* @template R - The type of the merged result data.
|
|
198
108
|
*/
|
|
199
|
-
collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
|
|
109
|
+
collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
|
|
200
110
|
/**
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
* Internally, adjacent workers are connected via `MessageChannel` ports.
|
|
205
|
-
* Only the final result is sent back to the main thread, minimising
|
|
206
|
-
* serialisation overhead for large intermediate data.
|
|
111
|
+
* Executes a sequential pipeline of worker steps.
|
|
112
|
+
* Passes the output of one step as the input to the next step.
|
|
207
113
|
*
|
|
208
|
-
* @
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
* @
|
|
212
|
-
* step must include `srcData`; subsequent steps receive the previous
|
|
213
|
-
* step's output as `{ data: previousOutput, index: 0 }`.
|
|
214
|
-
*
|
|
215
|
-
* @returns A promise that resolves with the final step's output.
|
|
216
|
-
* @throws {Error} When `steps` is empty or a worker name is not found.
|
|
217
|
-
*
|
|
218
|
-
* @example
|
|
219
|
-
* const result = await foreman.pipeline<FilteredPost[]>([
|
|
220
|
-
* { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
|
|
221
|
-
* { worker: 'transformPosts' },
|
|
222
|
-
* { worker: 'filterPosts' },
|
|
223
|
-
* ]);
|
|
224
|
-
* console.log(result); // final transformed + filtered data
|
|
114
|
+
* @param steps - An array of PipelineStep configurations.
|
|
115
|
+
* @returns A CollectedResult containing the final pipeline output and shard stats,
|
|
116
|
+
* consistent with the shape returned by `runWorker()`.
|
|
117
|
+
* @template TResult - The type of the pipeline result.
|
|
225
118
|
*/
|
|
226
|
-
pipeline<TResult = unknown>(steps:
|
|
119
|
+
pipeline<TResult = unknown>(steps: {
|
|
120
|
+
worker: keyof WorkerConfigMap<TConfigs> & string;
|
|
121
|
+
srcData?: unknown;
|
|
122
|
+
[key: string]: unknown;
|
|
123
|
+
}[]): Promise<CollectedResult<TResult>>;
|
|
227
124
|
/**
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
* the dataset in memory. On subsequent calls, only `config` is needed — the
|
|
232
|
-
* worker reuses the cached dataset and reprocesses it with the new config.
|
|
233
|
-
*
|
|
234
|
-
* The worker stays alive until {@link release} is called.
|
|
235
|
-
*
|
|
236
|
-
* @param workerName - Name of the registered worker.
|
|
237
|
-
* @param params - Object with optional `dataset` and required `config`.
|
|
238
|
-
* @returns The worker function's return value.
|
|
125
|
+
* Executes a task on a persistent (long-lived) worker thread.
|
|
126
|
+
* Useful for workers that maintain local state (like WebAssembly modules or databases)
|
|
127
|
+
* across multiple invocations.
|
|
239
128
|
*
|
|
240
|
-
* @
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
* config: { multiplier: 2 },
|
|
245
|
-
* });
|
|
246
|
-
*
|
|
247
|
-
* // Subsequent calls: only config, dataset is cached
|
|
248
|
-
* const r2 = await factory.runPersistent('transform', {
|
|
249
|
-
* config: { multiplier: 5 },
|
|
250
|
-
* });
|
|
251
|
-
*
|
|
252
|
-
* // Update dataset when needed
|
|
253
|
-
* const r3 = await factory.runPersistent('transform', {
|
|
254
|
-
* dataset: newArray,
|
|
255
|
-
* config: { multiplier: 3 },
|
|
256
|
-
* });
|
|
257
|
-
*
|
|
258
|
-
* // Release when done
|
|
259
|
-
* factory.release('transform');
|
|
129
|
+
* @param workerName - The name of the persistent worker to run.
|
|
130
|
+
* @param params - The input data and configuration for the task.
|
|
131
|
+
* @returns The result from the persistent worker thread.
|
|
132
|
+
* @template TResult - The expected type of the result from the worker thread.
|
|
260
133
|
*/
|
|
261
|
-
runPersistent<TResult =
|
|
134
|
+
runPersistent<TName extends keyof WorkerConfigMap<TConfigs> & string, TResult = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>(workerName: TName, params: {
|
|
262
135
|
dataset?: unknown;
|
|
263
136
|
config: unknown;
|
|
264
137
|
}): Promise<TResult>;
|
|
265
138
|
/**
|
|
266
|
-
* Releases a persistent worker
|
|
267
|
-
* the thread.
|
|
139
|
+
* Releases and terminates a persistent worker by name.
|
|
268
140
|
*
|
|
269
|
-
*
|
|
270
|
-
* will create a fresh instance (requiring a new dataset).
|
|
271
|
-
*
|
|
272
|
-
* @param workerName - Name of the persistent worker to release.
|
|
141
|
+
* @param workerName - The name of the persistent worker to release.
|
|
273
142
|
*/
|
|
274
143
|
release(workerName: string): void;
|
|
275
144
|
/**
|
|
276
|
-
* Terminates
|
|
277
|
-
*
|
|
278
|
-
* Calling `terminate()` immediately stops all running worker threads, releases
|
|
279
|
-
* cached persistent workers, and clears all internal worker state.
|
|
145
|
+
* Terminates all active worker threads, persistent workers, MemoryWorker, and clears the memory store.
|
|
146
|
+
* Marks this factory instance as terminated.
|
|
280
147
|
*/
|
|
281
148
|
terminate(): void;
|
|
282
149
|
/**
|
|
283
|
-
* Alias for {@link terminate}. Terminates
|
|
150
|
+
* Alias for {@link terminate}. Terminates all active resources.
|
|
284
151
|
*/
|
|
285
152
|
destroy(): void;
|
|
286
153
|
/**
|
|
287
|
-
*
|
|
288
|
-
* and resetting the factory state, allowing new worker instances to be initiated.
|
|
154
|
+
* Terminates all resources and resets the factory to an active state.
|
|
289
155
|
*/
|
|
290
156
|
reset(): void;
|
|
291
157
|
/**
|
|
292
|
-
* Alias for {@link reset}.
|
|
158
|
+
* Alias for {@link reset}. Terminates and reactivates the factory.
|
|
293
159
|
*/
|
|
294
160
|
restart(): void;
|
|
295
161
|
}
|