@offmain/workerkit 0.14.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +208 -40
  2. package/dist/define-worker.cjs +1 -0
  3. package/dist/define-worker.js +4 -0
  4. package/dist/index-2AONniOz.js +59 -0
  5. package/dist/index-CXKVsLvY.cjs +1 -0
  6. package/dist/index.cjs +196 -44
  7. package/dist/index.js +913 -631
  8. package/dist/types/tools/collect-results/collect-results.d.ts +26 -0
  9. package/dist/types/tools/collect-results/index.d.ts +2 -0
  10. package/dist/types/tools/collect-results/types.d.ts +10 -0
  11. package/dist/types/tools/define-worker/define-worker.d.ts +31 -0
  12. package/dist/types/tools/define-worker/define-worker.test.d.ts +1 -0
  13. package/dist/types/tools/define-worker/index.d.ts +1 -0
  14. package/dist/types/tools/define-worker-config/define-worker-config.d.ts +23 -0
  15. package/dist/types/tools/define-worker-config/index.d.ts +1 -0
  16. package/dist/types/tools/extract-transferable/extract-transferable.d.ts +15 -0
  17. package/dist/types/tools/extract-transferable/extract-transferable.test.d.ts +1 -0
  18. package/dist/types/tools/extract-transferable/index.d.ts +1 -0
  19. package/dist/types/tools/index.d.ts +2 -0
  20. package/dist/types/tools/logger/index.d.ts +2 -0
  21. package/dist/types/tools/logger/logger.d.ts +21 -0
  22. package/dist/types/tools/logger/logger.test.d.ts +1 -0
  23. package/dist/types/tools/logger/types.d.ts +6 -0
  24. package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +87 -244
  25. package/dist/types/tools/main-worker-factory/types.d.ts +54 -4
  26. package/dist/types/tools/memory-store/index.d.ts +3 -0
  27. package/dist/types/tools/memory-store/memory-store.d.ts +50 -0
  28. package/dist/types/tools/memory-store/memory-store.test.d.ts +1 -0
  29. package/dist/types/tools/memory-store/memory-worker-proxy.d.ts +75 -0
  30. package/dist/types/tools/memory-store/memory-worker.d.ts +11 -0
  31. package/dist/types/tools/orchestrator/index.d.ts +2 -0
  32. package/dist/types/tools/orchestrator/orchestrator.d.ts +21 -0
  33. package/dist/types/tools/orchestrator/orchestrator.test.d.ts +1 -0
  34. package/dist/types/tools/orchestrator/types.d.ts +10 -0
  35. package/dist/types/tools/partition-array/index.d.ts +1 -0
  36. package/dist/types/tools/partition-array/partition-array.d.ts +14 -0
  37. package/dist/types/tools/partition-array/partition-array.test.d.ts +1 -0
  38. package/dist/types/tools/persistent-manager/index.d.ts +2 -0
  39. package/dist/types/tools/persistent-manager/persistent-manager.d.ts +25 -0
  40. package/dist/types/tools/persistent-manager/persistent-manager.test.d.ts +1 -0
  41. package/dist/types/tools/persistent-manager/types.d.ts +9 -0
  42. package/dist/types/tools/pipeline/index.d.ts +2 -0
  43. package/dist/types/tools/pipeline/pipeline.d.ts +17 -0
  44. package/dist/types/tools/pipeline/pipeline.test.d.ts +1 -0
  45. package/dist/types/tools/pipeline/types.d.ts +13 -0
  46. package/dist/types/tools/run-worker/index.d.ts +2 -0
  47. package/dist/types/tools/run-worker/run-worker.d.ts +26 -0
  48. package/dist/types/tools/run-worker/run-worker.test.d.ts +1 -0
  49. package/dist/types/tools/run-worker/types.d.ts +14 -0
  50. package/dist/types/tools/worker-factory/index.d.ts +1 -1
  51. package/dist/types/tools/worker-factory/worker-factory.d.ts +7 -1
  52. package/dist/types/workers/initiator.d.ts +1 -1
  53. package/dist/types/workers/initiator.test.d.ts +1 -0
  54. package/package.json +11 -5
  55. package/dist/types/tools/define-worker.d.ts +0 -21
  56. package/dist/types/tools/main-worker-factory/memory-store.d.ts +0 -45
  57. package/dist/types/tools/main-worker-factory/memory-worker.d.ts +0 -7
  58. /package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts} +0 -0
@@ -0,0 +1,26 @@
1
+ import { CollectOptions, CollectedResult, TypedSettledResults } from '../main-worker-factory/types';
2
+ import { CollectResultsContext } from './types';
3
+ /**
4
+ * Collects and merges the results of multiple worker execution promises.
5
+ *
6
+ * This function processes an array of settled promises from worker executions,
7
+ * segregates them into fulfilled and rejected results, and then reduces (merges)
8
+ * the fulfilled data into a single combined output.
9
+ *
10
+ * If a `reducer` is provided in `options`, it will be used to merge the shards.
11
+ * Otherwise, a default reducer that flats the array will be used.
12
+ *
13
+ * The merge operation is offloaded to an ephemeral Web Worker. The reducer worker
14
+ * receives a direct `MessagePort` to the MemoryWorker, so it fetches shard data
15
+ * directly without routing through the main thread.
16
+ *
17
+ * @template T - The expected type of the data returned from an individual worker.
18
+ * @template R - The expected type of the merged data.
19
+ *
20
+ * @param settled - The settled results (`Promise.allSettled` output) from the workers.
21
+ * @param options - Options for collection, such as a custom `reducer` function.
22
+ * @param context - Context object providing logger, worker tracking, termination checks, and MemoryWorkerProxy.
23
+ * @returns A promise resolving to the collected results containing the merged data, success count, failure count, and errors.
24
+ * @throws Will throw an error if the `MainWorkerFactory` context has been terminated prior to collection.
25
+ */
26
+ export declare function collectWorkerResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options: CollectOptions<T, R> | undefined, context: CollectResultsContext): Promise<CollectedResult<R>>;
@@ -0,0 +1,2 @@
1
+ export * from './types';
2
+ export * from './collect-results';
@@ -0,0 +1,10 @@
1
+ import { ILogger } from '../logger';
2
+ import { MemoryWorkerProxy } from '../memory-store';
3
+ export interface CollectResultsContext {
4
+ isTerminated: () => boolean;
5
+ trackWorker: (worker: Worker) => Worker;
6
+ terminateWorker: (worker: Worker) => void;
7
+ logger: ILogger;
8
+ memoryWorkerProxy: MemoryWorkerProxy;
9
+ factoryToken: string;
10
+ }
@@ -0,0 +1,31 @@
1
+ import { WorkerFunction } from '../main-worker-factory/types';
2
+ /**
3
+ * Defines a function to run inside a native Web Worker script, establishing
4
+ * a standard messaging interface that provides full compatibility with `MainWorkerFactory`.
5
+ *
6
+ * This helper wraps your worker logic and automatically handles:
7
+ * - Standard single-execution runs (`foreman.run()`).
8
+ * - Worker-to-worker message passing in pipelines (`foreman.pipeline()`).
9
+ * - Dataset caching and persistent state (`foreman.runPersistent()`).
10
+ * - Extracting and passing transferable objects (like `ArrayBuffer` or `MessagePort`)
11
+ * automatically to optimize memory usage without structured cloning overhead.
12
+ * - Catching synchronous and asynchronous errors and formatting them for the main thread.
13
+ *
14
+ * @typeParam TParams - The type of the payload/parameters sent to the worker.
15
+ * It typically includes a `data` field alongside optional configurations.
16
+ * @typeParam TResult - The return type of the worker function. Can be a promise or a direct value.
17
+ *
18
+ * @param workerFn - The function containing the worker's execution logic. It takes the parsed
19
+ * payload and returns the computed result (or a Promise resolving to it).
20
+ *
21
+ * @example
22
+ * // my-native-worker.ts
23
+ * import { defineWorker } from '@offmain/workerkit';
24
+ *
25
+ * // This worker function is compatible with standard runs and pipelines
26
+ * export default defineWorker(async ({ data, options }: { data: number[], options?: { mult?: number } }) => {
27
+ * const mult = options?.mult ?? 2;
28
+ * return data.map((x) => x * mult);
29
+ * });
30
+ */
31
+ export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;
@@ -0,0 +1 @@
1
+ export * from './define-worker';
@@ -0,0 +1,23 @@
1
+ import { WorkerConfig, WorkerFunction } from '../main-worker-factory/types';
2
+ /**
3
+ * Helper to strictly type a single WorkerConfig.
4
+ * Especially useful for `createWorker` configs where the function type is not natively inferrable.
5
+ *
6
+ * @example
7
+ * defineWorkerConfig<typeof myFunc>({ name: 'myWorker', createWorker: () => new Worker(...) })
8
+ */
9
+ export declare function defineWorkerConfig<TFunc extends WorkerFunction<any, any>>(): <const TConfig extends WorkerConfig<TFunc>>(config: TConfig) => TConfig & {
10
+ _typeHint: TFunc;
11
+ };
12
+ export declare function defineWorkerConfig<const TConfig extends WorkerConfig<any>>(config: TConfig): TConfig;
13
+ /**
14
+ * Helper to strictly type an array of WorkerConfigs.
15
+ * Eliminates the need for `as const` while preserving the exact generics of each worker.
16
+ *
17
+ * @example
18
+ * const workers = defineWorkerConfigs(
19
+ * defineWorkerConfig({ name: 'a', func: funcA }),
20
+ * defineWorkerConfig<typeof b>()({ name: 'b', createWorker: () => new Worker(...) })
21
+ * );
22
+ */
23
+ export declare function defineWorkerConfigs<T extends readonly WorkerConfig<any>[]>(...workers: T): T;
@@ -0,0 +1 @@
1
+ export * from './define-worker-config';
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Recursively extracts transferable objects from a given payload to optimize Web Worker messages.
3
+ *
4
+ * Transferable objects (like `ArrayBuffer`, `MessagePort`, `ImageBitmap`, `OffscreenCanvas`)
5
+ * are transferred by reference rather than being copied via Structured Clone, which significantly
6
+ * reduces memory usage and improves performance when sending large datasets.
7
+ *
8
+ * This utility traverses arrays, objects, and nested structures to collect all valid transferables.
9
+ * It also intelligently handles circular references by maintaining a `seen` Set.
10
+ *
11
+ * @param value - The input value to scan for transferable objects.
12
+ * @param seen - An optional Set used internally to track visited objects and prevent infinite recursion in cyclic structures.
13
+ * @returns An array containing all unique transferable objects found within the value.
14
+ */
15
+ export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
@@ -0,0 +1 @@
1
+ export * from './extract-transferable';
@@ -1,3 +1,5 @@
1
1
  export * from './worker-factory';
2
2
  export * from './main-worker-factory';
3
+ export * from './define-worker/index';
4
+ export * from './define-worker-config';
3
5
  export * from './define-worker';
@@ -0,0 +1,2 @@
1
+ export * from './types';
2
+ export * from './logger';
@@ -0,0 +1,21 @@
1
+ import { LogLevel, ILogger } from './types';
2
+ /**
3
+ * A lightweight logging utility designed for the Web Worker Manager.
4
+ *
5
+ * Supports various log levels to control the verbosity of the output:
6
+ * - `verbose`: Outputs debug, info, and error messages.
7
+ * - `info`: Outputs info and error messages (suppresses debug).
8
+ * - `error`: Outputs only error messages (suppresses debug and info).
9
+ * - `silent`: Suppresses all messages.
10
+ *
11
+ * All log messages are automatically prefixed with `[WorkerManager]` to
12
+ * distinguish them easily in the console.
13
+ */
14
+ export declare class Logger implements ILogger {
15
+ private level;
16
+ constructor(level?: LogLevel);
17
+ setLevel(level: LogLevel): void;
18
+ verbose(...args: unknown[]): void;
19
+ info(...args: unknown[]): void;
20
+ error(...args: unknown[]): void;
21
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,6 @@
1
+ export type LogLevel = 'verbose' | 'info' | 'error' | 'silent';
2
+ export interface ILogger {
3
+ verbose(...args: unknown[]): void;
4
+ info(...args: unknown[]): void;
5
+ error(...args: unknown[]): void;
6
+ }
@@ -1,318 +1,161 @@
1
- import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults, MemoryStats } from './types.ts';
1
+ import { CollectOptions, CollectedResult, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerReturnType, TypedSettledResults, MemoryStats, RunWorkerOptions } from './types';
2
+ import { Logger, LogLevel } from '../logger';
2
3
  /**
3
- * Recursively collects all Transferable objects from a value.
4
- * Transferable (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
5
- * are zero-copy they are moved to the worker instead of cloned.
6
- */
7
- export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
8
- /**
9
- * Central orchestrator for running typed Web Workers in parallel.
10
- *
11
- * `MainWorkerFactory` manages a registry of named worker configurations and
12
- * handles the full lifecycle of each worker: spawning, partitioning input
13
- * data across threads, retrying on failure, and collecting results.
14
- *
15
- * Also supports:
16
- * - **Pipelines** — chain workers via `MessageChannel` so intermediate data
17
- * never crosses back to the main thread ({@link pipeline}).
18
- * - **Persistent workers** — keep a worker alive with a cached dataset,
19
- * re-running it with different configs without re-sending the data
20
- * ({@link runPersistent}, {@link release}).
4
+ * Main orchestration class for Web Worker management.
5
+ * Provides high-level APIs to configure, dispatch, and monitor workers,
6
+ * handling memory isolation, sharding, and concurrency.
21
7
  *
22
- * @typeParam TConfigs - A readonly tuple of {@link WorkerConfig} objects that
23
- * defines the set of available workers and their typed signatures.
8
+ * Workers store their results directly into the dedicated `MemoryWorker` thread
9
+ * via a pre-allocated `MessagePort`. Only lightweight `__memory_ref__` tokens
10
+ * cross the main thread boundary. `runWorker` auto-collects shards and returns
11
+ * a `CollectedResult<R>` directly — no manual `collectResults` call needed.
24
12
  *
25
- * @example
26
- * const foreman = new MainWorkerFactory({
27
- * workers: [
28
- * { name: 'sum', role: 'compute', func: sumWorker, partition: true },
29
- * ] as const,
30
- * });
31
- *
32
- * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3, 4] });
33
- * const { data } = await foreman.collectResults(settled);
13
+ * @typeParam TConfigs - A tuple of WorkerConfig configurations.
34
14
  */
35
- declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFunction<any, any>>[]> {
15
+ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<any>[]> {
36
16
  private readonly _workers;
37
17
  private readonly _threads;
38
- private readonly _persistentWorkers;
39
18
  private readonly _activeWorkers;
40
19
  private readonly _memoryStore;
20
+ private readonly _memoryWorkerProxy;
21
+ private readonly _factoryToken;
41
22
  private _isTerminated;
23
+ private readonly _persistentManager;
24
+ private readonly _orchestrator;
25
+ /**
26
+ * The logger instance used by this factory.
27
+ */
28
+ readonly logger: Logger;
42
29
  /**
43
- * Creates a new `MainWorkerFactory`.
30
+ * Initializes a new MainWorkerFactory.
44
31
  *
45
- * @param options - Configuration object containing the `workers` registry.
32
+ * @param options - Configuration options.
33
+ * @param options.workers - The list of worker configurations.
34
+ * @param options.logLevel - Optional logging level (defaults to 'error').
46
35
  */
47
36
  constructor(options: {
48
37
  workers: TConfigs;
38
+ logLevel?: LogLevel;
49
39
  });
50
40
  /**
51
- * Returns `true` if the factory has been terminated.
41
+ * Indicates whether the factory has been terminated.
52
42
  */
53
43
  get isTerminated(): boolean;
54
- /**
55
- * Registers an active worker instance for lifecycle tracking.
56
- */
57
44
  private trackWorker;
58
- /**
59
- * Terminates a worker instance and removes it from tracking.
60
- */
61
45
  private terminateWorker;
62
46
  /**
63
- * Instantiates a {@link WorkerFactory} for the given worker configuration.
64
- *
65
- * @param config - The worker configuration containing `func` or `createWorker`.
66
- * @returns A new `WorkerFactory` wrapping the worker.
67
- */
68
- private initWorker;
69
- /**
70
- * Splits an array into up to `numChunks` evenly-sized sub-arrays.
47
+ * Helper utility to partition an array into a specified number of chunks.
71
48
  *
72
- * When the array length is not evenly divisible, the first `remainder`
73
- * chunks receive one extra element so no data is lost.
74
- *
75
- * @param array - The source array to partition.
76
- * @param numChunks - Maximum number of chunks to produce.
77
- * Clamped to `array.length` so you never get empty chunks.
78
- * @returns An array of sub-arrays. Returns `[]` when `array` is empty.
79
- * @throws {Error} When `numChunks` is not a positive integer.
80
- *
81
- * @example
82
- * partitionArray([1, 2, 3, 4, 5], 3);
83
- * // → [[1, 2], [3, 4], [5]]
49
+ * @param array - The array to partition.
50
+ * @param numChunks - The desired number of chunks.
51
+ * @returns An array of array chunks.
52
+ * @template T - The type of elements in the array.
84
53
  */
85
54
  partitionArray<T>(array: T[], numChunks: number): T[][];
86
- /**
87
- * Looks up a registered worker configuration by name.
88
- *
89
- * @param name - The `name` field of the target {@link WorkerConfig}.
90
- * @returns The matching config, or `undefined` if not found.
91
- */
92
55
  private findWorkerByName;
93
56
  /**
94
- * Runs a named worker against the provided data, distributing work across
95
- * threads when the worker is configured for partitioning.
96
- *
97
- * When `config.partition` is `true` and `srcData` is an array with more
98
- * than one element, the array is split into up to `maxConcurrency` (or
99
- * `navigator.hardwareConcurrency`) shards and each shard is processed by
100
- * a separate worker thread in parallel.
101
- *
102
- * All threads are awaited with `Promise.allSettled`, so a failure in one
103
- * shard does not cancel the others. Use {@link collectResults} to merge
104
- * the settled output.
57
+ * Dispatches a worker task with the given parameters.
105
58
  *
106
- * @typeParam TName - The literal name of the worker to run (inferred from
107
- * the registered `workers` tuple).
59
+ * Workers store their results directly in the `MemoryWorker` thread large data
60
+ * never touches the main thread heap. Results are auto-collected and merged in a
61
+ * dedicated reducer worker. Returns a `CollectedResult<R>` directly.
108
62
  *
109
- * @param workerName - Name of the worker as declared in the `workers` config.
110
- * @param params - Object containing `srcData` (the payload) plus any
111
- * additional key/value pairs forwarded to the worker verbatim.
63
+ * Pass `autoCollect: false` in `rawParams` to skip auto-collection and receive
64
+ * the raw settled state (escape hatch for advanced custom reducers).
112
65
  *
113
- * @returns A {@link TypedSettledResults} wrapping the settled promises from
114
- * all spawned worker threads.
115
- *
116
- * @example
117
- * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
66
+ * @param workerName - The name of the registered worker to execute.
67
+ * @param rawParams - The payload, options, and reducer for the worker.
68
+ * @returns A promise resolving to a CollectedResult with merged data and shard stats.
69
+ * @template TName - The name of the registered worker.
70
+ * @template T - The return type of the worker function.
71
+ * @template R - The merged return type after reducing shards.
118
72
  */
119
- runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, rawParams: {
73
+ runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string, T = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>, R = T extends (infer Item)[] ? Item[] : T[]>(workerName: TName, rawParams: {
120
74
  srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
121
75
  __memory_ref__?: string;
122
76
  deleteMemory?: boolean;
123
- } & Record<string, unknown>): Promise<TypedSettledResults<WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>>;
77
+ } & RunWorkerOptions<T, R> & Record<string, unknown>): Promise<CollectedResult<R>>;
124
78
  /**
125
- * Deletes a specific memory reference from the factory's memory store.
79
+ * Deletes a specific reference from the memory store and MemoryWorker.
126
80
  *
127
- * @param ref - The `__memory_ref__` token string to delete.
128
- * @returns A promise that resolves to `true` if deleted, `false` otherwise.
81
+ * @param ref - The memory reference ID to delete.
82
+ * @returns True if the reference existed and was deleted, false otherwise.
129
83
  */
130
84
  deleteMemory(ref: string): Promise<boolean>;
131
85
  /**
132
- * Clears all stored dataset references from the factory's memory store.
86
+ * Clears all references from the memory store and MemoryWorker.
133
87
  */
134
88
  clearMemory(): Promise<void>;
135
89
  /**
136
- * Saves the output of a completed worker execution into MemoryStore
137
- * and modifies the returned settled results with a `__memory_ref__` token.
138
- */
139
- private storeWorkerMemoryResult;
140
- /**
141
- * Returns statistics about active memory references in the factory.
142
- */
143
- getMemoryStats(): Promise<MemoryStats>;
144
- /**
145
- * Builds the array of per-thread worker promises for a single `runWorker`
146
- * call.
147
- *
148
- * When `isPartitioned` is `true`, each promise receives its own slice of
149
- * `srcData`; otherwise every thread receives the full payload.
90
+ * Retrieves statistics about the currently stored memory handles.
150
91
  *
151
- * @param config - The resolved {@link WorkerConfig} for this run.
152
- * @param workerName - Name used in error/retry logging.
153
- * @param srcWorkerData - Combined `{ data, ...otherParams }` payload.
154
- * @param threadCount - Number of parallel worker threads to spawn.
155
- * @param isPartitioned - Whether `data` is a pre-split array of shards.
156
- * @returns An array of promises, one per thread.
92
+ * @returns The MemoryStats containing count and active reference IDs.
157
93
  */
158
- private createWorkerPromises;
159
- /**
160
- * Runs a single worker instance, retrying on failure up to `retryCount`
161
- * times before re-throwing the last error.
162
- *
163
- * Each retry is logged to `console.error` with the remaining attempt count
164
- * so failures are visible during development.
165
- *
166
- * @param instanceConfig - Full configuration for the worker instance.
167
- * @param retryCount - Remaining retry attempts (default `2`).
168
- * @returns The successful {@link WorkerResult} once the worker resolves.
169
- * @throws The last caught error when all retries are exhausted.
170
- */
171
- private runWorkerWithRetry;
94
+ getMemoryStats(): Promise<MemoryStats>;
172
95
  /**
173
- * Spawns a single worker thread, posts the payload, and resolves or rejects
174
- * based on the message the worker sends back.
175
- *
176
- * The worker is expected to respond with either:
177
- * - `{ ok: true, data: T }` — success; resolves with a {@link WorkerResult}.
178
- * - `{ ok: false, error: string }` — logical failure; rejects with a
179
- * structured error object.
180
- *
181
- * Any transferable objects found in the payload are moved (not copied) to
182
- * the worker via the `transfer` list of `postMessage`.
96
+ * Collects and reduces the results from a `runWorker` execution.
183
97
  *
184
- * The underlying `Worker` is always terminated after the message completes.
98
+ * @deprecated `runWorker` now auto-collects results. This method is kept as
99
+ * an escape hatch for advanced cases where `autoCollect: false` was passed to
100
+ * `runWorker`. In the common case, the return value of `runWorker` already
101
+ * contains the merged `CollectedResult`.
185
102
  *
186
- * @param instanceConfig - Worker function, factory, name, shard index, and data.
187
- * @returns A promise that resolves with the worker's result.
188
- */
189
- private initiateWorker;
190
- /**
191
- * Collects and merges the settled results from {@link runWorker} — off the
192
- * main thread.
193
- *
194
- * Fulfilled shards are extracted and passed to the `reducer` function, which
195
- * runs inside a dedicated inline worker so the merge itself never blocks the
196
- * main thread. Failed shards are counted and their raw rejection reasons are
197
- * preserved in `errors`.
198
- *
199
- * @typeParam T - The per-shard data type (inferred from `settled`).
200
- * @typeParam R - The final merged output type (defaults to a flat array of
201
- * `T` items when no custom reducer is provided).
202
- *
203
- * @param settled - The {@link TypedSettledResults} returned by `runWorker`.
204
- * @param options - Optional {@link CollectOptions}. Supply a `reducer` to
205
- * control how shards are merged. The reducer **must be self-contained**
206
- * (no closures over external variables) because it is serialised and run
207
- * inside a worker.
208
- *
209
- * @returns A {@link CollectedResult} with the merged `data`, counts of
210
- * `succeeded`/`failed` shards, and the raw `errors` array.
211
- *
212
- * @example
213
- * // default: flat array of all shard data
214
- * const { data, succeeded, failed } = await foreman.collectResults(res);
215
- *
216
- * @example
217
- * // custom reducer: sum numbers across shards
218
- * const { data } = await foreman.collectResults<number[], number>(res, {
219
- * reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
220
- * });
103
+ * @param settled - The settled results from `runWorker` (when `autoCollect: false`).
104
+ * @param options - Options containing the reducer function.
105
+ * @returns A structured CollectedResult object.
106
+ * @template T - The type of the worker result data.
107
+ * @template R - The type of the merged result data.
221
108
  */
222
109
  collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
223
110
  /**
224
- * Runs a chain of workers where each step's output feeds directly into the
225
- * next step **without passing through the main thread**.
226
- *
227
- * Internally, adjacent workers are connected via `MessageChannel` ports.
228
- * Only the final result is sent back to the main thread, minimising
229
- * serialisation overhead for large intermediate data.
230
- *
231
- * @typeParam TResult - The expected type of the final pipeline output.
232
- * Defaults to `unknown` if not specified.
233
- *
234
- * @param steps - An ordered array of {@link PipelineStep} objects. The first
235
- * step must include `srcData`; subsequent steps receive the previous
236
- * step's output as `{ data: previousOutput, index: 0 }`.
111
+ * Executes a sequential pipeline of worker steps.
112
+ * Passes the output of one step as the input to the next step.
237
113
  *
238
- * @returns A promise that resolves with the final step's output.
239
- * @throws {Error} When `steps` is empty or a worker name is not found.
240
- *
241
- * @example
242
- * const result = await foreman.pipeline<FilteredPost[]>([
243
- * { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
244
- * { worker: 'transformPosts' },
245
- * { worker: 'filterPosts' },
246
- * ]);
247
- * console.log(result); // final transformed + filtered data
114
+ * @param steps - An array of PipelineStep configurations.
115
+ * @returns A CollectedResult containing the final pipeline output and shard stats,
116
+ * consistent with the shape returned by `runWorker()`.
117
+ * @template TResult - The type of the pipeline result.
248
118
  */
249
- pipeline<TResult = unknown>(steps: PipelineStep[]): Promise<TResult>;
119
+ pipeline<TResult = unknown>(steps: {
120
+ worker: keyof WorkerConfigMap<TConfigs> & string;
121
+ srcData?: unknown;
122
+ [key: string]: unknown;
123
+ }[]): Promise<CollectedResult<TResult>>;
250
124
  /**
251
- * Runs a persistent worker that caches its dataset between calls.
252
- *
253
- * On the first call, provide both `dataset` and `config`. The worker stores
254
- * the dataset in memory. On subsequent calls, only `config` is needed — the
255
- * worker reuses the cached dataset and reprocesses it with the new config.
256
- *
257
- * The worker stays alive until {@link release} is called.
125
+ * Executes a task on a persistent (long-lived) worker thread.
126
+ * Useful for workers that maintain local state (like WebAssembly modules or databases)
127
+ * across multiple invocations.
258
128
  *
259
- * @param workerName - Name of the registered worker.
260
- * @param params - Object with optional `dataset` and required `config`.
261
- * @returns The worker function's return value.
262
- *
263
- * @example
264
- * // First call: send dataset + config
265
- * const r1 = await factory.runPersistent('transform', {
266
- * dataset: largeArray,
267
- * config: { multiplier: 2 },
268
- * });
269
- *
270
- * // Subsequent calls: only config, dataset is cached
271
- * const r2 = await factory.runPersistent('transform', {
272
- * config: { multiplier: 5 },
273
- * });
274
- *
275
- * // Update dataset when needed
276
- * const r3 = await factory.runPersistent('transform', {
277
- * dataset: newArray,
278
- * config: { multiplier: 3 },
279
- * });
280
- *
281
- * // Release when done
282
- * factory.release('transform');
129
+ * @param workerName - The name of the persistent worker to run.
130
+ * @param params - The input data and configuration for the task.
131
+ * @returns The result from the persistent worker thread.
132
+ * @template TResult - The expected type of the result from the worker thread.
283
133
  */
284
- runPersistent<TResult = unknown>(workerName: string, params: {
134
+ runPersistent<TName extends keyof WorkerConfigMap<TConfigs> & string, TResult = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>(workerName: TName, params: {
285
135
  dataset?: unknown;
286
136
  config: unknown;
287
137
  }): Promise<TResult>;
288
138
  /**
289
- * Releases a persistent worker, freeing its cached dataset and terminating
290
- * the thread.
291
- *
292
- * After calling `release`, subsequent `runPersistent` calls for this worker
293
- * will create a fresh instance (requiring a new dataset).
139
+ * Releases and terminates a persistent worker by name.
294
140
  *
295
- * @param workerName - Name of the persistent worker to release.
141
+ * @param workerName - The name of the persistent worker to release.
296
142
  */
297
143
  release(workerName: string): void;
298
144
  /**
299
- * Terminates the factory and all active and persistent worker instances.
300
- *
301
- * Calling `terminate()` immediately stops all running worker threads, releases
302
- * cached persistent workers, and clears all internal worker state.
145
+ * Terminates all active worker threads, persistent workers, MemoryWorker, and clears the memory store.
146
+ * Marks this factory instance as terminated.
303
147
  */
304
148
  terminate(): void;
305
149
  /**
306
- * Alias for {@link terminate}. Terminates the factory and all worker instances.
150
+ * Alias for {@link terminate}. Terminates all active resources.
307
151
  */
308
152
  destroy(): void;
309
153
  /**
310
- * Resets the factory by terminating all active and persistent workers
311
- * and resetting the factory state, allowing new worker instances to be initiated.
154
+ * Terminates all resources and resets the factory to an active state.
312
155
  */
313
156
  reset(): void;
314
157
  /**
315
- * Alias for {@link reset}. Resets the factory state to initiate new worker instances.
158
+ * Alias for {@link reset}. Terminates and reactivates the factory.
316
159
  */
317
160
  restart(): void;
318
161
  }