@offmain/workerkit 0.14.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +225 -46
  2. package/dist/define-worker.cjs +1 -0
  3. package/dist/define-worker.js +4 -0
  4. package/dist/index-2AONniOz.js +59 -0
  5. package/dist/index-CXKVsLvY.cjs +1 -0
  6. package/dist/index.cjs +195 -43
  7. package/dist/index.js +945 -487
  8. package/dist/types/tools/collect-results/collect-results.d.ts +26 -0
  9. package/dist/types/tools/collect-results/index.d.ts +2 -0
  10. package/dist/types/tools/collect-results/types.d.ts +10 -0
  11. package/dist/types/tools/define-worker/define-worker.d.ts +31 -0
  12. package/dist/types/tools/define-worker/define-worker.test.d.ts +1 -0
  13. package/dist/types/tools/define-worker/index.d.ts +1 -0
  14. package/dist/types/tools/define-worker-config/define-worker-config.d.ts +23 -0
  15. package/dist/types/tools/define-worker-config/index.d.ts +1 -0
  16. package/dist/types/tools/extract-transferable/extract-transferable.d.ts +15 -0
  17. package/dist/types/tools/extract-transferable/extract-transferable.test.d.ts +1 -0
  18. package/dist/types/tools/extract-transferable/index.d.ts +1 -0
  19. package/dist/types/tools/index.d.ts +2 -0
  20. package/dist/types/tools/logger/index.d.ts +2 -0
  21. package/dist/types/tools/logger/logger.d.ts +21 -0
  22. package/dist/types/tools/logger/logger.test.d.ts +1 -0
  23. package/dist/types/tools/logger/types.d.ts +6 -0
  24. package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts +94 -228
  25. package/dist/types/tools/main-worker-factory/types.d.ts +71 -4
  26. package/dist/types/tools/memory-store/index.d.ts +3 -0
  27. package/dist/types/tools/memory-store/memory-store.d.ts +50 -0
  28. package/dist/types/tools/memory-store/memory-store.test.d.ts +1 -0
  29. package/dist/types/tools/memory-store/memory-worker-proxy.d.ts +75 -0
  30. package/dist/types/tools/memory-store/memory-worker.d.ts +11 -0
  31. package/dist/types/tools/orchestrator/index.d.ts +2 -0
  32. package/dist/types/tools/orchestrator/orchestrator.d.ts +21 -0
  33. package/dist/types/tools/orchestrator/orchestrator.test.d.ts +1 -0
  34. package/dist/types/tools/orchestrator/types.d.ts +10 -0
  35. package/dist/types/tools/partition-array/index.d.ts +1 -0
  36. package/dist/types/tools/partition-array/partition-array.d.ts +14 -0
  37. package/dist/types/tools/partition-array/partition-array.test.d.ts +1 -0
  38. package/dist/types/tools/persistent-manager/index.d.ts +2 -0
  39. package/dist/types/tools/persistent-manager/persistent-manager.d.ts +25 -0
  40. package/dist/types/tools/persistent-manager/persistent-manager.test.d.ts +1 -0
  41. package/dist/types/tools/persistent-manager/types.d.ts +9 -0
  42. package/dist/types/tools/pipeline/index.d.ts +2 -0
  43. package/dist/types/tools/pipeline/pipeline.d.ts +17 -0
  44. package/dist/types/tools/pipeline/pipeline.test.d.ts +1 -0
  45. package/dist/types/tools/pipeline/types.d.ts +13 -0
  46. package/dist/types/tools/run-worker/index.d.ts +2 -0
  47. package/dist/types/tools/run-worker/run-worker.d.ts +26 -0
  48. package/dist/types/tools/run-worker/run-worker.test.d.ts +1 -0
  49. package/dist/types/tools/run-worker/types.d.ts +14 -0
  50. package/dist/types/tools/worker-factory/index.d.ts +1 -1
  51. package/dist/types/tools/worker-factory/worker-factory.d.ts +7 -1
  52. package/dist/types/workers/initiator.d.ts +1 -1
  53. package/dist/types/workers/initiator.test.d.ts +1 -0
  54. package/package.json +11 -5
  55. package/dist/types/tools/define-worker.d.ts +0 -21
  56. /package/dist/types/tools/{define-worker.test.d.ts → collect-results/collect-results.test.d.ts} +0 -0
@@ -0,0 +1,26 @@
1
+ import { CollectOptions, CollectedResult, TypedSettledResults } from '../main-worker-factory/types';
2
+ import { CollectResultsContext } from './types';
3
+ /**
4
+ * Collects and merges the results of multiple worker execution promises.
5
+ *
6
+ * This function processes an array of settled promises from worker executions,
7
+ * segregates them into fulfilled and rejected results, and then reduces (merges)
8
+ * the fulfilled data into a single combined output.
9
+ *
10
+ * If a `reducer` is provided in `options`, it will be used to merge the shards.
11
+ * Otherwise, a default reducer that flats the array will be used.
12
+ *
13
+ * The merge operation is offloaded to an ephemeral Web Worker. The reducer worker
14
+ * receives a direct `MessagePort` to the MemoryWorker, so it fetches shard data
15
+ * directly without routing through the main thread.
16
+ *
17
+ * @template T - The expected type of the data returned from an individual worker.
18
+ * @template R - The expected type of the merged data.
19
+ *
20
+ * @param settled - The settled results (`Promise.allSettled` output) from the workers.
21
+ * @param options - Options for collection, such as a custom `reducer` function.
22
+ * @param context - Context object providing logger, worker tracking, termination checks, and MemoryWorkerProxy.
23
+ * @returns A promise resolving to the collected results containing the merged data, success count, failure count, and errors.
24
+ * @throws Will throw an error if the `MainWorkerFactory` context has been terminated prior to collection.
25
+ */
26
+ export declare function collectWorkerResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options: CollectOptions<T, R> | undefined, context: CollectResultsContext): Promise<CollectedResult<R>>;
@@ -0,0 +1,2 @@
1
+ export * from './types';
2
+ export * from './collect-results';
@@ -0,0 +1,10 @@
1
+ import { ILogger } from '../logger';
2
+ import { MemoryWorkerProxy } from '../memory-store';
3
+ export interface CollectResultsContext {
4
+ isTerminated: () => boolean;
5
+ trackWorker: (worker: Worker) => Worker;
6
+ terminateWorker: (worker: Worker) => void;
7
+ logger: ILogger;
8
+ memoryWorkerProxy: MemoryWorkerProxy;
9
+ factoryToken: string;
10
+ }
@@ -0,0 +1,31 @@
1
+ import { WorkerFunction } from '../main-worker-factory/types';
2
+ /**
3
+ * Defines a function to run inside a native Web Worker script, establishing
4
+ * a standard messaging interface that provides full compatibility with `MainWorkerFactory`.
5
+ *
6
+ * This helper wraps your worker logic and automatically handles:
7
+ * - Standard single-execution runs (`foreman.run()`).
8
+ * - Worker-to-worker message passing in pipelines (`foreman.pipeline()`).
9
+ * - Dataset caching and persistent state (`foreman.runPersistent()`).
10
+ * - Extracting and passing transferable objects (like `ArrayBuffer` or `MessagePort`)
11
+ * automatically to optimize memory usage without structured cloning overhead.
12
+ * - Catching synchronous and asynchronous errors and formatting them for the main thread.
13
+ *
14
+ * @typeParam TParams - The type of the payload/parameters sent to the worker.
15
+ * It typically includes a `data` field alongside optional configurations.
16
+ * @typeParam TResult - The return type of the worker function. Can be a promise or a direct value.
17
+ *
18
+ * @param workerFn - The function containing the worker's execution logic. It takes the parsed
19
+ * payload and returns the computed result (or a Promise resolving to it).
20
+ *
21
+ * @example
22
+ * // my-native-worker.ts
23
+ * import { defineWorker } from '@offmain/workerkit';
24
+ *
25
+ * // This worker function is compatible with standard runs and pipelines
26
+ * export default defineWorker(async ({ data, options }: { data: number[], options?: { mult?: number } }) => {
27
+ * const mult = options?.mult ?? 2;
28
+ * return data.map((x) => x * mult);
29
+ * });
30
+ */
31
+ export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;
@@ -0,0 +1 @@
1
+ export * from './define-worker';
@@ -0,0 +1,23 @@
1
+ import { WorkerConfig, WorkerFunction } from '../main-worker-factory/types';
2
+ /**
3
+ * Helper to strictly type a single WorkerConfig.
4
+ * Especially useful for `createWorker` configs where the function type is not natively inferrable.
5
+ *
6
+ * @example
7
+ * defineWorkerConfig<typeof myFunc>({ name: 'myWorker', createWorker: () => new Worker(...) })
8
+ */
9
+ export declare function defineWorkerConfig<TFunc extends WorkerFunction<any, any>>(): <const TConfig extends WorkerConfig<TFunc>>(config: TConfig) => TConfig & {
10
+ _typeHint: TFunc;
11
+ };
12
+ export declare function defineWorkerConfig<const TConfig extends WorkerConfig<any>>(config: TConfig): TConfig;
13
+ /**
14
+ * Helper to strictly type an array of WorkerConfigs.
15
+ * Eliminates the need for `as const` while preserving the exact generics of each worker.
16
+ *
17
+ * @example
18
+ * const workers = defineWorkerConfigs(
19
+ * defineWorkerConfig({ name: 'a', func: funcA }),
20
+ * defineWorkerConfig<typeof b>()({ name: 'b', createWorker: () => new Worker(...) })
21
+ * );
22
+ */
23
+ export declare function defineWorkerConfigs<T extends readonly WorkerConfig<any>[]>(...workers: T): T;
@@ -0,0 +1 @@
1
+ export * from './define-worker-config';
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Recursively extracts transferable objects from a given payload to optimize Web Worker messages.
3
+ *
4
+ * Transferable objects (like `ArrayBuffer`, `MessagePort`, `ImageBitmap`, `OffscreenCanvas`)
5
+ * are transferred by reference rather than being copied via Structured Clone, which significantly
6
+ * reduces memory usage and improves performance when sending large datasets.
7
+ *
8
+ * This utility traverses arrays, objects, and nested structures to collect all valid transferables.
9
+ * It also intelligently handles circular references by maintaining a `seen` Set.
10
+ *
11
+ * @param value - The input value to scan for transferable objects.
12
+ * @param seen - An optional Set used internally to track visited objects and prevent infinite recursion in cyclic structures.
13
+ * @returns An array containing all unique transferable objects found within the value.
14
+ */
15
+ export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
@@ -0,0 +1 @@
1
+ export * from './extract-transferable';
@@ -1,3 +1,5 @@
1
1
  export * from './worker-factory';
2
2
  export * from './main-worker-factory';
3
+ export * from './define-worker/index';
4
+ export * from './define-worker-config';
3
5
  export * from './define-worker';
@@ -0,0 +1,2 @@
1
+ export * from './types';
2
+ export * from './logger';
@@ -0,0 +1,21 @@
1
+ import { LogLevel, ILogger } from './types';
2
+ /**
3
+ * A lightweight logging utility designed for the Web Worker Manager.
4
+ *
5
+ * Supports various log levels to control the verbosity of the output:
6
+ * - `verbose`: Outputs debug, info, and error messages.
7
+ * - `info`: Outputs info and error messages (suppresses debug).
8
+ * - `error`: Outputs only error messages (suppresses debug and info).
9
+ * - `silent`: Suppresses all messages.
10
+ *
11
+ * All log messages are automatically prefixed with `[WorkerManager]` to
12
+ * distinguish them easily in the console.
13
+ */
14
+ export declare class Logger implements ILogger {
15
+ private level;
16
+ constructor(level?: LogLevel);
17
+ setLevel(level: LogLevel): void;
18
+ verbose(...args: unknown[]): void;
19
+ info(...args: unknown[]): void;
20
+ error(...args: unknown[]): void;
21
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,6 @@
1
+ export type LogLevel = 'verbose' | 'info' | 'error' | 'silent';
2
+ export interface ILogger {
3
+ verbose(...args: unknown[]): void;
4
+ info(...args: unknown[]): void;
5
+ error(...args: unknown[]): void;
6
+ }
@@ -1,295 +1,161 @@
1
- import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
1
+ import { CollectOptions, CollectedResult, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerReturnType, TypedSettledResults, MemoryStats, RunWorkerOptions } from './types';
2
+ import { Logger, LogLevel } from '../logger';
2
3
  /**
3
- * Recursively collects all Transferable objects from a value.
4
- * Transferable (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
5
- * are zero-copy they are moved to the worker instead of cloned.
6
- */
7
- export declare function extractTransferable(value: unknown, seen?: Set<object>): Transferable[];
8
- /**
9
- * Central orchestrator for running typed Web Workers in parallel.
10
- *
11
- * `MainWorkerFactory` manages a registry of named worker configurations and
12
- * handles the full lifecycle of each worker: spawning, partitioning input
13
- * data across threads, retrying on failure, and collecting results.
4
+ * Main orchestration class for Web Worker management.
5
+ * Provides high-level APIs to configure, dispatch, and monitor workers,
6
+ * handling memory isolation, sharding, and concurrency.
14
7
  *
15
- * Also supports:
16
- * - **Pipelines** chain workers via `MessageChannel` so intermediate data
17
- * never crosses back to the main thread ({@link pipeline}).
18
- * - **Persistent workers**keep a worker alive with a cached dataset,
19
- * re-running it with different configs without re-sending the data
20
- * ({@link runPersistent}, {@link release}).
8
+ * Workers store their results directly into the dedicated `MemoryWorker` thread
9
+ * via a pre-allocated `MessagePort`. Only lightweight `__memory_ref__` tokens
10
+ * cross the main thread boundary. `runWorker` auto-collects shards and returns
11
+ * a `CollectedResult<R>` directlyno manual `collectResults` call needed.
21
12
  *
22
- * @typeParam TConfigs - A readonly tuple of {@link WorkerConfig} objects that
23
- * defines the set of available workers and their typed signatures.
24
- *
25
- * @example
26
- * const foreman = new MainWorkerFactory({
27
- * workers: [
28
- * { name: 'sum', role: 'compute', func: sumWorker, partition: true },
29
- * ] as const,
30
- * });
31
- *
32
- * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3, 4] });
33
- * const { data } = await foreman.collectResults(settled);
13
+ * @typeParam TConfigs - A tuple of WorkerConfig configurations.
34
14
  */
35
- declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFunction<any, any>>[]> {
15
+ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<any>[]> {
36
16
  private readonly _workers;
37
17
  private readonly _threads;
38
- private readonly _persistentWorkers;
39
18
  private readonly _activeWorkers;
19
+ private readonly _memoryStore;
20
+ private readonly _memoryWorkerProxy;
21
+ private readonly _factoryToken;
40
22
  private _isTerminated;
23
+ private readonly _persistentManager;
24
+ private readonly _orchestrator;
41
25
  /**
42
- * Creates a new `MainWorkerFactory`.
26
+ * The logger instance used by this factory.
27
+ */
28
+ readonly logger: Logger;
29
+ /**
30
+ * Initializes a new MainWorkerFactory.
43
31
  *
44
- * @param options - Configuration object containing the `workers` registry.
32
+ * @param options - Configuration options.
33
+ * @param options.workers - The list of worker configurations.
34
+ * @param options.logLevel - Optional logging level (defaults to 'error').
45
35
  */
46
36
  constructor(options: {
47
37
  workers: TConfigs;
38
+ logLevel?: LogLevel;
48
39
  });
49
40
  /**
50
- * Returns `true` if the factory has been terminated.
41
+ * Indicates whether the factory has been terminated.
51
42
  */
52
43
  get isTerminated(): boolean;
53
- /**
54
- * Registers an active worker instance for lifecycle tracking.
55
- */
56
44
  private trackWorker;
57
- /**
58
- * Terminates a worker instance and removes it from tracking.
59
- */
60
45
  private terminateWorker;
61
46
  /**
62
- * Instantiates a {@link WorkerFactory} for the given worker configuration.
47
+ * Helper utility to partition an array into a specified number of chunks.
63
48
  *
64
- * @param config - The worker configuration containing `func` or `createWorker`.
65
- * @returns A new `WorkerFactory` wrapping the worker.
66
- */
67
- private initWorker;
68
- /**
69
- * Splits an array into up to `numChunks` evenly-sized sub-arrays.
70
- *
71
- * When the array length is not evenly divisible, the first `remainder`
72
- * chunks receive one extra element so no data is lost.
73
- *
74
- * @param array - The source array to partition.
75
- * @param numChunks - Maximum number of chunks to produce.
76
- * Clamped to `array.length` so you never get empty chunks.
77
- * @returns An array of sub-arrays. Returns `[]` when `array` is empty.
78
- * @throws {Error} When `numChunks` is not a positive integer.
79
- *
80
- * @example
81
- * partitionArray([1, 2, 3, 4, 5], 3);
82
- * // → [[1, 2], [3, 4], [5]]
49
+ * @param array - The array to partition.
50
+ * @param numChunks - The desired number of chunks.
51
+ * @returns An array of array chunks.
52
+ * @template T - The type of elements in the array.
83
53
  */
84
54
  partitionArray<T>(array: T[], numChunks: number): T[][];
85
- /**
86
- * Looks up a registered worker configuration by name.
87
- *
88
- * @param name - The `name` field of the target {@link WorkerConfig}.
89
- * @returns The matching config, or `undefined` if not found.
90
- */
91
55
  private findWorkerByName;
92
56
  /**
93
- * Runs a named worker against the provided data, distributing work across
94
- * threads when the worker is configured for partitioning.
95
- *
96
- * When `config.partition` is `true` and `srcData` is an array with more
97
- * than one element, the array is split into up to `maxConcurrency` (or
98
- * `navigator.hardwareConcurrency`) shards and each shard is processed by
99
- * a separate worker thread in parallel.
100
- *
101
- * All threads are awaited with `Promise.allSettled`, so a failure in one
102
- * shard does not cancel the others. Use {@link collectResults} to merge
103
- * the settled output.
104
- *
105
- * @typeParam TName - The literal name of the worker to run (inferred from
106
- * the registered `workers` tuple).
57
+ * Dispatches a worker task with the given parameters.
107
58
  *
108
- * @param workerName - Name of the worker as declared in the `workers` config.
109
- * @param params - Object containing `srcData` (the payload) plus any
110
- * additional key/value pairs forwarded to the worker verbatim.
59
+ * Workers store their results directly in the `MemoryWorker` thread — large data
60
+ * never touches the main thread heap. Results are auto-collected and merged in a
61
+ * dedicated reducer worker. Returns a `CollectedResult<R>` directly.
111
62
  *
112
- * @returns A {@link TypedSettledResults} wrapping the settled promises from
113
- * all spawned worker threads.
63
+ * Pass `autoCollect: false` in `rawParams` to skip auto-collection and receive
64
+ * the raw settled state (escape hatch for advanced custom reducers).
114
65
  *
115
- * @example
116
- * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
66
+ * @param workerName - The name of the registered worker to execute.
67
+ * @param rawParams - The payload, options, and reducer for the worker.
68
+ * @returns A promise resolving to a CollectedResult with merged data and shard stats.
69
+ * @template TName - The name of the registered worker.
70
+ * @template T - The return type of the worker function.
71
+ * @template R - The merged return type after reducing shards.
117
72
  */
118
- runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, { srcData, ...otherParams }: {
119
- srcData: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
120
- } & Record<string, unknown>): Promise<TypedSettledResults<WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>>;
73
+ runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string, T = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>, R = T extends (infer Item)[] ? Item[] : T[]>(workerName: TName, rawParams: {
74
+ srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
75
+ __memory_ref__?: string;
76
+ deleteMemory?: boolean;
77
+ } & RunWorkerOptions<T, R> & Record<string, unknown>): Promise<CollectedResult<R>>;
121
78
  /**
122
- * Builds the array of per-thread worker promises for a single `runWorker`
123
- * call.
79
+ * Deletes a specific reference from the memory store and MemoryWorker.
124
80
  *
125
- * When `isPartitioned` is `true`, each promise receives its own slice of
126
- * `srcData`; otherwise every thread receives the full payload.
127
- *
128
- * @param config - The resolved {@link WorkerConfig} for this run.
129
- * @param workerName - Name used in error/retry logging.
130
- * @param srcWorkerData - Combined `{ data, ...otherParams }` payload.
131
- * @param threadCount - Number of parallel worker threads to spawn.
132
- * @param isPartitioned - Whether `data` is a pre-split array of shards.
133
- * @returns An array of promises, one per thread.
81
+ * @param ref - The memory reference ID to delete.
82
+ * @returns True if the reference existed and was deleted, false otherwise.
134
83
  */
135
- private createWorkerPromises;
84
+ deleteMemory(ref: string): Promise<boolean>;
136
85
  /**
137
- * Runs a single worker instance, retrying on failure up to `retryCount`
138
- * times before re-throwing the last error.
139
- *
140
- * Each retry is logged to `console.error` with the remaining attempt count
141
- * so failures are visible during development.
142
- *
143
- * @param instanceConfig - Full configuration for the worker instance.
144
- * @param retryCount - Remaining retry attempts (default `2`).
145
- * @returns The successful {@link WorkerResult} once the worker resolves.
146
- * @throws The last caught error when all retries are exhausted.
86
+ * Clears all references from the memory store and MemoryWorker.
147
87
  */
148
- private runWorkerWithRetry;
88
+ clearMemory(): Promise<void>;
149
89
  /**
150
- * Spawns a single worker thread, posts the payload, and resolves or rejects
151
- * based on the message the worker sends back.
152
- *
153
- * The worker is expected to respond with either:
154
- * - `{ ok: true, data: T }` — success; resolves with a {@link WorkerResult}.
155
- * - `{ ok: false, error: string }` — logical failure; rejects with a
156
- * structured error object.
157
- *
158
- * Any transferable objects found in the payload are moved (not copied) to
159
- * the worker via the `transfer` list of `postMessage`.
160
- *
161
- * The underlying `Worker` is always terminated after the message completes.
90
+ * Retrieves statistics about the currently stored memory handles.
162
91
  *
163
- * @param instanceConfig - Worker function, factory, name, shard index, and data.
164
- * @returns A promise that resolves with the worker's result.
92
+ * @returns The MemoryStats containing count and active reference IDs.
165
93
  */
166
- private initiateWorker;
94
+ getMemoryStats(): Promise<MemoryStats>;
167
95
  /**
168
- * Collects and merges the settled results from {@link runWorker} — off the
169
- * main thread.
96
+ * Collects and reduces the results from a `runWorker` execution.
170
97
  *
171
- * Fulfilled shards are extracted and passed to the `reducer` function, which
172
- * runs inside a dedicated inline worker so the merge itself never blocks the
173
- * main thread. Failed shards are counted and their raw rejection reasons are
174
- * preserved in `errors`.
98
+ * @deprecated `runWorker` now auto-collects results. This method is kept as
99
+ * an escape hatch for advanced cases where `autoCollect: false` was passed to
100
+ * `runWorker`. In the common case, the return value of `runWorker` already
101
+ * contains the merged `CollectedResult`.
175
102
  *
176
- * @typeParam T - The per-shard data type (inferred from `settled`).
177
- * @typeParam R - The final merged output type (defaults to a flat array of
178
- * `T` items when no custom reducer is provided).
179
- *
180
- * @param settled - The {@link TypedSettledResults} returned by `runWorker`.
181
- * @param options - Optional {@link CollectOptions}. Supply a `reducer` to
182
- * control how shards are merged. The reducer **must be self-contained**
183
- * (no closures over external variables) because it is serialised and run
184
- * inside a worker.
185
- *
186
- * @returns A {@link CollectedResult} with the merged `data`, counts of
187
- * `succeeded`/`failed` shards, and the raw `errors` array.
188
- *
189
- * @example
190
- * // default: flat array of all shard data
191
- * const { data, succeeded, failed } = await foreman.collectResults(res);
192
- *
193
- * @example
194
- * // custom reducer: sum numbers across shards
195
- * const { data } = await foreman.collectResults<number[], number>(res, {
196
- * reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
197
- * });
103
+ * @param settled - The settled results from `runWorker` (when `autoCollect: false`).
104
+ * @param options - Options containing the reducer function.
105
+ * @returns A structured CollectedResult object.
106
+ * @template T - The type of the worker result data.
107
+ * @template R - The type of the merged result data.
198
108
  */
199
- collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
109
+ collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
200
110
  /**
201
- * Runs a chain of workers where each step's output feeds directly into the
202
- * next step **without passing through the main thread**.
203
- *
204
- * Internally, adjacent workers are connected via `MessageChannel` ports.
205
- * Only the final result is sent back to the main thread, minimising
206
- * serialisation overhead for large intermediate data.
111
+ * Executes a sequential pipeline of worker steps.
112
+ * Passes the output of one step as the input to the next step.
207
113
  *
208
- * @typeParam TResult - The expected type of the final pipeline output.
209
- * Defaults to `unknown` if not specified.
210
- *
211
- * @param steps - An ordered array of {@link PipelineStep} objects. The first
212
- * step must include `srcData`; subsequent steps receive the previous
213
- * step's output as `{ data: previousOutput, index: 0 }`.
214
- *
215
- * @returns A promise that resolves with the final step's output.
216
- * @throws {Error} When `steps` is empty or a worker name is not found.
217
- *
218
- * @example
219
- * const result = await foreman.pipeline<FilteredPost[]>([
220
- * { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
221
- * { worker: 'transformPosts' },
222
- * { worker: 'filterPosts' },
223
- * ]);
224
- * console.log(result); // final transformed + filtered data
114
+ * @param steps - An array of PipelineStep configurations.
115
+ * @returns A CollectedResult containing the final pipeline output and shard stats,
116
+ * consistent with the shape returned by `runWorker()`.
117
+ * @template TResult - The type of the pipeline result.
225
118
  */
226
- pipeline<TResult = unknown>(steps: PipelineStep[]): Promise<TResult>;
119
+ pipeline<TResult = unknown>(steps: {
120
+ worker: keyof WorkerConfigMap<TConfigs> & string;
121
+ srcData?: unknown;
122
+ [key: string]: unknown;
123
+ }[]): Promise<CollectedResult<TResult>>;
227
124
  /**
228
- * Runs a persistent worker that caches its dataset between calls.
229
- *
230
- * On the first call, provide both `dataset` and `config`. The worker stores
231
- * the dataset in memory. On subsequent calls, only `config` is needed — the
232
- * worker reuses the cached dataset and reprocesses it with the new config.
233
- *
234
- * The worker stays alive until {@link release} is called.
235
- *
236
- * @param workerName - Name of the registered worker.
237
- * @param params - Object with optional `dataset` and required `config`.
238
- * @returns The worker function's return value.
125
+ * Executes a task on a persistent (long-lived) worker thread.
126
+ * Useful for workers that maintain local state (like WebAssembly modules or databases)
127
+ * across multiple invocations.
239
128
  *
240
- * @example
241
- * // First call: send dataset + config
242
- * const r1 = await factory.runPersistent('transform', {
243
- * dataset: largeArray,
244
- * config: { multiplier: 2 },
245
- * });
246
- *
247
- * // Subsequent calls: only config, dataset is cached
248
- * const r2 = await factory.runPersistent('transform', {
249
- * config: { multiplier: 5 },
250
- * });
251
- *
252
- * // Update dataset when needed
253
- * const r3 = await factory.runPersistent('transform', {
254
- * dataset: newArray,
255
- * config: { multiplier: 3 },
256
- * });
257
- *
258
- * // Release when done
259
- * factory.release('transform');
129
+ * @param workerName - The name of the persistent worker to run.
130
+ * @param params - The input data and configuration for the task.
131
+ * @returns The result from the persistent worker thread.
132
+ * @template TResult - The expected type of the result from the worker thread.
260
133
  */
261
- runPersistent<TResult = unknown>(workerName: string, params: {
134
+ runPersistent<TName extends keyof WorkerConfigMap<TConfigs> & string, TResult = WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>(workerName: TName, params: {
262
135
  dataset?: unknown;
263
136
  config: unknown;
264
137
  }): Promise<TResult>;
265
138
  /**
266
- * Releases a persistent worker, freeing its cached dataset and terminating
267
- * the thread.
139
+ * Releases and terminates a persistent worker by name.
268
140
  *
269
- * After calling `release`, subsequent `runPersistent` calls for this worker
270
- * will create a fresh instance (requiring a new dataset).
271
- *
272
- * @param workerName - Name of the persistent worker to release.
141
+ * @param workerName - The name of the persistent worker to release.
273
142
  */
274
143
  release(workerName: string): void;
275
144
  /**
276
- * Terminates the factory and all active and persistent worker instances.
277
- *
278
- * Calling `terminate()` immediately stops all running worker threads, releases
279
- * cached persistent workers, and clears all internal worker state.
145
+ * Terminates all active worker threads, persistent workers, MemoryWorker, and clears the memory store.
146
+ * Marks this factory instance as terminated.
280
147
  */
281
148
  terminate(): void;
282
149
  /**
283
- * Alias for {@link terminate}. Terminates the factory and all worker instances.
150
+ * Alias for {@link terminate}. Terminates all active resources.
284
151
  */
285
152
  destroy(): void;
286
153
  /**
287
- * Resets the factory by terminating all active and persistent workers
288
- * and resetting the factory state, allowing new worker instances to be initiated.
154
+ * Terminates all resources and resets the factory to an active state.
289
155
  */
290
156
  reset(): void;
291
157
  /**
292
- * Alias for {@link reset}. Resets the factory state to initiate new worker instances.
158
+ * Alias for {@link reset}. Terminates and reactivates the factory.
293
159
  */
294
160
  restart(): void;
295
161
  }