@offmain/workerkit 0.13.0 → 0.14.1

package/README.md

@@ -75,27 +75,32 @@ When worker logic relies on external npm packages (such as Luxon, `date-fns`, `i
 
 Passing `createWorker` enables modern module bundlers (Webpack 5, Vite, Rollup, Parcel) to analyze and bundle the worker file along with all of its dependencies into a dedicated ES module worker chunk.
 
-### Dedicated Worker Script
+### Dedicated Worker Script with `defineWorker`
+
+When writing standalone worker files, wrap your export in `defineWorker`. This automatically handles standard runs, worker-to-worker pipelines (`pipeline()`), and persistent dataset caching without manual `postMessage` / `onmessage` boilerplate:
 
 ```ts
 // transform-data.worker.ts
+import { defineWorker } from '@offmain/workerkit';
 import { format } from 'date-fns';
 import { t } from './i18n.ts';
 
-self.addEventListener('message', (event) => {
-  try {
-    const { items, locale } = event.data.data;
-    const result = items.map((item: any) => ({
+export default defineWorker(
+  async ({
+    data,
+    options,
+  }: {
+    data: { items: any[]; locale: string };
+    options?: { prefix?: string };
+  }) => {
+    const { items, locale } = data;
+    return items.map((item: any) => ({
       ...item,
       formattedDate: format(new Date(item.timestamp), 'yyyy-MM-dd'),
-      label: t('transaction', locale),
+      label: (options?.prefix ?? '') + t('transaction', locale),
     }));
-
-    self.postMessage({ ok: true, data: result });
-  } catch (err) {
-    self.postMessage({ ok: false, error: (err as Error).message });
-  }
-});
+  },
+);
 ```
 
 ### Webpack 5 & Vite Static Analysis (`createWorker`)
@@ -150,6 +155,14 @@ const { data } = await factory.collectResults(settled, {
 
 > **Note:** The reducer runs inside a worker thread and must be self-contained — it cannot reference variables from the outer scope.
 
+### Dynamic Thread Scaling & Partitioning Behavior
+
+When `partition: true` is enabled on a worker:
+
+- **Dynamic Thread Allocation:** The library calculates worker thread count as `Math.min(maxConcurrency, srcData.length)`. For instance, if an array has 2 items and `maxConcurrency` is 20, the factory will spawn **only 2 worker threads** (instead of 20), eliminating idle thread overhead and memory pressure.
+- **No Data Duplication:** Each thread receives only its assigned chunk (e.g. Worker 0 gets `[item1]`, Worker 1 gets `[item2]`), ensuring results are processed once without duplication.
+- **Non-Partitioned Workers (`partition: false` / omitted):** If `partition` is not enabled, the input payload is not split, and up to `maxConcurrency` threads will each execute the full payload independently in parallel.
+
 ---
 
 ## Pipeline
@@ -176,7 +189,7 @@ Only the final result crosses back to the main thread. If your pipeline generate
 
 ### Usage
 
-```ts
+````ts
 import { MainWorkerFactory } from '@offmain/workerkit';
 import { fetchData, transform, aggregate } from './workers.ts';
 
@@ -188,29 +201,45 @@ const factory = new MainWorkerFactory({
   ] as const,
 });
 
+### Step-Specific Options and Configs in Pipeline
+
+You can pass step-specific parameters (such as `options`, `configs`, etc.) directly to each pipeline step:
+
+```ts
 const result = await factory.pipeline<AggregateResult>([
-  { worker: 'fetchData', srcData: { url: '/api/records' } },
-  { worker: 'transform' }, // receives fetchData output directly
-  { worker: 'aggregate' }, // receives transform output directly
+  {
+    worker: 'fetchData',
+    srcData: { url: '/api/records' },
+    options: { timeout: 5000 },
+  },
+  {
+    worker: 'transform',
+    configs: { multiplier: 2 },
+  },
+  {
+    worker: 'aggregate',
+    options: { threshold: 10 },
+  },
 ]);
+````
 
-console.log(result); // only this small result crossed to main thread
-```
-
-### How each step receives data
+### How each step receives data and parameters
 
-- The first step receives `srcData` as `{ data: srcData, index: 0 }` — same as `runWorker`.
-- Each subsequent step receives the previous step's output as `{ data: previousOutput, index: 0 }`.
-- Worker functions don't need any special handling — they use the same `{ data }` parameter signature as regular workers.
+- The first step receives `srcData` merged with its step parameters as `{ data: srcData, options: { timeout: 5000 }, index: 0 }`.
+- Each subsequent step receives the previous step's output merged with its step parameters as `{ data: previousOutput, configs: { multiplier: 2 }, index: 0 }`.
+- Worker functions (both inline functions and native scripts using `defineWorker`) receive all step parameters in their first argument.
 
 ### When to use pipeline vs runWorker
 
-| Scenario                                             | Use                     |
-| ---------------------------------------------------- | ----------------------- |
-| Single step, or steps that need partitioning/retries | `runWorker`             |
-| Multi-step chain where intermediate data is large    | `pipeline`              |
-| Steps that are independent (not sequential)          | `runWorker` in parallel |
-| Steps where only the final result matters to the UI  | `pipeline`              |
+| Scenario                                                      | Use                     |
+| ------------------------------------------------------------- | ----------------------- |
+| Single step, or steps that need multi-core array partitioning | `runWorker`             |
+| Multi-step chain where intermediate data is large             | `pipeline`              |
+| Steps that are independent (not sequential)                   | `runWorker` in parallel |
+| Steps where only the final result matters to the UI           | `pipeline`              |
+
+> **Note on `partition: true` in Pipelines:**  
+> `pipeline()` creates **1 worker thread per step** in a linear 1:1 `MessageChannel` chain. If a worker in a pipeline step has `partition: true`, `pipeline()` processes it as a single streaming step without splitting it across parallel worker threads. For parallel multi-core array partitioning across CPU threads, use `runWorker()`.
 
 ---
 
@@ -310,11 +339,42 @@ The framework handles the caching transparently — your function always receive
 
 The cached dataset lives in worker memory until `release()` is called. For large datasets, always call `release()` when you're done to free the memory:
 
+After releasing, the next `runPersistent` call will create a fresh worker instance (requiring a new dataset).
+
+---
+
+## Lifecycle Management (`terminate`, `destroy`, `reset`, `restart`)
+
+`MainWorkerFactory` provides built-in lifecycle management to terminate running workers and clean up browser resources:
+
+### `terminate()` / `destroy()`
+
+Immediately stops all active workers (one-shot tasks, pipelines, reducers) and releases all cached persistent workers:
+
 ```ts
-factory.release('transform');
+// Stop all active threads and release persistent workers
+factory.terminate();
+// or
+factory.destroy(); // Alias for terminate()
+
+console.log(factory.isTerminated); // true
 ```
 
-After releasing, the next `runPersistent` call will create a fresh worker instance (requiring a new dataset).
+After calling `terminate()`, any attempt to run workers on the factory instance will immediately reject.
+
+### `reset()` / `restart()`
+
+Terminates all active/persistent worker instances and restores the factory to an active state (`isTerminated = false`), allowing new worker instances to be initiated cleanly:
+
+```ts
+// Stop existing workers and reset factory state
+factory.reset(); // or factory.restart()
+
+console.log(factory.isTerminated); // false
+
+// Factory is ready to initiate fresh worker instances again
+const settled = await factory.runWorker('sum', { srcData: [1, 2, 3] });
+```
 
 ---
 

package/dist/index.cjs

@@ -1,4 +1,4 @@
-"use strict";var y=Object.defineProperty;var w=(n,e,r)=>e in n?y(n,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):n[e]=r;var h=(n,e,r)=>w(n,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const v=n=>`
+"use strict";var O=Object.defineProperty;var x=(l,e,r)=>e in l?O(l,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):l[e]=r;var _=(l,e,r)=>x(l,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const j=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -15,13 +15,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${n}(event.data);
+    const output = await ${l}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`,W=n=>`
+`,F=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -36,14 +36,19 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
+let stepParams = {};
 
 async function processData(data) {
   try {
-    const output = await workerFn(data);
+    const payload =
+      typeof data === 'object' && data !== null && 'data' in data
+        ? { ...stepParams, ...data }
+        : { data, ...stepParams, index: 0 };
+    const output = await workerFn(payload);
     const result = { ok: true, data: output };
     const transfers = extractTransferables(output);
     if (outputPort) {
@@ -63,6 +68,9 @@ async function processData(data) {
 
 self.addEventListener('message', (event) => {
   if (event.data && event.data.__pipeline_ports__) {
+    if (event.data.stepParams) {
+      stepParams = event.data.stepParams;
+    }
     if (event.data.outputPort) {
       outputPort = event.data.outputPort;
     }
@@ -74,7 +82,7 @@ self.addEventListener('message', (event) => {
           if (outputPort) outputPort.postMessage(e.data);
           else self.postMessage(e.data);
         } else {
-          processData({ data: e.data.data, index: 0 });
+          processData({ data: e.data.data, ...stepParams, index: 0 });
         }
       };
     }
@@ -93,7 +101,7 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
-`,M=n=>`
+`,L=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -108,7 +116,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let cachedDataset = null;
 
 self.addEventListener('message', async (event) => {
@@ -149,14 +157,14 @@ self.addEventListener('message', async (event) => {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 });
-`;var m=(n=>(n.Default="default",n.Pipeline="pipeline",n.Persistent="persistent",n))(m||{});const P=Object.freeze({persistent:M,pipeline:W,default:v});class g{constructor(e,r){h(this,"_worker");if(r!=null&&r.createWorker)this._worker=r.createWorker();else if(e){const o=(r==null?void 0:r.mode)??"default",a=P[o](e.toString()),i=new Blob([a],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(i))}else throw new Error("Either workerFunction or options.createWorker must be provided to WorkerFactory.")}get getWorker(){return this._worker}}class b{constructor(e){this.results=e}}function k(n,e=new Set){return n===null||typeof n!="object"?[]:e.has(n)?[]:(e.add(n),n instanceof ArrayBuffer||n instanceof MessagePort||typeof ImageBitmap<"u"&&n instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&n instanceof OffscreenCanvas?[n]:ArrayBuffer.isView(n)?[n.buffer]:Array.isArray(n)?n.flatMap(r=>k(r,e)):Object.values(n).flatMap(r=>k(r,e)))}class _{constructor(e){h(this,"_workers");h(this,"_threads");h(this,"_persistentWorkers",new Map);h(this,"_activeWorkers",new Set);h(this,"_isTerminated",!1);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}get isTerminated(){return this._isTerminated}trackWorker(e){if(this._isTerminated)throw e.terminate(),new Error("MainWorkerFactory has been terminated");return this._activeWorkers.add(e),e}terminateWorker(e){this._activeWorkers.delete(e);try{e.terminate()}catch{}}initWorker(e){const r=new g(e.func,{createWorker:e.createWorker});return this.trackWorker(r.getWorker),r}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const o=Math.min(r,e.length),a=Math.floor(e.length/o),i=e.length%o,c=[];let u=0;for(let t=0;t<o;t++){const s=a+(t<i?1:0);c.push(e.slice(u,u+s)),u+=s}return c}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,{srcData:r,...o}){if(this._isTerminated)return Promise.reject(new Error("MainWorkerFactory has been terminated"));const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const i=a.maxConcurrency??this._threads,c=!!(Array.isArray(r)&&r.length>1&&a.partition),u=c?this.partitionArray(r,i):r,t=this.createWorkerPromises(a,e,{data:u,...o},i,c),s=await Promise.allSettled(t);return new b(s)}createWorkerPromises(e,r,o,a,i){const{data:c,...u}=o;return Array.from({length:a},(t,s)=>{const f=i&&Array.isArray(c)?c[s]:c;return this.runWorkerWithRetry({workerFunc:e.func,createWorker:e.createWorker,workerName:r,index:s,data:{data:f,...u}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(o){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,o),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",o),o}}initiateWorker({workerFunc:e,createWorker:r,workerName:o,index:a,data:i}){return new Promise((c,u)=>{const s=this.initWorker({name:o,role:"",func:e,createWorker:r}).getWorker;s.onerror=l=>{this.terminateWorker(s),u({index:a,workerConfigs:{workerFunc:e,createWorker:r,workerName:o,index:a,data:i},failedResult:l})},s.onmessage=l=>{var d,p;if(((d=l.data)==null?void 0:d.ok)===!1){this.terminateWorker(s),u({index:a,workerConfigs:{workerFunc:e,createWorker:r,workerName:o,index:a,data:i},failedResult:new ErrorEvent("error",{message:l.data.error})});return}c({index:a,workerConfigs:{workerFunc:e,createWorker:r,workerName:o,index:a,data:i},successResult:new MessageEvent("message",{data:(p=l.data)==null?void 0:p.data})}),this.terminateWorker(s)};const f={index:a,...Array.isArray(i)?{data:i}:i};s.postMessage(f,k(f))})}async collectResults(e,r={}){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const o=e.results.filter(t=>t.status==="fulfilled"),a=e.results.filter(t=>t.status==="rejected"),i=o.map(t=>t.value.successResult.data),c=r.reducer?r.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((t,s)=>{const f=`
-        const reducer = ${c};
-        self.addEventListener('message', (event) => {
-          try {
-            const result = reducer(event.data);
-            self.postMessage({ ok: true, data: result });
-          } catch (err) {
-            self.postMessage({ ok: false, error: String(err) });
-          }
-        });
-      `,l=new Blob([f],{type:"application/javascript"}),d=this.trackWorker(new Worker(URL.createObjectURL(l)));d.onmessage=p=>{this.terminateWorker(d),p.data.ok?t(p.data.data):s(new Error(p.data.error))},d.onerror=p=>{this.terminateWorker(d),s(p)},d.postMessage(i)}),succeeded:o.length,failed:a.length,errors:a}}async pipeline(e){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const r=e[0],o=this.findWorkerByName(r.worker);if(!o)throw new Error(`Worker "${r.worker}" not found`);const i=this.initWorker(o).getWorker;return new Promise((c,u)=>{i.onmessage=s=>{var f,l;this.terminateWorker(i),((f=s.data)==null?void 0:f.ok)===!1?u(new Error(s.data.error)):c((l=s.data)==null?void 0:l.data)},i.onerror=s=>{this.terminateWorker(i),u(s)};const t=r.srcData??{};i.postMessage({data:t,index:0},k(t))})}return new Promise((r,o)=>{const a=[],i=[];for(const t of e){const s=this.findWorkerByName(t.worker);if(!s){o(new Error(`Worker "${t.worker}" not found`));return}const f=new g(s.func,{mode:m.Pipeline,createWorker:s.createWorker}),l=this.trackWorker(f.getWorker);a.push(l)}for(let t=0;t<a.length-1;t++)i.push(new MessageChannel);for(let t=0;t<a.length;t++){const s=[],f={};t>0&&(f.inputPort=i[t-1].port1,s.push(f.inputPort)),t<a.length-1&&(f.outputPort=i[t].port2,s.push(f.outputPort)),a[t].postMessage({__pipeline_ports__:!0,...f},s)}const c=a[a.length-1];c.onmessage=t=>{var s,f;a.forEach(l=>this.terminateWorker(l)),((s=t.data)==null?void 0:s.ok)===!1?o(new Error(t.data.error)):r((f=t.data)==null?void 0:f.data)},c.onerror=t=>{a.forEach(s=>this.terminateWorker(s)),o(t)};const u=e[0].srcData??{};a[0].postMessage({data:u,index:0},k(u))})}async runPersistent(e,r){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const o=this.findWorkerByName(e);if(!o)throw new Error(`Worker "${e}" not found`);let a=this._persistentWorkers.get(e);if(!a){const i=new g(o.func,{mode:m.Persistent,createWorker:o.createWorker});a=this.trackWorker(i.getWorker),this._persistentWorkers.set(e,a)}return new Promise((i,c)=>{a.onmessage=t=>{var s,f;((s=t.data)==null?void 0:s.ok)===!1?c(new Error(t.data.error)):i((f=t.data)==null?void 0:f.data)},a.onerror=t=>{c(t)};const u={type:"run",config:r.config};r.dataset!==void 0&&(u.dataset=r.dataset),a.postMessage(u,k(u))})}release(e){const r=this._persistentWorkers.get(e);if(r){try{r.postMessage({type:"release"})}catch{}this.terminateWorker(r),this._persistentWorkers.delete(e)}}terminate(){this._isTerminated=!0;for(const e of this._persistentWorkers.values()){try{e.postMessage({type:"release"})}catch{}this.terminateWorker(e)}this._persistentWorkers.clear();for(const e of Array.from(this._activeWorkers))this.terminateWorker(e);this._activeWorkers.clear()}destroy(){this.terminate()}reset(){this.terminate(),this._isTerminated=!1}restart(){this.reset()}}exports.MainWorkerFactory=_;exports.WorkerFactory=g;
+`;var P=(l=>(l.Default="default",l.Pipeline="pipeline",l.Persistent="persistent",l))(P||{});const C=Object.freeze({persistent:L,pipeline:F,default:j});class v{constructor(e,r){_(this,"_worker");if(r!=null&&r.createWorker)this._worker=r.createWorker();else if(e){const a=(r==null?void 0:r.mode)??"default",t=C[a](e.toString()),i=new Blob([t],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(i))}else throw new Error("Either workerFunction or options.createWorker must be provided to WorkerFactory.")}get getWorker(){return this._worker}}class I{constructor(e){this.results=e}}class ${constructor(){_(this,"store",new Map)}set(e,r){const a=r??`mem_${crypto.randomUUID()}`;return this.store.set(a,e),a}get(e){return this.store.get(e)}delete(e){return this.store.delete(e)}clear(){this.store.clear()}has(e){return this.store.has(e)}stats(){return{count:this.store.size,refs:Array.from(this.store.keys())}}}function w(l,e=new Set){return l===null||typeof l!="object"?[]:e.has(l)?[]:(e.add(l),l instanceof ArrayBuffer||l instanceof MessagePort||typeof ImageBitmap<"u"&&l instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&l instanceof OffscreenCanvas?[l]:ArrayBuffer.isView(l)?[l.buffer]:Array.isArray(l)?l.flatMap(r=>w(r,e)):Object.values(l).flatMap(r=>w(r,e)))}class U{constructor(e){_(this,"_workers");_(this,"_threads");_(this,"_persistentWorkers",new Map);_(this,"_activeWorkers",new Set);_(this,"_memoryStore",new $);_(this,"_isTerminated",!1);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}get isTerminated(){return this._isTerminated}trackWorker(e){if(this._isTerminated)throw e.terminate(),new Error("MainWorkerFactory has been terminated");return this._activeWorkers.add(e),e}terminateWorker(e){this._activeWorkers.delete(e);try{e.terminate()}catch{}}initWorker(e){const r=new v(e.func,{createWorker:e.createWorker});return this.trackWorker(r.getWorker),r}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const a=Math.min(r,e.length),t=Math.floor(e.length/a),i=e.length%a,f=[];let d=0;for(let s=0;s<a;s++){const o=t+(s<i?1:0);f.push(e.slice(d,d+o)),d+=o}return f}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,r){if(this._isTerminated)return Promise.reject(new Error("MainWorkerFactory has been terminated"));const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));let{srcData:t,...i}=r||{};const f=i.__memory_ref__,d=!!i.deleteMemory;if(t===void 0&&f){if(!this._memoryStore.has(f))return Promise.reject(new Error(`Memory reference "${f}" not found in MemoryStore`));t=this._memoryStore.get(f),delete i.__memory_ref__,delete i.deleteMemory}else delete i.deleteMemory;const s=a.maxConcurrency??this._threads,o=!!(Array.isArray(t)&&a.partition),u=o?this.partitionArray(t,s):t,p=o&&Array.isArray(u)?u.length:s,m=this.createWorkerPromises(a,e,{data:u,...i},p,o),c=await Promise.allSettled(m);return d&&f&&this._memoryStore.delete(f),(a.memoryOnly||a.memory)&&this.storeWorkerMemoryResult(c,{memoryOnly:!!a.memoryOnly,shouldPartition:o}),new I(c)}async deleteMemory(e){return this._memoryStore.delete(e)}async clearMemory(){this._memoryStore.clear()}storeWorkerMemoryResult(e,{memoryOnly:r,shouldPartition:a}){const t=e.filter(s=>s.status==="fulfilled"&&!!s.value.successResult);if(t.length===0)return;const i=t.map(s=>s.value.successResult.data),f=a&&i.every(Array.isArray)?i.flat():i.length===1?i[0]:i,d=this._memoryStore.set(f);for(const s of t){const o=r?{__memory_ref__:d}:{data:s.value.successResult.data,__memory_ref__:d};s.value.successResult=new MessageEvent("message",{data:o})}}async getMemoryStats(){return this._memoryStore.stats()}createWorkerPromises(e,r,a,t,i){const{data:f,...d}=a;return Array.from({length:t},(s,o)=>{const u=i&&Array.isArray(f)?f[o]:f;return this.runWorkerWithRetry({workerFunc:e.func,createWorker:e.createWorker,workerName:r,index:o,data:{data:u,...d}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(a){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,a),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",a),a}}initiateWorker({workerFunc:e,createWorker:r,workerName:a,index:t,data:i}){return new Promise((f,d)=>{const o=this.initWorker({name:a,role:"",func:e,createWorker:r}).getWorker;o.onerror=p=>{this.terminateWorker(o),d({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},failedResult:p})},o.onmessage=p=>{var c,n;if(((c=p.data)==null?void 0:c.ok)===!1){this.terminateWorker(o),d({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},failedResult:new ErrorEvent("error",{message:p.data.error})});return}const m=((n=p.data)==null?void 0:n.ok)!==void 0?p.data.data:p.data;f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},successResult:new MessageEvent("message",{data:m})}),this.terminateWorker(o)};const u={index:t,...Array.isArray(i)?{data:i}:i};o.postMessage(u,w(u))})}async collectResults(e,r={}){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const a=e.results.filter(m=>m.status==="fulfilled"),t=e.results.filter(m=>m.status==="rejected");if(a.length>0&&a.every(m=>{var n;const c=(n=m.value.successResult)==null?void 0:n.data;return c&&typeof c=="object"&&"__memory_ref__"in c&&!("data"in c)}))return{data:a[0].value.successResult.data,succeeded:a.length,failed:t.length,errors:t};const f=a.length>0&&a.every(m=>{var n;const c=(n=m.value.successResult)==null?void 0:n.data;return c&&typeof c=="object"&&"__memory_ref__"in c&&"data"in c}),d=f?a[0].value.successResult.data.__memory_ref__:void 0,s=a.map(m=>{const c=m.value.successResult.data;return f&&c&&typeof c=="object"&&"data"in c?c.data:c}),o=r.reducer?r.reducer.toString():"(shards) => shards.flat()";let u;if(typeof Worker<"u"&&typeof Blob<"u"&&typeof URL<"u"&&typeof URL.createObjectURL=="function")try{u=await new Promise((m,c)=>{const n=`
+            const reducer = ${o};
+            self.addEventListener('message', (event) => {
+              try {
+                const result = reducer(event.data);
+                self.postMessage({ ok: true, data: result });
+              } catch (error) {
+                self.postMessage({ ok: false, error: String(error) });
+              }
+            });
+          `,h=new Blob([n],{type:"application/javascript"}),y=this.trackWorker(new Worker(URL.createObjectURL(h)));y.onmessage=k=>{this.terminateWorker(y),k.data.ok?m(k.data.data):c(new Error(k.data.error))},y.onerror=k=>{this.terminateWorker(y),c(k)},y.postMessage(s)})}catch{const m=n=>n.flat();u=(r.reducer??m)(s)}else{const m=n=>n.flat();u=(r.reducer??m)(s)}return{data:f?{data:u,__memory_ref__:d}:u,succeeded:a.length,failed:t.length,errors:t}}async pipeline(e){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const r=e[0],{worker:a,srcData:t,...i}=r,f=this.findWorkerByName(r.worker);if(!f)throw new Error(`Worker "${r.worker}" not found`);const s=this.initWorker(f).getWorker;return new Promise((o,u)=>{s.onmessage=c=>{var n,h;this.terminateWorker(s),((n=c.data)==null?void 0:n.ok)===!1?u(new Error(c.data.error)):o((h=c.data)==null?void 0:h.data)},s.onerror=c=>{this.terminateWorker(s),u(c)};const m={data:t??{},...i,index:0};s.postMessage(m,w(m))})}return new Promise((r,a)=>{const t=[],i=[];for(const n of e){const h=this.findWorkerByName(n.worker);if(!h){a(new Error(`Worker "${n.worker}" not found`));return}const y=new v(h.func,{mode:P.Pipeline,createWorker:h.createWorker}),k=this.trackWorker(y.getWorker);t.push(k)}for(let n=0;n<t.length-1;n++)i.push(new MessageChannel);for(let n=0;n<t.length;n++){const{worker:h,srcData:y,...k}=e[n],M=[],W={};if(n>0&&(W.inputPort=i[n-1].port1,M.push(W.inputPort)),n<t.length-1&&(W.outputPort=i[n].port2,M.push(W.outputPort)),t[n].postMessage({__pipeline_ports__:!0,stepParams:k,...W},M),n<t.length-1){const b=t[n],T=t[n+1],{worker:V,srcData:q,...A}=e[n+1];b.onmessage=g=>{var S,R;if(g.data&&g.data.__pipeline_ports__)return;if(((S=g.data)==null?void 0:S.ok)===!1){t.forEach(B=>this.terminateWorker(B)),a(new Error(g.data.error));return}const E={data:((R=g.data)==null?void 0:R.ok)!==void 0?g.data.data:g.data,...A,index:0};T.postMessage(E,w(E))},b.onerror=g=>{t.forEach(D=>this.terminateWorker(D)),a(g)}}}const f=t[t.length-1];f.onmessage=n=>{var h,y;t.forEach(k=>this.terminateWorker(k)),((h=n.data)==null?void 0:h.ok)===!1?a(new Error(n.data.error)):r((y=n.data)==null?void 0:y.data)},f.onerror=n=>{t.forEach(h=>this.terminateWorker(h)),a(n)};const{worker:d,srcData:s,...o}=e[0];let u=s;const p=o.__memory_ref__,m=!!o.deleteMemory;if(u===void 0&&p){if(!this._memoryStore.has(p)){t.forEach(n=>this.terminateWorker(n)),a(new Error(`Memory reference "${p}" not found in MemoryStore`));return}u=this._memoryStore.get(p),m&&this._memoryStore.delete(p),delete o.__memory_ref__,delete o.deleteMemory}u===void 0&&(u={});const c={data:u,...o,index:0};t[0].postMessage(c,w(c))})}async runPersistent(e,r){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const a=this.findWorkerByName(e);if(!a)throw new Error(`Worker "${e}" not found`);let t=this._persistentWorkers.get(e);if(!t){const i=new v(a.func,{mode:P.Persistent,createWorker:a.createWorker});t=this.trackWorker(i.getWorker),this._persistentWorkers.set(e,t)}return new Promise((i,f)=>{t.onmessage=s=>{var o,u;((o=s.data)==null?void 0:o.ok)===!1?f(new Error(s.data.error)):i((u=s.data)==null?void 0:u.data)},t.onerror=s=>{f(s)};const d={type:"run",config:r.config};r.dataset!==void 0&&(d.dataset=r.dataset),t.postMessage(d,w(d))})}release(e){const r=this._persistentWorkers.get(e);if(r){try{r.postMessage({type:"release"})}catch{}this.terminateWorker(r),this._persistentWorkers.delete(e)}}terminate(){this._isTerminated=!0;for(const e of this._persistentWorkers.values()){try{e.postMessage({type:"release"})}catch{}this.terminateWorker(e)}this._persistentWorkers.clear();for(const e of Array.from(this._activeWorkers))this.terminateWorker(e);this._activeWorkers.clear(),this._memoryStore.clear()}destroy(){this.terminate()}reset(){this.terminate(),this._isTerminated=!1}restart(){this.reset()}}function z(l){if(typeof self>"u")return;let e=null,r=null,a=null,t={};const i=(d,s)=>{self.postMessage(d,s)};async function f(d){try{const s=typeof d=="object"&&d!==null&&"data"in d?{...t,...d}:{data:d,...t,index:0},o=await l(s),u={ok:!0,data:o},p=w(o);e?e.postMessage(u,p):i(u,p)}catch(s){const o={ok:!1,error:s instanceof Error?s.message:String(s)};e?e.postMessage(o):i(o)}}self.addEventListener("message",d=>{const s=d.data;if(s&&s.__pipeline_ports__){s.stepParams&&(t=s.stepParams),s.outputPort&&(e=s.outputPort),s.inputPort&&(r=s.inputPort,r.onmessage=o=>{var u;o.data&&o.data.ok===!1?e?e.postMessage(o.data):i(o.data):f({data:(u=o.data)==null?void 0:u.data,...t,index:0})}),a!==null&&(f(a),a=null);return}r?a=s:f(s)})}exports.MainWorkerFactory=U;exports.WorkerFactory=v;exports.defineWorker=z;

package/dist/index.js

@@ -1,7 +1,7 @@
-var w = Object.defineProperty;
-var v = (n, e, r) => e in n ? w(n, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : n[e] = r;
-var h = (n, e, r) => v(n, typeof e != "symbol" ? e + "" : e, r);
-const y = (n) => `
+var x = Object.defineProperty;
+var O = (l, e, r) => e in l ? x(l, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : l[e] = r;
+var _ = (l, e, r) => O(l, typeof e != "symbol" ? e + "" : e, r);
+const j = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -18,13 +18,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${n}(event.data);
+    const output = await ${l}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`, W = (n) => `
+`, L = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -39,14 +39,19 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
+let stepParams = {};
 
 async function processData(data) {
   try {
-    const output = await workerFn(data);
+    const payload =
+      typeof data === 'object' && data !== null && 'data' in data
+        ? { ...stepParams, ...data }
+        : { data, ...stepParams, index: 0 };
+    const output = await workerFn(payload);
     const result = { ok: true, data: output };
     const transfers = extractTransferables(output);
     if (outputPort) {
@@ -66,6 +71,9 @@ async function processData(data) {
 
 self.addEventListener('message', (event) => {
   if (event.data && event.data.__pipeline_ports__) {
+    if (event.data.stepParams) {
+      stepParams = event.data.stepParams;
+    }
     if (event.data.outputPort) {
       outputPort = event.data.outputPort;
     }
@@ -77,7 +85,7 @@ self.addEventListener('message', (event) => {
           if (outputPort) outputPort.postMessage(e.data);
           else self.postMessage(e.data);
         } else {
-          processData({ data: e.data.data, index: 0 });
+          processData({ data: e.data.data, ...stepParams, index: 0 });
         }
       };
     }
@@ -96,7 +104,7 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
-`, M = (n) => `
+`, C = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -111,7 +119,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let cachedDataset = null;
 
 self.addEventListener('message', async (event) => {
@@ -153,13 +161,13 @@ self.addEventListener('message', async (event) => {
   }
 });
 `;
-var m = /* @__PURE__ */ ((n) => (n.Default = "default", n.Pipeline = "pipeline", n.Persistent = "persistent", n))(m || {});
-const P = Object.freeze({
-  persistent: M,
-  pipeline: W,
-  default: y
+var P = /* @__PURE__ */ ((l) => (l.Default = "default", l.Pipeline = "pipeline", l.Persistent = "persistent", l))(P || {});
+const F = Object.freeze({
+  persistent: C,
+  pipeline: L,
+  default: j
 });
-class g {
+class M {
   /**
    * Creates a new `Worker` from the given function or factory option.
    *
@@ -169,11 +177,11 @@ class g {
    * @param options - Optional configuration containing `createWorker` or `mode`.
    */
   constructor(e, r) {
-    h(this, "_worker");
+    _(this, "_worker");
     if (r != null && r.createWorker)
       this._worker = r.createWorker();
     else if (e) {
-      const o = (r == null ? void 0 : r.mode) ?? "default", a = P[o](e.toString()), i = new Blob([a], {
+      const a = (r == null ? void 0 : r.mode) ?? "default", t = F[a](e.toString()), i = new Blob([t], {
         type: "application/javascript"
       });
       this._worker = new Worker(URL.createObjectURL(i));
@@ -192,28 +200,84 @@ class g {
     return this._worker;
   }
 }
-class b {
+class I {
   constructor(e) {
     this.results = e;
   }
 }
-function k(n, e = /* @__PURE__ */ new Set()) {
-  return n === null || typeof n != "object" ? [] : e.has(n) ? [] : (e.add(n), n instanceof ArrayBuffer || n instanceof MessagePort || typeof ImageBitmap < "u" && n instanceof ImageBitmap || typeof OffscreenCanvas < "u" && n instanceof OffscreenCanvas ? [n] : ArrayBuffer.isView(n) ? [n.buffer] : Array.isArray(n) ? n.flatMap((r) => k(r, e)) : Object.values(n).flatMap(
-    (r) => k(r, e)
+class $ {
+  constructor() {
+    _(this, "store", /* @__PURE__ */ new Map());
+  }
+  /**
+   * Stores a dataset in RAM under an unguessable reference ID.
+   *
+   * @param data - The dataset to store.
+   * @param refId - Optional reference ID; if omitted, a UUID will be generated.
+   * @returns The reference ID under which the dataset is stored.
+   */
+  set(e, r) {
+    const a = r ?? `mem_${crypto.randomUUID()}`;
+    return this.store.set(a, e), a;
+  }
+  /**
+   * Retrieves a dataset from RAM by its reference ID.
+   *
+   * @param refId - The reference ID to retrieve.
+   * @returns The stored dataset, or undefined if not found.
+   */
+  get(e) {
+    return this.store.get(e);
+  }
+  /**
+   * Deletes a dataset reference from RAM.
+   *
+   * @param refId - The reference ID to delete.
+   * @returns `true` if the key existed and was removed, `false` otherwise.
+   */
+  delete(e) {
+    return this.store.delete(e);
+  }
+  /**
+   * Clears all dataset handles from RAM.
+   */
+  clear() {
+    this.store.clear();
+  }
+  /**
+   * Checks if a reference ID exists in RAM.
+   */
+  has(e) {
+    return this.store.has(e);
+  }
+  /**
+   * Returns statistics about current memory handles.
+   */
+  stats() {
+    return {
+      count: this.store.size,
+      refs: Array.from(this.store.keys())
+    };
+  }
+}
+function w(l, e = /* @__PURE__ */ new Set()) {
+  return l === null || typeof l != "object" ? [] : e.has(l) ? [] : (e.add(l), l instanceof ArrayBuffer || l instanceof MessagePort || typeof ImageBitmap < "u" && l instanceof ImageBitmap || typeof OffscreenCanvas < "u" && l instanceof OffscreenCanvas ? [l] : ArrayBuffer.isView(l) ? [l.buffer] : Array.isArray(l) ? l.flatMap((r) => w(r, e)) : Object.values(l).flatMap(
+    (r) => w(r, e)
   ));
 }
-class E {
+class q {
   /**
    * Creates a new `MainWorkerFactory`.
    *
    * @param options - Configuration object containing the `workers` registry.
    */
   constructor(e) {
-    h(this, "_workers");
-    h(this, "_threads");
-    h(this, "_persistentWorkers", /* @__PURE__ */ new Map());
-    h(this, "_activeWorkers", /* @__PURE__ */ new Set());
-    h(this, "_isTerminated", !1);
+    _(this, "_workers");
+    _(this, "_threads");
+    _(this, "_persistentWorkers", /* @__PURE__ */ new Map());
+    _(this, "_activeWorkers", /* @__PURE__ */ new Set());
+    _(this, "_memoryStore", new $());
+    _(this, "_isTerminated", !1);
     this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
   }
   /**
@@ -247,7 +311,7 @@ class E {
    * @returns A new `WorkerFactory` wrapping the worker.
    */
   initWorker(e) {
-    const r = new g(e.func, {
+    const r = new M(e.func, {
       createWorker: e.createWorker
     });
     return this.trackWorker(r.getWorker), r;
@@ -271,13 +335,13 @@ class E {
   partitionArray(e, r) {
     if (!e.length) return [];
     if (r <= 0) throw new Error("numChunks must be positive");
-    const o = Math.min(r, e.length), a = Math.floor(e.length / o), i = e.length % o, c = [];
-    let u = 0;
-    for (let t = 0; t < o; t++) {
-      const s = a + (t < i ? 1 : 0);
-      c.push(e.slice(u, u + s)), u += s;
+    const a = Math.min(r, e.length), t = Math.floor(e.length / a), i = e.length % a, f = [];
+    let d = 0;
+    for (let s = 0; s < a; s++) {
+      const o = t + (s < i ? 1 : 0);
+      f.push(e.slice(d, d + o)), d += o;
     }
-    return c;
+    return f;
   }
   /**
    * Looks up a registered worker configuration by name.
@@ -314,23 +378,76 @@ class E {
    * @example
    * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
    */
-  async runWorker(e, {
-    srcData: r,
-    ...o
-  }) {
+  async runWorker(e, r) {
     if (this._isTerminated)
       return Promise.reject(new Error("MainWorkerFactory has been terminated"));
     const a = this.findWorkerByName(e);
     if (!a)
       return Promise.reject(new Error(`Worker "${e}" not found`));
-    const i = a.maxConcurrency ?? this._threads, c = !!(Array.isArray(r) && r.length > 1 && a.partition), u = c ? this.partitionArray(r, i) : r, t = this.createWorkerPromises(
+    let { srcData: t, ...i } = r || {};
+    const f = i.__memory_ref__, d = !!i.deleteMemory;
+    if (t === void 0 && f) {
+      if (!this._memoryStore.has(f))
+        return Promise.reject(
+          new Error(`Memory reference "${f}" not found in MemoryStore`)
+        );
+      t = this._memoryStore.get(f), delete i.__memory_ref__, delete i.deleteMemory;
+    } else
+      delete i.deleteMemory;
+    const s = a.maxConcurrency ?? this._threads, o = !!(Array.isArray(t) && a.partition), u = o ? this.partitionArray(t, s) : t, m = o && Array.isArray(u) ? u.length : s, p = this.createWorkerPromises(
       a,
       e,
-      { data: u, ...o },
-      i,
-      c
-    ), s = await Promise.allSettled(t);
-    return new b(s);
+      { data: u, ...i },
+      m,
+      o
+    ), c = await Promise.allSettled(p);
+    return d && f && this._memoryStore.delete(f), (a.memoryOnly || a.memory) && this.storeWorkerMemoryResult(c, {
+      memoryOnly: !!a.memoryOnly,
+      shouldPartition: o
+    }), new I(c);
+  }
+  /**
+   * Deletes a specific memory reference from the factory's memory store.
+   *
+   * @param ref - The `__memory_ref__` token string to delete.
+   * @returns A promise that resolves to `true` if deleted, `false` otherwise.
+   */
+  async deleteMemory(e) {
+    return this._memoryStore.delete(e);
+  }
+  /**
+   * Clears all stored dataset references from the factory's memory store.
+   */
+  async clearMemory() {
+    this._memoryStore.clear();
+  }
+  /**
+   * Saves the output of a completed worker execution into MemoryStore
+   * and modifies the returned settled results with a `__memory_ref__` token.
+   */
+  storeWorkerMemoryResult(e, {
+    memoryOnly: r,
+    shouldPartition: a
+  }) {
+    const t = e.filter(
+      (s) => s.status === "fulfilled" && !!s.value.successResult
+    );
+    if (t.length === 0) return;
+    const i = t.map(
+      (s) => s.value.successResult.data
+    ), f = a && i.every(Array.isArray) ? i.flat() : i.length === 1 ? i[0] : i, d = this._memoryStore.set(f);
+    for (const s of t) {
+      const o = r ? { __memory_ref__: d } : { data: s.value.successResult.data, __memory_ref__: d };
+      s.value.successResult = new MessageEvent("message", {
+        data: o
+      });
+    }
+  }
+  /**
+   * Returns statistics about active memory references in the factory.
+   */
+  async getMemoryStats() {
+    return this._memoryStore.stats();
   }
   /**
    * Builds the array of per-thread worker promises for a single `runWorker`
@@ -346,17 +463,17 @@ class E {
    * @param isPartitioned  - Whether `data` is a pre-split array of shards.
    * @returns An array of promises, one per thread.
    */
-  createWorkerPromises(e, r, o, a, i) {
-    const { data: c, ...u } = o;
-    return Array.from({ length: a }, (t, s) => {
-      const f = i && Array.isArray(c) ? c[s] : c;
+  createWorkerPromises(e, r, a, t, i) {
+    const { data: f, ...d } = a;
+    return Array.from({ length: t }, (s, o) => {
+      const u = i && Array.isArray(f) ? f[o] : f;
       return this.runWorkerWithRetry(
         {
           workerFunc: e.func,
           createWorker: e.createWorker,
           workerName: r,
-          index: s,
-          data: { data: f, ...u }
+          index: o,
+          data: { data: u, ...d }
         },
         e.retries
       );
@@ -377,13 +494,13 @@ class E {
   async runWorkerWithRetry(e, r = 2) {
     try {
       return await this.initiateWorker(e);
-    } catch (o) {
+    } catch (a) {
       if (r > 0)
         return console.error(
           `Worker ${e.index} failed, retrying (${r} left):`,
-          o
+          a
         ), this.runWorkerWithRetry(e, r - 1);
-      throw console.error("Worker failed after all retries:", o), o;
+      throw console.error("Worker failed after all retries:", a), a;
     }
   }
   /**
@@ -406,66 +523,67 @@ class E {
   initiateWorker({
     workerFunc: e,
     createWorker: r,
-    workerName: o,
-    index: a,
+    workerName: a,
+    index: t,
     data: i
   }) {
-    return new Promise((c, u) => {
-      const s = this.initWorker({
-        name: o,
+    return new Promise((f, d) => {
+      const o = this.initWorker({
+        name: a,
         role: "",
         func: e,
         createWorker: r
       }).getWorker;
-      s.onerror = (l) => {
-        this.terminateWorker(s), u({
-          index: a,
+      o.onerror = (m) => {
+        this.terminateWorker(o), d({
+          index: t,
           workerConfigs: {
             workerFunc: e,
             createWorker: r,
-            workerName: o,
-            index: a,
+            workerName: a,
+            index: t,
             data: i
           },
-          failedResult: l
+          failedResult: m
         });
-      }, s.onmessage = (l) => {
-        var d, p;
-        if (((d = l.data) == null ? void 0 : d.ok) === !1) {
-          this.terminateWorker(s), u({
-            index: a,
+      }, o.onmessage = (m) => {
+        var c, n;
+        if (((c = m.data) == null ? void 0 : c.ok) === !1) {
+          this.terminateWorker(o), d({
+            index: t,
             workerConfigs: {
               workerFunc: e,
               createWorker: r,
-              workerName: o,
-              index: a,
+              workerName: a,
+              index: t,
               data: i
             },
             failedResult: new ErrorEvent("error", {
-              message: l.data.error
+              message: m.data.error
             })
           });
           return;
         }
-        c({
-          index: a,
+        const p = ((n = m.data) == null ? void 0 : n.ok) !== void 0 ? m.data.data : m.data;
+        f({
+          index: t,
           workerConfigs: {
             workerFunc: e,
             createWorker: r,
-            workerName: o,
-            index: a,
+            workerName: a,
+            index: t,
             data: i
           },
           successResult: new MessageEvent("message", {
-            data: (p = l.data) == null ? void 0 : p.data
+            data: p
           })
-        }), this.terminateWorker(s);
+        }), this.terminateWorker(o);
       };
-      const f = {
-        index: a,
+      const u = {
+        index: t,
         ...Array.isArray(i) ? { data: i } : i
       };
-      s.postMessage(f, k(f));
+      o.postMessage(u, w(u));
     });
   }
   /**
@@ -503,33 +621,68 @@ class E {
   async collectResults(e, r = {}) {
     if (this._isTerminated)
       throw new Error("MainWorkerFactory has been terminated");
-    const o = e.results.filter(
-      (t) => t.status === "fulfilled"
-    ), a = e.results.filter(
-      (t) => t.status === "rejected"
-    ), i = o.map((t) => t.value.successResult.data), c = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
-    return {
-      data: await new Promise((t, s) => {
-        const f = `
-        const reducer = ${c};
-        self.addEventListener('message', (event) => {
-          try {
-            const result = reducer(event.data);
-            self.postMessage({ ok: true, data: result });
-          } catch (err) {
-            self.postMessage({ ok: false, error: String(err) });
-          }
+    const a = e.results.filter(
+      (p) => p.status === "fulfilled"
+    ), t = e.results.filter(
+      (p) => p.status === "rejected"
+    );
+    if (a.length > 0 && a.every((p) => {
+      var n;
+      const c = (n = p.value.successResult) == null ? void 0 : n.data;
+      return c && typeof c == "object" && "__memory_ref__" in c && !("data" in c);
+    }))
+      return {
+        data: a[0].value.successResult.data,
+        succeeded: a.length,
+        failed: t.length,
+        errors: t
+      };
+    const f = a.length > 0 && a.every((p) => {
+      var n;
+      const c = (n = p.value.successResult) == null ? void 0 : n.data;
+      return c && typeof c == "object" && "__memory_ref__" in c && "data" in c;
+    }), d = f ? a[0].value.successResult.data.__memory_ref__ : void 0, s = a.map((p) => {
+      const c = p.value.successResult.data;
+      return f && c && typeof c == "object" && "data" in c ? c.data : c;
+    }), o = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
+    let u;
+    if (typeof Worker < "u" && typeof Blob < "u" && typeof URL < "u" && typeof URL.createObjectURL == "function")
+      try {
+        u = await new Promise((p, c) => {
+          const n = `
+            const reducer = ${o};
+            self.addEventListener('message', (event) => {
+              try {
+                const result = reducer(event.data);
+                self.postMessage({ ok: true, data: result });
+              } catch (error) {
+                self.postMessage({ ok: false, error: String(error) });
+              }
+            });
+          `, h = new Blob([n], {
+            type: "application/javascript"
+          }), y = this.trackWorker(
+            new Worker(URL.createObjectURL(h))
+          );
+          y.onmessage = (k) => {
+            this.terminateWorker(y), k.data.ok ? p(k.data.data) : c(new Error(k.data.error));
+          }, y.onerror = (k) => {
+            this.terminateWorker(y), c(k);
+          }, y.postMessage(s);
         });
-      `, l = new Blob([f], { type: "application/javascript" }), d = this.trackWorker(new Worker(URL.createObjectURL(l)));
-        d.onmessage = (p) => {
-          this.terminateWorker(d), p.data.ok ? t(p.data.data) : s(new Error(p.data.error));
-        }, d.onerror = (p) => {
-          this.terminateWorker(d), s(p);
-        }, d.postMessage(i);
-      }),
-      succeeded: o.length,
-      failed: a.length,
-      errors: a
+      } catch {
+        const p = (n) => n.flat();
+        u = (r.reducer ?? p)(s);
+      }
+    else {
+      const p = (n) => n.flat();
+      u = (r.reducer ?? p)(s);
+    }
+    return {
+      data: f ? { data: u, __memory_ref__: d } : u,
+      succeeded: a.length,
+      failed: t.length,
+      errors: t
     };
   }
   /**
@@ -564,58 +717,97 @@ class E {
     if (e.length === 0)
       throw new Error("Pipeline requires at least one step");
     if (e.length === 1) {
-      const r = e[0], o = this.findWorkerByName(r.worker);
-      if (!o) throw new Error(`Worker "${r.worker}" not found`);
-      const i = this.initWorker(o).getWorker;
-      return new Promise((c, u) => {
-        i.onmessage = (s) => {
-          var f, l;
-          this.terminateWorker(i), ((f = s.data) == null ? void 0 : f.ok) === !1 ? u(new Error(s.data.error)) : c((l = s.data) == null ? void 0 : l.data);
-        }, i.onerror = (s) => {
-          this.terminateWorker(i), u(s);
+      const r = e[0], { worker: a, srcData: t, ...i } = r, f = this.findWorkerByName(r.worker);
+      if (!f) throw new Error(`Worker "${r.worker}" not found`);
+      const s = this.initWorker(f).getWorker;
+      return new Promise((o, u) => {
+        s.onmessage = (c) => {
+          var n, h;
+          this.terminateWorker(s), ((n = c.data) == null ? void 0 : n.ok) === !1 ? u(new Error(c.data.error)) : o((h = c.data) == null ? void 0 : h.data);
+        }, s.onerror = (c) => {
+          this.terminateWorker(s), u(c);
         };
-        const t = r.srcData ?? {};
-        i.postMessage(
-          { data: t, index: 0 },
-          k(t)
-        );
+        const p = { data: t ?? {}, ...i, index: 0 };
+        s.postMessage(p, w(p));
       });
     }
-    return new Promise((r, o) => {
-      const a = [], i = [];
-      for (const t of e) {
-        const s = this.findWorkerByName(t.worker);
-        if (!s) {
-          o(new Error(`Worker "${t.worker}" not found`));
+    return new Promise((r, a) => {
+      const t = [], i = [];
+      for (const n of e) {
+        const h = this.findWorkerByName(n.worker);
+        if (!h) {
+          a(new Error(`Worker "${n.worker}" not found`));
           return;
         }
-        const f = new g(s.func, {
-          mode: m.Pipeline,
-          createWorker: s.createWorker
-        }), l = this.trackWorker(f.getWorker);
-        a.push(l);
+        const y = new M(h.func, {
+          mode: P.Pipeline,
+          createWorker: h.createWorker
+        }), k = this.trackWorker(y.getWorker);
+        t.push(k);
       }
-      for (let t = 0; t < a.length - 1; t++)
+      for (let n = 0; n < t.length - 1; n++)
         i.push(new MessageChannel());
-      for (let t = 0; t < a.length; t++) {
-        const s = [], f = {};
-        t > 0 && (f.inputPort = i[t - 1].port1, s.push(f.inputPort)), t < a.length - 1 && (f.outputPort = i[t].port2, s.push(f.outputPort)), a[t].postMessage(
-          { __pipeline_ports__: !0, ...f },
-          s
-        );
+      for (let n = 0; n < t.length; n++) {
+        const {
+          worker: h,
+          srcData: y,
+          ...k
+        } = e[n], v = [], W = {};
+        if (n > 0 && (W.inputPort = i[n - 1].port1, v.push(W.inputPort)), n < t.length - 1 && (W.outputPort = i[n].port2, v.push(W.outputPort)), t[n].postMessage(
+          { __pipeline_ports__: !0, stepParams: k, ...W },
+          v
+        ), n < t.length - 1) {
+          const b = t[n], A = t[n + 1], {
+            worker: U,
+            srcData: z,
+            ...T
+          } = e[n + 1];
+          b.onmessage = (g) => {
+            var S, R;
+            if (g.data && g.data.__pipeline_ports__) return;
+            if (((S = g.data) == null ? void 0 : S.ok) === !1) {
+              t.forEach((B) => this.terminateWorker(B)), a(new Error(g.data.error));
+              return;
+            }
+            const E = { data: ((R = g.data) == null ? void 0 : R.ok) !== void 0 ? g.data.data : g.data, ...T, index: 0 };
+            A.postMessage(E, w(E));
+          }, b.onerror = (g) => {
+            t.forEach((D) => this.terminateWorker(D)), a(g);
+          };
+        }
       }
-      const c = a[a.length - 1];
-      c.onmessage = (t) => {
-        var s, f;
-        a.forEach((l) => this.terminateWorker(l)), ((s = t.data) == null ? void 0 : s.ok) === !1 ? o(new Error(t.data.error)) : r((f = t.data) == null ? void 0 : f.data);
-      }, c.onerror = (t) => {
-        a.forEach((s) => this.terminateWorker(s)), o(t);
+      const f = t[t.length - 1];
+      f.onmessage = (n) => {
+        var h, y;
+        t.forEach((k) => this.terminateWorker(k)), ((h = n.data) == null ? void 0 : h.ok) === !1 ? a(new Error(n.data.error)) : r((y = n.data) == null ? void 0 : y.data);
+      }, f.onerror = (n) => {
+        t.forEach((h) => this.terminateWorker(h)), a(n);
       };
-      const u = e[0].srcData ?? {};
-      a[0].postMessage(
-        { data: u, index: 0 },
-        k(u)
-      );
+      const {
+        worker: d,
+        srcData: s,
+        ...o
+      } = e[0];
+      let u = s;
+      const m = o.__memory_ref__, p = !!o.deleteMemory;
+      if (u === void 0 && m) {
+        if (!this._memoryStore.has(m)) {
+          t.forEach((n) => this.terminateWorker(n)), a(
+            new Error(
+              `Memory reference "${m}" not found in MemoryStore`
+            )
+          );
+          return;
+        }
+        u = this._memoryStore.get(m), p && this._memoryStore.delete(m), delete o.__memory_ref__, delete o.deleteMemory;
+      }
+      u === void 0 && (u = {});
+      const c = {
+        data: u,
+        ...o,
+        index: 0
+      };
+      t[0].postMessage(c, w(c));
     });
   }
   /**
@@ -655,28 +847,28 @@ class E {
   async runPersistent(e, r) {
     if (this._isTerminated)
       throw new Error("MainWorkerFactory has been terminated");
-    const o = this.findWorkerByName(e);
-    if (!o) throw new Error(`Worker "${e}" not found`);
-    let a = this._persistentWorkers.get(e);
-    if (!a) {
-      const i = new g(o.func, {
-        mode: m.Persistent,
-        createWorker: o.createWorker
+    const a = this.findWorkerByName(e);
+    if (!a) throw new Error(`Worker "${e}" not found`);
+    let t = this._persistentWorkers.get(e);
+    if (!t) {
+      const i = new M(a.func, {
+        mode: P.Persistent,
+        createWorker: a.createWorker
       });
-      a = this.trackWorker(i.getWorker), this._persistentWorkers.set(e, a);
+      t = this.trackWorker(i.getWorker), this._persistentWorkers.set(e, t);
     }
-    return new Promise((i, c) => {
-      a.onmessage = (t) => {
-        var s, f;
-        ((s = t.data) == null ? void 0 : s.ok) === !1 ? c(new Error(t.data.error)) : i((f = t.data) == null ? void 0 : f.data);
-      }, a.onerror = (t) => {
-        c(t);
+    return new Promise((i, f) => {
+      t.onmessage = (s) => {
+        var o, u;
+        ((o = s.data) == null ? void 0 : o.ok) === !1 ? f(new Error(s.data.error)) : i((u = s.data) == null ? void 0 : u.data);
+      }, t.onerror = (s) => {
+        f(s);
       };
-      const u = {
+      const d = {
         type: "run",
         config: r.config
       };
-      r.dataset !== void 0 && (u.dataset = r.dataset), a.postMessage(u, k(u));
+      r.dataset !== void 0 && (d.dataset = r.dataset), t.postMessage(d, w(d));
     });
   }
   /**
@@ -716,7 +908,7 @@ class E {
     this._persistentWorkers.clear();
     for (const e of Array.from(this._activeWorkers))
       this.terminateWorker(e);
-    this._activeWorkers.clear();
+    this._activeWorkers.clear(), this._memoryStore.clear();
   }
   /**
    * Alias for {@link terminate}. Terminates the factory and all worker instances.
@@ -738,7 +930,38 @@ class E {
     this.reset();
   }
 }
+function N(l) {
+  if (typeof self > "u") return;
+  let e = null, r = null, a = null, t = {};
+  const i = (d, s) => {
+    self.postMessage(d, s);
+  };
+  async function f(d) {
+    try {
+      const s = typeof d == "object" && d !== null && "data" in d ? { ...t, ...d } : { data: d, ...t, index: 0 }, o = await l(s), u = { ok: !0, data: o }, m = w(o);
+      e ? e.postMessage(u, m) : i(u, m);
+    } catch (s) {
+      const o = {
+        ok: !1,
+        error: s instanceof Error ? s.message : String(s)
+      };
+      e ? e.postMessage(o) : i(o);
+    }
+  }
+  self.addEventListener("message", (d) => {
+    const s = d.data;
+    if (s && s.__pipeline_ports__) {
+      s.stepParams && (t = s.stepParams), s.outputPort && (e = s.outputPort), s.inputPort && (r = s.inputPort, r.onmessage = (o) => {
+        var u;
+        o.data && o.data.ok === !1 ? e ? e.postMessage(o.data) : i(o.data) : f({ data: (u = o.data) == null ? void 0 : u.data, ...t, index: 0 });
+      }), a !== null && (f(a), a = null);
+      return;
+    }
+    r ? a = s : f(s);
+  });
+}
 export {
-  E as MainWorkerFactory,
-  g as WorkerFactory
+  q as MainWorkerFactory,
+  M as WorkerFactory,
+  N as defineWorker
 };

package/dist/types/tools/define-worker.d.ts

@@ -0,0 +1,21 @@
+import { WorkerFunction } from './main-worker-factory/types';
+/**
+ * Defines and exports a function to run inside a native Web Worker script,
+ * providing full compatibility with `MainWorkerFactory` features including
+ * standard runs, worker-to-worker pipelines (`foreman.pipeline()`), and
+ * dataset caching (`foreman.runPersistent()`).
+ *
+ * @typeParam TParams - The type of the payload sent to the worker.
+ * @typeParam TResult - The return type of the worker function.
+ *
+ * @param workerFn - The worker execution function.
+ *
+ * @example
+ * // my-native-worker.ts
+ * import { defineWorker } from '@offmain/workerkit';
+ *
+ * export default defineWorker(async ({ data }: { data: number[] }) => {
+ *   return data.map((x) => x * 2);
+ * });
+ */
+export declare function defineWorker<TParams = unknown, TResult = unknown>(workerFn: WorkerFunction<TParams, TResult | Promise<TResult>>): void;

package/dist/types/tools/define-worker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/tools/index.d.ts

@@ -1,2 +1,3 @@
 export * from './worker-factory';
 export * from './main-worker-factory';
+export * from './define-worker';

package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts

@@ -1,4 +1,4 @@
-import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
+import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults, MemoryStats } from './types.ts';
 /**
  * Recursively collects all Transferable objects from a value.
  * Transferable (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
@@ -37,6 +37,7 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
     private readonly _threads;
     private readonly _persistentWorkers;
     private readonly _activeWorkers;
+    private readonly _memoryStore;
     private _isTerminated;
     /**
      * Creates a new `MainWorkerFactory`.
@@ -115,9 +116,31 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      * @example
      * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
      */
-    runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, { srcData, ...otherParams }: {
-        srcData: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
+    runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, rawParams: {
+        srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
+        __memory_ref__?: string;
+        deleteMemory?: boolean;
     } & Record<string, unknown>): Promise<TypedSettledResults<WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>>;
+    /**
+     * Deletes a specific memory reference from the factory's memory store.
+     *
+     * @param ref - The `__memory_ref__` token string to delete.
+     * @returns A promise that resolves to `true` if deleted, `false` otherwise.
+     */
+    deleteMemory(ref: string): Promise<boolean>;
+    /**
+     * Clears all stored dataset references from the factory's memory store.
+     */
+    clearMemory(): Promise<void>;
+    /**
+     * Saves the output of a completed worker execution into MemoryStore
+     * and modifies the returned settled results with a `__memory_ref__` token.
+     */
+    private storeWorkerMemoryResult;
+    /**
+     * Returns statistics about active memory references in the factory.
+     */
+    getMemoryStats(): Promise<MemoryStats>;
     /**
      * Builds the array of per-thread worker promises for a single `runWorker`
      * call.
@@ -196,7 +219,7 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      *   reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
      * });
      */
-    collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
+    collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
     /**
      * Runs a chain of workers where each step's output feeds directly into the
      * next step — **without passing through the main thread**.

package/dist/types/tools/main-worker-factory/memory-store.d.ts

@@ -0,0 +1,45 @@
+/**
+ * MemoryStore manages dataset handles stored in volatile RAM inside the MemoryWorker.
+ *
+ * Each dataset is indexed by a cryptographically generated UUID (`__memory_ref__`).
+ */
+export declare class MemoryStore {
+    private readonly store;
+    /**
+     * Stores a dataset in RAM under an unguessable reference ID.
+     *
+     * @param data - The dataset to store.
+     * @param refId - Optional reference ID; if omitted, a UUID will be generated.
+     * @returns The reference ID under which the dataset is stored.
+     */
+    set(data: unknown, refId?: string): string;
+    /**
+     * Retrieves a dataset from RAM by its reference ID.
+     *
+     * @param refId - The reference ID to retrieve.
+     * @returns The stored dataset, or undefined if not found.
+     */
+    get(refId: string): unknown;
+    /**
+     * Deletes a dataset reference from RAM.
+     *
+     * @param refId - The reference ID to delete.
+     * @returns `true` if the key existed and was removed, `false` otherwise.
+     */
+    delete(refId: string): boolean;
+    /**
+     * Clears all dataset handles from RAM.
+     */
+    clear(): void;
+    /**
+     * Checks if a reference ID exists in RAM.
+     */
+    has(refId: string): boolean;
+    /**
+     * Returns statistics about current memory handles.
+     */
+    stats(): {
+        count: number;
+        refs: string[];
+    };
+}

package/dist/types/tools/main-worker-factory/memory-worker.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Script source for the dedicated MemoryWorker thread.
+ *
+ * Runs inside an isolated Web Worker thread and maintains the MemoryStore.
+ * Handshakes and authenticates all incoming MessagePort requests using a secret `factoryToken`.
+ */
+export declare const memoryWorkerScript = "\nconst store = new Map();\n\nself.addEventListener('message', (event) => {\n  const msg = event.data;\n  if (!msg || typeof msg !== 'object') return;\n\n  const { action, factoryToken, expectedToken, ref, data, id } = msg;\n\n  // Initial handshake to set expected token if needed\n  if (action === 'INIT_TOKEN') {\n    self.__expectedToken = expectedToken;\n    self.postMessage({ ok: true, action: 'INIT_TOKEN_ACK' });\n    return;\n  }\n\n  // Validate factory token\n  if (self.__expectedToken && factoryToken !== self.__expectedToken) {\n    self.postMessage({ ok: false, error: 'Unauthorized: invalid factory token', id });\n    return;\n  }\n\n  try {\n    switch (action) {\n      case 'SET': {\n        const refId = ref || ('mem_' + crypto.randomUUID());\n        store.set(refId, data);\n        self.postMessage({ ok: true, ref: refId, id });\n        break;\n      }\n      case 'GET': {\n        const resultData = store.get(ref);\n        const exists = store.has(ref);\n        self.postMessage({ ok: true, exists, data: resultData, ref, id });\n        break;\n      }\n      case 'DELETE': {\n        const deleted = store.delete(ref);\n        self.postMessage({ ok: true, deleted, ref, id });\n        break;\n      }\n      case 'CLEAR': {\n        store.clear();\n        self.postMessage({ ok: true, action: 'CLEAR_ACK', id });\n        break;\n      }\n      case 'STATS': {\n        self.postMessage({\n          ok: true,\n          stats: {\n            count: store.size,\n            refs: Array.from(store.keys()),\n          },\n          id,\n        });\n        break;\n      }\n      default:\n        self.postMessage({ ok: false, error: 'Unknown action: ' + action, id });\n    }\n  } catch (err) {\n    self.postMessage({\n      ok: false,\n      error: err instanceof Error ? err.message : String(err),\n      id,\n    });\n  }\n});\n";

package/dist/types/tools/main-worker-factory/types.d.ts

@@ -52,6 +52,23 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
      * If any dependency fails, this worker will not run.
      */
     dependencies?: Array<() => void>;
+    /**
+     * When `true`, the worker's output is saved in the isolated `MemoryWorker`
+     * and returned alongside a `__memory_ref__` token.
+     */
+    memory?: boolean;
+    /**
+     * When `true`, the worker's output is saved in the isolated `MemoryWorker`
+     * and **only** the `__memory_ref__` token is returned (0 bytes data transferred back to main thread).
+     */
+    memoryOnly?: boolean;
+}
+/** Statistics about stored memory handles in MemoryWorker. */
+export interface MemoryStats {
+    /** Total number of active memory references. */
+    count: number;
+    /** Array of active reference IDs. */
+    refs: string[];
 }
 /**
  * Derives a `name → function` map from a readonly tuple of
@@ -168,4 +185,6 @@ export interface PipelineStep {
     worker: string;
     /** Input data for the first step (subsequent steps receive previous output) */
     srcData?: unknown;
+    /** Any additional step parameters, configs, or options forwarded to the worker payload. */
+    [key: string]: unknown;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@offmain/workerkit",
-  "version": "0.13.0",
+  "version": "0.14.1",
   "description": "A lightweight manager for running functions in Web Workers with partitioning, retries, and concurrency control",
   "type": "module",
   "main": "./dist/index.cjs",