@offmain/workerkit 0.14.0 → 0.14.1

package/README.md

@@ -155,6 +155,14 @@ const { data } = await factory.collectResults(settled, {
 
 > **Note:** The reducer runs inside a worker thread and must be self-contained — it cannot reference variables from the outer scope.
 
+### Dynamic Thread Scaling & Partitioning Behavior
+
+When `partition: true` is enabled on a worker:
+
+- **Dynamic Thread Allocation:** The library calculates worker thread count as `Math.min(maxConcurrency, srcData.length)`. For instance, if an array has 2 items and `maxConcurrency` is 20, the factory will spawn **only 2 worker threads** (instead of 20), eliminating idle thread overhead and memory pressure.
+- **No Data Duplication:** Each thread receives only its assigned chunk (e.g. Worker 0 gets `[item1]`, Worker 1 gets `[item2]`), ensuring results are processed once without duplication.
+- **Non-Partitioned Workers (`partition: false` / omitted):** If `partition` is not enabled, the input payload is not split, and up to `maxConcurrency` threads will each execute the full payload independently in parallel.
+
 ---
 
 ## Pipeline
@@ -223,12 +231,15 @@ const result = await factory.pipeline<AggregateResult>([
 
 ### When to use pipeline vs runWorker
 
-| Scenario                                             | Use                     |
-| ---------------------------------------------------- | ----------------------- |
-| Single step, or steps that need partitioning/retries | `runWorker`             |
-| Multi-step chain where intermediate data is large    | `pipeline`              |
-| Steps that are independent (not sequential)          | `runWorker` in parallel |
-| Steps where only the final result matters to the UI  | `pipeline`              |
+| Scenario                                                      | Use                     |
+| ------------------------------------------------------------- | ----------------------- |
+| Single step, or steps that need multi-core array partitioning | `runWorker`             |
+| Multi-step chain where intermediate data is large             | `pipeline`              |
+| Steps that are independent (not sequential)                   | `runWorker` in parallel |
+| Steps where only the final result matters to the UI           | `pipeline`              |
+
+> **Note on `partition: true` in Pipelines:**  
+> `pipeline()` creates **1 worker thread per step** in a linear 1:1 `MessageChannel` chain. If a worker in a pipeline step has `partition: true`, `pipeline()` processes it as a single streaming step without splitting it across parallel worker threads. For parallel multi-core array partitioning across CPU threads, use `runWorker()`.
 
 ---
 

package/dist/index.cjs

@@ -1,4 +1,4 @@
-"use strict";var S=Object.defineProperty;var x=(n,e,r)=>e in n?S(n,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):n[e]=r;var y=(n,e,r)=>x(n,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const O=n=>`
+"use strict";var O=Object.defineProperty;var x=(l,e,r)=>e in l?O(l,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):l[e]=r;var _=(l,e,r)=>x(l,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const j=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -15,13 +15,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${n}(event.data);
+    const output = await ${l}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`,F=n=>`
+`,F=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -36,7 +36,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
@@ -101,7 +101,7 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
-`,R=n=>`
+`,L=l=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -116,7 +116,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let cachedDataset = null;
 
 self.addEventListener('message', async (event) => {
@@ -157,14 +157,14 @@ self.addEventListener('message', async (event) => {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 });
-`;var P=(n=>(n.Default="default",n.Pipeline="pipeline",n.Persistent="persistent",n))(P||{});const C=Object.freeze({persistent:R,pipeline:F,default:O});class v{constructor(e,r){y(this,"_worker");if(r!=null&&r.createWorker)this._worker=r.createWorker();else if(e){const s=(r==null?void 0:r.mode)??"default",t=C[s](e.toString()),l=new Blob([t],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(l))}else throw new Error("Either workerFunction or options.createWorker must be provided to WorkerFactory.")}get getWorker(){return this._worker}}class j{constructor(e){this.results=e}}function m(n,e=new Set){return n===null||typeof n!="object"?[]:e.has(n)?[]:(e.add(n),n instanceof ArrayBuffer||n instanceof MessagePort||typeof ImageBitmap<"u"&&n instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&n instanceof OffscreenCanvas?[n]:ArrayBuffer.isView(n)?[n.buffer]:Array.isArray(n)?n.flatMap(r=>m(r,e)):Object.values(n).flatMap(r=>m(r,e)))}class L{constructor(e){y(this,"_workers");y(this,"_threads");y(this,"_persistentWorkers",new Map);y(this,"_activeWorkers",new Set);y(this,"_isTerminated",!1);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}get isTerminated(){return this._isTerminated}trackWorker(e){if(this._isTerminated)throw e.terminate(),new Error("MainWorkerFactory has been terminated");return this._activeWorkers.add(e),e}terminateWorker(e){this._activeWorkers.delete(e);try{e.terminate()}catch{}}initWorker(e){const r=new v(e.func,{createWorker:e.createWorker});return this.trackWorker(r.getWorker),r}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const s=Math.min(r,e.length),t=Math.floor(e.length/s),l=e.length%s,c=[];let f=0;for(let a=0;a<s;a++){const o=t+(a<l?1:0);c.push(e.slice(f,f+o)),f+=o}return c}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,{srcData:r,...s}){if(this._isTerminated)return Promise.reject(new Error("MainWorkerFactory has been terminated"));const t=this.findWorkerByName(e);if(!t)return Promise.reject(new Error(`Worker "${e}" not found`));const l=t.maxConcurrency??this._threads,c=!!(Array.isArray(r)&&r.length>1&&t.partition),f=c?this.partitionArray(r,l):r,a=this.createWorkerPromises(t,e,{data:f,...s},l,c),o=await Promise.allSettled(a);return new j(o)}createWorkerPromises(e,r,s,t,l){const{data:c,...f}=s;return Array.from({length:t},(a,o)=>{const d=l&&Array.isArray(c)?c[o]:c;return this.runWorkerWithRetry({workerFunc:e.func,createWorker:e.createWorker,workerName:r,index:o,data:{data:d,...f}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(s){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,s),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",s),s}}initiateWorker({workerFunc:e,createWorker:r,workerName:s,index:t,data:l}){return new Promise((c,f)=>{const o=this.initWorker({name:s,role:"",func:e,createWorker:r}).getWorker;o.onerror=p=>{this.terminateWorker(o),f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},failedResult:p})},o.onmessage=p=>{var i,u;if(((i=p.data)==null?void 0:i.ok)===!1){this.terminateWorker(o),f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},failedResult:new ErrorEvent("error",{message:p.data.error})});return}c({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:s,index:t,data:l},successResult:new MessageEvent("message",{data:(u=p.data)==null?void 0:u.data})}),this.terminateWorker(o)};const d={index:t,...Array.isArray(l)?{data:l}:l};o.postMessage(d,m(d))})}async collectResults(e,r={}){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const s=e.results.filter(a=>a.status==="fulfilled"),t=e.results.filter(a=>a.status==="rejected"),l=s.map(a=>a.value.successResult.data),c=r.reducer?r.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((a,o)=>{const d=`
-        const reducer = ${c};
-        self.addEventListener('message', (event) => {
-          try {
-            const result = reducer(event.data);
-            self.postMessage({ ok: true, data: result });
-          } catch (err) {
-            self.postMessage({ ok: false, error: String(err) });
-          }
-        });
-      `,p=new Blob([d],{type:"application/javascript"}),i=this.trackWorker(new Worker(URL.createObjectURL(p)));i.onmessage=u=>{this.terminateWorker(i),u.data.ok?a(u.data.data):o(new Error(u.data.error))},i.onerror=u=>{this.terminateWorker(i),o(u)},i.postMessage(l)}),succeeded:s.length,failed:t.length,errors:t}}async pipeline(e){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const r=e[0],{worker:s,srcData:t,...l}=r,c=this.findWorkerByName(r.worker);if(!c)throw new Error(`Worker "${r.worker}" not found`);const a=this.initWorker(c).getWorker;return new Promise((o,d)=>{a.onmessage=u=>{var k,g;this.terminateWorker(a),((k=u.data)==null?void 0:k.ok)===!1?d(new Error(u.data.error)):o((g=u.data)==null?void 0:g.data)},a.onerror=u=>{this.terminateWorker(a),d(u)};const i={data:t??{},...l,index:0};a.postMessage(i,m(i))})}return new Promise((r,s)=>{const t=[],l=[];for(const i of e){const u=this.findWorkerByName(i.worker);if(!u){s(new Error(`Worker "${i.worker}" not found`));return}const k=new v(u.func,{mode:P.Pipeline,createWorker:u.createWorker}),g=this.trackWorker(k.getWorker);t.push(g)}for(let i=0;i<t.length-1;i++)l.push(new MessageChannel);for(let i=0;i<t.length;i++){const{worker:u,srcData:k,...g}=e[i],W=[],w={};if(i>0&&(w.inputPort=l[i-1].port1,W.push(w.inputPort)),i<t.length-1&&(w.outputPort=l[i].port2,W.push(w.outputPort)),t[i].postMessage({__pipeline_ports__:!0,stepParams:g,...w},W),i<t.length-1){const _=t[i],A=t[i+1],{worker:I,srcData:U,...D}=e[i+1];_.onmessage=h=>{var E,T;if(h.data&&h.data.__pipeline_ports__)return;if(((E=h.data)==null?void 0:E.ok)===!1){t.forEach(B=>this.terminateWorker(B)),s(new Error(h.data.error));return}const b={data:((T=h.data)==null?void 0:T.ok)!==void 0?h.data.data:h.data,...D,index:0};A.postMessage(b,m(b))},_.onerror=h=>{t.forEach(M=>this.terminateWorker(M)),s(h)}}}const c=t[t.length-1];c.onmessage=i=>{var u,k;t.forEach(g=>this.terminateWorker(g)),((u=i.data)==null?void 0:u.ok)===!1?s(new Error(i.data.error)):r((k=i.data)==null?void 0:k.data)},c.onerror=i=>{t.forEach(u=>this.terminateWorker(u)),s(i)};const{worker:f,srcData:a,...o}=e[0],p={data:a??{},...o,index:0};t[0].postMessage(p,m(p))})}async runPersistent(e,r){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const s=this.findWorkerByName(e);if(!s)throw new Error(`Worker "${e}" not found`);let t=this._persistentWorkers.get(e);if(!t){const l=new v(s.func,{mode:P.Persistent,createWorker:s.createWorker});t=this.trackWorker(l.getWorker),this._persistentWorkers.set(e,t)}return new Promise((l,c)=>{t.onmessage=a=>{var o,d;((o=a.data)==null?void 0:o.ok)===!1?c(new Error(a.data.error)):l((d=a.data)==null?void 0:d.data)},t.onerror=a=>{c(a)};const f={type:"run",config:r.config};r.dataset!==void 0&&(f.dataset=r.dataset),t.postMessage(f,m(f))})}release(e){const r=this._persistentWorkers.get(e);if(r){try{r.postMessage({type:"release"})}catch{}this.terminateWorker(r),this._persistentWorkers.delete(e)}}terminate(){this._isTerminated=!0;for(const e of this._persistentWorkers.values()){try{e.postMessage({type:"release"})}catch{}this.terminateWorker(e)}this._persistentWorkers.clear();for(const e of Array.from(this._activeWorkers))this.terminateWorker(e);this._activeWorkers.clear()}destroy(){this.terminate()}reset(){this.terminate(),this._isTerminated=!1}restart(){this.reset()}}function $(n){if(typeof self>"u")return;let e=null,r=null,s=null,t={};const l=(f,a)=>{self.postMessage(f,a)};async function c(f){try{const a=typeof f=="object"&&f!==null&&"data"in f?{...t,...f}:{data:f,...t,index:0},o=await n(a),d={ok:!0,data:o},p=m(o);e?e.postMessage(d,p):l(d,p)}catch(a){const o={ok:!1,error:a instanceof Error?a.message:String(a)};e?e.postMessage(o):l(o)}}self.addEventListener("message",f=>{const a=f.data;if(a&&a.__pipeline_ports__){a.stepParams&&(t=a.stepParams),a.outputPort&&(e=a.outputPort),a.inputPort&&(r=a.inputPort,r.onmessage=o=>{var d;o.data&&o.data.ok===!1?e?e.postMessage(o.data):l(o.data):c({data:(d=o.data)==null?void 0:d.data,...t,index:0})}),s!==null&&(c(s),s=null);return}r?s=a:c(a)})}exports.MainWorkerFactory=L;exports.WorkerFactory=v;exports.defineWorker=$;
+`;var P=(l=>(l.Default="default",l.Pipeline="pipeline",l.Persistent="persistent",l))(P||{});const C=Object.freeze({persistent:L,pipeline:F,default:j});class v{constructor(e,r){_(this,"_worker");if(r!=null&&r.createWorker)this._worker=r.createWorker();else if(e){const a=(r==null?void 0:r.mode)??"default",t=C[a](e.toString()),i=new Blob([t],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(i))}else throw new Error("Either workerFunction or options.createWorker must be provided to WorkerFactory.")}get getWorker(){return this._worker}}class I{constructor(e){this.results=e}}class ${constructor(){_(this,"store",new Map)}set(e,r){const a=r??`mem_${crypto.randomUUID()}`;return this.store.set(a,e),a}get(e){return this.store.get(e)}delete(e){return this.store.delete(e)}clear(){this.store.clear()}has(e){return this.store.has(e)}stats(){return{count:this.store.size,refs:Array.from(this.store.keys())}}}function w(l,e=new Set){return l===null||typeof l!="object"?[]:e.has(l)?[]:(e.add(l),l instanceof ArrayBuffer||l instanceof MessagePort||typeof ImageBitmap<"u"&&l instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&l instanceof OffscreenCanvas?[l]:ArrayBuffer.isView(l)?[l.buffer]:Array.isArray(l)?l.flatMap(r=>w(r,e)):Object.values(l).flatMap(r=>w(r,e)))}class U{constructor(e){_(this,"_workers");_(this,"_threads");_(this,"_persistentWorkers",new Map);_(this,"_activeWorkers",new Set);_(this,"_memoryStore",new $);_(this,"_isTerminated",!1);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}get isTerminated(){return this._isTerminated}trackWorker(e){if(this._isTerminated)throw e.terminate(),new Error("MainWorkerFactory has been terminated");return this._activeWorkers.add(e),e}terminateWorker(e){this._activeWorkers.delete(e);try{e.terminate()}catch{}}initWorker(e){const r=new v(e.func,{createWorker:e.createWorker});return this.trackWorker(r.getWorker),r}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const a=Math.min(r,e.length),t=Math.floor(e.length/a),i=e.length%a,f=[];let d=0;for(let s=0;s<a;s++){const o=t+(s<i?1:0);f.push(e.slice(d,d+o)),d+=o}return f}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,r){if(this._isTerminated)return Promise.reject(new Error("MainWorkerFactory has been terminated"));const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));let{srcData:t,...i}=r||{};const f=i.__memory_ref__,d=!!i.deleteMemory;if(t===void 0&&f){if(!this._memoryStore.has(f))return Promise.reject(new Error(`Memory reference "${f}" not found in MemoryStore`));t=this._memoryStore.get(f),delete i.__memory_ref__,delete i.deleteMemory}else delete i.deleteMemory;const s=a.maxConcurrency??this._threads,o=!!(Array.isArray(t)&&a.partition),u=o?this.partitionArray(t,s):t,p=o&&Array.isArray(u)?u.length:s,m=this.createWorkerPromises(a,e,{data:u,...i},p,o),c=await Promise.allSettled(m);return d&&f&&this._memoryStore.delete(f),(a.memoryOnly||a.memory)&&this.storeWorkerMemoryResult(c,{memoryOnly:!!a.memoryOnly,shouldPartition:o}),new I(c)}async deleteMemory(e){return this._memoryStore.delete(e)}async clearMemory(){this._memoryStore.clear()}storeWorkerMemoryResult(e,{memoryOnly:r,shouldPartition:a}){const t=e.filter(s=>s.status==="fulfilled"&&!!s.value.successResult);if(t.length===0)return;const i=t.map(s=>s.value.successResult.data),f=a&&i.every(Array.isArray)?i.flat():i.length===1?i[0]:i,d=this._memoryStore.set(f);for(const s of t){const o=r?{__memory_ref__:d}:{data:s.value.successResult.data,__memory_ref__:d};s.value.successResult=new MessageEvent("message",{data:o})}}async getMemoryStats(){return this._memoryStore.stats()}createWorkerPromises(e,r,a,t,i){const{data:f,...d}=a;return Array.from({length:t},(s,o)=>{const u=i&&Array.isArray(f)?f[o]:f;return this.runWorkerWithRetry({workerFunc:e.func,createWorker:e.createWorker,workerName:r,index:o,data:{data:u,...d}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(a){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,a),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",a),a}}initiateWorker({workerFunc:e,createWorker:r,workerName:a,index:t,data:i}){return new Promise((f,d)=>{const o=this.initWorker({name:a,role:"",func:e,createWorker:r}).getWorker;o.onerror=p=>{this.terminateWorker(o),d({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},failedResult:p})},o.onmessage=p=>{var c,n;if(((c=p.data)==null?void 0:c.ok)===!1){this.terminateWorker(o),d({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},failedResult:new ErrorEvent("error",{message:p.data.error})});return}const m=((n=p.data)==null?void 0:n.ok)!==void 0?p.data.data:p.data;f({index:t,workerConfigs:{workerFunc:e,createWorker:r,workerName:a,index:t,data:i},successResult:new MessageEvent("message",{data:m})}),this.terminateWorker(o)};const u={index:t,...Array.isArray(i)?{data:i}:i};o.postMessage(u,w(u))})}async collectResults(e,r={}){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const a=e.results.filter(m=>m.status==="fulfilled"),t=e.results.filter(m=>m.status==="rejected");if(a.length>0&&a.every(m=>{var n;const c=(n=m.value.successResult)==null?void 0:n.data;return c&&typeof c=="object"&&"__memory_ref__"in c&&!("data"in c)}))return{data:a[0].value.successResult.data,succeeded:a.length,failed:t.length,errors:t};const f=a.length>0&&a.every(m=>{var n;const c=(n=m.value.successResult)==null?void 0:n.data;return c&&typeof c=="object"&&"__memory_ref__"in c&&"data"in c}),d=f?a[0].value.successResult.data.__memory_ref__:void 0,s=a.map(m=>{const c=m.value.successResult.data;return f&&c&&typeof c=="object"&&"data"in c?c.data:c}),o=r.reducer?r.reducer.toString():"(shards) => shards.flat()";let u;if(typeof Worker<"u"&&typeof Blob<"u"&&typeof URL<"u"&&typeof URL.createObjectURL=="function")try{u=await new Promise((m,c)=>{const n=`
+            const reducer = ${o};
+            self.addEventListener('message', (event) => {
+              try {
+                const result = reducer(event.data);
+                self.postMessage({ ok: true, data: result });
+              } catch (error) {
+                self.postMessage({ ok: false, error: String(error) });
+              }
+            });
+          `,h=new Blob([n],{type:"application/javascript"}),y=this.trackWorker(new Worker(URL.createObjectURL(h)));y.onmessage=k=>{this.terminateWorker(y),k.data.ok?m(k.data.data):c(new Error(k.data.error))},y.onerror=k=>{this.terminateWorker(y),c(k)},y.postMessage(s)})}catch{const m=n=>n.flat();u=(r.reducer??m)(s)}else{const m=n=>n.flat();u=(r.reducer??m)(s)}return{data:f?{data:u,__memory_ref__:d}:u,succeeded:a.length,failed:t.length,errors:t}}async pipeline(e){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const r=e[0],{worker:a,srcData:t,...i}=r,f=this.findWorkerByName(r.worker);if(!f)throw new Error(`Worker "${r.worker}" not found`);const s=this.initWorker(f).getWorker;return new Promise((o,u)=>{s.onmessage=c=>{var n,h;this.terminateWorker(s),((n=c.data)==null?void 0:n.ok)===!1?u(new Error(c.data.error)):o((h=c.data)==null?void 0:h.data)},s.onerror=c=>{this.terminateWorker(s),u(c)};const m={data:t??{},...i,index:0};s.postMessage(m,w(m))})}return new Promise((r,a)=>{const t=[],i=[];for(const n of e){const h=this.findWorkerByName(n.worker);if(!h){a(new Error(`Worker "${n.worker}" not found`));return}const y=new v(h.func,{mode:P.Pipeline,createWorker:h.createWorker}),k=this.trackWorker(y.getWorker);t.push(k)}for(let n=0;n<t.length-1;n++)i.push(new MessageChannel);for(let n=0;n<t.length;n++){const{worker:h,srcData:y,...k}=e[n],M=[],W={};if(n>0&&(W.inputPort=i[n-1].port1,M.push(W.inputPort)),n<t.length-1&&(W.outputPort=i[n].port2,M.push(W.outputPort)),t[n].postMessage({__pipeline_ports__:!0,stepParams:k,...W},M),n<t.length-1){const b=t[n],T=t[n+1],{worker:V,srcData:q,...A}=e[n+1];b.onmessage=g=>{var S,R;if(g.data&&g.data.__pipeline_ports__)return;if(((S=g.data)==null?void 0:S.ok)===!1){t.forEach(B=>this.terminateWorker(B)),a(new Error(g.data.error));return}const E={data:((R=g.data)==null?void 0:R.ok)!==void 0?g.data.data:g.data,...A,index:0};T.postMessage(E,w(E))},b.onerror=g=>{t.forEach(D=>this.terminateWorker(D)),a(g)}}}const f=t[t.length-1];f.onmessage=n=>{var h,y;t.forEach(k=>this.terminateWorker(k)),((h=n.data)==null?void 0:h.ok)===!1?a(new Error(n.data.error)):r((y=n.data)==null?void 0:y.data)},f.onerror=n=>{t.forEach(h=>this.terminateWorker(h)),a(n)};const{worker:d,srcData:s,...o}=e[0];let u=s;const p=o.__memory_ref__,m=!!o.deleteMemory;if(u===void 0&&p){if(!this._memoryStore.has(p)){t.forEach(n=>this.terminateWorker(n)),a(new Error(`Memory reference "${p}" not found in MemoryStore`));return}u=this._memoryStore.get(p),m&&this._memoryStore.delete(p),delete o.__memory_ref__,delete o.deleteMemory}u===void 0&&(u={});const c={data:u,...o,index:0};t[0].postMessage(c,w(c))})}async runPersistent(e,r){if(this._isTerminated)throw new Error("MainWorkerFactory has been terminated");const a=this.findWorkerByName(e);if(!a)throw new Error(`Worker "${e}" not found`);let t=this._persistentWorkers.get(e);if(!t){const i=new v(a.func,{mode:P.Persistent,createWorker:a.createWorker});t=this.trackWorker(i.getWorker),this._persistentWorkers.set(e,t)}return new Promise((i,f)=>{t.onmessage=s=>{var o,u;((o=s.data)==null?void 0:o.ok)===!1?f(new Error(s.data.error)):i((u=s.data)==null?void 0:u.data)},t.onerror=s=>{f(s)};const d={type:"run",config:r.config};r.dataset!==void 0&&(d.dataset=r.dataset),t.postMessage(d,w(d))})}release(e){const r=this._persistentWorkers.get(e);if(r){try{r.postMessage({type:"release"})}catch{}this.terminateWorker(r),this._persistentWorkers.delete(e)}}terminate(){this._isTerminated=!0;for(const e of this._persistentWorkers.values()){try{e.postMessage({type:"release"})}catch{}this.terminateWorker(e)}this._persistentWorkers.clear();for(const e of Array.from(this._activeWorkers))this.terminateWorker(e);this._activeWorkers.clear(),this._memoryStore.clear()}destroy(){this.terminate()}reset(){this.terminate(),this._isTerminated=!1}restart(){this.reset()}}function z(l){if(typeof self>"u")return;let e=null,r=null,a=null,t={};const i=(d,s)=>{self.postMessage(d,s)};async function f(d){try{const s=typeof d=="object"&&d!==null&&"data"in d?{...t,...d}:{data:d,...t,index:0},o=await l(s),u={ok:!0,data:o},p=w(o);e?e.postMessage(u,p):i(u,p)}catch(s){const o={ok:!1,error:s instanceof Error?s.message:String(s)};e?e.postMessage(o):i(o)}}self.addEventListener("message",d=>{const s=d.data;if(s&&s.__pipeline_ports__){s.stepParams&&(t=s.stepParams),s.outputPort&&(e=s.outputPort),s.inputPort&&(r=s.inputPort,r.onmessage=o=>{var u;o.data&&o.data.ok===!1?e?e.postMessage(o.data):i(o.data):f({data:(u=o.data)==null?void 0:u.data,...t,index:0})}),a!==null&&(f(a),a=null);return}r?a=s:f(s)})}exports.MainWorkerFactory=U;exports.WorkerFactory=v;exports.defineWorker=z;

package/dist/index.js

@@ -1,7 +1,7 @@
-var S = Object.defineProperty;
-var x = (n, e, r) => e in n ? S(n, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : n[e] = r;
-var y = (n, e, r) => x(n, typeof e != "symbol" ? e + "" : e, r);
-const O = (n) => `
+var x = Object.defineProperty;
+var O = (l, e, r) => e in l ? x(l, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : l[e] = r;
+var _ = (l, e, r) => O(l, typeof e != "symbol" ? e + "" : e, r);
+const j = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -18,13 +18,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${n}(event.data);
+    const output = await ${l}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`, R = (n) => `
+`, L = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -39,7 +39,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
@@ -104,7 +104,7 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
-`, F = (n) => `
+`, C = (l) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -119,7 +119,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${n};
+const workerFn = ${l};
 let cachedDataset = null;
 
 self.addEventListener('message', async (event) => {
@@ -161,13 +161,13 @@ self.addEventListener('message', async (event) => {
   }
 });
 `;
-var P = /* @__PURE__ */ ((n) => (n.Default = "default", n.Pipeline = "pipeline", n.Persistent = "persistent", n))(P || {});
-const C = Object.freeze({
-  persistent: F,
-  pipeline: R,
-  default: O
+var P = /* @__PURE__ */ ((l) => (l.Default = "default", l.Pipeline = "pipeline", l.Persistent = "persistent", l))(P || {});
+const F = Object.freeze({
+  persistent: C,
+  pipeline: L,
+  default: j
 });
-class W {
+class M {
   /**
    * Creates a new `Worker` from the given function or factory option.
    *
@@ -177,14 +177,14 @@ class W {
    * @param options - Optional configuration containing `createWorker` or `mode`.
    */
   constructor(e, r) {
-    y(this, "_worker");
+    _(this, "_worker");
     if (r != null && r.createWorker)
       this._worker = r.createWorker();
     else if (e) {
-      const s = (r == null ? void 0 : r.mode) ?? "default", t = C[s](e.toString()), l = new Blob([t], {
+      const a = (r == null ? void 0 : r.mode) ?? "default", t = F[a](e.toString()), i = new Blob([t], {
         type: "application/javascript"
       });
-      this._worker = new Worker(URL.createObjectURL(l));
+      this._worker = new Worker(URL.createObjectURL(i));
     } else
       throw new Error(
         "Either workerFunction or options.createWorker must be provided to WorkerFactory."
@@ -200,28 +200,84 @@ class W {
     return this._worker;
   }
 }
-class L {
+class I {
   constructor(e) {
     this.results = e;
   }
 }
-function m(n, e = /* @__PURE__ */ new Set()) {
-  return n === null || typeof n != "object" ? [] : e.has(n) ? [] : (e.add(n), n instanceof ArrayBuffer || n instanceof MessagePort || typeof ImageBitmap < "u" && n instanceof ImageBitmap || typeof OffscreenCanvas < "u" && n instanceof OffscreenCanvas ? [n] : ArrayBuffer.isView(n) ? [n.buffer] : Array.isArray(n) ? n.flatMap((r) => m(r, e)) : Object.values(n).flatMap(
-    (r) => m(r, e)
+class $ {
+  constructor() {
+    _(this, "store", /* @__PURE__ */ new Map());
+  }
+  /**
+   * Stores a dataset in RAM under an unguessable reference ID.
+   *
+   * @param data - The dataset to store.
+   * @param refId - Optional reference ID; if omitted, a UUID will be generated.
+   * @returns The reference ID under which the dataset is stored.
+   */
+  set(e, r) {
+    const a = r ?? `mem_${crypto.randomUUID()}`;
+    return this.store.set(a, e), a;
+  }
+  /**
+   * Retrieves a dataset from RAM by its reference ID.
+   *
+   * @param refId - The reference ID to retrieve.
+   * @returns The stored dataset, or undefined if not found.
+   */
+  get(e) {
+    return this.store.get(e);
+  }
+  /**
+   * Deletes a dataset reference from RAM.
+   *
+   * @param refId - The reference ID to delete.
+   * @returns `true` if the key existed and was removed, `false` otherwise.
+   */
+  delete(e) {
+    return this.store.delete(e);
+  }
+  /**
+   * Clears all dataset handles from RAM.
+   */
+  clear() {
+    this.store.clear();
+  }
+  /**
+   * Checks if a reference ID exists in RAM.
+   */
+  has(e) {
+    return this.store.has(e);
+  }
+  /**
+   * Returns statistics about current memory handles.
+   */
+  stats() {
+    return {
+      count: this.store.size,
+      refs: Array.from(this.store.keys())
+    };
+  }
+}
+function w(l, e = /* @__PURE__ */ new Set()) {
+  return l === null || typeof l != "object" ? [] : e.has(l) ? [] : (e.add(l), l instanceof ArrayBuffer || l instanceof MessagePort || typeof ImageBitmap < "u" && l instanceof ImageBitmap || typeof OffscreenCanvas < "u" && l instanceof OffscreenCanvas ? [l] : ArrayBuffer.isView(l) ? [l.buffer] : Array.isArray(l) ? l.flatMap((r) => w(r, e)) : Object.values(l).flatMap(
+    (r) => w(r, e)
   ));
 }
-class U {
+class q {
   /**
    * Creates a new `MainWorkerFactory`.
    *
    * @param options - Configuration object containing the `workers` registry.
    */
   constructor(e) {
-    y(this, "_workers");
-    y(this, "_threads");
-    y(this, "_persistentWorkers", /* @__PURE__ */ new Map());
-    y(this, "_activeWorkers", /* @__PURE__ */ new Set());
-    y(this, "_isTerminated", !1);
+    _(this, "_workers");
+    _(this, "_threads");
+    _(this, "_persistentWorkers", /* @__PURE__ */ new Map());
+    _(this, "_activeWorkers", /* @__PURE__ */ new Set());
+    _(this, "_memoryStore", new $());
+    _(this, "_isTerminated", !1);
     this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
   }
   /**
@@ -255,7 +311,7 @@ class U {
    * @returns A new `WorkerFactory` wrapping the worker.
    */
   initWorker(e) {
-    const r = new W(e.func, {
+    const r = new M(e.func, {
       createWorker: e.createWorker
     });
     return this.trackWorker(r.getWorker), r;
@@ -279,13 +335,13 @@ class U {
   partitionArray(e, r) {
     if (!e.length) return [];
     if (r <= 0) throw new Error("numChunks must be positive");
-    const s = Math.min(r, e.length), t = Math.floor(e.length / s), l = e.length % s, c = [];
-    let f = 0;
-    for (let a = 0; a < s; a++) {
-      const o = t + (a < l ? 1 : 0);
-      c.push(e.slice(f, f + o)), f += o;
+    const a = Math.min(r, e.length), t = Math.floor(e.length / a), i = e.length % a, f = [];
+    let d = 0;
+    for (let s = 0; s < a; s++) {
+      const o = t + (s < i ? 1 : 0);
+      f.push(e.slice(d, d + o)), d += o;
     }
-    return c;
+    return f;
   }
   /**
    * Looks up a registered worker configuration by name.
@@ -322,23 +378,76 @@ class U {
    * @example
    * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
    */
-  async runWorker(e, {
-    srcData: r,
-    ...s
-  }) {
+  async runWorker(e, r) {
     if (this._isTerminated)
       return Promise.reject(new Error("MainWorkerFactory has been terminated"));
-    const t = this.findWorkerByName(e);
-    if (!t)
+    const a = this.findWorkerByName(e);
+    if (!a)
       return Promise.reject(new Error(`Worker "${e}" not found`));
-    const l = t.maxConcurrency ?? this._threads, c = !!(Array.isArray(r) && r.length > 1 && t.partition), f = c ? this.partitionArray(r, l) : r, a = this.createWorkerPromises(
-      t,
+    let { srcData: t, ...i } = r || {};
+    const f = i.__memory_ref__, d = !!i.deleteMemory;
+    if (t === void 0 && f) {
+      if (!this._memoryStore.has(f))
+        return Promise.reject(
+          new Error(`Memory reference "${f}" not found in MemoryStore`)
+        );
+      t = this._memoryStore.get(f), delete i.__memory_ref__, delete i.deleteMemory;
+    } else
+      delete i.deleteMemory;
+    const s = a.maxConcurrency ?? this._threads, o = !!(Array.isArray(t) && a.partition), u = o ? this.partitionArray(t, s) : t, m = o && Array.isArray(u) ? u.length : s, p = this.createWorkerPromises(
+      a,
       e,
-      { data: f, ...s },
-      l,
-      c
-    ), o = await Promise.allSettled(a);
-    return new L(o);
+      { data: u, ...i },
+      m,
+      o
+    ), c = await Promise.allSettled(p);
+    return d && f && this._memoryStore.delete(f), (a.memoryOnly || a.memory) && this.storeWorkerMemoryResult(c, {
+      memoryOnly: !!a.memoryOnly,
+      shouldPartition: o
+    }), new I(c);
+  }
+  /**
+   * Deletes a specific memory reference from the factory's memory store.
+   *
+   * @param ref - The `__memory_ref__` token string to delete.
+   * @returns A promise that resolves to `true` if deleted, `false` otherwise.
+   */
+  async deleteMemory(e) {
+    return this._memoryStore.delete(e);
+  }
+  /**
+   * Clears all stored dataset references from the factory's memory store.
+   */
+  async clearMemory() {
+    this._memoryStore.clear();
+  }
+  /**
+   * Saves the output of a completed worker execution into MemoryStore
+   * and modifies the returned settled results with a `__memory_ref__` token.
+   */
+  storeWorkerMemoryResult(e, {
+    memoryOnly: r,
+    shouldPartition: a
+  }) {
+    const t = e.filter(
+      (s) => s.status === "fulfilled" && !!s.value.successResult
+    );
+    if (t.length === 0) return;
+    const i = t.map(
+      (s) => s.value.successResult.data
+    ), f = a && i.every(Array.isArray) ? i.flat() : i.length === 1 ? i[0] : i, d = this._memoryStore.set(f);
+    for (const s of t) {
+      const o = r ? { __memory_ref__: d } : { data: s.value.successResult.data, __memory_ref__: d };
+      s.value.successResult = new MessageEvent("message", {
+        data: o
+      });
+    }
+  }
+  /**
+   * Returns statistics about active memory references in the factory.
+   */
+  async getMemoryStats() {
+    return this._memoryStore.stats();
   }
   /**
    * Builds the array of per-thread worker promises for a single `runWorker`
@@ -354,17 +463,17 @@ class U {
    * @param isPartitioned  - Whether `data` is a pre-split array of shards.
    * @returns An array of promises, one per thread.
    */
-  createWorkerPromises(e, r, s, t, l) {
-    const { data: c, ...f } = s;
-    return Array.from({ length: t }, (a, o) => {
-      const d = l && Array.isArray(c) ? c[o] : c;
+  createWorkerPromises(e, r, a, t, i) {
+    const { data: f, ...d } = a;
+    return Array.from({ length: t }, (s, o) => {
+      const u = i && Array.isArray(f) ? f[o] : f;
       return this.runWorkerWithRetry(
         {
           workerFunc: e.func,
           createWorker: e.createWorker,
           workerName: r,
           index: o,
-          data: { data: d, ...f }
+          data: { data: u, ...d }
         },
         e.retries
       );
@@ -385,13 +494,13 @@ class U {
   async runWorkerWithRetry(e, r = 2) {
     try {
       return await this.initiateWorker(e);
-    } catch (s) {
+    } catch (a) {
       if (r > 0)
         return console.error(
           `Worker ${e.index} failed, retrying (${r} left):`,
-          s
+          a
         ), this.runWorkerWithRetry(e, r - 1);
-      throw console.error("Worker failed after all retries:", s), s;
+      throw console.error("Worker failed after all retries:", a), a;
     }
   }
   /**
@@ -414,66 +523,67 @@ class U {
   initiateWorker({
     workerFunc: e,
     createWorker: r,
-    workerName: s,
+    workerName: a,
     index: t,
-    data: l
+    data: i
   }) {
-    return new Promise((c, f) => {
+    return new Promise((f, d) => {
       const o = this.initWorker({
-        name: s,
+        name: a,
         role: "",
         func: e,
         createWorker: r
       }).getWorker;
-      o.onerror = (p) => {
-        this.terminateWorker(o), f({
+      o.onerror = (m) => {
+        this.terminateWorker(o), d({
           index: t,
           workerConfigs: {
             workerFunc: e,
             createWorker: r,
-            workerName: s,
+            workerName: a,
             index: t,
-            data: l
+            data: i
           },
-          failedResult: p
+          failedResult: m
         });
-      }, o.onmessage = (p) => {
-        var i, u;
-        if (((i = p.data) == null ? void 0 : i.ok) === !1) {
-          this.terminateWorker(o), f({
+      }, o.onmessage = (m) => {
+        var c, n;
+        if (((c = m.data) == null ? void 0 : c.ok) === !1) {
+          this.terminateWorker(o), d({
             index: t,
             workerConfigs: {
               workerFunc: e,
               createWorker: r,
-              workerName: s,
+              workerName: a,
               index: t,
-              data: l
+              data: i
             },
             failedResult: new ErrorEvent("error", {
-              message: p.data.error
+              message: m.data.error
             })
           });
           return;
         }
-        c({
+        const p = ((n = m.data) == null ? void 0 : n.ok) !== void 0 ? m.data.data : m.data;
+        f({
           index: t,
           workerConfigs: {
             workerFunc: e,
             createWorker: r,
-            workerName: s,
+            workerName: a,
             index: t,
-            data: l
+            data: i
           },
           successResult: new MessageEvent("message", {
-            data: (u = p.data) == null ? void 0 : u.data
+            data: p
           })
         }), this.terminateWorker(o);
       };
-      const d = {
+      const u = {
         index: t,
-        ...Array.isArray(l) ? { data: l } : l
+        ...Array.isArray(i) ? { data: i } : i
       };
-      o.postMessage(d, m(d));
+      o.postMessage(u, w(u));
     });
   }
   /**
@@ -511,31 +621,66 @@ class U {
   async collectResults(e, r = {}) {
     if (this._isTerminated)
       throw new Error("MainWorkerFactory has been terminated");
-    const s = e.results.filter(
-      (a) => a.status === "fulfilled"
+    const a = e.results.filter(
+      (p) => p.status === "fulfilled"
     ), t = e.results.filter(
-      (a) => a.status === "rejected"
-    ), l = s.map((a) => a.value.successResult.data), c = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
-    return {
-      data: await new Promise((a, o) => {
-        const d = `
-        const reducer = ${c};
-        self.addEventListener('message', (event) => {
-          try {
-            const result = reducer(event.data);
-            self.postMessage({ ok: true, data: result });
-          } catch (err) {
-            self.postMessage({ ok: false, error: String(err) });
-          }
+      (p) => p.status === "rejected"
+    );
+    if (a.length > 0 && a.every((p) => {
+      var n;
+      const c = (n = p.value.successResult) == null ? void 0 : n.data;
+      return c && typeof c == "object" && "__memory_ref__" in c && !("data" in c);
+    }))
+      return {
+        data: a[0].value.successResult.data,
+        succeeded: a.length,
+        failed: t.length,
+        errors: t
+      };
+    const f = a.length > 0 && a.every((p) => {
+      var n;
+      const c = (n = p.value.successResult) == null ? void 0 : n.data;
+      return c && typeof c == "object" && "__memory_ref__" in c && "data" in c;
+    }), d = f ? a[0].value.successResult.data.__memory_ref__ : void 0, s = a.map((p) => {
+      const c = p.value.successResult.data;
+      return f && c && typeof c == "object" && "data" in c ? c.data : c;
+    }), o = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
+    let u;
+    if (typeof Worker < "u" && typeof Blob < "u" && typeof URL < "u" && typeof URL.createObjectURL == "function")
+      try {
+        u = await new Promise((p, c) => {
+          const n = `
+            const reducer = ${o};
+            self.addEventListener('message', (event) => {
+              try {
+                const result = reducer(event.data);
+                self.postMessage({ ok: true, data: result });
+              } catch (error) {
+                self.postMessage({ ok: false, error: String(error) });
+              }
+            });
+          `, h = new Blob([n], {
+            type: "application/javascript"
+          }), y = this.trackWorker(
+            new Worker(URL.createObjectURL(h))
+          );
+          y.onmessage = (k) => {
+            this.terminateWorker(y), k.data.ok ? p(k.data.data) : c(new Error(k.data.error));
+          }, y.onerror = (k) => {
+            this.terminateWorker(y), c(k);
+          }, y.postMessage(s);
         });
-      `, p = new Blob([d], { type: "application/javascript" }), i = this.trackWorker(new Worker(URL.createObjectURL(p)));
-        i.onmessage = (u) => {
-          this.terminateWorker(i), u.data.ok ? a(u.data.data) : o(new Error(u.data.error));
-        }, i.onerror = (u) => {
-          this.terminateWorker(i), o(u);
-        }, i.postMessage(l);
-      }),
-      succeeded: s.length,
+      } catch {
+        const p = (n) => n.flat();
+        u = (r.reducer ?? p)(s);
+      }
+    else {
+      const p = (n) => n.flat();
+      u = (r.reducer ?? p)(s);
+    }
+    return {
+      data: f ? { data: u, __memory_ref__: d } : u,
+      succeeded: a.length,
       failed: t.length,
       errors: t
     };
@@ -572,66 +717,97 @@ class U {
     if (e.length === 0)
       throw new Error("Pipeline requires at least one step");
     if (e.length === 1) {
-      const r = e[0], { worker: s, srcData: t, ...l } = r, c = this.findWorkerByName(r.worker);
-      if (!c) throw new Error(`Worker "${r.worker}" not found`);
-      const a = this.initWorker(c).getWorker;
-      return new Promise((o, d) => {
-        a.onmessage = (u) => {
-          var k, g;
-          this.terminateWorker(a), ((k = u.data) == null ? void 0 : k.ok) === !1 ? d(new Error(u.data.error)) : o((g = u.data) == null ? void 0 : g.data);
-        }, a.onerror = (u) => {
-          this.terminateWorker(a), d(u);
+      const r = e[0], { worker: a, srcData: t, ...i } = r, f = this.findWorkerByName(r.worker);
+      if (!f) throw new Error(`Worker "${r.worker}" not found`);
+      const s = this.initWorker(f).getWorker;
+      return new Promise((o, u) => {
+        s.onmessage = (c) => {
+          var n, h;
+          this.terminateWorker(s), ((n = c.data) == null ? void 0 : n.ok) === !1 ? u(new Error(c.data.error)) : o((h = c.data) == null ? void 0 : h.data);
+        }, s.onerror = (c) => {
+          this.terminateWorker(s), u(c);
         };
-        const i = { data: t ?? {}, ...l, index: 0 };
-        a.postMessage(i, m(i));
+        const p = { data: t ?? {}, ...i, index: 0 };
+        s.postMessage(p, w(p));
       });
     }
-    return new Promise((r, s) => {
-      const t = [], l = [];
-      for (const i of e) {
-        const u = this.findWorkerByName(i.worker);
-        if (!u) {
-          s(new Error(`Worker "${i.worker}" not found`));
+    return new Promise((r, a) => {
+      const t = [], i = [];
+      for (const n of e) {
+        const h = this.findWorkerByName(n.worker);
+        if (!h) {
+          a(new Error(`Worker "${n.worker}" not found`));
           return;
         }
-        const k = new W(u.func, {
+        const y = new M(h.func, {
           mode: P.Pipeline,
-          createWorker: u.createWorker
-        }), g = this.trackWorker(k.getWorker);
-        t.push(g);
+          createWorker: h.createWorker
+        }), k = this.trackWorker(y.getWorker);
+        t.push(k);
       }
-      for (let i = 0; i < t.length - 1; i++)
-        l.push(new MessageChannel());
-      for (let i = 0; i < t.length; i++) {
-        const { worker: u, srcData: k, ...g } = e[i], v = [], w = {};
-        if (i > 0 && (w.inputPort = l[i - 1].port1, v.push(w.inputPort)), i < t.length - 1 && (w.outputPort = l[i].port2, v.push(w.outputPort)), t[i].postMessage(
-          { __pipeline_ports__: !0, stepParams: g, ...w },
+      for (let n = 0; n < t.length - 1; n++)
+        i.push(new MessageChannel());
+      for (let n = 0; n < t.length; n++) {
+        const {
+          worker: h,
+          srcData: y,
+          ...k
+        } = e[n], v = [], W = {};
+        if (n > 0 && (W.inputPort = i[n - 1].port1, v.push(W.inputPort)), n < t.length - 1 && (W.outputPort = i[n].port2, v.push(W.outputPort)), t[n].postMessage(
+          { __pipeline_ports__: !0, stepParams: k, ...W },
           v
-        ), i < t.length - 1) {
-          const _ = t[i], A = t[i + 1], { worker: j, srcData: $, ...D } = e[i + 1];
-          _.onmessage = (h) => {
-            var E, T;
-            if (h.data && h.data.__pipeline_ports__) return;
-            if (((E = h.data) == null ? void 0 : E.ok) === !1) {
-              t.forEach((B) => this.terminateWorker(B)), s(new Error(h.data.error));
+        ), n < t.length - 1) {
+          const b = t[n], A = t[n + 1], {
+            worker: U,
+            srcData: z,
+            ...T
+          } = e[n + 1];
+          b.onmessage = (g) => {
+            var S, R;
+            if (g.data && g.data.__pipeline_ports__) return;
+            if (((S = g.data) == null ? void 0 : S.ok) === !1) {
+              t.forEach((B) => this.terminateWorker(B)), a(new Error(g.data.error));
               return;
             }
-            const b = { data: ((T = h.data) == null ? void 0 : T.ok) !== void 0 ? h.data.data : h.data, ...D, index: 0 };
-            A.postMessage(b, m(b));
-          }, _.onerror = (h) => {
-            t.forEach((M) => this.terminateWorker(M)), s(h);
+            const E = { data: ((R = g.data) == null ? void 0 : R.ok) !== void 0 ? g.data.data : g.data, ...T, index: 0 };
+            A.postMessage(E, w(E));
+          }, b.onerror = (g) => {
+            t.forEach((D) => this.terminateWorker(D)), a(g);
           };
         }
       }
-      const c = t[t.length - 1];
-      c.onmessage = (i) => {
-        var u, k;
-        t.forEach((g) => this.terminateWorker(g)), ((u = i.data) == null ? void 0 : u.ok) === !1 ? s(new Error(i.data.error)) : r((k = i.data) == null ? void 0 : k.data);
-      }, c.onerror = (i) => {
-        t.forEach((u) => this.terminateWorker(u)), s(i);
+      const f = t[t.length - 1];
+      f.onmessage = (n) => {
+        var h, y;
+        t.forEach((k) => this.terminateWorker(k)), ((h = n.data) == null ? void 0 : h.ok) === !1 ? a(new Error(n.data.error)) : r((y = n.data) == null ? void 0 : y.data);
+      }, f.onerror = (n) => {
+        t.forEach((h) => this.terminateWorker(h)), a(n);
       };
-      const { worker: f, srcData: a, ...o } = e[0], p = { data: a ?? {}, ...o, index: 0 };
-      t[0].postMessage(p, m(p));
+      const {
+        worker: d,
+        srcData: s,
+        ...o
+      } = e[0];
+      let u = s;
+      const m = o.__memory_ref__, p = !!o.deleteMemory;
+      if (u === void 0 && m) {
+        if (!this._memoryStore.has(m)) {
+          t.forEach((n) => this.terminateWorker(n)), a(
+            new Error(
+              `Memory reference "${m}" not found in MemoryStore`
+            )
+          );
+          return;
+        }
+        u = this._memoryStore.get(m), p && this._memoryStore.delete(m), delete o.__memory_ref__, delete o.deleteMemory;
+      }
+      u === void 0 && (u = {});
+      const c = {
+        data: u,
+        ...o,
+        index: 0
+      };
+      t[0].postMessage(c, w(c));
     });
   }
   /**
@@ -671,28 +847,28 @@ class U {
   async runPersistent(e, r) {
     if (this._isTerminated)
       throw new Error("MainWorkerFactory has been terminated");
-    const s = this.findWorkerByName(e);
-    if (!s) throw new Error(`Worker "${e}" not found`);
+    const a = this.findWorkerByName(e);
+    if (!a) throw new Error(`Worker "${e}" not found`);
     let t = this._persistentWorkers.get(e);
     if (!t) {
-      const l = new W(s.func, {
+      const i = new M(a.func, {
         mode: P.Persistent,
-        createWorker: s.createWorker
+        createWorker: a.createWorker
       });
-      t = this.trackWorker(l.getWorker), this._persistentWorkers.set(e, t);
+      t = this.trackWorker(i.getWorker), this._persistentWorkers.set(e, t);
     }
-    return new Promise((l, c) => {
-      t.onmessage = (a) => {
-        var o, d;
-        ((o = a.data) == null ? void 0 : o.ok) === !1 ? c(new Error(a.data.error)) : l((d = a.data) == null ? void 0 : d.data);
-      }, t.onerror = (a) => {
-        c(a);
+    return new Promise((i, f) => {
+      t.onmessage = (s) => {
+        var o, u;
+        ((o = s.data) == null ? void 0 : o.ok) === !1 ? f(new Error(s.data.error)) : i((u = s.data) == null ? void 0 : u.data);
+      }, t.onerror = (s) => {
+        f(s);
       };
-      const f = {
+      const d = {
         type: "run",
         config: r.config
       };
-      r.dataset !== void 0 && (f.dataset = r.dataset), t.postMessage(f, m(f));
+      r.dataset !== void 0 && (d.dataset = r.dataset), t.postMessage(d, w(d));
     });
   }
   /**
@@ -732,7 +908,7 @@ class U {
     this._persistentWorkers.clear();
     for (const e of Array.from(this._activeWorkers))
       this.terminateWorker(e);
-    this._activeWorkers.clear();
+    this._activeWorkers.clear(), this._memoryStore.clear();
   }
   /**
    * Alias for {@link terminate}. Terminates the factory and all worker instances.
@@ -754,38 +930,38 @@ class U {
     this.reset();
   }
 }
-function V(n) {
+function N(l) {
   if (typeof self > "u") return;
-  let e = null, r = null, s = null, t = {};
-  const l = (f, a) => {
-    self.postMessage(f, a);
+  let e = null, r = null, a = null, t = {};
+  const i = (d, s) => {
+    self.postMessage(d, s);
   };
-  async function c(f) {
+  async function f(d) {
     try {
-      const a = typeof f == "object" && f !== null && "data" in f ? { ...t, ...f } : { data: f, ...t, index: 0 }, o = await n(a), d = { ok: !0, data: o }, p = m(o);
-      e ? e.postMessage(d, p) : l(d, p);
-    } catch (a) {
+      const s = typeof d == "object" && d !== null && "data" in d ? { ...t, ...d } : { data: d, ...t, index: 0 }, o = await l(s), u = { ok: !0, data: o }, m = w(o);
+      e ? e.postMessage(u, m) : i(u, m);
+    } catch (s) {
       const o = {
         ok: !1,
-        error: a instanceof Error ? a.message : String(a)
+        error: s instanceof Error ? s.message : String(s)
       };
-      e ? e.postMessage(o) : l(o);
+      e ? e.postMessage(o) : i(o);
     }
   }
-  self.addEventListener("message", (f) => {
-    const a = f.data;
-    if (a && a.__pipeline_ports__) {
-      a.stepParams && (t = a.stepParams), a.outputPort && (e = a.outputPort), a.inputPort && (r = a.inputPort, r.onmessage = (o) => {
-        var d;
-        o.data && o.data.ok === !1 ? e ? e.postMessage(o.data) : l(o.data) : c({ data: (d = o.data) == null ? void 0 : d.data, ...t, index: 0 });
-      }), s !== null && (c(s), s = null);
+  self.addEventListener("message", (d) => {
+    const s = d.data;
+    if (s && s.__pipeline_ports__) {
+      s.stepParams && (t = s.stepParams), s.outputPort && (e = s.outputPort), s.inputPort && (r = s.inputPort, r.onmessage = (o) => {
+        var u;
+        o.data && o.data.ok === !1 ? e ? e.postMessage(o.data) : i(o.data) : f({ data: (u = o.data) == null ? void 0 : u.data, ...t, index: 0 });
+      }), a !== null && (f(a), a = null);
       return;
     }
-    r ? s = a : c(a);
+    r ? a = s : f(s);
   });
 }
 export {
-  U as MainWorkerFactory,
-  W as WorkerFactory,
-  V as defineWorker
+  q as MainWorkerFactory,
+  M as WorkerFactory,
+  N as defineWorker
 };

package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts

@@ -1,4 +1,4 @@
-import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
+import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults, MemoryStats } from './types.ts';
 /**
  * Recursively collects all Transferable objects from a value.
  * Transferable (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
@@ -37,6 +37,7 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
     private readonly _threads;
     private readonly _persistentWorkers;
     private readonly _activeWorkers;
+    private readonly _memoryStore;
     private _isTerminated;
     /**
      * Creates a new `MainWorkerFactory`.
@@ -115,9 +116,31 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      * @example
      * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
      */
-    runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, { srcData, ...otherParams }: {
-        srcData: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
+    runWorker<TName extends keyof WorkerConfigMap<TConfigs> & string>(workerName: TName, rawParams: {
+        srcData?: WorkerDataParam<WorkerConfigMap<TConfigs>[TName]>;
+        __memory_ref__?: string;
+        deleteMemory?: boolean;
     } & Record<string, unknown>): Promise<TypedSettledResults<WorkerReturnType<WorkerConfigMap<TConfigs>[TName]>>>;
+    /**
+     * Deletes a specific memory reference from the factory's memory store.
+     *
+     * @param ref - The `__memory_ref__` token string to delete.
+     * @returns A promise that resolves to `true` if deleted, `false` otherwise.
+     */
+    deleteMemory(ref: string): Promise<boolean>;
+    /**
+     * Clears all stored dataset references from the factory's memory store.
+     */
+    clearMemory(): Promise<void>;
+    /**
+     * Saves the output of a completed worker execution into MemoryStore
+     * and modifies the returned settled results with a `__memory_ref__` token.
+     */
+    private storeWorkerMemoryResult;
+    /**
+     * Returns statistics about active memory references in the factory.
+     */
+    getMemoryStats(): Promise<MemoryStats>;
     /**
      * Builds the array of per-thread worker promises for a single `runWorker`
      * call.
@@ -196,7 +219,7 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      *   reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
      * });
      */
-    collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
+    collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T> | TypedSettledResults<unknown>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
     /**
      * Runs a chain of workers where each step's output feeds directly into the
      * next step — **without passing through the main thread**.

package/dist/types/tools/main-worker-factory/memory-store.d.ts

@@ -0,0 +1,45 @@
+/**
+ * MemoryStore manages dataset handles stored in volatile RAM inside the MemoryWorker.
+ *
+ * Each dataset is indexed by a cryptographically generated UUID (`__memory_ref__`).
+ */
+export declare class MemoryStore {
+    private readonly store;
+    /**
+     * Stores a dataset in RAM under an unguessable reference ID.
+     *
+     * @param data - The dataset to store.
+     * @param refId - Optional reference ID; if omitted, a UUID will be generated.
+     * @returns The reference ID under which the dataset is stored.
+     */
+    set(data: unknown, refId?: string): string;
+    /**
+     * Retrieves a dataset from RAM by its reference ID.
+     *
+     * @param refId - The reference ID to retrieve.
+     * @returns The stored dataset, or undefined if not found.
+     */
+    get(refId: string): unknown;
+    /**
+     * Deletes a dataset reference from RAM.
+     *
+     * @param refId - The reference ID to delete.
+     * @returns `true` if the key existed and was removed, `false` otherwise.
+     */
+    delete(refId: string): boolean;
+    /**
+     * Clears all dataset handles from RAM.
+     */
+    clear(): void;
+    /**
+     * Checks if a reference ID exists in RAM.
+     */
+    has(refId: string): boolean;
+    /**
+     * Returns statistics about current memory handles.
+     */
+    stats(): {
+        count: number;
+        refs: string[];
+    };
+}

package/dist/types/tools/main-worker-factory/memory-worker.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Script source for the dedicated MemoryWorker thread.
+ *
+ * Runs inside an isolated Web Worker thread and maintains the MemoryStore.
+ * Handshakes and authenticates all incoming MessagePort requests using a secret `factoryToken`.
+ */
+export declare const memoryWorkerScript = "\nconst store = new Map();\n\nself.addEventListener('message', (event) => {\n  const msg = event.data;\n  if (!msg || typeof msg !== 'object') return;\n\n  const { action, factoryToken, expectedToken, ref, data, id } = msg;\n\n  // Initial handshake to set expected token if needed\n  if (action === 'INIT_TOKEN') {\n    self.__expectedToken = expectedToken;\n    self.postMessage({ ok: true, action: 'INIT_TOKEN_ACK' });\n    return;\n  }\n\n  // Validate factory token\n  if (self.__expectedToken && factoryToken !== self.__expectedToken) {\n    self.postMessage({ ok: false, error: 'Unauthorized: invalid factory token', id });\n    return;\n  }\n\n  try {\n    switch (action) {\n      case 'SET': {\n        const refId = ref || ('mem_' + crypto.randomUUID());\n        store.set(refId, data);\n        self.postMessage({ ok: true, ref: refId, id });\n        break;\n      }\n      case 'GET': {\n        const resultData = store.get(ref);\n        const exists = store.has(ref);\n        self.postMessage({ ok: true, exists, data: resultData, ref, id });\n        break;\n      }\n      case 'DELETE': {\n        const deleted = store.delete(ref);\n        self.postMessage({ ok: true, deleted, ref, id });\n        break;\n      }\n      case 'CLEAR': {\n        store.clear();\n        self.postMessage({ ok: true, action: 'CLEAR_ACK', id });\n        break;\n      }\n      case 'STATS': {\n        self.postMessage({\n          ok: true,\n          stats: {\n            count: store.size,\n            refs: Array.from(store.keys()),\n          },\n          id,\n        });\n        break;\n      }\n      default:\n        self.postMessage({ ok: false, error: 'Unknown action: ' + action, id });\n    }\n  } catch (err) {\n    self.postMessage({\n      ok: false,\n      error: err instanceof Error ? err.message : String(err),\n      id,\n    });\n  }\n});\n";

package/dist/types/tools/main-worker-factory/types.d.ts

@@ -52,6 +52,23 @@ export interface WorkerConfig<TFunc extends WorkerFunction = WorkerFunction> {
      * If any dependency fails, this worker will not run.
      */
     dependencies?: Array<() => void>;
+    /**
+     * When `true`, the worker's output is saved in the isolated `MemoryWorker`
+     * and returned alongside a `__memory_ref__` token.
+     */
+    memory?: boolean;
+    /**
+     * When `true`, the worker's output is saved in the isolated `MemoryWorker`
+     * and **only** the `__memory_ref__` token is returned (0 bytes data transferred back to main thread).
+     */
+    memoryOnly?: boolean;
+}
+/** Statistics about stored memory handles in MemoryWorker. */
+export interface MemoryStats {
+    /** Total number of active memory references. */
+    count: number;
+    /** Array of active reference IDs. */
+    refs: string[];
 }
 /**
  * Derives a `name → function` map from a readonly tuple of

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@offmain/workerkit",
-  "version": "0.14.0",
+  "version": "0.14.1",
   "description": "A lightweight manager for running functions in Web Workers with partitioning, retries, and concurrency control",
   "type": "module",
   "main": "./dist/index.cjs",