@offmain/workerkit 0.9.2 → 0.11.0

package/README.md

@@ -92,6 +92,172 @@ const { data } = await factory.collectResults(settled, {
 
 ---
 
+## Pipeline
+
+Chain multiple workers together so data flows directly between them via `MessageChannel` — without passing through the main thread between steps.
+
+### Why use a pipeline?
+
+In a traditional multi-step workflow, intermediate data is serialized back to the main thread after each step:
+
+```
+Main → Worker A → Main → Worker B → Main → Worker C → Main
+         ↑ serialize    ↑ serialize    ↑ serialize
+```
+
+With large datasets (100k+ records), each serialization round-trip adds significant overhead — both in time and memory pressure on the main thread. The pipeline eliminates this:
+
+```
+Main → Worker A → Worker B → Worker C → Main
+                ↑ MessageChannel      ↑ only final result
+```
+
+Only the final result crosses back to the main thread. If your pipeline generates 20 MB of intermediate data but produces a 1 KB summary, you save ~40 MB of serialization (two round-trips avoided).
+
+### Usage
+
+```ts
+import { MainWorkerFactory } from '@offmain/workerkit';
+import { fetchData, transform, aggregate } from './workers.ts';
+
+const factory = new MainWorkerFactory({
+  workers: [
+    { name: 'fetchData', role: 'io', func: fetchData },
+    { name: 'transform', role: 'compute', func: transform },
+    { name: 'aggregate', role: 'compute', func: aggregate },
+  ] as const,
+});
+
+const result = await factory.pipeline<AggregateResult>([
+  { worker: 'fetchData', srcData: { url: '/api/records' } },
+  { worker: 'transform' }, // receives fetchData output directly
+  { worker: 'aggregate' }, // receives transform output directly
+]);
+
+console.log(result); // only this small result crossed to main thread
+```
+
+### How each step receives data
+
+- The first step receives `srcData` as `{ data: srcData, index: 0 }` — same as `runWorker`.
+- Each subsequent step receives the previous step's output as `{ data: previousOutput, index: 0 }`.
+- Worker functions don't need any special handling — they use the same `{ data }` parameter signature as regular workers.
+
+### When to use pipeline vs runWorker
+
+| Scenario                                             | Use                     |
+| ---------------------------------------------------- | ----------------------- |
+| Single step, or steps that need partitioning/retries | `runWorker`             |
+| Multi-step chain where intermediate data is large    | `pipeline`              |
+| Steps that are independent (not sequential)          | `runWorker` in parallel |
+| Steps where only the final result matters to the UI  | `pipeline`              |
+
+---
+
+## Persistent Workers
+
+Keep a worker alive with a cached dataset, then re-run it with different configs without re-sending the data.
+
+### Why use persistent workers?
+
+In a typical workflow where you apply multiple transformations to the same dataset, the standard `runWorker` approach re-serializes the entire dataset on every call:
+
+```
+Call 1: Main ──[200k items]──→ Worker → Main
+Call 2: Main ──[200k items]──→ Worker → Main   ← same data, different config
+Call 3: Main ──[200k items]──→ Worker → Main   ← same data again
+```
+
+With 5 config variations on a 1.6 MB dataset, that's ~8 MB of redundant serialization. Persistent workers eliminate this by caching the dataset inside the worker:
+
+```
+Call 1: Main ──[200k items + config]──→ Worker → Main   ← dataset cached
+Call 2: Main ──[config only]──────────→ Worker → Main   ← reuses cache
+Call 3: Main ──[config only]──────────→ Worker → Main   ← reuses cache
+```
+
+Only the first call transfers the dataset. Subsequent calls send just the config object (typically a few bytes), saving both serialization time and memory pressure.
+
+### Usage
+
+```ts
+import { MainWorkerFactory } from '@offmain/workerkit';
+import { transformArray } from './transform.worker.ts';
+
+const factory = new MainWorkerFactory({
+  workers: [
+    { name: 'transform', role: 'computation', func: transformArray },
+  ] as const,
+});
+
+// First call: send dataset + config (dataset gets cached in worker memory)
+const r1 = await factory.runPersistent('transform', {
+  dataset: largeArray,
+  config: { multiplier: 2, filter: 'even' },
+});
+
+// Subsequent calls: only config — dataset is reused from cache
+const r2 = await factory.runPersistent('transform', {
+  config: { multiplier: 5, filter: 'odd' },
+});
+
+const r3 = await factory.runPersistent('transform', {
+  config: { multiplier: 1, filter: 'none', limit: 1000 },
+});
+
+// Update the dataset when it changes
+const r4 = await factory.runPersistent('transform', {
+  dataset: newArray, // replaces cached dataset
+  config: { multiplier: 3, filter: 'even' },
+});
+
+// Release the worker when done — frees memory
+factory.release('transform');
+```
+
+### How the worker function receives data
+
+The worker function signature stays the same as a regular worker — it receives `{ data, config }`:
+
+```ts
+// transform.worker.ts
+export function transformArray({
+  data,
+  config,
+}: {
+  data: number[];
+  config: { multiplier: number; filter: string };
+}) {
+  return data
+    .filter((n) => /* apply filter */)
+    .map((n) => n * config.multiplier);
+}
+```
+
+The framework handles the caching transparently — your function always receives the full `data` (from cache or freshly provided) plus the current `config`.
+
+### When to use persistent vs runWorker
+
+| Scenario                                               | Use             |
+| ------------------------------------------------------ | --------------- |
+| One-off computation                                    | `runWorker`     |
+| Same dataset, multiple config variations               | `runPersistent` |
+| Interactive UI where user tweaks params on static data | `runPersistent` |
+| Dataset changes frequently                             | `runWorker`     |
+| Need partitioning across multiple threads              | `runWorker`     |
+
+### Memory management
+
+The cached dataset lives in worker memory until `release()` is called. For large datasets, always call `release()` when you're done to free the memory:
+
+```ts
+factory.release('transform');
+```
+
+After releasing, the next `runPersistent` call will create a fresh worker instance (requiring a new dataset).
+
+---
+
 ## ESLint Plugin
 
 The package ships with two ESLint rules to catch common worker mistakes at lint time.

package/dist/index.cjs

@@ -1,4 +1,4 @@
-"use strict";var y=Object.defineProperty;var m=(t,e,r)=>e in t?y(t,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):t[e]=r;var g=(t,e,r)=>m(t,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const w=t=>`
+"use strict";var y=Object.defineProperty;var m=(s,e,t)=>e in s?y(s,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):s[e]=t;var h=(s,e,t)=>m(s,typeof e!="symbol"?e+"":e,t);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const w=s=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -15,14 +15,142 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${t}(event.data);
+    const output = await ${s}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`;class p{constructor(e){g(this,"_worker");const r=w(e.toString()),s=new Blob([r],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(s))}get getWorker(){return this._worker}}class W{constructor(e){this.results=e}}function k(t,e=new Set){return t===null||typeof t!="object"?[]:e.has(t)?[]:(e.add(t),t instanceof ArrayBuffer||t instanceof MessagePort||typeof ImageBitmap<"u"&&t instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&t instanceof OffscreenCanvas?[t]:ArrayBuffer.isView(t)?[t.buffer]:Array.isArray(t)?t.flatMap(r=>k(r,e)):Object.values(t).flatMap(r=>k(r,e)))}class v{constructor(e){g(this,"_workers");g(this,"_threads");this._workers=e.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new p(e)}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const s=Math.min(r,e.length),n=Math.floor(e.length/s),c=e.length%s,o=[];let f=0;for(let a=0;a<s;a++){const i=n+(a<c?1:0);o.push(e.slice(f,f+i)),f+=i}return o}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,{srcData:r,...s}){const n=this.findWorkerByName(e);if(!n)return Promise.reject(new Error(`Worker "${e}" not found`));const c=n.maxConcurrency??this._threads,o=!!(Array.isArray(r)&&r.length>1&&n.partition),f=o?this.partitionArray(r,c):r,a=this.createWorkerPromises(n,e,{data:f,...s},c,o),i=await Promise.allSettled(a);return new W(i)}createWorkerPromises(e,r,s,n,c){const{data:o,...f}=s;return Array.from({length:n},(a,i)=>{const l=c&&Array.isArray(o)?o[i]:o;return this.runWorkerWithRetry({workerFunc:e.func,workerName:r,index:i,data:{data:l,...f}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(s){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,s),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",s),s}}initiateWorker({workerFunc:e,workerName:r,index:s,data:n}){return new Promise((c,o)=>{const a=this.initWorker(e).getWorker;a.onerror=l=>{a.terminate(),o({index:s,workerConfigs:{workerFunc:e,workerName:r,index:s,data:n},failedResult:l})},a.onmessage=l=>{var h,u;if(((h=l.data)==null?void 0:h.ok)===!1){a.terminate(),o({index:s,workerConfigs:{workerFunc:e,workerName:r,index:s,data:n},failedResult:new ErrorEvent("error",{message:l.data.error})});return}c({index:s,workerConfigs:{workerFunc:e,workerName:r,index:s,data:n},successResult:new MessageEvent("message",{data:(u=l.data)==null?void 0:u.data})}),a.terminate()};const i={index:s,...Array.isArray(n)?{data:n}:n};a.postMessage(i,k(i))})}async collectResults(e,r={}){const s=e.results.filter(a=>a.status==="fulfilled"),n=e.results.filter(a=>a.status==="rejected"),c=s.map(a=>a.value.successResult.data),o=r.reducer?r.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((a,i)=>{const l=`
-        const reducer = ${o};
+`,P=s=>`
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let outputPort = null;
+let inputPort = null;
+let pendingData = null;
+
+async function processData(data) {
+  try {
+    const output = await workerFn(data);
+    const result = { ok: true, data: output };
+    const transfers = extractTransferables(output);
+    if (outputPort) {
+      outputPort.postMessage(result, transfers);
+    } else {
+      self.postMessage(result, transfers);
+    }
+  } catch (err) {
+    const result = { ok: false, error: err instanceof Error ? err.message : String(err) };
+    if (outputPort) {
+      outputPort.postMessage(result);
+    } else {
+      self.postMessage(result);
+    }
+  }
+}
+
+self.addEventListener('message', (event) => {
+  if (event.data && event.data.__pipeline_ports__) {
+    if (event.data.outputPort) {
+      outputPort = event.data.outputPort;
+    }
+    if (event.data.inputPort) {
+      inputPort = event.data.inputPort;
+      inputPort.onmessage = (e) => {
+        if (e.data && e.data.ok === false) {
+          // Propagate errors through the pipeline
+          if (outputPort) outputPort.postMessage(e.data);
+          else self.postMessage(e.data);
+        } else {
+          processData({ data: e.data.data, index: 0 });
+        }
+      };
+    }
+    // If we already received data before ports, process it now
+    if (pendingData !== null) {
+      processData(pendingData);
+      pendingData = null;
+    }
+    return;
+  }
+  // First worker in pipeline or standalone — process directly
+  if (!inputPort) {
+    processData(event.data);
+  } else {
+    // Store data until ports are configured
+    pendingData = event.data;
+  }
+});
+`,M=s=>`
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let cachedDataset = null;
+
+self.addEventListener('message', async (event) => {
+  const msg = event.data;
+
+  if (msg && msg.type === 'release') {
+    cachedDataset = null;
+    self.postMessage({ ok: true, data: null, type: 'released' });
+    self.close();
+    return;
+  }
+
+  if (msg && msg.type === 'run') {
+    // Update cache if new dataset provided
+    if (msg.dataset !== undefined) {
+      cachedDataset = msg.dataset;
+    }
+
+    if (cachedDataset === null) {
+      self.postMessage({ ok: false, error: 'No dataset cached. Provide a dataset on the first call.' });
+      return;
+    }
+
+    try {
+      const output = await workerFn({ data: cachedDataset, config: msg.config });
+      self.postMessage({ ok: true, data: output }, extractTransferables(output));
+    } catch (err) {
+      self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+    }
+    return;
+  }
+
+  // Fallback: treat as a regular one-shot call for backwards compat
+  try {
+    const output = await workerFn(msg);
+    self.postMessage({ ok: true, data: output }, extractTransferables(output));
+  } catch (err) {
+    self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+});
+`;var v=(s=>(s.Default="default",s.Pipeline="pipeline",s.Persistent="persistent",s))(v||{});const W=Object.freeze({persistent:M,pipeline:P,default:w});class k{constructor(e,t){h(this,"_worker");const n=(t==null?void 0:t.mode)??"default",a=W[n](e.toString()),u=new Blob([a],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(u))}get getWorker(){return this._worker}}class b{constructor(e){this.results=e}}function p(s,e=new Set){return s===null||typeof s!="object"?[]:e.has(s)?[]:(e.add(s),s instanceof ArrayBuffer||s instanceof MessagePort||typeof ImageBitmap<"u"&&s instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&s instanceof OffscreenCanvas?[s]:ArrayBuffer.isView(s)?[s.buffer]:Array.isArray(s)?s.flatMap(t=>p(t,e)):Object.values(s).flatMap(t=>p(t,e)))}class A{constructor(e){h(this,"_workers");h(this,"_threads");h(this,"_persistentWorkers",new Map);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new k(e)}partitionArray(e,t){if(!e.length)return[];if(t<=0)throw new Error("numChunks must be positive");const n=Math.min(t,e.length),a=Math.floor(e.length/n),u=e.length%n,f=[];let l=0;for(let r=0;r<n;r++){const o=a+(r<u?1:0);f.push(e.slice(l,l+o)),l+=o}return f}findWorkerByName(e){return this._workers.find(t=>t.name===e)}async runWorker(e,{srcData:t,...n}){const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const u=a.maxConcurrency??this._threads,f=!!(Array.isArray(t)&&t.length>1&&a.partition),l=f?this.partitionArray(t,u):t,r=this.createWorkerPromises(a,e,{data:l,...n},u,f),o=await Promise.allSettled(r);return new b(o)}createWorkerPromises(e,t,n,a,u){const{data:f,...l}=n;return Array.from({length:a},(r,o)=>{const i=u&&Array.isArray(f)?f[o]:f;return this.runWorkerWithRetry({workerFunc:e.func,workerName:t,index:o,data:{data:i,...l}},e.retries)})}async runWorkerWithRetry(e,t=2){try{return await this.initiateWorker(e)}catch(n){if(t>0)return console.error(`Worker ${e.index} failed, retrying (${t} left):`,n),this.runWorkerWithRetry(e,t-1);throw console.error("Worker failed after all retries:",n),n}}initiateWorker({workerFunc:e,workerName:t,index:n,data:a}){return new Promise((u,f)=>{const r=this.initWorker(e).getWorker;r.onerror=i=>{r.terminate(),f({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:i})},r.onmessage=i=>{var c,d;if(((c=i.data)==null?void 0:c.ok)===!1){r.terminate(),f({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:new ErrorEvent("error",{message:i.data.error})});return}u({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},successResult:new MessageEvent("message",{data:(d=i.data)==null?void 0:d.data})}),r.terminate()};const o={index:n,...Array.isArray(a)?{data:a}:a};r.postMessage(o,p(o))})}async collectResults(e,t={}){const n=e.results.filter(r=>r.status==="fulfilled"),a=e.results.filter(r=>r.status==="rejected"),u=n.map(r=>r.value.successResult.data),f=t.reducer?t.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((r,o)=>{const i=`
+        const reducer = ${f};
         self.addEventListener('message', (event) => {
           try {
             const result = reducer(event.data);
@@ -31,4 +159,4 @@ self.addEventListener('message', async (event) => {
             self.postMessage({ ok: false, error: String(err) });
           }
         });
-      `,h=new Blob([l],{type:"application/javascript"}),u=new Worker(URL.createObjectURL(h));u.onmessage=d=>{u.terminate(),d.data.ok?a(d.data.data):i(new Error(d.data.error))},u.onerror=d=>{u.terminate(),i(d)},u.postMessage(c)}),succeeded:s.length,failed:n.length,errors:n}}}exports.MainWorkerFactory=v;exports.WorkerFactory=p;
+      `,c=new Blob([i],{type:"application/javascript"}),d=new Worker(URL.createObjectURL(c));d.onmessage=g=>{d.terminate(),g.data.ok?r(g.data.data):o(new Error(g.data.error))},d.onerror=g=>{d.terminate(),o(g)},d.postMessage(u)}),succeeded:n.length,failed:a.length,errors:a}}async pipeline(e){if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const t=e[0],n=this.findWorkerByName(t.worker);if(!n)throw new Error(`Worker "${t.worker}" not found`);const u=this.initWorker(n.func).getWorker;return new Promise((f,l)=>{u.onmessage=o=>{var i,c;u.terminate(),((i=o.data)==null?void 0:i.ok)===!1?l(new Error(o.data.error)):f((c=o.data)==null?void 0:c.data)},u.onerror=o=>{u.terminate(),l(o)};const r=t.srcData??{};u.postMessage({data:r,index:0},p(r))})}return new Promise((t,n)=>{const a=[],u=[];for(const r of e){const o=this.findWorkerByName(r.worker);if(!o){n(new Error(`Worker "${r.worker}" not found`));return}const i=new k(o.func,{mode:v.Pipeline});a.push(i.getWorker)}for(let r=0;r<a.length-1;r++)u.push(new MessageChannel);for(let r=0;r<a.length;r++){const o=[],i={};r>0&&(i.inputPort=u[r-1].port1,o.push(i.inputPort)),r<a.length-1&&(i.outputPort=u[r].port2,o.push(i.outputPort)),a[r].postMessage({__pipeline_ports__:!0,...i},o)}const f=a[a.length-1];f.onmessage=r=>{var o,i;a.forEach(c=>c.terminate()),((o=r.data)==null?void 0:o.ok)===!1?n(new Error(r.data.error)):t((i=r.data)==null?void 0:i.data)},f.onerror=r=>{a.forEach(o=>o.terminate()),n(r)};const l=e[0].srcData??{};a[0].postMessage({data:l,index:0},p(l))})}async runPersistent(e,t){const n=this.findWorkerByName(e);if(!n)throw new Error(`Worker "${e}" not found`);let a=this._persistentWorkers.get(e);return a||(a=new k(n.func,{mode:v.Persistent}).getWorker,this._persistentWorkers.set(e,a)),new Promise((u,f)=>{a.onmessage=r=>{var o,i;((o=r.data)==null?void 0:o.ok)===!1?f(new Error(r.data.error)):u((i=r.data)==null?void 0:i.data)},a.onerror=r=>{f(r)};const l={type:"run",config:t.config};t.dataset!==void 0&&(l.dataset=t.dataset),a.postMessage(l,p(l))})}release(e){const t=this._persistentWorkers.get(e);t&&(t.postMessage({type:"release"}),t.terminate(),this._persistentWorkers.delete(e))}}exports.MainWorkerFactory=A;exports.WorkerFactory=k;

package/dist/index.js

@@ -1,7 +1,7 @@
-var k = Object.defineProperty;
-var y = (t, e, r) => e in t ? k(t, e, { enumerable: !0, configurable: !0, writable: !0, value: r }) : t[e] = r;
-var p = (t, e, r) => y(t, typeof e != "symbol" ? e + "" : e, r);
-const m = (t) => `
+var w = Object.defineProperty;
+var m = (s, e, t) => e in s ? w(s, e, { enumerable: !0, configurable: !0, writable: !0, value: t }) : s[e] = t;
+var h = (s, e, t) => m(s, typeof e != "symbol" ? e + "" : e, t);
+const y = (s) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -18,14 +18,148 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${t}(event.data);
+    const output = await ${s}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
+`, P = (s) => `
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let outputPort = null;
+let inputPort = null;
+let pendingData = null;
+
+async function processData(data) {
+  try {
+    const output = await workerFn(data);
+    const result = { ok: true, data: output };
+    const transfers = extractTransferables(output);
+    if (outputPort) {
+      outputPort.postMessage(result, transfers);
+    } else {
+      self.postMessage(result, transfers);
+    }
+  } catch (err) {
+    const result = { ok: false, error: err instanceof Error ? err.message : String(err) };
+    if (outputPort) {
+      outputPort.postMessage(result);
+    } else {
+      self.postMessage(result);
+    }
+  }
+}
+
+self.addEventListener('message', (event) => {
+  if (event.data && event.data.__pipeline_ports__) {
+    if (event.data.outputPort) {
+      outputPort = event.data.outputPort;
+    }
+    if (event.data.inputPort) {
+      inputPort = event.data.inputPort;
+      inputPort.onmessage = (e) => {
+        if (e.data && e.data.ok === false) {
+          // Propagate errors through the pipeline
+          if (outputPort) outputPort.postMessage(e.data);
+          else self.postMessage(e.data);
+        } else {
+          processData({ data: e.data.data, index: 0 });
+        }
+      };
+    }
+    // If we already received data before ports, process it now
+    if (pendingData !== null) {
+      processData(pendingData);
+      pendingData = null;
+    }
+    return;
+  }
+  // First worker in pipeline or standalone — process directly
+  if (!inputPort) {
+    processData(event.data);
+  } else {
+    // Store data until ports are configured
+    pendingData = event.data;
+  }
+});
+`, M = (s) => `
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let cachedDataset = null;
+
+self.addEventListener('message', async (event) => {
+  const msg = event.data;
+
+  if (msg && msg.type === 'release') {
+    cachedDataset = null;
+    self.postMessage({ ok: true, data: null, type: 'released' });
+    self.close();
+    return;
+  }
+
+  if (msg && msg.type === 'run') {
+    // Update cache if new dataset provided
+    if (msg.dataset !== undefined) {
+      cachedDataset = msg.dataset;
+    }
+
+    if (cachedDataset === null) {
+      self.postMessage({ ok: false, error: 'No dataset cached. Provide a dataset on the first call.' });
+      return;
+    }
+
+    try {
+      const output = await workerFn({ data: cachedDataset, config: msg.config });
+      self.postMessage({ ok: true, data: output }, extractTransferables(output));
+    } catch (err) {
+      self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+    }
+    return;
+  }
+
+  // Fallback: treat as a regular one-shot call for backwards compat
+  try {
+    const output = await workerFn(msg);
+    self.postMessage({ ok: true, data: output }, extractTransferables(output));
+  } catch (err) {
+    self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+});
 `;
-class w {
+var v = /* @__PURE__ */ ((s) => (s.Default = "default", s.Pipeline = "pipeline", s.Persistent = "persistent", s))(v || {});
+const W = Object.freeze({
+  persistent: M,
+  pipeline: P,
+  default: y
+});
+class k {
   /**
    * Creates a new `Worker` from the given function.
    *
@@ -35,13 +169,15 @@ class w {
    * @param workerFunction - The function to run inside the worker thread.
    *   Must be self-contained — it cannot reference variables from the outer
    *   scope because it is serialised via `.toString()`.
+   * @param options - Optional configuration. Set `mode` to control the
+   *   worker execution mode (default, pipeline, or persistent).
    */
-  constructor(e) {
-    p(this, "_worker");
-    const r = m(e.toString()), s = new Blob([r], {
+  constructor(e, t) {
+    h(this, "_worker");
+    const n = (t == null ? void 0 : t.mode) ?? "default", a = W[n](e.toString()), f = new Blob([a], {
       type: "application/javascript"
     });
-    this._worker = new Worker(URL.createObjectURL(s));
+    this._worker = new Worker(URL.createObjectURL(f));
   }
   /**
    * Returns the underlying native `Worker` instance.
@@ -53,25 +189,26 @@ class w {
     return this._worker;
   }
 }
-class W {
+class b {
   constructor(e) {
     this.results = e;
   }
 }
-function g(t, e = /* @__PURE__ */ new Set()) {
-  return t === null || typeof t != "object" ? [] : e.has(t) ? [] : (e.add(t), t instanceof ArrayBuffer || t instanceof MessagePort || typeof ImageBitmap < "u" && t instanceof ImageBitmap || typeof OffscreenCanvas < "u" && t instanceof OffscreenCanvas ? [t] : ArrayBuffer.isView(t) ? [t.buffer] : Array.isArray(t) ? t.flatMap((r) => g(r, e)) : Object.values(t).flatMap(
-    (r) => g(r, e)
+function p(s, e = /* @__PURE__ */ new Set()) {
+  return s === null || typeof s != "object" ? [] : e.has(s) ? [] : (e.add(s), s instanceof ArrayBuffer || s instanceof MessagePort || typeof ImageBitmap < "u" && s instanceof ImageBitmap || typeof OffscreenCanvas < "u" && s instanceof OffscreenCanvas ? [s] : ArrayBuffer.isView(s) ? [s.buffer] : Array.isArray(s) ? s.flatMap((t) => p(t, e)) : Object.values(s).flatMap(
+    (t) => p(t, e)
   ));
 }
-class b {
+class B {
   /**
    * Creates a new `MainWorkerFactory`.
    *
    * @param options - Configuration object containing the `workers` registry.
    */
   constructor(e) {
-    p(this, "_workers");
-    p(this, "_threads");
+    h(this, "_workers");
+    h(this, "_threads");
+    h(this, "_persistentWorkers", /* @__PURE__ */ new Map());
     this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
   }
   /**
@@ -81,7 +218,7 @@ class b {
    * @returns A new `WorkerFactory` wrapping the worker.
    */
   initWorker(e) {
-    return new w(e);
+    return new k(e);
   }
   /**
    * Splits an array into up to `numChunks` evenly-sized sub-arrays.
@@ -99,16 +236,16 @@ class b {
    * partitionArray([1, 2, 3, 4, 5], 3);
    * // → [[1, 2], [3, 4], [5]]
    */
-  partitionArray(e, r) {
+  partitionArray(e, t) {
     if (!e.length) return [];
-    if (r <= 0) throw new Error("numChunks must be positive");
-    const s = Math.min(r, e.length), n = Math.floor(e.length / s), c = e.length % s, o = [];
-    let f = 0;
-    for (let a = 0; a < s; a++) {
-      const i = n + (a < c ? 1 : 0);
-      o.push(e.slice(f, f + i)), f += i;
+    if (t <= 0) throw new Error("numChunks must be positive");
+    const n = Math.min(t, e.length), a = Math.floor(e.length / n), f = e.length % n, u = [];
+    let l = 0;
+    for (let r = 0; r < n; r++) {
+      const o = a + (r < f ? 1 : 0);
+      u.push(e.slice(l, l + o)), l += o;
     }
-    return o;
+    return u;
   }
   /**
    * Looks up a registered worker configuration by name.
@@ -117,7 +254,7 @@ class b {
    * @returns The matching config, or `undefined` if not found.
    */
   findWorkerByName(e) {
-    return this._workers.find((r) => r.name === e);
+    return this._workers.find((t) => t.name === e);
   }
   /**
    * Runs a named worker against the provided data, distributing work across
@@ -146,20 +283,20 @@ class b {
    * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3] });
    */
   async runWorker(e, {
-    srcData: r,
-    ...s
+    srcData: t,
+    ...n
   }) {
-    const n = this.findWorkerByName(e);
-    if (!n)
+    const a = this.findWorkerByName(e);
+    if (!a)
       return Promise.reject(new Error(`Worker "${e}" not found`));
-    const c = n.maxConcurrency ?? this._threads, o = !!(Array.isArray(r) && r.length > 1 && n.partition), f = o ? this.partitionArray(r, c) : r, a = this.createWorkerPromises(
-      n,
+    const f = a.maxConcurrency ?? this._threads, u = !!(Array.isArray(t) && t.length > 1 && a.partition), l = u ? this.partitionArray(t, f) : t, r = this.createWorkerPromises(
+      a,
       e,
-      { data: f, ...s },
-      c,
-      o
-    ), i = await Promise.allSettled(a);
-    return new W(i);
+      { data: l, ...n },
+      f,
+      u
+    ), o = await Promise.allSettled(r);
+    return new b(o);
   }
   /**
    * Builds the array of per-thread worker promises for a single `runWorker`
@@ -175,16 +312,16 @@ class b {
    * @param isPartitioned  - Whether `data` is a pre-split array of shards.
    * @returns An array of promises, one per thread.
    */
-  createWorkerPromises(e, r, s, n, c) {
-    const { data: o, ...f } = s;
-    return Array.from({ length: n }, (a, i) => {
-      const l = c && Array.isArray(o) ? o[i] : o;
+  createWorkerPromises(e, t, n, a, f) {
+    const { data: u, ...l } = n;
+    return Array.from({ length: a }, (r, o) => {
+      const i = f && Array.isArray(u) ? u[o] : u;
       return this.runWorkerWithRetry(
         {
           workerFunc: e.func,
-          workerName: r,
-          index: i,
-          data: { data: l, ...f }
+          workerName: t,
+          index: o,
+          data: { data: i, ...l }
         },
         e.retries
       );
@@ -202,16 +339,16 @@ class b {
    * @returns The successful {@link WorkerResult} once the worker resolves.
    * @throws The last caught error when all retries are exhausted.
    */
-  async runWorkerWithRetry(e, r = 2) {
+  async runWorkerWithRetry(e, t = 2) {
     try {
       return await this.initiateWorker(e);
-    } catch (s) {
-      if (r > 0)
+    } catch (n) {
+      if (t > 0)
         return console.error(
-          `Worker ${e.index} failed, retrying (${r} left):`,
-          s
-        ), this.runWorkerWithRetry(e, r - 1);
-      throw console.error("Worker failed after all retries:", s), s;
+          `Worker ${e.index} failed, retrying (${t} left):`,
+          n
+        ), this.runWorkerWithRetry(e, t - 1);
+      throw console.error("Worker failed after all retries:", n), n;
     }
   }
   /**
@@ -234,43 +371,43 @@ class b {
    */
   initiateWorker({
     workerFunc: e,
-    workerName: r,
-    index: s,
-    data: n
+    workerName: t,
+    index: n,
+    data: a
   }) {
-    return new Promise((c, o) => {
-      const a = this.initWorker(e).getWorker;
-      a.onerror = (l) => {
-        a.terminate(), o({
-          index: s,
-          workerConfigs: { workerFunc: e, workerName: r, index: s, data: n },
-          failedResult: l
+    return new Promise((f, u) => {
+      const r = this.initWorker(e).getWorker;
+      r.onerror = (i) => {
+        r.terminate(), u({
+          index: n,
+          workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
+          failedResult: i
         });
-      }, a.onmessage = (l) => {
-        var h, u;
-        if (((h = l.data) == null ? void 0 : h.ok) === !1) {
-          a.terminate(), o({
-            index: s,
-            workerConfigs: { workerFunc: e, workerName: r, index: s, data: n },
+      }, r.onmessage = (i) => {
+        var c, d;
+        if (((c = i.data) == null ? void 0 : c.ok) === !1) {
+          r.terminate(), u({
+            index: n,
+            workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
             failedResult: new ErrorEvent("error", {
-              message: l.data.error
+              message: i.data.error
             })
           });
           return;
         }
-        c({
-          index: s,
-          workerConfigs: { workerFunc: e, workerName: r, index: s, data: n },
+        f({
+          index: n,
+          workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
           successResult: new MessageEvent("message", {
-            data: (u = l.data) == null ? void 0 : u.data
+            data: (d = i.data) == null ? void 0 : d.data
           })
-        }), a.terminate();
+        }), r.terminate();
       };
-      const i = {
-        index: s,
-        ...Array.isArray(n) ? { data: n } : n
+      const o = {
+        index: n,
+        ...Array.isArray(a) ? { data: a } : a
       };
-      a.postMessage(i, g(i));
+      r.postMessage(o, p(o));
     });
   }
   /**
@@ -305,16 +442,16 @@ class b {
    *   reducer: (shards) => shards.flat().reduce((a, b) => a + b, 0),
    * });
    */
-  async collectResults(e, r = {}) {
-    const s = e.results.filter(
-      (a) => a.status === "fulfilled"
-    ), n = e.results.filter(
-      (a) => a.status === "rejected"
-    ), c = s.map((a) => a.value.successResult.data), o = r.reducer ? r.reducer.toString() : "(shards) => shards.flat()";
+  async collectResults(e, t = {}) {
+    const n = e.results.filter(
+      (r) => r.status === "fulfilled"
+    ), a = e.results.filter(
+      (r) => r.status === "rejected"
+    ), f = n.map((r) => r.value.successResult.data), u = t.reducer ? t.reducer.toString() : "(shards) => shards.flat()";
     return {
-      data: await new Promise((a, i) => {
-        const l = `
-        const reducer = ${o};
+      data: await new Promise((r, o) => {
+        const i = `
+        const reducer = ${u};
         self.addEventListener('message', (event) => {
           try {
             const result = reducer(event.data);
@@ -323,20 +460,170 @@ class b {
             self.postMessage({ ok: false, error: String(err) });
           }
         });
-      `, h = new Blob([l], { type: "application/javascript" }), u = new Worker(URL.createObjectURL(h));
-        u.onmessage = (d) => {
-          u.terminate(), d.data.ok ? a(d.data.data) : i(new Error(d.data.error));
-        }, u.onerror = (d) => {
-          u.terminate(), i(d);
-        }, u.postMessage(c);
+      `, c = new Blob([i], { type: "application/javascript" }), d = new Worker(URL.createObjectURL(c));
+        d.onmessage = (g) => {
+          d.terminate(), g.data.ok ? r(g.data.data) : o(new Error(g.data.error));
+        }, d.onerror = (g) => {
+          d.terminate(), o(g);
+        }, d.postMessage(f);
       }),
-      succeeded: s.length,
-      failed: n.length,
-      errors: n
+      succeeded: n.length,
+      failed: a.length,
+      errors: a
     };
   }
+  /**
+   * Runs a chain of workers where each step's output feeds directly into the
+   * next step — **without passing through the main thread**.
+   *
+   * Internally, adjacent workers are connected via `MessageChannel` ports.
+   * Only the final result is sent back to the main thread, minimising
+   * serialisation overhead for large intermediate data.
+   *
+   * @typeParam TResult - The expected type of the final pipeline output.
+   *   Defaults to `unknown` if not specified.
+   *
+   * @param steps - An ordered array of {@link PipelineStep} objects. The first
+   *   step must include `srcData`; subsequent steps receive the previous
+   *   step's output as `{ data: previousOutput, index: 0 }`.
+   *
+   * @returns A promise that resolves with the final step's output.
+   * @throws {Error} When `steps` is empty or a worker name is not found.
+   *
+   * @example
+   * const result = await foreman.pipeline<FilteredPost[]>([
+   *   { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
+   *   { worker: 'transformPosts' },
+   *   { worker: 'filterPosts' },
+   * ]);
+   * console.log(result); // final transformed + filtered data
+   */
+  async pipeline(e) {
+    if (e.length === 0)
+      throw new Error("Pipeline requires at least one step");
+    if (e.length === 1) {
+      const t = e[0], n = this.findWorkerByName(t.worker);
+      if (!n) throw new Error(`Worker "${t.worker}" not found`);
+      const f = this.initWorker(n.func).getWorker;
+      return new Promise((u, l) => {
+        f.onmessage = (o) => {
+          var i, c;
+          f.terminate(), ((i = o.data) == null ? void 0 : i.ok) === !1 ? l(new Error(o.data.error)) : u((c = o.data) == null ? void 0 : c.data);
+        }, f.onerror = (o) => {
+          f.terminate(), l(o);
+        };
+        const r = t.srcData ?? {};
+        f.postMessage(
+          { data: r, index: 0 },
+          p(r)
+        );
+      });
+    }
+    return new Promise((t, n) => {
+      const a = [], f = [];
+      for (const r of e) {
+        const o = this.findWorkerByName(r.worker);
+        if (!o) {
+          n(new Error(`Worker "${r.worker}" not found`));
+          return;
+        }
+        const i = new k(o.func, {
+          mode: v.Pipeline
+        });
+        a.push(i.getWorker);
+      }
+      for (let r = 0; r < a.length - 1; r++)
+        f.push(new MessageChannel());
+      for (let r = 0; r < a.length; r++) {
+        const o = [], i = {};
+        r > 0 && (i.inputPort = f[r - 1].port1, o.push(i.inputPort)), r < a.length - 1 && (i.outputPort = f[r].port2, o.push(i.outputPort)), a[r].postMessage(
+          { __pipeline_ports__: !0, ...i },
+          o
+        );
+      }
+      const u = a[a.length - 1];
+      u.onmessage = (r) => {
+        var o, i;
+        a.forEach((c) => c.terminate()), ((o = r.data) == null ? void 0 : o.ok) === !1 ? n(new Error(r.data.error)) : t((i = r.data) == null ? void 0 : i.data);
+      }, u.onerror = (r) => {
+        a.forEach((o) => o.terminate()), n(r);
+      };
+      const l = e[0].srcData ?? {};
+      a[0].postMessage(
+        { data: l, index: 0 },
+        p(l)
+      );
+    });
+  }
+  /**
+   * Runs a persistent worker that caches its dataset between calls.
+   *
+   * On the first call, provide both `dataset` and `config`. The worker stores
+   * the dataset in memory. On subsequent calls, only `config` is needed — the
+   * worker reuses the cached dataset and reprocesses it with the new config.
+   *
+   * The worker stays alive until {@link release} is called.
+   *
+   * @param workerName - Name of the registered worker.
+   * @param params - Object with optional `dataset` and required `config`.
+   * @returns The worker function's return value.
+   *
+   * @example
+   * // First call: send dataset + config
+   * const r1 = await factory.runPersistent('transform', {
+   *   dataset: largeArray,
+   *   config: { multiplier: 2 },
+   * });
+   *
+   * // Subsequent calls: only config, dataset is cached
+   * const r2 = await factory.runPersistent('transform', {
+   *   config: { multiplier: 5 },
+   * });
+   *
+   * // Update dataset when needed
+   * const r3 = await factory.runPersistent('transform', {
+   *   dataset: newArray,
+   *   config: { multiplier: 3 },
+   * });
+   *
+   * // Release when done
+   * factory.release('transform');
+   */
+  async runPersistent(e, t) {
+    const n = this.findWorkerByName(e);
+    if (!n) throw new Error(`Worker "${e}" not found`);
+    let a = this._persistentWorkers.get(e);
+    return a || (a = new k(n.func, {
+      mode: v.Persistent
+    }).getWorker, this._persistentWorkers.set(e, a)), new Promise((f, u) => {
+      a.onmessage = (r) => {
+        var o, i;
+        ((o = r.data) == null ? void 0 : o.ok) === !1 ? u(new Error(r.data.error)) : f((i = r.data) == null ? void 0 : i.data);
+      }, a.onerror = (r) => {
+        u(r);
+      };
+      const l = {
+        type: "run",
+        config: t.config
+      };
+      t.dataset !== void 0 && (l.dataset = t.dataset), a.postMessage(l, p(l));
+    });
+  }
+  /**
+   * Releases a persistent worker, freeing its cached dataset and terminating
+   * the thread.
+   *
+   * After calling `release`, subsequent `runPersistent` calls for this worker
+   * will create a fresh instance (requiring a new dataset).
+   *
+   * @param workerName - Name of the persistent worker to release.
+   */
+  release(e) {
+    const t = this._persistentWorkers.get(e);
+    t && (t.postMessage({ type: "release" }), t.terminate(), this._persistentWorkers.delete(e));
+  }
 }
 export {
-  b as MainWorkerFactory,
-  w as WorkerFactory
+  B as MainWorkerFactory,
+  k as WorkerFactory
 };

package/dist/types/tools/main-worker-factory/index.d.ts

@@ -1,2 +1,2 @@
 export { default as MainWorkerFactory } from './main-worker-factory.ts';
-export type { WorkerFunction, MainWorkerFactoryWorker, MainWorkerFactoryOptions, WorkerConfig, WorkerName, WorkerRole, } from './types.ts';
+export type { WorkerFunction, MainWorkerFactoryWorker, MainWorkerFactoryOptions, PipelineStep, WorkerConfig, WorkerName, WorkerRole, } from './types.ts';

package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts

@@ -1,4 +1,4 @@
-import { CollectOptions, CollectedResult, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
+import { CollectOptions, CollectedResult, PipelineStep, WorkerConfig, WorkerConfigMap, WorkerDataParam, WorkerFunction, WorkerReturnType, TypedSettledResults } from './types.ts';
 /**
  * Recursively collects all Transferable objects from a value.
  * Transferables (ArrayBuffer, MessagePort, ImageBitmap, OffscreenCanvas)
@@ -12,6 +12,13 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
  * handles the full lifecycle of each worker: spawning, partitioning input
  * data across threads, retrying on failure, and collecting results.
  *
+ * Also supports:
+ * - **Pipelines** — chain workers via `MessageChannel` so intermediate data
+ *   never crosses back to the main thread ({@link pipeline}).
+ * - **Persistent workers** — keep a worker alive with a cached dataset,
+ *   re-running it with different configs without re-sending the data
+ *   ({@link runPersistent}, {@link release}).
+ *
  * @typeParam TConfigs - A readonly tuple of {@link WorkerConfig} objects that
  *   defines the set of available workers and their typed signatures.
  *
@@ -19,7 +26,7 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
  * const foreman = new MainWorkerFactory({
  *   workers: [
  *     { name: 'sum', role: 'compute', func: sumWorker, partition: true },
- *   ],
+ *   ] as const,
  * });
  *
  * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3, 4] });
@@ -28,6 +35,7 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
 declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFunction<any, any>>[]> {
     private readonly _workers;
     private readonly _threads;
+    private readonly _persistentWorkers;
     /**
      * Creates a new `MainWorkerFactory`.
      *
@@ -176,5 +184,80 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      * });
      */
     collectResults<T = unknown, R = T extends (infer Item)[] ? Item[] : T[]>(settled: TypedSettledResults<T>, options?: CollectOptions<T, R>): Promise<CollectedResult<R>>;
+    /**
+     * Runs a chain of workers where each step's output feeds directly into the
+     * next step — **without passing through the main thread**.
+     *
+     * Internally, adjacent workers are connected via `MessageChannel` ports.
+     * Only the final result is sent back to the main thread, minimising
+     * serialisation overhead for large intermediate data.
+     *
+     * @typeParam TResult - The expected type of the final pipeline output.
+     *   Defaults to `unknown` if not specified.
+     *
+     * @param steps - An ordered array of {@link PipelineStep} objects. The first
+     *   step must include `srcData`; subsequent steps receive the previous
+     *   step's output as `{ data: previousOutput, index: 0 }`.
+     *
+     * @returns A promise that resolves with the final step's output.
+     * @throws {Error} When `steps` is empty or a worker name is not found.
+     *
+     * @example
+     * const result = await foreman.pipeline<FilteredPost[]>([
+     *   { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
+     *   { worker: 'transformPosts' },
+     *   { worker: 'filterPosts' },
+     * ]);
+     * console.log(result); // final transformed + filtered data
+     */
+    pipeline<TResult = unknown>(steps: PipelineStep[]): Promise<TResult>;
+    /**
+     * Runs a persistent worker that caches its dataset between calls.
+     *
+     * On the first call, provide both `dataset` and `config`. The worker stores
+     * the dataset in memory. On subsequent calls, only `config` is needed — the
+     * worker reuses the cached dataset and reprocesses it with the new config.
+     *
+     * The worker stays alive until {@link release} is called.
+     *
+     * @param workerName - Name of the registered worker.
+     * @param params - Object with optional `dataset` and required `config`.
+     * @returns The worker function's return value.
+     *
+     * @example
+     * // First call: send dataset + config
+     * const r1 = await factory.runPersistent('transform', {
+     *   dataset: largeArray,
+     *   config: { multiplier: 2 },
+     * });
+     *
+     * // Subsequent calls: only config, dataset is cached
+     * const r2 = await factory.runPersistent('transform', {
+     *   config: { multiplier: 5 },
+     * });
+     *
+     * // Update dataset when needed
+     * const r3 = await factory.runPersistent('transform', {
+     *   dataset: newArray,
+     *   config: { multiplier: 3 },
+     * });
+     *
+     * // Release when done
+     * factory.release('transform');
+     */
+    runPersistent<TResult = unknown>(workerName: string, params: {
+        dataset?: unknown;
+        config: unknown;
+    }): Promise<TResult>;
+    /**
+     * Releases a persistent worker, freeing its cached dataset and terminating
+     * the thread.
+     *
+     * After calling `release`, subsequent `runPersistent` calls for this worker
+     * will create a fresh instance (requiring a new dataset).
+     *
+     * @param workerName - Name of the persistent worker to release.
+     */
+    release(workerName: string): void;
 }
 export default MainWorkerFactory;

package/dist/types/tools/main-worker-factory/types.d.ts

@@ -147,3 +147,10 @@ export interface CollectedResult<R> {
     /** Raw rejected results, if any */
     errors: PromiseRejectedResult[];
 }
+/** A single step in a worker pipeline */
+export interface PipelineStep {
+    /** Name of the registered worker to run */
+    worker: string;
+    /** Input data for the first step (subsequent steps receive previous output) */
+    srcData?: unknown;
+}

package/dist/types/tools/worker-factory/worker-factory.d.ts

@@ -1,4 +1,13 @@
 import { WorkerFunction } from '../main-worker-factory/types';
+export declare enum WorkerMode {
+    Default = "default",
+    Pipeline = "pipeline",
+    Persistent = "persistent"
+}
+export interface WorkerFactoryOptions {
+    /** The worker execution mode. Defaults to `WorkerMode.Default`. */
+    mode?: WorkerMode;
+}
 /**
  * Low-level factory that serialises a {@link WorkerFunction} into a Blob URL
  * and spawns a native `Worker` from it.
@@ -7,8 +16,18 @@ import { WorkerFunction } from '../main-worker-factory/types';
  * It handles the mechanics of turning a plain TypeScript function into a
  * runnable worker thread — you rarely need to use it directly.
  *
- * The worker script is generated by {@link workerTemplate}, which wraps the
- * function with a message listener and transferable-extraction logic.
+ * Supports three execution modes via {@link WorkerMode}:
+ * - **Default** — one-shot worker that processes a single message and is
+ *   terminated after responding.
+ * - **Pipeline** — stays alive and forwards output to the next worker via
+ *   `MessagePort`, enabling worker-to-worker data flow without main-thread
+ *   round-trips.
+ * - **Persistent** — stays alive indefinitely, caches a dataset in memory,
+ *   and re-processes it with different configs on subsequent messages.
+ *
+ * The worker script is generated by the template corresponding to the chosen
+ * mode, which wraps the function with a message listener and
+ * transferable-extraction logic.
  */
 declare class WorkerFactory {
     readonly _worker: Worker;
@@ -21,8 +40,10 @@ declare class WorkerFactory {
      * @param workerFunction - The function to run inside the worker thread.
      *   Must be self-contained — it cannot reference variables from the outer
      *   scope because it is serialised via `.toString()`.
+     * @param options - Optional configuration. Set `mode` to control the
+     *   worker execution mode (default, pipeline, or persistent).
      */
-    constructor(workerFunction: WorkerFunction);
+    constructor(workerFunction: WorkerFunction, options?: WorkerFactoryOptions);
     /**
      * Returns the underlying native `Worker` instance.
      *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@offmain/workerkit",
-  "version": "0.9.2",
+  "version": "0.11.0",
   "description": "A lightweight manager for running functions in Web Workers with partitioning, retries, and concurrency control",
   "type": "module",
   "main": "./dist/index.cjs",