@offmain/workerkit 0.10.0 → 0.11.0

package/README.md

@@ -154,6 +154,110 @@ console.log(result); // only this small result crossed to main thread
 
 ---
 
+## Persistent Workers
+
+Keep a worker alive with a cached dataset, then re-run it with different configs without re-sending the data.
+
+### Why use persistent workers?
+
+In a typical workflow where you apply multiple transformations to the same dataset, the standard `runWorker` approach re-serializes the entire dataset on every call:
+
+```
+Call 1: Main ──[200k items]──→ Worker → Main
+Call 2: Main ──[200k items]──→ Worker → Main   ← same data, different config
+Call 3: Main ──[200k items]──→ Worker → Main   ← same data again
+```
+
+With 5 config variations on a 1.6 MB dataset, that's ~8 MB of redundant serialization. Persistent workers eliminate this by caching the dataset inside the worker:
+
+```
+Call 1: Main ──[200k items + config]──→ Worker → Main   ← dataset cached
+Call 2: Main ──[config only]──────────→ Worker → Main   ← reuses cache
+Call 3: Main ──[config only]──────────→ Worker → Main   ← reuses cache
+```
+
+Only the first call transfers the dataset. Subsequent calls send just the config object (typically a few bytes), saving both serialization time and memory pressure.
+
+### Usage
+
+```ts
+import { MainWorkerFactory } from '@offmain/workerkit';
+import { transformArray } from './transform.worker.ts';
+
+const factory = new MainWorkerFactory({
+  workers: [
+    { name: 'transform', role: 'computation', func: transformArray },
+  ] as const,
+});
+
+// First call: send dataset + config (dataset gets cached in worker memory)
+const r1 = await factory.runPersistent('transform', {
+  dataset: largeArray,
+  config: { multiplier: 2, filter: 'even' },
+});
+
+// Subsequent calls: only config — dataset is reused from cache
+const r2 = await factory.runPersistent('transform', {
+  config: { multiplier: 5, filter: 'odd' },
+});
+
+const r3 = await factory.runPersistent('transform', {
+  config: { multiplier: 1, filter: 'none', limit: 1000 },
+});
+
+// Update the dataset when it changes
+const r4 = await factory.runPersistent('transform', {
+  dataset: newArray, // replaces cached dataset
+  config: { multiplier: 3, filter: 'even' },
+});
+
+// Release the worker when done — frees memory
+factory.release('transform');
+```
+
+### How the worker function receives data
+
+The worker function signature stays the same as a regular worker — it receives `{ data, config }`:
+
+```ts
+// transform.worker.ts
+export function transformArray({
+  data,
+  config,
+}: {
+  data: number[];
+  config: { multiplier: number; filter: string };
+}) {
+  return data
+    .filter((n) => /* apply filter */)
+    .map((n) => n * config.multiplier);
+}
+```
+
+The framework handles the caching transparently — your function always receives the full `data` (from cache or freshly provided) plus the current `config`.
+
+### When to use persistent vs runWorker
+
+| Scenario                                               | Use             |
+| ------------------------------------------------------ | --------------- |
+| One-off computation                                    | `runWorker`     |
+| Same dataset, multiple config variations               | `runPersistent` |
+| Interactive UI where user tweaks params on static data | `runPersistent` |
+| Dataset changes frequently                             | `runWorker`     |
+| Need partitioning across multiple threads              | `runWorker`     |
+
+### Memory management
+
+The cached dataset lives in worker memory until `release()` is called. For large datasets, always call `release()` when you're done to free the memory:
+
+```ts
+factory.release('transform');
+```
+
+After releasing, the next `runPersistent` call will create a fresh worker instance (requiring a new dataset).
+
+---
+
 ## ESLint Plugin
 
 The package ships with two ESLint rules to catch common worker mistakes at lint time.

package/dist/index.cjs

@@ -1,4 +1,4 @@
-"use strict";var w=Object.defineProperty;var y=(o,e,t)=>e in o?w(o,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):o[e]=t;var h=(o,e,t)=>y(o,typeof e!="symbol"?e+"":e,t);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const m=o=>`
+"use strict";var y=Object.defineProperty;var m=(s,e,t)=>e in s?y(s,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):s[e]=t;var h=(s,e,t)=>m(s,typeof e!="symbol"?e+"":e,t);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const w=s=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -15,13 +15,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${o}(event.data);
+    const output = await ${s}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`,v=o=>`
+`,P=s=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -36,7 +36,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${o};
+const workerFn = ${s};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
@@ -93,8 +93,64 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
-`;class k{constructor(e,t){h(this,"_worker");const a=(t!=null&&t.pipeline?v:m)(e.toString()),u=new Blob([a],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(u))}get getWorker(){return this._worker}}class P{constructor(e){this.results=e}}function g(o,e=new Set){return o===null||typeof o!="object"?[]:e.has(o)?[]:(e.add(o),o instanceof ArrayBuffer||o instanceof MessagePort||typeof ImageBitmap<"u"&&o instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&o instanceof OffscreenCanvas?[o]:ArrayBuffer.isView(o)?[o.buffer]:Array.isArray(o)?o.flatMap(t=>g(t,e)):Object.values(o).flatMap(t=>g(t,e)))}class W{constructor(e){h(this,"_workers");h(this,"_threads");this._workers=e.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new k(e)}partitionArray(e,t){if(!e.length)return[];if(t<=0)throw new Error("numChunks must be positive");const n=Math.min(t,e.length),a=Math.floor(e.length/n),u=e.length%n,l=[];let f=0;for(let r=0;r<n;r++){const s=a+(r<u?1:0);l.push(e.slice(f,f+s)),f+=s}return l}findWorkerByName(e){return this._workers.find(t=>t.name===e)}async runWorker(e,{srcData:t,...n}){const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const u=a.maxConcurrency??this._threads,l=!!(Array.isArray(t)&&t.length>1&&a.partition),f=l?this.partitionArray(t,u):t,r=this.createWorkerPromises(a,e,{data:f,...n},u,l),s=await Promise.allSettled(r);return new P(s)}createWorkerPromises(e,t,n,a,u){const{data:l,...f}=n;return Array.from({length:a},(r,s)=>{const i=u&&Array.isArray(l)?l[s]:l;return this.runWorkerWithRetry({workerFunc:e.func,workerName:t,index:s,data:{data:i,...f}},e.retries)})}async runWorkerWithRetry(e,t=2){try{return await this.initiateWorker(e)}catch(n){if(t>0)return console.error(`Worker ${e.index} failed, retrying (${t} left):`,n),this.runWorkerWithRetry(e,t-1);throw console.error("Worker failed after all retries:",n),n}}initiateWorker({workerFunc:e,workerName:t,index:n,data:a}){return new Promise((u,l)=>{const r=this.initWorker(e).getWorker;r.onerror=i=>{r.terminate(),l({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:i})},r.onmessage=i=>{var c,p;if(((c=i.data)==null?void 0:c.ok)===!1){r.terminate(),l({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:new ErrorEvent("error",{message:i.data.error})});return}u({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},successResult:new MessageEvent("message",{data:(p=i.data)==null?void 0:p.data})}),r.terminate()};const s={index:n,...Array.isArray(a)?{data:a}:a};r.postMessage(s,g(s))})}async collectResults(e,t={}){const n=e.results.filter(r=>r.status==="fulfilled"),a=e.results.filter(r=>r.status==="rejected"),u=n.map(r=>r.value.successResult.data),l=t.reducer?t.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((r,s)=>{const i=`
-        const reducer = ${l};
+`,M=s=>`
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let cachedDataset = null;
+
+self.addEventListener('message', async (event) => {
+  const msg = event.data;
+
+  if (msg && msg.type === 'release') {
+    cachedDataset = null;
+    self.postMessage({ ok: true, data: null, type: 'released' });
+    self.close();
+    return;
+  }
+
+  if (msg && msg.type === 'run') {
+    // Update cache if new dataset provided
+    if (msg.dataset !== undefined) {
+      cachedDataset = msg.dataset;
+    }
+
+    if (cachedDataset === null) {
+      self.postMessage({ ok: false, error: 'No dataset cached. Provide a dataset on the first call.' });
+      return;
+    }
+
+    try {
+      const output = await workerFn({ data: cachedDataset, config: msg.config });
+      self.postMessage({ ok: true, data: output }, extractTransferables(output));
+    } catch (err) {
+      self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+    }
+    return;
+  }
+
+  // Fallback: treat as a regular one-shot call for backwards compat
+  try {
+    const output = await workerFn(msg);
+    self.postMessage({ ok: true, data: output }, extractTransferables(output));
+  } catch (err) {
+    self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+});
+`;var v=(s=>(s.Default="default",s.Pipeline="pipeline",s.Persistent="persistent",s))(v||{});const W=Object.freeze({persistent:M,pipeline:P,default:w});class k{constructor(e,t){h(this,"_worker");const n=(t==null?void 0:t.mode)??"default",a=W[n](e.toString()),u=new Blob([a],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(u))}get getWorker(){return this._worker}}class b{constructor(e){this.results=e}}function p(s,e=new Set){return s===null||typeof s!="object"?[]:e.has(s)?[]:(e.add(s),s instanceof ArrayBuffer||s instanceof MessagePort||typeof ImageBitmap<"u"&&s instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&s instanceof OffscreenCanvas?[s]:ArrayBuffer.isView(s)?[s.buffer]:Array.isArray(s)?s.flatMap(t=>p(t,e)):Object.values(s).flatMap(t=>p(t,e)))}class A{constructor(e){h(this,"_workers");h(this,"_threads");h(this,"_persistentWorkers",new Map);this._workers=e.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new k(e)}partitionArray(e,t){if(!e.length)return[];if(t<=0)throw new Error("numChunks must be positive");const n=Math.min(t,e.length),a=Math.floor(e.length/n),u=e.length%n,f=[];let l=0;for(let r=0;r<n;r++){const o=a+(r<u?1:0);f.push(e.slice(l,l+o)),l+=o}return f}findWorkerByName(e){return this._workers.find(t=>t.name===e)}async runWorker(e,{srcData:t,...n}){const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const u=a.maxConcurrency??this._threads,f=!!(Array.isArray(t)&&t.length>1&&a.partition),l=f?this.partitionArray(t,u):t,r=this.createWorkerPromises(a,e,{data:l,...n},u,f),o=await Promise.allSettled(r);return new b(o)}createWorkerPromises(e,t,n,a,u){const{data:f,...l}=n;return Array.from({length:a},(r,o)=>{const i=u&&Array.isArray(f)?f[o]:f;return this.runWorkerWithRetry({workerFunc:e.func,workerName:t,index:o,data:{data:i,...l}},e.retries)})}async runWorkerWithRetry(e,t=2){try{return await this.initiateWorker(e)}catch(n){if(t>0)return console.error(`Worker ${e.index} failed, retrying (${t} left):`,n),this.runWorkerWithRetry(e,t-1);throw console.error("Worker failed after all retries:",n),n}}initiateWorker({workerFunc:e,workerName:t,index:n,data:a}){return new Promise((u,f)=>{const r=this.initWorker(e).getWorker;r.onerror=i=>{r.terminate(),f({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:i})},r.onmessage=i=>{var c,d;if(((c=i.data)==null?void 0:c.ok)===!1){r.terminate(),f({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:new ErrorEvent("error",{message:i.data.error})});return}u({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},successResult:new MessageEvent("message",{data:(d=i.data)==null?void 0:d.data})}),r.terminate()};const o={index:n,...Array.isArray(a)?{data:a}:a};r.postMessage(o,p(o))})}async collectResults(e,t={}){const n=e.results.filter(r=>r.status==="fulfilled"),a=e.results.filter(r=>r.status==="rejected"),u=n.map(r=>r.value.successResult.data),f=t.reducer?t.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((r,o)=>{const i=`
+        const reducer = ${f};
         self.addEventListener('message', (event) => {
           try {
             const result = reducer(event.data);
@@ -103,4 +159,4 @@ self.addEventListener('message', (event) => {
             self.postMessage({ ok: false, error: String(err) });
           }
         });
-      `,c=new Blob([i],{type:"application/javascript"}),p=new Worker(URL.createObjectURL(c));p.onmessage=d=>{p.terminate(),d.data.ok?r(d.data.data):s(new Error(d.data.error))},p.onerror=d=>{p.terminate(),s(d)},p.postMessage(u)}),succeeded:n.length,failed:a.length,errors:a}}async pipeline(e){if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const t=e[0],n=this.findWorkerByName(t.worker);if(!n)throw new Error(`Worker "${t.worker}" not found`);const u=this.initWorker(n.func).getWorker;return new Promise((l,f)=>{u.onmessage=s=>{var i,c;u.terminate(),((i=s.data)==null?void 0:i.ok)===!1?f(new Error(s.data.error)):l((c=s.data)==null?void 0:c.data)},u.onerror=s=>{u.terminate(),f(s)};const r=t.srcData??{};u.postMessage({data:r,index:0},g(r))})}return new Promise((t,n)=>{const a=[],u=[];for(const r of e){const s=this.findWorkerByName(r.worker);if(!s){n(new Error(`Worker "${r.worker}" not found`));return}const i=new k(s.func,{pipeline:!0});a.push(i.getWorker)}for(let r=0;r<a.length-1;r++)u.push(new MessageChannel);for(let r=0;r<a.length;r++){const s=[],i={};r>0&&(i.inputPort=u[r-1].port1,s.push(i.inputPort)),r<a.length-1&&(i.outputPort=u[r].port2,s.push(i.outputPort)),a[r].postMessage({__pipeline_ports__:!0,...i},s)}const l=a[a.length-1];l.onmessage=r=>{var s,i;a.forEach(c=>c.terminate()),((s=r.data)==null?void 0:s.ok)===!1?n(new Error(r.data.error)):t((i=r.data)==null?void 0:i.data)},l.onerror=r=>{a.forEach(s=>s.terminate()),n(r)};const f=e[0].srcData??{};a[0].postMessage({data:f,index:0},g(f))})}}exports.MainWorkerFactory=W;exports.WorkerFactory=k;
+      `,c=new Blob([i],{type:"application/javascript"}),d=new Worker(URL.createObjectURL(c));d.onmessage=g=>{d.terminate(),g.data.ok?r(g.data.data):o(new Error(g.data.error))},d.onerror=g=>{d.terminate(),o(g)},d.postMessage(u)}),succeeded:n.length,failed:a.length,errors:a}}async pipeline(e){if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const t=e[0],n=this.findWorkerByName(t.worker);if(!n)throw new Error(`Worker "${t.worker}" not found`);const u=this.initWorker(n.func).getWorker;return new Promise((f,l)=>{u.onmessage=o=>{var i,c;u.terminate(),((i=o.data)==null?void 0:i.ok)===!1?l(new Error(o.data.error)):f((c=o.data)==null?void 0:c.data)},u.onerror=o=>{u.terminate(),l(o)};const r=t.srcData??{};u.postMessage({data:r,index:0},p(r))})}return new Promise((t,n)=>{const a=[],u=[];for(const r of e){const o=this.findWorkerByName(r.worker);if(!o){n(new Error(`Worker "${r.worker}" not found`));return}const i=new k(o.func,{mode:v.Pipeline});a.push(i.getWorker)}for(let r=0;r<a.length-1;r++)u.push(new MessageChannel);for(let r=0;r<a.length;r++){const o=[],i={};r>0&&(i.inputPort=u[r-1].port1,o.push(i.inputPort)),r<a.length-1&&(i.outputPort=u[r].port2,o.push(i.outputPort)),a[r].postMessage({__pipeline_ports__:!0,...i},o)}const f=a[a.length-1];f.onmessage=r=>{var o,i;a.forEach(c=>c.terminate()),((o=r.data)==null?void 0:o.ok)===!1?n(new Error(r.data.error)):t((i=r.data)==null?void 0:i.data)},f.onerror=r=>{a.forEach(o=>o.terminate()),n(r)};const l=e[0].srcData??{};a[0].postMessage({data:l,index:0},p(l))})}async runPersistent(e,t){const n=this.findWorkerByName(e);if(!n)throw new Error(`Worker "${e}" not found`);let a=this._persistentWorkers.get(e);return a||(a=new k(n.func,{mode:v.Persistent}).getWorker,this._persistentWorkers.set(e,a)),new Promise((u,f)=>{a.onmessage=r=>{var o,i;((o=r.data)==null?void 0:o.ok)===!1?f(new Error(r.data.error)):u((i=r.data)==null?void 0:i.data)},a.onerror=r=>{f(r)};const l={type:"run",config:t.config};t.dataset!==void 0&&(l.dataset=t.dataset),a.postMessage(l,p(l))})}release(e){const t=this._persistentWorkers.get(e);t&&(t.postMessage({type:"release"}),t.terminate(),this._persistentWorkers.delete(e))}}exports.MainWorkerFactory=A;exports.WorkerFactory=k;

package/dist/index.js

@@ -1,7 +1,7 @@
 var w = Object.defineProperty;
-var m = (o, e, t) => e in o ? w(o, e, { enumerable: !0, configurable: !0, writable: !0, value: t }) : o[e] = t;
-var h = (o, e, t) => m(o, typeof e != "symbol" ? e + "" : e, t);
-const y = (o) => `
+var m = (s, e, t) => e in s ? w(s, e, { enumerable: !0, configurable: !0, writable: !0, value: t }) : s[e] = t;
+var h = (s, e, t) => m(s, typeof e != "symbol" ? e + "" : e, t);
+const y = (s) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -18,13 +18,13 @@ const extractTransferables = (value, seen = new Set()) => {
 
 self.addEventListener('message', async (event) => {
   try {
-    const output = await ${o}(event.data);
+    const output = await ${s}(event.data);
     self.postMessage({ ok: true, data: output }, extractTransferables(output));
   } catch (err) {
     self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
   }
 })
-`, v = (o) => `
+`, P = (s) => `
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -39,7 +39,7 @@ const extractTransferables = (value, seen = new Set()) => {
   return Object.values(value).flatMap(v => extractTransferables(v, seen));
 };
 
-const workerFn = ${o};
+const workerFn = ${s};
 let outputPort = null;
 let inputPort = null;
 let pendingData = null;
@@ -96,7 +96,69 @@ self.addEventListener('message', (event) => {
     pendingData = event.data;
   }
 });
+`, M = (s) => `
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${s};
+let cachedDataset = null;
+
+self.addEventListener('message', async (event) => {
+  const msg = event.data;
+
+  if (msg && msg.type === 'release') {
+    cachedDataset = null;
+    self.postMessage({ ok: true, data: null, type: 'released' });
+    self.close();
+    return;
+  }
+
+  if (msg && msg.type === 'run') {
+    // Update cache if new dataset provided
+    if (msg.dataset !== undefined) {
+      cachedDataset = msg.dataset;
+    }
+
+    if (cachedDataset === null) {
+      self.postMessage({ ok: false, error: 'No dataset cached. Provide a dataset on the first call.' });
+      return;
+    }
+
+    try {
+      const output = await workerFn({ data: cachedDataset, config: msg.config });
+      self.postMessage({ ok: true, data: output }, extractTransferables(output));
+    } catch (err) {
+      self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+    }
+    return;
+  }
+
+  // Fallback: treat as a regular one-shot call for backwards compat
+  try {
+    const output = await workerFn(msg);
+    self.postMessage({ ok: true, data: output }, extractTransferables(output));
+  } catch (err) {
+    self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+});
 `;
+var v = /* @__PURE__ */ ((s) => (s.Default = "default", s.Pipeline = "pipeline", s.Persistent = "persistent", s))(v || {});
+const W = Object.freeze({
+  persistent: M,
+  pipeline: P,
+  default: y
+});
 class k {
   /**
    * Creates a new `Worker` from the given function.
@@ -107,15 +169,15 @@ class k {
    * @param workerFunction - The function to run inside the worker thread.
    *   Must be self-contained — it cannot reference variables from the outer
    *   scope because it is serialised via `.toString()`.
-   * @param options - Optional configuration. Set `pipeline: true` for
-   *   pipeline-aware workers that support MessagePort forwarding.
+   * @param options - Optional configuration. Set `mode` to control the
+   *   worker execution mode (default, pipeline, or persistent).
    */
   constructor(e, t) {
     h(this, "_worker");
-    const a = (t != null && t.pipeline ? v : y)(e.toString()), u = new Blob([a], {
+    const n = (t == null ? void 0 : t.mode) ?? "default", a = W[n](e.toString()), f = new Blob([a], {
       type: "application/javascript"
     });
-    this._worker = new Worker(URL.createObjectURL(u));
+    this._worker = new Worker(URL.createObjectURL(f));
   }
   /**
    * Returns the underlying native `Worker` instance.
@@ -127,17 +189,17 @@ class k {
     return this._worker;
   }
 }
-class P {
+class b {
   constructor(e) {
     this.results = e;
   }
 }
-function g(o, e = /* @__PURE__ */ new Set()) {
-  return o === null || typeof o != "object" ? [] : e.has(o) ? [] : (e.add(o), o instanceof ArrayBuffer || o instanceof MessagePort || typeof ImageBitmap < "u" && o instanceof ImageBitmap || typeof OffscreenCanvas < "u" && o instanceof OffscreenCanvas ? [o] : ArrayBuffer.isView(o) ? [o.buffer] : Array.isArray(o) ? o.flatMap((t) => g(t, e)) : Object.values(o).flatMap(
-    (t) => g(t, e)
+function p(s, e = /* @__PURE__ */ new Set()) {
+  return s === null || typeof s != "object" ? [] : e.has(s) ? [] : (e.add(s), s instanceof ArrayBuffer || s instanceof MessagePort || typeof ImageBitmap < "u" && s instanceof ImageBitmap || typeof OffscreenCanvas < "u" && s instanceof OffscreenCanvas ? [s] : ArrayBuffer.isView(s) ? [s.buffer] : Array.isArray(s) ? s.flatMap((t) => p(t, e)) : Object.values(s).flatMap(
+    (t) => p(t, e)
   ));
 }
-class M {
+class B {
   /**
    * Creates a new `MainWorkerFactory`.
    *
@@ -146,6 +208,7 @@ class M {
   constructor(e) {
     h(this, "_workers");
     h(this, "_threads");
+    h(this, "_persistentWorkers", /* @__PURE__ */ new Map());
     this._workers = e.workers, this._threads = navigator.hardwareConcurrency;
   }
   /**
@@ -176,13 +239,13 @@ class M {
   partitionArray(e, t) {
     if (!e.length) return [];
     if (t <= 0) throw new Error("numChunks must be positive");
-    const n = Math.min(t, e.length), a = Math.floor(e.length / n), u = e.length % n, l = [];
-    let f = 0;
+    const n = Math.min(t, e.length), a = Math.floor(e.length / n), f = e.length % n, u = [];
+    let l = 0;
     for (let r = 0; r < n; r++) {
-      const s = a + (r < u ? 1 : 0);
-      l.push(e.slice(f, f + s)), f += s;
+      const o = a + (r < f ? 1 : 0);
+      u.push(e.slice(l, l + o)), l += o;
     }
-    return l;
+    return u;
   }
   /**
    * Looks up a registered worker configuration by name.
@@ -226,14 +289,14 @@ class M {
     const a = this.findWorkerByName(e);
     if (!a)
       return Promise.reject(new Error(`Worker "${e}" not found`));
-    const u = a.maxConcurrency ?? this._threads, l = !!(Array.isArray(t) && t.length > 1 && a.partition), f = l ? this.partitionArray(t, u) : t, r = this.createWorkerPromises(
+    const f = a.maxConcurrency ?? this._threads, u = !!(Array.isArray(t) && t.length > 1 && a.partition), l = u ? this.partitionArray(t, f) : t, r = this.createWorkerPromises(
       a,
       e,
-      { data: f, ...n },
-      u,
-      l
-    ), s = await Promise.allSettled(r);
-    return new P(s);
+      { data: l, ...n },
+      f,
+      u
+    ), o = await Promise.allSettled(r);
+    return new b(o);
   }
   /**
    * Builds the array of per-thread worker promises for a single `runWorker`
@@ -249,16 +312,16 @@ class M {
    * @param isPartitioned  - Whether `data` is a pre-split array of shards.
    * @returns An array of promises, one per thread.
    */
-  createWorkerPromises(e, t, n, a, u) {
-    const { data: l, ...f } = n;
-    return Array.from({ length: a }, (r, s) => {
-      const i = u && Array.isArray(l) ? l[s] : l;
+  createWorkerPromises(e, t, n, a, f) {
+    const { data: u, ...l } = n;
+    return Array.from({ length: a }, (r, o) => {
+      const i = f && Array.isArray(u) ? u[o] : u;
       return this.runWorkerWithRetry(
         {
           workerFunc: e.func,
           workerName: t,
-          index: s,
-          data: { data: i, ...f }
+          index: o,
+          data: { data: i, ...l }
         },
         e.retries
       );
@@ -312,18 +375,18 @@ class M {
     index: n,
     data: a
   }) {
-    return new Promise((u, l) => {
+    return new Promise((f, u) => {
       const r = this.initWorker(e).getWorker;
       r.onerror = (i) => {
-        r.terminate(), l({
+        r.terminate(), u({
           index: n,
           workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
           failedResult: i
         });
       }, r.onmessage = (i) => {
-        var c, p;
+        var c, d;
         if (((c = i.data) == null ? void 0 : c.ok) === !1) {
-          r.terminate(), l({
+          r.terminate(), u({
             index: n,
             workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
             failedResult: new ErrorEvent("error", {
@@ -332,19 +395,19 @@ class M {
           });
           return;
         }
-        u({
+        f({
           index: n,
           workerConfigs: { workerFunc: e, workerName: t, index: n, data: a },
           successResult: new MessageEvent("message", {
-            data: (p = i.data) == null ? void 0 : p.data
+            data: (d = i.data) == null ? void 0 : d.data
           })
         }), r.terminate();
       };
-      const s = {
+      const o = {
         index: n,
         ...Array.isArray(a) ? { data: a } : a
       };
-      r.postMessage(s, g(s));
+      r.postMessage(o, p(o));
     });
   }
   /**
@@ -384,11 +447,11 @@ class M {
       (r) => r.status === "fulfilled"
     ), a = e.results.filter(
       (r) => r.status === "rejected"
-    ), u = n.map((r) => r.value.successResult.data), l = t.reducer ? t.reducer.toString() : "(shards) => shards.flat()";
+    ), f = n.map((r) => r.value.successResult.data), u = t.reducer ? t.reducer.toString() : "(shards) => shards.flat()";
     return {
-      data: await new Promise((r, s) => {
+      data: await new Promise((r, o) => {
         const i = `
-        const reducer = ${l};
+        const reducer = ${u};
         self.addEventListener('message', (event) => {
           try {
             const result = reducer(event.data);
@@ -397,12 +460,12 @@ class M {
             self.postMessage({ ok: false, error: String(err) });
           }
         });
-      `, c = new Blob([i], { type: "application/javascript" }), p = new Worker(URL.createObjectURL(c));
-        p.onmessage = (d) => {
-          p.terminate(), d.data.ok ? r(d.data.data) : s(new Error(d.data.error));
-        }, p.onerror = (d) => {
-          p.terminate(), s(d);
-        }, p.postMessage(u);
+      `, c = new Blob([i], { type: "application/javascript" }), d = new Worker(URL.createObjectURL(c));
+        d.onmessage = (g) => {
+          d.terminate(), g.data.ok ? r(g.data.data) : o(new Error(g.data.error));
+        }, d.onerror = (g) => {
+          d.terminate(), o(g);
+        }, d.postMessage(f);
       }),
       succeeded: n.length,
       failed: a.length,
@@ -417,13 +480,18 @@ class M {
    * Only the final result is sent back to the main thread, minimising
    * serialisation overhead for large intermediate data.
    *
-   * @param steps - An ordered array of pipeline steps. The first step must
-   *   include `srcData`; subsequent steps receive the previous step's output.
+   * @typeParam TResult - The expected type of the final pipeline output.
+   *   Defaults to `unknown` if not specified.
+   *
+   * @param steps - An ordered array of {@link PipelineStep} objects. The first
+   *   step must include `srcData`; subsequent steps receive the previous
+   *   step's output as `{ data: previousOutput, index: 0 }`.
    *
    * @returns A promise that resolves with the final step's output.
+   * @throws {Error} When `steps` is empty or a worker name is not found.
    *
    * @example
-   * const result = await foreman.pipeline([
+   * const result = await foreman.pipeline<FilteredPost[]>([
    *   { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
    *   { worker: 'transformPosts' },
    *   { worker: 'filterPosts' },
@@ -436,57 +504,126 @@ class M {
     if (e.length === 1) {
       const t = e[0], n = this.findWorkerByName(t.worker);
       if (!n) throw new Error(`Worker "${t.worker}" not found`);
-      const u = this.initWorker(n.func).getWorker;
-      return new Promise((l, f) => {
-        u.onmessage = (s) => {
+      const f = this.initWorker(n.func).getWorker;
+      return new Promise((u, l) => {
+        f.onmessage = (o) => {
           var i, c;
-          u.terminate(), ((i = s.data) == null ? void 0 : i.ok) === !1 ? f(new Error(s.data.error)) : l((c = s.data) == null ? void 0 : c.data);
-        }, u.onerror = (s) => {
-          u.terminate(), f(s);
+          f.terminate(), ((i = o.data) == null ? void 0 : i.ok) === !1 ? l(new Error(o.data.error)) : u((c = o.data) == null ? void 0 : c.data);
+        }, f.onerror = (o) => {
+          f.terminate(), l(o);
         };
         const r = t.srcData ?? {};
-        u.postMessage(
+        f.postMessage(
           { data: r, index: 0 },
-          g(r)
+          p(r)
         );
       });
     }
     return new Promise((t, n) => {
-      const a = [], u = [];
+      const a = [], f = [];
       for (const r of e) {
-        const s = this.findWorkerByName(r.worker);
-        if (!s) {
+        const o = this.findWorkerByName(r.worker);
+        if (!o) {
           n(new Error(`Worker "${r.worker}" not found`));
           return;
         }
-        const i = new k(s.func, { pipeline: !0 });
+        const i = new k(o.func, {
+          mode: v.Pipeline
+        });
         a.push(i.getWorker);
       }
       for (let r = 0; r < a.length - 1; r++)
-        u.push(new MessageChannel());
+        f.push(new MessageChannel());
       for (let r = 0; r < a.length; r++) {
-        const s = [], i = {};
-        r > 0 && (i.inputPort = u[r - 1].port1, s.push(i.inputPort)), r < a.length - 1 && (i.outputPort = u[r].port2, s.push(i.outputPort)), a[r].postMessage(
+        const o = [], i = {};
+        r > 0 && (i.inputPort = f[r - 1].port1, o.push(i.inputPort)), r < a.length - 1 && (i.outputPort = f[r].port2, o.push(i.outputPort)), a[r].postMessage(
           { __pipeline_ports__: !0, ...i },
-          s
+          o
         );
       }
-      const l = a[a.length - 1];
-      l.onmessage = (r) => {
-        var s, i;
-        a.forEach((c) => c.terminate()), ((s = r.data) == null ? void 0 : s.ok) === !1 ? n(new Error(r.data.error)) : t((i = r.data) == null ? void 0 : i.data);
-      }, l.onerror = (r) => {
-        a.forEach((s) => s.terminate()), n(r);
+      const u = a[a.length - 1];
+      u.onmessage = (r) => {
+        var o, i;
+        a.forEach((c) => c.terminate()), ((o = r.data) == null ? void 0 : o.ok) === !1 ? n(new Error(r.data.error)) : t((i = r.data) == null ? void 0 : i.data);
+      }, u.onerror = (r) => {
+        a.forEach((o) => o.terminate()), n(r);
       };
-      const f = e[0].srcData ?? {};
+      const l = e[0].srcData ?? {};
       a[0].postMessage(
-        { data: f, index: 0 },
-        g(f)
+        { data: l, index: 0 },
+        p(l)
       );
     });
   }
+  /**
+   * Runs a persistent worker that caches its dataset between calls.
+   *
+   * On the first call, provide both `dataset` and `config`. The worker stores
+   * the dataset in memory. On subsequent calls, only `config` is needed — the
+   * worker reuses the cached dataset and reprocesses it with the new config.
+   *
+   * The worker stays alive until {@link release} is called.
+   *
+   * @param workerName - Name of the registered worker.
+   * @param params - Object with optional `dataset` and required `config`.
+   * @returns The worker function's return value.
+   *
+   * @example
+   * // First call: send dataset + config
+   * const r1 = await factory.runPersistent('transform', {
+   *   dataset: largeArray,
+   *   config: { multiplier: 2 },
+   * });
+   *
+   * // Subsequent calls: only config, dataset is cached
+   * const r2 = await factory.runPersistent('transform', {
+   *   config: { multiplier: 5 },
+   * });
+   *
+   * // Update dataset when needed
+   * const r3 = await factory.runPersistent('transform', {
+   *   dataset: newArray,
+   *   config: { multiplier: 3 },
+   * });
+   *
+   * // Release when done
+   * factory.release('transform');
+   */
+  async runPersistent(e, t) {
+    const n = this.findWorkerByName(e);
+    if (!n) throw new Error(`Worker "${e}" not found`);
+    let a = this._persistentWorkers.get(e);
+    return a || (a = new k(n.func, {
+      mode: v.Persistent
+    }).getWorker, this._persistentWorkers.set(e, a)), new Promise((f, u) => {
+      a.onmessage = (r) => {
+        var o, i;
+        ((o = r.data) == null ? void 0 : o.ok) === !1 ? u(new Error(r.data.error)) : f((i = r.data) == null ? void 0 : i.data);
+      }, a.onerror = (r) => {
+        u(r);
+      };
+      const l = {
+        type: "run",
+        config: t.config
+      };
+      t.dataset !== void 0 && (l.dataset = t.dataset), a.postMessage(l, p(l));
+    });
+  }
+  /**
+   * Releases a persistent worker, freeing its cached dataset and terminating
+   * the thread.
+   *
+   * After calling `release`, subsequent `runPersistent` calls for this worker
+   * will create a fresh instance (requiring a new dataset).
+   *
+   * @param workerName - Name of the persistent worker to release.
+   */
+  release(e) {
+    const t = this._persistentWorkers.get(e);
+    t && (t.postMessage({ type: "release" }), t.terminate(), this._persistentWorkers.delete(e));
+  }
 }
 export {
-  M as MainWorkerFactory,
+  B as MainWorkerFactory,
   k as WorkerFactory
 };

package/dist/types/tools/main-worker-factory/main-worker-factory.d.ts

@@ -12,6 +12,13 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
  * handles the full lifecycle of each worker: spawning, partitioning input
  * data across threads, retrying on failure, and collecting results.
  *
+ * Also supports:
+ * - **Pipelines** — chain workers via `MessageChannel` so intermediate data
+ *   never crosses back to the main thread ({@link pipeline}).
+ * - **Persistent workers** — keep a worker alive with a cached dataset,
+ *   re-running it with different configs without re-sending the data
+ *   ({@link runPersistent}, {@link release}).
+ *
  * @typeParam TConfigs - A readonly tuple of {@link WorkerConfig} objects that
  *   defines the set of available workers and their typed signatures.
  *
@@ -19,7 +26,7 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
  * const foreman = new MainWorkerFactory({
  *   workers: [
  *     { name: 'sum', role: 'compute', func: sumWorker, partition: true },
- *   ],
+ *   ] as const,
  * });
  *
  * const settled = await foreman.runWorker('sum', { srcData: [1, 2, 3, 4] });
@@ -28,6 +35,7 @@ export declare function extractTransferables(value: unknown, seen?: Set<object>)
 declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFunction<any, any>>[]> {
     private readonly _workers;
     private readonly _threads;
+    private readonly _persistentWorkers;
     /**
      * Creates a new `MainWorkerFactory`.
      *
@@ -184,13 +192,18 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      * Only the final result is sent back to the main thread, minimising
      * serialisation overhead for large intermediate data.
      *
-     * @param steps - An ordered array of pipeline steps. The first step must
-     *   include `srcData`; subsequent steps receive the previous step's output.
+     * @typeParam TResult - The expected type of the final pipeline output.
+     *   Defaults to `unknown` if not specified.
+     *
+     * @param steps - An ordered array of {@link PipelineStep} objects. The first
+     *   step must include `srcData`; subsequent steps receive the previous
+     *   step's output as `{ data: previousOutput, index: 0 }`.
      *
      * @returns A promise that resolves with the final step's output.
+     * @throws {Error} When `steps` is empty or a worker name is not found.
      *
      * @example
-     * const result = await foreman.pipeline([
+     * const result = await foreman.pipeline<FilteredPost[]>([
      *   { worker: 'fetchPosts', srcData: { url: '/api/posts' } },
      *   { worker: 'transformPosts' },
      *   { worker: 'filterPosts' },
@@ -198,5 +211,53 @@ declare class MainWorkerFactory<TConfigs extends readonly WorkerConfig<WorkerFun
      * console.log(result); // final transformed + filtered data
      */
     pipeline<TResult = unknown>(steps: PipelineStep[]): Promise<TResult>;
+    /**
+     * Runs a persistent worker that caches its dataset between calls.
+     *
+     * On the first call, provide both `dataset` and `config`. The worker stores
+     * the dataset in memory. On subsequent calls, only `config` is needed — the
+     * worker reuses the cached dataset and reprocesses it with the new config.
+     *
+     * The worker stays alive until {@link release} is called.
+     *
+     * @param workerName - Name of the registered worker.
+     * @param params - Object with optional `dataset` and required `config`.
+     * @returns The worker function's return value.
+     *
+     * @example
+     * // First call: send dataset + config
+     * const r1 = await factory.runPersistent('transform', {
+     *   dataset: largeArray,
+     *   config: { multiplier: 2 },
+     * });
+     *
+     * // Subsequent calls: only config, dataset is cached
+     * const r2 = await factory.runPersistent('transform', {
+     *   config: { multiplier: 5 },
+     * });
+     *
+     * // Update dataset when needed
+     * const r3 = await factory.runPersistent('transform', {
+     *   dataset: newArray,
+     *   config: { multiplier: 3 },
+     * });
+     *
+     * // Release when done
+     * factory.release('transform');
+     */
+    runPersistent<TResult = unknown>(workerName: string, params: {
+        dataset?: unknown;
+        config: unknown;
+    }): Promise<TResult>;
+    /**
+     * Releases a persistent worker, freeing its cached dataset and terminating
+     * the thread.
+     *
+     * After calling `release`, subsequent `runPersistent` calls for this worker
+     * will create a fresh instance (requiring a new dataset).
+     *
+     * @param workerName - Name of the persistent worker to release.
+     */
+    release(workerName: string): void;
 }
 export default MainWorkerFactory;

package/dist/types/tools/worker-factory/worker-factory.d.ts

@@ -1,7 +1,12 @@
 import { WorkerFunction } from '../main-worker-factory/types';
+export declare enum WorkerMode {
+    Default = "default",
+    Pipeline = "pipeline",
+    Persistent = "persistent"
+}
 export interface WorkerFactoryOptions {
-    /** When true, generates a pipeline-aware worker that supports MessagePort forwarding */
-    pipeline?: boolean;
+    /** The worker execution mode. Defaults to `WorkerMode.Default`. */
+    mode?: WorkerMode;
 }
 /**
  * Low-level factory that serialises a {@link WorkerFunction} into a Blob URL
@@ -11,8 +16,18 @@ export interface WorkerFactoryOptions {
  * It handles the mechanics of turning a plain TypeScript function into a
  * runnable worker thread — you rarely need to use it directly.
  *
- * The worker script is generated by {@link workerTemplate}, which wraps the
- * function with a message listener and transferable-extraction logic.
+ * Supports three execution modes via {@link WorkerMode}:
+ * - **Default** — one-shot worker that processes a single message and is
+ *   terminated after responding.
+ * - **Pipeline** — stays alive and forwards output to the next worker via
+ *   `MessagePort`, enabling worker-to-worker data flow without main-thread
+ *   round-trips.
+ * - **Persistent** — stays alive indefinitely, caches a dataset in memory,
+ *   and re-processes it with different configs on subsequent messages.
+ *
+ * The worker script is generated by the template corresponding to the chosen
+ * mode, which wraps the function with a message listener and
+ * transferable-extraction logic.
  */
 declare class WorkerFactory {
     readonly _worker: Worker;
@@ -25,8 +40,8 @@ declare class WorkerFactory {
      * @param workerFunction - The function to run inside the worker thread.
      *   Must be self-contained — it cannot reference variables from the outer
      *   scope because it is serialised via `.toString()`.
-     * @param options - Optional configuration. Set `pipeline: true` for
-     *   pipeline-aware workers that support MessagePort forwarding.
+     * @param options - Optional configuration. Set `mode` to control the
+     *   worker execution mode (default, pipeline, or persistent).
      */
     constructor(workerFunction: WorkerFunction, options?: WorkerFactoryOptions);
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@offmain/workerkit",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "A lightweight manager for running functions in Web Workers with partitioning, retries, and concurrency control",
   "type": "module",
   "main": "./dist/index.cjs",