@offmain/workerkit 0.8.9 → 0.10.0

package/README.md

@@ -2,16 +2,16 @@
 
 A lightweight TypeScript library for running functions in Web Workers with support for partitioning, retries, and concurrency control — all without the boilerplate.
 
-Instead of manually creating worker scripts and wiring up `postMessage` / `onmessage`, you write plain exported functions and hand them to `MainWorkerFactory`. The library serialises them into Blob workers, manages a pool per function, handles retries on failure, and merges results back on the main thread.
+Instead of manually creating worker scripts and wiring up `postMessage` / `onmessage`, you write plain exported functions and hand them to `MainWorkerFactory`. The library serializes them into Blob workers, manages threads per function, handles retries on failure, and merges results back on the main thread.
 
 ---
 
 ## Installation
 
 ```bash
-npm install workerkit
+npm install @offmain/workerkit
 # or
-pnpm add workerkit
+pnpm add @offmain/workerkit
 ```
 
 ---
@@ -32,10 +32,10 @@ export function sum({ data }: { data: number[] }): number {
 ### 2. Register and run it
 
 ```ts
-import { MainWorkerFactory } from 'workerkit';
+import { MainWorkerFactory } from '@offmain/workerkit';
 import { sum } from './sum.worker.ts';
 
-const factory = new MainWorkerFactory(initiator, {
+const factory = new MainWorkerFactory({
   workers: [
     {
       name: 'sum',
@@ -47,10 +47,10 @@ const factory = new MainWorkerFactory(initiator, {
   ],
 });
 
-const result = await factory.runWorker('sum', { srcData: [1, 2, 3, 4, 5] });
-const { data } = await factory.collectResults(result);
+const settled = await factory.runWorker('sum', { srcData: [1, 2, 3, 4, 5] });
+const { data } = await factory.collectResults(settled);
 
-console.log(data); // 15
+console.log(data); // [15]
 ```
 
 ---
@@ -63,7 +63,7 @@ console.log(data); // 15
 | `role`           | `string`   | —                               | Logical grouping label                                                                              |
 | `func`           | `Function` | —                               | The exported worker function to run                                                                 |
 | `maxConcurrency` | `number`   | `navigator.hardwareConcurrency` | Max parallel worker instances — defaults to the number of logical CPU cores reported by the browser |
-| `retries`        | `number`   | `0`                             | How many times to retry a failed shard                                                              |
+| `retries`        | `number`   | `0`                             | How many times to retry a failed shard before marking it as rejected                                |
 | `partition`      | `boolean`  | `false`                         | Split array input across multiple workers automatically                                             |
 
 ---
@@ -73,21 +73,85 @@ console.log(data); // 15
 When `partition: true`, an array passed as `srcData` is automatically split across worker instances and results are merged back.
 
 ```ts
-const result = await factory.runWorker('sum', {
+const settled = await factory.runWorker('sum', {
   srcData: largeArray, // split across workers
 });
 
-const { data, succeeded, failed } = await factory.collectResults(result);
+const { data, succeeded, failed } = await factory.collectResults(settled);
 ```
 
 You can also provide a custom reducer to control how shard results are merged:
 
 ```ts
-const { data } = await factory.collectResults(result, {
+const { data } = await factory.collectResults(settled, {
   reducer: (shards) => shards.flat().sort((a, b) => b.score - a.score),
 });
 ```
 
+> **Note:** The reducer runs inside a worker thread and must be self-contained — it cannot reference variables from the outer scope.
+
+---
+
+## Pipeline
+
+Chain multiple workers together so data flows directly between them via `MessageChannel` — without passing through the main thread between steps.
+
+### Why use a pipeline?
+
+In a traditional multi-step workflow, intermediate data is serialized back to the main thread after each step:
+
+```
+Main → Worker A → Main → Worker B → Main → Worker C → Main
+         ↑ serialize    ↑ serialize    ↑ serialize
+```
+
+With large datasets (100k+ records), each serialization round-trip adds significant overhead — both in time and memory pressure on the main thread. The pipeline eliminates this:
+
+```
+Main → Worker A → Worker B → Worker C → Main
+                ↑ MessageChannel      ↑ only final result
+```
+
+Only the final result crosses back to the main thread. If your pipeline generates 20 MB of intermediate data but produces a 1 KB summary, you save ~40 MB of serialization (two round-trips avoided).
+
+### Usage
+
+```ts
+import { MainWorkerFactory } from '@offmain/workerkit';
+import { fetchData, transform, aggregate } from './workers.ts';
+
+const factory = new MainWorkerFactory({
+  workers: [
+    { name: 'fetchData', role: 'io', func: fetchData },
+    { name: 'transform', role: 'compute', func: transform },
+    { name: 'aggregate', role: 'compute', func: aggregate },
+  ] as const,
+});
+
+const result = await factory.pipeline<AggregateResult>([
+  { worker: 'fetchData', srcData: { url: '/api/records' } },
+  { worker: 'transform' }, // receives fetchData output directly
+  { worker: 'aggregate' }, // receives transform output directly
+]);
+
+console.log(result); // only this small result crossed to main thread
+```
+
+### How each step receives data
+
+- The first step receives `srcData` as `{ data: srcData, index: 0 }` — same as `runWorker`.
+- Each subsequent step receives the previous step's output as `{ data: previousOutput, index: 0 }`.
+- Worker functions don't need any special handling — they use the same `{ data }` parameter signature as regular workers.
+
+### When to use pipeline vs runWorker
+
+| Scenario                                             | Use                     |
+| ---------------------------------------------------- | ----------------------- |
+| Single step, or steps that need partitioning/retries | `runWorker`             |
+| Multi-step chain where intermediate data is large    | `pipeline`              |
+| Steps that are independent (not sequential)          | `runWorker` in parallel |
+| Steps where only the final result matters to the UI  | `pipeline`              |
+
 ---
 
 ## ESLint Plugin
@@ -98,7 +162,7 @@ The package ships with two ESLint rules to catch common worker mistakes at lint
 
 ```js
 // eslint.config.js
-import workerPlugin from 'workerkit/eslint-plugin';
+import workerPlugin from '@offmain/workerkit/eslint-plugin';
 
 export default [...workerPlugin.configs.recommended];
 ```
@@ -113,16 +177,16 @@ Flags usage of browser main-thread-only APIs that are unavailable inside Web Wor
 
 ```ts
 // sum.worker.ts ❌ — will be flagged
-export function sum({ data }: MessageEvent) {
+export function sum({ data }: { data: number[] }) {
   document.title = 'working...'; // Error: 'document' is not available inside Web Workers
-  return data.reduce((a: number, b: number) => a + b, 0);
+  return data.reduce((a, b) => a + b, 0);
 }
 ```
 
 ```ts
 // sum.worker.ts ✅
-export function sum({ data }: MessageEvent) {
-  return data.reduce((a: number, b: number) => a + b, 0);
+export function sum({ data }: { data: number[] }) {
+  return data.reduce((a, b) => a + b, 0);
 }
 ```
 
@@ -150,8 +214,8 @@ You can also import rules individually if you don't want the full recommended co
 
 ```js
 // eslint.config.js
-import noDomInWorker from 'workerkit/eslint-rules/no-dom-in-worker';
-import workerExportable from 'workerkit/eslint-rules/worker-exportable';
+import noDomInWorker from '@offmain/workerkit/eslint-rules/no-dom-in-worker';
+import workerExportable from '@offmain/workerkit/eslint-rules/worker-exportable';
 
 export default [
   {

package/dist/index.cjs

@@ -1,4 +1,4 @@
-"use strict";var g=Object.defineProperty;var m=(t,e,r)=>e in t?g(t,e,{enumerable:!0,configurable:!0,writable:!0,value:r}):t[e]=r;var h=(t,e,r)=>m(t,typeof e!="symbol"?e+"":e,r);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const w=t=>`
+"use strict";var w=Object.defineProperty;var y=(o,e,t)=>e in o?w(o,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):o[e]=t;var h=(o,e,t)=>y(o,typeof e!="symbol"?e+"":e,t);Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const m=o=>`
 const extractTransferables = (value, seen = new Set()) => {
   if (value === null || typeof value !== 'object') return [];
   if (seen.has(value)) return [];
@@ -14,12 +14,87 @@ const extractTransferables = (value, seen = new Set()) => {
 };
 
 self.addEventListener('message', async (event) => {
-    const begin = performance.now();
-    const output = await ${t}(event.data);
-    self.postMessage(output, extractTransferables(output));
-  })
-`;class k{constructor(e){h(this,"_worker");const r=w(e.toString()),s=new Blob([r],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(s))}get getWorker(){return this._worker}}function p(t,e=new Set){return t===null||typeof t!="object"?[]:e.has(t)?[]:(e.add(t),t instanceof ArrayBuffer||t instanceof MessagePort||typeof ImageBitmap<"u"&&t instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&t instanceof OffscreenCanvas?[t]:ArrayBuffer.isView(t)?[t.buffer]:Array.isArray(t)?t.flatMap(r=>p(r,e)):Object.values(t).flatMap(r=>p(r,e)))}class W{constructor(e,r){h(this,"_workers");h(this,"_threads");this._workers=r.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new k(e)}partitionArray(e,r){if(!e.length)return[];if(r<=0)throw new Error("numChunks must be positive");const s=Math.min(r,e.length),a=Math.floor(e.length/s),c=e.length%s,o=[];let f=0;for(let n=0;n<s;n++){const i=a+(n<c?1:0);o.push(e.slice(f,f+i)),f+=i}return o}findWorkerByName(e){return this._workers.find(r=>r.name===e)}async runWorker(e,{srcData:r,...s}){const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const c=a.maxConcurrency??this._threads,o=!!(Array.isArray(r)&&r.length>1&&a.partition),f=o?this.partitionArray(r,c):r,n=this.createWorkerPromises(a,e,{data:f,...s},c,o);return Promise.allSettled(n)}createWorkerPromises(e,r,s,a,c){const{data:o,...f}=s;return Array.from({length:a},(n,i)=>{const u=c&&Array.isArray(o)?o[i]:o;return this.runWorkerWithRetry({workerFunc:e.func,workerName:r,index:i,data:{data:u,...f}},e.retries)})}async runWorkerWithRetry(e,r=2){try{return await this.initiateWorker(e)}catch(s){if(r>0)return console.error(`Worker ${e.index} failed, retrying (${r} left):`,s),this.runWorkerWithRetry(e,r-1);throw console.error("Worker failed after all retries:",s),s}}initiateWorker({workerFunc:e,workerName:r,index:s,data:a}){return new Promise((c,o)=>{const n=this.initWorker(e).getWorker;n.onerror=u=>{n.terminate(),o({index:s,workerConfigs:{workerFunc:e,workerName:r,index:s,data:a},failedResult:u})},n.onmessage=u=>{c({index:s,workerConfigs:{workerFunc:e,workerName:r,index:s,data:a},successResult:u}),n.terminate()};const i={index:s,...Array.isArray(a)?{data:a}:a};n.postMessage(i,p(i))})}async collectResults(e,r={}){const s=e.filter(n=>n.status==="fulfilled"),a=e.filter(n=>n.status==="rejected"),c=s.map(n=>n.value.successResult.data),o=r.reducer?r.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((n,i)=>{const u=`
-        const reducer = ${o};
+  try {
+    const output = await ${o}(event.data);
+    self.postMessage({ ok: true, data: output }, extractTransferables(output));
+  } catch (err) {
+    self.postMessage({ ok: false, error: err instanceof Error ? err.message : String(err) });
+  }
+})
+`,v=o=>`
+const extractTransferables = (value, seen = new Set()) => {
+  if (value === null || typeof value !== 'object') return [];
+  if (seen.has(value)) return [];
+  seen.add(value);
+  if (value instanceof ArrayBuffer || value instanceof MessagePort ||
+      (typeof ImageBitmap !== 'undefined' && value instanceof ImageBitmap) ||
+      (typeof OffscreenCanvas !== 'undefined' && value instanceof OffscreenCanvas)) {
+    return [value];
+  }
+  if (ArrayBuffer.isView(value)) return [value.buffer];
+  if (Array.isArray(value)) return value.flatMap(i => extractTransferables(i, seen));
+  return Object.values(value).flatMap(v => extractTransferables(v, seen));
+};
+
+const workerFn = ${o};
+let outputPort = null;
+let inputPort = null;
+let pendingData = null;
+
+async function processData(data) {
+  try {
+    const output = await workerFn(data);
+    const result = { ok: true, data: output };
+    const transfers = extractTransferables(output);
+    if (outputPort) {
+      outputPort.postMessage(result, transfers);
+    } else {
+      self.postMessage(result, transfers);
+    }
+  } catch (err) {
+    const result = { ok: false, error: err instanceof Error ? err.message : String(err) };
+    if (outputPort) {
+      outputPort.postMessage(result);
+    } else {
+      self.postMessage(result);
+    }
+  }
+}
+
+self.addEventListener('message', (event) => {
+  if (event.data && event.data.__pipeline_ports__) {
+    if (event.data.outputPort) {
+      outputPort = event.data.outputPort;
+    }
+    if (event.data.inputPort) {
+      inputPort = event.data.inputPort;
+      inputPort.onmessage = (e) => {
+        if (e.data && e.data.ok === false) {
+          // Propagate errors through the pipeline
+          if (outputPort) outputPort.postMessage(e.data);
+          else self.postMessage(e.data);
+        } else {
+          processData({ data: e.data.data, index: 0 });
+        }
+      };
+    }
+    // If we already received data before ports, process it now
+    if (pendingData !== null) {
+      processData(pendingData);
+      pendingData = null;
+    }
+    return;
+  }
+  // First worker in pipeline or standalone — process directly
+  if (!inputPort) {
+    processData(event.data);
+  } else {
+    // Store data until ports are configured
+    pendingData = event.data;
+  }
+});
+`;class k{constructor(e,t){h(this,"_worker");const a=(t!=null&&t.pipeline?v:m)(e.toString()),u=new Blob([a],{type:"application/javascript"});this._worker=new Worker(URL.createObjectURL(u))}get getWorker(){return this._worker}}class P{constructor(e){this.results=e}}function g(o,e=new Set){return o===null||typeof o!="object"?[]:e.has(o)?[]:(e.add(o),o instanceof ArrayBuffer||o instanceof MessagePort||typeof ImageBitmap<"u"&&o instanceof ImageBitmap||typeof OffscreenCanvas<"u"&&o instanceof OffscreenCanvas?[o]:ArrayBuffer.isView(o)?[o.buffer]:Array.isArray(o)?o.flatMap(t=>g(t,e)):Object.values(o).flatMap(t=>g(t,e)))}class W{constructor(e){h(this,"_workers");h(this,"_threads");this._workers=e.workers,this._threads=navigator.hardwareConcurrency}initWorker(e){return new k(e)}partitionArray(e,t){if(!e.length)return[];if(t<=0)throw new Error("numChunks must be positive");const n=Math.min(t,e.length),a=Math.floor(e.length/n),u=e.length%n,l=[];let f=0;for(let r=0;r<n;r++){const s=a+(r<u?1:0);l.push(e.slice(f,f+s)),f+=s}return l}findWorkerByName(e){return this._workers.find(t=>t.name===e)}async runWorker(e,{srcData:t,...n}){const a=this.findWorkerByName(e);if(!a)return Promise.reject(new Error(`Worker "${e}" not found`));const u=a.maxConcurrency??this._threads,l=!!(Array.isArray(t)&&t.length>1&&a.partition),f=l?this.partitionArray(t,u):t,r=this.createWorkerPromises(a,e,{data:f,...n},u,l),s=await Promise.allSettled(r);return new P(s)}createWorkerPromises(e,t,n,a,u){const{data:l,...f}=n;return Array.from({length:a},(r,s)=>{const i=u&&Array.isArray(l)?l[s]:l;return this.runWorkerWithRetry({workerFunc:e.func,workerName:t,index:s,data:{data:i,...f}},e.retries)})}async runWorkerWithRetry(e,t=2){try{return await this.initiateWorker(e)}catch(n){if(t>0)return console.error(`Worker ${e.index} failed, retrying (${t} left):`,n),this.runWorkerWithRetry(e,t-1);throw console.error("Worker failed after all retries:",n),n}}initiateWorker({workerFunc:e,workerName:t,index:n,data:a}){return new Promise((u,l)=>{const r=this.initWorker(e).getWorker;r.onerror=i=>{r.terminate(),l({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:i})},r.onmessage=i=>{var c,p;if(((c=i.data)==null?void 0:c.ok)===!1){r.terminate(),l({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},failedResult:new ErrorEvent("error",{message:i.data.error})});return}u({index:n,workerConfigs:{workerFunc:e,workerName:t,index:n,data:a},successResult:new MessageEvent("message",{data:(p=i.data)==null?void 0:p.data})}),r.terminate()};const s={index:n,...Array.isArray(a)?{data:a}:a};r.postMessage(s,g(s))})}async collectResults(e,t={}){const n=e.results.filter(r=>r.status==="fulfilled"),a=e.results.filter(r=>r.status==="rejected"),u=n.map(r=>r.value.successResult.data),l=t.reducer?t.reducer.toString():"(shards) => shards.flat()";return{data:await new Promise((r,s)=>{const i=`
+        const reducer = ${l};
         self.addEventListener('message', (event) => {
           try {
             const result = reducer(event.data);
@@ -28,4 +103,4 @@ self.addEventListener('message', async (event) => {
             self.postMessage({ ok: false, error: String(err) });
           }
         });
-      `,y=new Blob([u],{type:"application/javascript"}),l=new Worker(URL.createObjectURL(y));l.onmessage=d=>{l.terminate(),d.data.ok?n(d.data.data):i(new Error(d.data.error))},l.onerror=d=>{l.terminate(),i(d)},l.postMessage(c)}),succeeded:s.length,failed:a.length,errors:a}}}exports.MainWorkerFactory=W;exports.WorkerFactory=k;
+      `,c=new Blob([i],{type:"application/javascript"}),p=new Worker(URL.createObjectURL(c));p.onmessage=d=>{p.terminate(),d.data.ok?r(d.data.data):s(new Error(d.data.error))},p.onerror=d=>{p.terminate(),s(d)},p.postMessage(u)}),succeeded:n.length,failed:a.length,errors:a}}async pipeline(e){if(e.length===0)throw new Error("Pipeline requires at least one step");if(e.length===1){const t=e[0],n=this.findWorkerByName(t.worker);if(!n)throw new Error(`Worker "${t.worker}" not found`);const u=this.initWorker(n.func).getWorker;return new Promise((l,f)=>{u.onmessage=s=>{var i,c;u.terminate(),((i=s.data)==null?void 0:i.ok)===!1?f(new Error(s.data.error)):l((c=s.data)==null?void 0:c.data)},u.onerror=s=>{u.terminate(),f(s)};const r=t.srcData??{};u.postMessage({data:r,index:0},g(r))})}return new Promise((t,n)=>{const a=[],u=[];for(const r of e){const s=this.findWorkerByName(r.worker);if(!s){n(new Error(`Worker "${r.worker}" not found`));return}const i=new k(s.func,{pipeline:!0});a.push(i.getWorker)}for(let r=0;r<a.length-1;r++)u.push(new MessageChannel);for(let r=0;r<a.length;r++){const s=[],i={};r>0&&(i.inputPort=u[r-1].port1,s.push(i.inputPort)),r<a.length-1&&(i.outputPort=u[r].port2,s.push(i.outputPort)),a[r].postMessage({__pipeline_ports__:!0,...i},s)}const l=a[a.length-1];l.onmessage=r=>{var s,i;a.forEach(c=>c.terminate()),((s=r.data)==null?void 0:s.ok)===!1?n(new Error(r.data.error)):t((i=r.data)==null?void 0:i.data)},l.onerror=r=>{a.forEach(s=>s.terminate()),n(r)};const f=e[0].srcData??{};a[0].postMessage({data:f,index:0},g(f))})}}exports.MainWorkerFactory=W;exports.WorkerFactory=k;