@xylabs/threads 4.3.1 → 4.3.3

package/README.md

@@ -11,7 +11,7 @@
 
 Offload CPU-intensive tasks to worker threads in node.js, web browsers and electron using one uniform API.
 
-Uses web workers in the browser and `worker_threads` in node 12+
+Uses web workers in the browser, `worker_threads` in node 12+ and [`tiny-worker`](https://github.com/avoidwork/tiny-worker) in node 8 to 11.
 
 ### Features
 
@@ -28,9 +28,11 @@ You can find the old version 0.12 of threads.js on the [`v0` branch](https://git
 ## Installation
 
 ```
-npm install threads
+npm install threads tiny-worker
 ```
 
+*You only need to install the `tiny-worker` package to support node.js < 12. It's an optional dependency and used as a fallback if `worker_threads` are not available.*
+
 ## Platform support
 
 <details>
@@ -71,6 +73,22 @@ Then add it to your `webpack.config.js`:
   }
 ```
 
+#### Node.js bundles
+
+If you are using webpack to create a bundle that will be run in node (webpack config `target: "node"`), you also need to specify that the `tiny-worker` package used for node < 12 should not be bundled:
+
+```diff
+  module.exports = {
+    // ...
++   externals: {
++     "tiny-worker": "tiny-worker"
++   }
+    // ...
+}
+```
+
+Make sure that `tiny-worker` is listed in your `package.json` `dependencies` in that case.
+
 #### When using TypeScript
 
 Note: You'll need to be using Typescript version 4+, as the types generated by threads.js are not supported in Typescript 3.

package/dist/esm/master/implementation.node.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.defaultPoolSize = void 0;
 exports.getWorkerImplementation = getWorkerImplementation;
 exports.isWorkerRuntime = isWorkerRuntime;
+const node_events_1 = require("node:events");
 const node_os_1 = require("node:os");
 const node_path_1 = __importDefault(require("node:path"));
 const node_url_1 = require("node:url");
@@ -113,13 +114,76 @@ function initWorkerThreadsWorker() {
         default: Worker,
     };
 }
+function initTinyWorker() {
+    const TinyWorker = require('tiny-worker');
+    let allWorkers = [];
+    class Worker extends TinyWorker {
+        emitter;
+        constructor(scriptPath, options) {
+            const resolvedScriptPath = options && options.fromSource
+                ? null
+                : process.platform === 'win32'
+                    ? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
+                    : resolveScriptPath(scriptPath);
+            if (!resolvedScriptPath) {
+                const sourceCode = scriptPath;
+                super(new Function(sourceCode), [], { esm: true });
+            }
+            else if (/\.tsx?$/i.test(resolvedScriptPath) && detectTsNode()) {
+                super(new Function(createTsNodeModule(resolveScriptPath(scriptPath))), [], { esm: true });
+            }
+            else if (/\.asar[/\\]/.test(resolvedScriptPath)) {
+                super(resolvedScriptPath.replace(/\.asar([/\\])/, '.asar.unpacked$1'), [], { esm: true });
+            }
+            else {
+                super(resolvedScriptPath, [], { esm: true });
+            }
+            allWorkers.push(this);
+            this.emitter = new node_events_1.EventEmitter();
+            this.onerror = (error) => this.emitter.emit('error', error);
+            this.onmessage = (message) => this.emitter.emit('message', message);
+        }
+        addEventListener(eventName, listener) {
+            this.emitter.addListener(eventName, listener);
+        }
+        removeEventListener(eventName, listener) {
+            this.emitter.removeListener(eventName, listener);
+        }
+        terminate() {
+            allWorkers = allWorkers.filter(worker => worker !== this);
+            return super.terminate();
+        }
+    }
+    const terminateWorkersAndMaster = () => {
+        Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
+        allWorkers = [];
+    };
+    process.on('SIGINT', () => terminateWorkersAndMaster());
+    process.on('SIGTERM', () => terminateWorkersAndMaster());
+    class BlobWorker extends Worker {
+        constructor(blob, options) {
+            super(Buffer.from(blob).toString('utf-8'), { ...options, fromSource: true });
+        }
+        static fromText(source, options) {
+            return new Worker(source, { ...options, fromSource: true });
+        }
+    }
+    return {
+        blob: BlobWorker,
+        default: Worker,
+    };
+}
 let implementation;
+let isTinyWorker;
 function selectWorkerImplementation() {
     try {
+        isTinyWorker = false;
         return initWorkerThreadsWorker();
     }
     catch {
-        throw new Error('Node worker_threads not available...');
+        console.debug('Node worker_threads not available. Trying to fall back to tiny-worker polyfill...');
+        isTinyWorker = true;
+        return initTinyWorker();
     }
 }
 function getWorkerImplementation() {
@@ -129,8 +193,13 @@ function getWorkerImplementation() {
     return implementation;
 }
 function isWorkerRuntime() {
-    const isMainThread = typeof __non_webpack_require__ === 'function'
-        ? __non_webpack_require__('worker_threads').isMainThread
-        : eval('require')('worker_threads').isMainThread;
-    return !isMainThread;
+    if (isTinyWorker) {
+        return self !== undefined && self['postMessage'] ? true : false;
+    }
+    else {
+        const isMainThread = typeof __non_webpack_require__ === 'function'
+            ? __non_webpack_require__('worker_threads').isMainThread
+            : eval('require')('worker_threads').isMainThread;
+        return !isMainThread;
+    }
 }

package/dist/esm/worker/implementation.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const implementation_browser_1 = __importDefault(require("./implementation.browser"));
+const implementation_tiny_worker_1 = __importDefault(require("./implementation.tiny-worker"));
 const implementation_worker_threads_1 = __importDefault(require("./implementation.worker_threads"));
 const runningInNode = typeof process !== 'undefined' && process.arch !== 'browser' && 'pid' in process;
 function selectNodeImplementation() {
@@ -12,7 +13,7 @@ function selectNodeImplementation() {
         return implementation_worker_threads_1.default;
     }
     catch {
-        throw new Error('No worker implementation available in this environment');
+        return implementation_tiny_worker_1.default;
     }
 }
 exports.default = runningInNode ? selectNodeImplementation() : implementation_browser_1.default;

package/dist/master/implementation.node.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.defaultPoolSize = void 0;
 exports.getWorkerImplementation = getWorkerImplementation;
 exports.isWorkerRuntime = isWorkerRuntime;
+const node_events_1 = require("node:events");
 const node_os_1 = require("node:os");
 const node_path_1 = __importDefault(require("node:path"));
 const node_url_1 = require("node:url");
@@ -113,13 +114,76 @@ function initWorkerThreadsWorker() {
         default: Worker,
     };
 }
+function initTinyWorker() {
+    const TinyWorker = require('tiny-worker');
+    let allWorkers = [];
+    class Worker extends TinyWorker {
+        emitter;
+        constructor(scriptPath, options) {
+            const resolvedScriptPath = options && options.fromSource
+                ? null
+                : process.platform === 'win32'
+                    ? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
+                    : resolveScriptPath(scriptPath);
+            if (!resolvedScriptPath) {
+                const sourceCode = scriptPath;
+                super(new Function(sourceCode), [], { esm: true });
+            }
+            else if (/\.tsx?$/i.test(resolvedScriptPath) && detectTsNode()) {
+                super(new Function(createTsNodeModule(resolveScriptPath(scriptPath))), [], { esm: true });
+            }
+            else if (/\.asar[/\\]/.test(resolvedScriptPath)) {
+                super(resolvedScriptPath.replace(/\.asar([/\\])/, '.asar.unpacked$1'), [], { esm: true });
+            }
+            else {
+                super(resolvedScriptPath, [], { esm: true });
+            }
+            allWorkers.push(this);
+            this.emitter = new node_events_1.EventEmitter();
+            this.onerror = (error) => this.emitter.emit('error', error);
+            this.onmessage = (message) => this.emitter.emit('message', message);
+        }
+        addEventListener(eventName, listener) {
+            this.emitter.addListener(eventName, listener);
+        }
+        removeEventListener(eventName, listener) {
+            this.emitter.removeListener(eventName, listener);
+        }
+        terminate() {
+            allWorkers = allWorkers.filter(worker => worker !== this);
+            return super.terminate();
+        }
+    }
+    const terminateWorkersAndMaster = () => {
+        Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
+        allWorkers = [];
+    };
+    process.on('SIGINT', () => terminateWorkersAndMaster());
+    process.on('SIGTERM', () => terminateWorkersAndMaster());
+    class BlobWorker extends Worker {
+        constructor(blob, options) {
+            super(Buffer.from(blob).toString('utf-8'), { ...options, fromSource: true });
+        }
+        static fromText(source, options) {
+            return new Worker(source, { ...options, fromSource: true });
+        }
+    }
+    return {
+        blob: BlobWorker,
+        default: Worker,
+    };
+}
 let implementation;
+let isTinyWorker;
 function selectWorkerImplementation() {
     try {
+        isTinyWorker = false;
         return initWorkerThreadsWorker();
     }
     catch {
-        throw new Error('Node worker_threads not available...');
+        console.debug('Node worker_threads not available. Trying to fall back to tiny-worker polyfill...');
+        isTinyWorker = true;
+        return initTinyWorker();
     }
 }
 function getWorkerImplementation() {
@@ -129,8 +193,13 @@ function getWorkerImplementation() {
     return implementation;
 }
 function isWorkerRuntime() {
-    const isMainThread = typeof __non_webpack_require__ === 'function'
-        ? __non_webpack_require__('worker_threads').isMainThread
-        : eval('require')('worker_threads').isMainThread;
-    return !isMainThread;
+    if (isTinyWorker) {
+        return self !== undefined && self['postMessage'] ? true : false;
+    }
+    else {
+        const isMainThread = typeof __non_webpack_require__ === 'function'
+            ? __non_webpack_require__('worker_threads').isMainThread
+            : eval('require')('worker_threads').isMainThread;
+        return !isMainThread;
+    }
 }

package/dist/worker/implementation.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const implementation_browser_1 = __importDefault(require("./implementation.browser"));
+const implementation_tiny_worker_1 = __importDefault(require("./implementation.tiny-worker"));
 const implementation_worker_threads_1 = __importDefault(require("./implementation.worker_threads"));
 const runningInNode = typeof process !== 'undefined' && process.arch !== 'browser' && 'pid' in process;
 function selectNodeImplementation() {
@@ -12,7 +13,7 @@ function selectNodeImplementation() {
         return implementation_worker_threads_1.default;
     }
     catch {
-        throw new Error('No worker implementation available in this environment');
+        return implementation_tiny_worker_1.default;
     }
 }
 exports.default = runningInNode ? selectNodeImplementation() : implementation_browser_1.default;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xylabs/threads",
-  "version": "4.3.1",
+  "version": "4.3.3",
   "description": "Web workers & worker threads as simple as a function call",
   "license": "MIT",
   "main": "dist/index.js",
@@ -95,11 +95,12 @@
     "@rollup/plugin-node-resolve": "^15.3.0",
     "@types/chai": "^5.0.0",
     "@types/debug": "^4.1.12",
-    "@types/node": "^22.7.7",
+    "@types/execa": "^2.0.2",
+    "@types/node": "^22.8.0",
     "@types/webpack": "^5.28.5",
     "@xylabs/ts-scripts-yarn3": "^4.2.3",
     "ava": "^6.1.3",
-    "chai": "^5.1.1",
+    "chai": "^5.1.2",
     "cross-env": "^7.0.3",
     "mocha": "^10.7.3",
     "puppet-run": "^0.11.4",
@@ -108,6 +109,7 @@
     "rimraf": "^5.0.10",
     "rollup": "^4.24.0",
     "threads-plugin": "^1.4.0",
+    "tiny-worker": "^2.3.0",
     "ts-loader": "^9.5.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.6.3",
@@ -116,6 +118,9 @@
     "webpack": "^5.95.0",
     "worker-plugin": "^5.0.1"
   },
+  "optionalDependencies": {
+    "tiny-worker": "^2.3.0"
+  },
   "ava": {
     "extensions": [
       "ts"
@@ -133,12 +138,15 @@
     "./dist/esm/master/implementation.js": "./dist/esm/master/implementation.browser.js",
     "./dist/esm/master/implementation.node.js": false,
     "./dist/esm/worker/implementation.js": "./dist/esm/worker/implementation.browser.js",
+    "./dist/esm/worker/implementation.tiny-worker.js": false,
     "./dist/esm/worker/implementation.worker_threads.js": false,
     "./dist/master/implementation.js": "./dist/master/implementation.browser.js",
     "./dist/master/implementation.node.js": false,
     "./dist/worker/implementation.js": "./dist/worker/implementation.browser.js",
+    "./dist/worker/implementation.tiny-worker.js": false,
     "./dist/worker/implementation.worker_threads.js": false,
     "callsites": false,
+    "tiny-worker": false,
     "ts-node": false,
     "ts-node/register": false,
     "worker_threads": false

package/src/master/implementation.node.ts

@@ -1,12 +1,16 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
 /* eslint-disable import-x/no-internal-modules */
 /* eslint-disable unicorn/no-process-exit */
 /* eslint-disable unicorn/prefer-logical-operator-over-ternary */
 /* eslint-disable unicorn/prefer-regexp-test */
 
+/* eslint-disable unicorn/prefer-add-event-listener */
+/* eslint-disable unicorn/prefer-event-target */
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable unicorn/text-encoding-identifier-case */
 /// <reference lib="dom" />
 
+import { EventEmitter } from 'node:events'
 import { cpus } from 'node:os'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
@@ -25,9 +29,10 @@ interface WorkerGlobalScope {
 }
 
 declare const __non_webpack_require__: typeof require
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 declare const self: WorkerGlobalScope
 
+type WorkerEventName = 'error' | 'message'
+
 let tsNodeAvailable: boolean | undefined
 
 export const defaultPoolSize = cpus().length
@@ -162,14 +167,100 @@ function initWorkerThreadsWorker(): ImplementationExport {
   }
 }
 
+function initTinyWorker(): ImplementationExport {
+  const TinyWorker = require('tiny-worker')
+
+  let allWorkers: Array<typeof TinyWorker> = []
+
+  class Worker extends TinyWorker {
+    private emitter: EventEmitter
+
+    constructor(scriptPath: string, options?: ThreadsWorkerOptions & { fromSource?: boolean }) {
+      // Need to apply a work-around for Windows or it will choke upon the absolute path
+      // (`Error [ERR_INVALID_PROTOCOL]: Protocol 'c:' not supported`)
+      const resolvedScriptPath
+        = options && options.fromSource
+          ? null
+          : process.platform === 'win32'
+            ? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
+            : resolveScriptPath(scriptPath)
+
+      if (!resolvedScriptPath) {
+        // `options.fromSource` is true
+        const sourceCode = scriptPath
+        super(new Function(sourceCode), [], { esm: true })
+      } else if (/\.tsx?$/i.test(resolvedScriptPath) && detectTsNode()) {
+        super(new Function(createTsNodeModule(resolveScriptPath(scriptPath))), [], { esm: true })
+      } else if (/\.asar[/\\]/.test(resolvedScriptPath)) {
+        // See <https://github.com/andywer/threads-plugin/issues/17>
+        super(resolvedScriptPath.replace(/\.asar([/\\])/, '.asar.unpacked$1'), [], { esm: true })
+      } else {
+        super(resolvedScriptPath, [], { esm: true })
+      }
+
+      allWorkers.push(this)
+
+      this.emitter = new EventEmitter()
+      this.onerror = (error: Error) => this.emitter.emit('error', error)
+      this.onmessage = (message: MessageEvent) => this.emitter.emit('message', message)
+    }
+
+    addEventListener(eventName: WorkerEventName, listener: EventListener) {
+      this.emitter.addListener(eventName, listener)
+    }
+
+    removeEventListener(eventName: WorkerEventName, listener: EventListener) {
+      this.emitter.removeListener(eventName, listener)
+    }
+
+    terminate() {
+      allWorkers = allWorkers.filter(worker => worker !== this)
+      return super.terminate()
+    }
+  }
+
+  const terminateWorkersAndMaster = () => {
+    // we should terminate all workers and then gracefully shutdown self process
+    Promise.all(allWorkers.map(worker => worker.terminate())).then(
+      () => process.exit(0),
+      () => process.exit(1),
+    )
+    allWorkers = []
+  }
+
+  // Take care to not leave orphaned processes behind
+  // See <https://github.com/avoidwork/tiny-worker#faq>
+  process.on('SIGINT', () => terminateWorkersAndMaster())
+  process.on('SIGTERM', () => terminateWorkersAndMaster())
+
+  class BlobWorker extends Worker {
+    constructor(blob: Uint8Array, options?: ThreadsWorkerOptions) {
+      super(Buffer.from(blob).toString('utf-8'), { ...options, fromSource: true })
+    }
+
+    static fromText(source: string, options?: ThreadsWorkerOptions): WorkerImplementation {
+      return new Worker(source, { ...options, fromSource: true }) as any
+    }
+  }
+
+  return {
+    blob: BlobWorker as any,
+    default: Worker as any,
+  }
+}
+
 let implementation: ImplementationExport
+let isTinyWorker: boolean
 
 function selectWorkerImplementation(): ImplementationExport {
   try {
+    isTinyWorker = false
     return initWorkerThreadsWorker()
   } catch {
     // tslint:disable-next-line no-console
-    throw new Error('Node worker_threads not available...')
+    console.debug('Node worker_threads not available. Trying to fall back to tiny-worker polyfill...')
+    isTinyWorker = true
+    return initTinyWorker()
   }
 }
 
@@ -181,10 +272,14 @@ export function getWorkerImplementation(): ImplementationExport {
 }
 
 export function isWorkerRuntime() {
-  // Webpack hack
-  const isMainThread
-    = typeof __non_webpack_require__ === 'function'
-      ? __non_webpack_require__('worker_threads').isMainThread
-      : eval('require')('worker_threads').isMainThread
-  return !isMainThread
+  if (isTinyWorker) {
+    return self !== undefined && self['postMessage'] ? true : false
+  } else {
+    // Webpack hack
+    const isMainThread
+      = typeof __non_webpack_require__ === 'function'
+        ? __non_webpack_require__('worker_threads').isMainThread
+        : eval('require')('worker_threads').isMainThread
+    return !isMainThread
+  }
 }

package/src/master/spawn.ts

@@ -138,7 +138,7 @@ function setPrivateThreadProps<T>(
  * abstraction layer to provide the transparent API and verifies that
  * the worker has initialized successfully.
  *
- * @param worker Instance of `Worker`. Either a web worker or `worker_threads` worker.
+ * @param worker Instance of `Worker`. Either a web worker, `worker_threads` worker or `tiny-worker` worker.
  * @param [options]
  * @param [options.timeout] Init message timeout. Default: 10000 or set by environment variable.
  */

package/src/types/master.ts

@@ -66,10 +66,10 @@ export type Thread = AnyFunctionThread | AnyModuleThread
 
 export type TransferList = Transferable[]
 
-/** Worker instance. Either a web worker or a node.js Worker provided by `worker_threads`. */
+/** Worker instance. Either a web worker or a node.js Worker provided by `worker_threads` or `tiny-worker`. */
 export interface Worker extends EventTarget {
   postMessage(value: any, transferList?: TransferList): void
-  /** In nodejs 10+ return type is Promise while in browser return type is void */
+  /** In nodejs 10+ return type is Promise while with tiny-worker and in browser return type is void */
   terminate(callback?: (error?: Error, exitCode?: number) => void): void | Promise<number>
 }
 export interface ThreadsWorkerOptions extends WorkerOptions {

package/src/worker/implementation.tiny-worker.ts

@@ -34,6 +34,8 @@ const messageHandlers = new Set<(data: any) => void>()
 
 const subscribeToMasterMessages: AbstractedWorkerAPI['subscribeToMasterMessages'] = function subscribeToMasterMessages(onMessage) {
   if (!muxingHandlerSetUp) {
+    // We have one multiplexing message handler as tiny-worker's
+    // addEventListener() only allows you to set a single message handler
     self.addEventListener('message', ((event: MessageEvent) => {
       for (const handler of messageHandlers) handler(event.data)
     }) as EventListener)

package/src/worker/implementation.ts

@@ -6,6 +6,7 @@
 
 import type { AbstractedWorkerAPI } from '../types/worker'
 import WebWorkerImplementation from './implementation.browser'
+import TinyWorkerImplementation from './implementation.tiny-worker'
 import WorkerThreadsImplementation from './implementation.worker_threads'
 
 const runningInNode = typeof process !== 'undefined' && (process.arch as string) !== 'browser' && 'pid' in process
@@ -15,7 +16,7 @@ function selectNodeImplementation(): AbstractedWorkerAPI {
     WorkerThreadsImplementation.testImplementation()
     return WorkerThreadsImplementation
   } catch {
-    throw new Error('No worker implementation available in this environment')
+    return TinyWorkerImplementation
   }
 }
 

package/test-tooling/webpack/webpack.node.config.js

@@ -6,6 +6,7 @@ module.exports = {
   context: __dirname,
   devtool: false,
   entry: require.resolve('./app.ts'),
+  externals: { 'tiny-worker': 'tiny-worker' },
   mode: 'development',
   module: {
     rules: [

package/types/tiny-worker.d.ts

@@ -0,0 +1,4 @@
+declare module 'tiny-worker' {
+  const TinyWorker: typeof Worker
+  export = TinyWorker
+}