multi-tasks 3.0.4 → 3.1.1

package/README.md

@@ -12,23 +12,36 @@ npm install multi-tasks
 
 ### API:
 
-- `multiTasks(config)` — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
-- `multiTasks.start(config)` — always start a fresh run.
-- `multiTasks.resume(config)` — resume an interrupted run.
-- `multiTasks.restart(config)` — resume and also retry failed tasks (needs `config.taskFolder`).
-- `helper.createNewTasks(tasks)` — create new tasks dynamically while processing.
-- Config options:
-  - `initialTasks` — array of task objects, or a folder path string to resume from.
-  - `processTask(task, helper)` — required; return a value or a Promise.
-  - `taskRootFolder` — root directory where progress and result files are stored.
-  - `taskId` — task folder name under `taskRootFolder`.
-  - `taskFolder` — full task folder path, required by `restart`.
-  - `numberOfWorkers` — how many worker processes run in parallel; also accepts a percentage string of CPU cores, e.g. `"50%"`.
-  - `taskTimeout` — optional, milliseconds; an overdue task fails with a timeout error.
-  - `maxTaskRetries` — optional, max times a failed task is auto-retried.
-  - `autoCloseAfterCompletion` — set `false` if you create new tasks dynamically.
-  - `shouldTerminate(info)` — return `true` to terminate the whole process.
-  - `onFinish(report)` — called once after all workers are done.
+The API has three parts: the entry functions, the config options, and the `helper` object injected into `processTask`.
+
+**Entry functions:**
+
+- **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
+- **`multiTasks.start(config)`** — always start a fresh run.
+- **`multiTasks.resume(config)`** — resume an interrupted run.
+- **`multiTasks.restart(config)`** — resume and also retry failed tasks (needs `config.taskFolder`).
+
+**Config options:**
+
+- **`initialTasks`** — array of task objects, or a folder path string to resume from.
+- **`processTask(task, helper)`** — required; return a value or a Promise.
+- **`taskRootFolder`** — root directory where progress and result files are stored.
+- **`taskId`** — task folder name under `taskRootFolder`.
+- **`taskFolder`** — full task folder path, required by `restart`.
+- **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
+- **`taskTimeout`** — optional, milliseconds; an overdue task fails with a timeout error.
+- **`maxTaskRetries`** — optional, max times a failed task is auto-retried.
+- **`autoCloseAfterCompletion`** — set `false` if you create new tasks dynamically.
+- **`shouldTerminate(info)`** — return `true` to terminate the whole process.
+- **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
+- **`onFinish(report)`** — called once after all workers are done.
+
+**Task helper** (the `helper` object passed as the second argument of `processTask`):
+
+- **`helper.createNewTasks(tasks)`** — create new tasks dynamically while processing.
+- **`helper.emit(event, payload, option?)`** — broadcast an event to all workers (relayed by the master, best-effort); `option.includingMe` defaults to `true` — pass `{includingMe: false}` to exclude the sender. Payload must be JSON-serializable.
+- **`helper.setListener(event, handler)`** — listen for broadcast events; one handler per event per worker process (re-registering replaces it), so calling it inside `processTask` is always safe; `handler(payload, meta)` gets `meta.fromWorkerId` and `meta.fromWorkerPid`. Broadcasts are runtime-only messages — not persisted, not replayed on resume.
+- **`helper.emitSys(event, payload?)`** — send a system event to the master (consumed by the master itself, not relayed). Built-in event: `'TERMINATE_ALL_WORKERS'` force-kills all workers and exits the master; any other event goes to `config.setSysListener`.
 
 ### How to use:
 
@@ -138,6 +151,51 @@ multiTasks({
     maxTaskRetries: 2,
 });
 
+//Example6, broadcast events between workers:
+//          a worker emits an event and the master relays it to every worker
+//          (best-effort, runtime only - not persisted, not replayed on
+//          resume); by default the sender also receives its own event, pass
+//          {includingMe: false} to exclude it; each event keeps only one
+//          listener per worker process, so calling setListener on every
+//          task is safe
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-broadcast`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    processTask: (task, helper) => {
+        helper.setListener('task-done', (payload, meta) => {
+            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
+        });
+        helper.emit('task-done', {seq: task.seq});
+        return {data:'succ'};
+    },
+});
+
+//Example7, system events from a worker to the master:
+//          helper.emitSys sends a system event that the master consumes
+//          itself (not relayed to workers); the built-in event
+//          TERMINATE_ALL_WORKERS force-kills all workers and exits the
+//          master (unfinished tasks stay for resume); any other event is
+//          passed to config.setSysListener on the master
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-sysevent`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    setSysListener: (event, payload, meta) => {
+        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
+    },
+    processTask: (task, helper) => {
+        helper.emitSys('task-started', {seq: task.seq});
+        if(task.fatal){
+            helper.emitSys('TERMINATE_ALL_WORKERS');//stop everything
+            return;
+        };
+        return {data:'succ'};
+    },
+});
+
 ```
 
 ### Resuming
@@ -161,6 +219,8 @@ The usage patterns documented above are covered by automated tests (unit tests i
 
 ### Changelog:
 
+- 3.1.1 Fix readme documentation
+- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
 - 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
 - 3.0.3 Support 'maxTaskRetries'
 - 3.0.2 Support 'taskTimeout'
@@ -203,4 +263,4 @@ The usage patterns documented above are covered by automated tests (unit tests i
 
 ### License:
 
-[MIT](https://opensource.org/licenses/MIT) (see [LICENSE](LICENSE))
+[MIT](https://opensource.org/license/MIT) (see [LICENSE](LICENSE))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.0.4",
+  "version": "3.1.1",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/utils/resolveWorkers.js

@@ -5,13 +5,17 @@ const numCPUs = require('os').cpus().length;
 const PERCENT_PATTERN = /^(\d+(?:\.\d+)?)%$/;
 
 // 解析 numberOfWorkers 配置:
-// - undefined -> CPU 核数减 1(默认);
+// - undefined -> 默认 "50%", 按核数四舍五入折算, 最小为 1;
 // - number -> 原样返回(现状行为不变);
 // - string -> 必须是 "数字%" 形式且百分比在 1-100 之间, 按核数四舍五入折算, 最小为 1;
 // - 其余 -> 抛 Error。
+const percentToWorkers = (percent) => {
+  return Math.max(1, Math.round(numCPUs * percent / 100));
+};
+
 const resolveNumberOfWorkers = (value) => {
   if (typeof value === 'undefined') {
-    return numCPUs - 1;
+    return percentToWorkers(50);
   };
   if (typeof value === 'number') {
     return value;
@@ -21,7 +25,7 @@ const resolveNumberOfWorkers = (value) => {
     if (match) {
       const percent = parseFloat(match[1]);
       if (percent >= 1 && percent <= 100) {
-        return Math.max(1, Math.round(numCPUs * percent / 100));
+        return percentToWorkers(percent);
       };
     };
   };

package/workers/Multithread.js

@@ -122,6 +122,37 @@ const start = (config)=>{
         let initWorker = (worker)=>{
             WorkerMgr.addWorker(worker);
             worker.on('message', function(message) {
+                if(message && message.__sysEvent){//worker 系统事件: master 自己消费, 不转发
+                    if(message.event === 'TERMINATE_ALL_WORKERS'){
+                        console.warn(`[Master]: TERMINATE_ALL_WORKERS from worker ${worker.id}, killing all workers`);
+                        killAllWorkers();
+                        return process.exit(0);
+                    }
+                    let meta = {fromWorkerId: worker.id, fromWorkerPid: worker.process.pid};
+                    if(typeof USER_CONFIG.setSysListener === 'function'){
+                        try{
+                            USER_CONFIG.setSysListener(message.event, message.payload, meta);
+                        }catch(e){
+                            console.error(`[Master]: setSysListener for event "${message.event}" threw:`, e);
+                        }
+                    }else{
+                        console.warn(`[Master]: unknown sys event "${message.event}" from worker ${worker.id} (no setSysListener configured)`);
+                    }
+                    return;
+                }
+                if(message && message.__broadcast){//worker 广播请求: master 中转给所有 worker
+                    let meta = {fromWorkerId: worker.id, fromWorkerPid: worker.process.pid};
+                    for (const id in cluster.workers) {
+                        let w = cluster.workers[id];
+                        if(w.id === worker.id && message.includingMe === false) continue;
+                        try{
+                            w.send({__broadcast: true, event: message.event, payload: message.payload, meta});
+                        }catch(e){
+                            console.warn(`[Master]: failed to relay broadcast to worker ${w.id}`, e);
+                        }
+                    }
+                    return;
+                }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
                     queue.enqueue(async ()=>{
@@ -215,6 +246,10 @@ const start = (config)=>{
 function subWorker(){
     let worker = cluster.worker;
     worker.on('message', function(message) {
+        if(message && message.__broadcast){//master 中转来的广播: 分发给 helper.on 注册的 handler
+            helper.__dispatchBroadcast(message);
+            return;
+        }
         if(message === MSG_NO_TASK_FOUND){//wait for new task
             console.log(`  (Worker #${worker.id}): waiting`);
             setTimeout(()=>{

package/workers/helper.js

@@ -1,22 +1,61 @@
-let fs = require('fs');
-let pathutil = require('path');
-
-let TaskMgr = require('./TaskMgr');
-
-let myconfig;
-const load = (config)=>{
-    myconfig = config;
-};
-const createNewTasks = (tasks)=>{
-    if (!myconfig) throw new Error('Please call helper.load(config) first!');
-    if (!Array.isArray(tasks)) tasks = [tasks];
-    TaskMgr.load(myconfig);
-    tasks.forEach((task)=>{
-        console.log('Creating new task', task)
-        TaskMgr.createNewTask(task);
-    });
-};
-module.exports = {
-    load,
-    createNewTasks
-};
+let fs = require('fs');
+let pathutil = require('path');
+
+let TaskMgr = require('./TaskMgr');
+
+let myconfig;
+let listeners = {};//event -> handler, worker 进程内模块级广播监听 registry(每个 event 一个 handler)
+
+const load = (config)=>{
+    myconfig = config;
+};
+const createNewTasks = (tasks)=>{
+    if (!myconfig) throw new Error('Please call helper.load(config) first!');
+    if (!Array.isArray(tasks)) tasks = [tasks];
+    TaskMgr.load(myconfig);
+    tasks.forEach((task)=>{
+        console.log('Creating new task', task)
+        TaskMgr.createNewTask(task);
+    });
+};
+
+//广播事件给所有 worker(经 master 中转); option.includingMe 默认 true(也发给自己)
+const emit = (event, payload, option)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emit: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emit: broadcast is only available in worker processes');
+    let includingMe = !option || option.includingMe !== false;
+    process.send({__broadcast: true, event, payload, includingMe});
+};
+//发送系统事件给 master(master 自己消费, 不转发); 内建事件 TERMINATE_ALL_WORKERS,
+//其余交给 config.setSysListener
+const emitSys = (event, payload)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emitSys: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emitSys: system events are only available in worker processes');
+    process.send({__sysEvent: true, event, payload});
+};
+//监听广播事件, 每个 event 每个 worker 进程只保留一个 handler(重复注册覆盖),
+//handler 签名 (payload, meta), meta 含 fromWorkerId/fromWorkerPid
+const setListener = (event, handler)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.setListener: event must be a non-empty string');
+    if (typeof handler !== 'function') throw new Error('helper.setListener: handler must be a function');
+    listeners[event] = handler;
+};
+//内部方法: 供 subWorker 收到 master 中转的广播时调用, 按 event 分发
+const __dispatchBroadcast = (message)=>{
+    let {event, payload, meta} = message;
+    let handler = listeners[event];
+    if (!handler) return;
+    try{
+        handler(payload, meta);
+    }catch(e){
+        console.error(`[Broadcast] handler for event "${event}" threw:`, e);
+    }
+};
+module.exports = {
+    load,
+    createNewTasks,
+    emit,
+    emitSys,
+    setListener,
+    __dispatchBroadcast
+};