multi-tasks 3.0.3 → 3.1.0

package/README.md

@@ -17,17 +17,21 @@ npm install multi-tasks
 - `multiTasks.resume(config)` — resume an interrupted run.
 - `multiTasks.restart(config)` — resume and also retry failed tasks (needs `config.taskFolder`).
 - `helper.createNewTasks(tasks)` — create new tasks dynamically while processing.
+- `helper.emit(event, payload, option?)` — broadcast an event to all workers (relayed by the master, best-effort); `option.includingMe` defaults to `true` — pass `{includingMe: false}` to exclude the sender. Payload must be JSON-serializable.
+- `helper.setListener(event, handler)` — listen for broadcast events; one handler per event per worker process (re-registering replaces it), so calling it inside `processTask` is always safe; `handler(payload, meta)` gets `meta.fromWorkerId` and `meta.fromWorkerPid`. Broadcasts are runtime-only messages — not persisted, not replayed on resume.
+- `helper.emitSys(event, payload?)` — send a system event to the master (consumed by the master itself, not relayed). Built-in event: `'TERMINATE_ALL_WORKERS'` force-kills all workers and exits the master; any other event goes to `config.setSysListener`.
 - Config options:
   - `initialTasks` — array of task objects, or a folder path string to resume from.
   - `processTask(task, helper)` — required; return a value or a Promise.
   - `taskRootFolder` — root directory where progress and result files are stored.
   - `taskId` — task folder name under `taskRootFolder`.
   - `taskFolder` — full task folder path, required by `restart`.
-  - `numberOfWorkers` — how many worker processes run in parallel.
+  - `numberOfWorkers` — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
   - `taskTimeout` — optional, milliseconds; an overdue task fails with a timeout error.
-  - `maxTaskRetries` — optional, max times a failed task is auto-retried (sent back to the queue immediately); covers worker crashes, exceptions/rejections from `processTask`, and `taskTimeout` timeouts; the retry count is kept in the task file as `__retryCount` — it survives `resume` and is reset by `restart`.
+  - `maxTaskRetries` — optional, max times a failed task is auto-retried.
   - `autoCloseAfterCompletion` — set `false` if you create new tasks dynamically.
   - `shouldTerminate(info)` — return `true` to terminate the whole process.
+  - `setSysListener(event, payload, meta)` — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
   - `onFinish(report)` — called once after all workers are done.
 
 ### How to use:
@@ -138,6 +142,51 @@ multiTasks({
     maxTaskRetries: 2,
 });
 
+//Example6, broadcast events between workers:
+//          a worker emits an event and the master relays it to every worker
+//          (best-effort, runtime only - not persisted, not replayed on
+//          resume); by default the sender also receives its own event, pass
+//          {includingMe: false} to exclude it; each event keeps only one
+//          listener per worker process, so calling setListener on every
+//          task is safe
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-broadcast`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    processTask: (task, helper) => {
+        helper.setListener('task-done', (payload, meta) => {
+            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
+        });
+        helper.emit('task-done', {seq: task.seq});
+        return {data:'succ'};
+    },
+});
+
+//Example7, system events from a worker to the master:
+//          helper.emitSys sends a system event that the master consumes
+//          itself (not relayed to workers); the built-in event
+//          TERMINATE_ALL_WORKERS force-kills all workers and exits the
+//          master (unfinished tasks stay for resume); any other event is
+//          passed to config.setSysListener on the master
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-sysevent`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    setSysListener: (event, payload, meta) => {
+        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
+    },
+    processTask: (task, helper) => {
+        helper.emitSys('task-started', {seq: task.seq});
+        if(task.fatal){
+            helper.emitSys('TERMINATE_ALL_WORKERS');//stop everything
+            return;
+        };
+        return {data:'succ'};
+    },
+});
+
 ```
 
 ### Resuming
@@ -161,6 +210,8 @@ The usage patterns documented above are covered by automated tests (unit tests i
 
 ### Changelog:
 
+- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
+- 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
 - 3.0.3 Support 'maxTaskRetries'
 - 3.0.2 Support 'taskTimeout'
 - 3.0.1 Versions 3.0.0 and above are maintained by AI
@@ -199,3 +250,7 @@ The usage patterns documented above are covered by automated tests (unit tests i
 ### Gitee:
 
 [https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)
+
+### License:
+
+[MIT](https://opensource.org/license/MIT) (see [LICENSE](LICENSE))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.0.3",
+  "version": "3.1.0",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/utils/resolveWorkers.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const numCPUs = require('os').cpus().length;
+
+const PERCENT_PATTERN = /^(\d+(?:\.\d+)?)%$/;
+
+// 解析 numberOfWorkers 配置:
+// - undefined -> 默认 "50%", 按核数四舍五入折算, 最小为 1;
+// - number -> 原样返回(现状行为不变);
+// - string -> 必须是 "数字%" 形式且百分比在 1-100 之间, 按核数四舍五入折算, 最小为 1;
+// - 其余 -> 抛 Error。
+const percentToWorkers = (percent) => {
+  return Math.max(1, Math.round(numCPUs * percent / 100));
+};
+
+const resolveNumberOfWorkers = (value) => {
+  if (typeof value === 'undefined') {
+    return percentToWorkers(50);
+  };
+  if (typeof value === 'number') {
+    return value;
+  };
+  if (typeof value === 'string') {
+    const match = value.trim().match(PERCENT_PATTERN);
+    if (match) {
+      const percent = parseFloat(match[1]);
+      if (percent >= 1 && percent <= 100) {
+        return percentToWorkers(percent);
+      };
+    };
+  };
+  throw new Error(`Invalid numberOfWorkers: ${JSON.stringify(value)}. Expected a number or a percentage string like "50%" (1-100).`);
+};
+
+module.exports = { resolveNumberOfWorkers };

package/workers/Multithread.js

@@ -8,6 +8,7 @@ const asMaster = require('./asMaster');
 const TaskMgr = require('./TaskMgr');
 const WorkerMgr = require('./WorkerMgr');
 const helper = require('./helper');
+const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
 
 let fmtdigit = (n)=>{
     return n > 9 ? "" + n: "0" + n;
@@ -50,6 +51,7 @@ const start = (config)=>{
     if (isMaster) {//init
         console.log('[Master]: Start!!');
         let PID = process.pid;
+        config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串; 须在 fork 循环读值前折算, 非法值在此抛错
         let numOfWorkers = config.numberOfWorkers;
         console.log(`  numCPUs=${numCPUs}`);
         console.log(`  numOfWorkers=${numOfWorkers}`);
@@ -120,6 +122,37 @@ const start = (config)=>{
         let initWorker = (worker)=>{
             WorkerMgr.addWorker(worker);
             worker.on('message', function(message) {
+                if(message && message.__sysEvent){//worker 系统事件: master 自己消费, 不转发
+                    if(message.event === 'TERMINATE_ALL_WORKERS'){
+                        console.warn(`[Master]: TERMINATE_ALL_WORKERS from worker ${worker.id}, killing all workers`);
+                        killAllWorkers();
+                        return process.exit(0);
+                    }
+                    let meta = {fromWorkerId: worker.id, fromWorkerPid: worker.process.pid};
+                    if(typeof USER_CONFIG.setSysListener === 'function'){
+                        try{
+                            USER_CONFIG.setSysListener(message.event, message.payload, meta);
+                        }catch(e){
+                            console.error(`[Master]: setSysListener for event "${message.event}" threw:`, e);
+                        }
+                    }else{
+                        console.warn(`[Master]: unknown sys event "${message.event}" from worker ${worker.id} (no setSysListener configured)`);
+                    }
+                    return;
+                }
+                if(message && message.__broadcast){//worker 广播请求: master 中转给所有 worker
+                    let meta = {fromWorkerId: worker.id, fromWorkerPid: worker.process.pid};
+                    for (const id in cluster.workers) {
+                        let w = cluster.workers[id];
+                        if(w.id === worker.id && message.includingMe === false) continue;
+                        try{
+                            w.send({__broadcast: true, event: message.event, payload: message.payload, meta});
+                        }catch(e){
+                            console.warn(`[Master]: failed to relay broadcast to worker ${w.id}`, e);
+                        }
+                    }
+                    return;
+                }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
                     queue.enqueue(async ()=>{
@@ -213,6 +246,10 @@ const start = (config)=>{
 function subWorker(){
     let worker = cluster.worker;
     worker.on('message', function(message) {
+        if(message && message.__broadcast){//master 中转来的广播: 分发给 helper.on 注册的 handler
+            helper.__dispatchBroadcast(message);
+            return;
+        }
         if(message === MSG_NO_TASK_FOUND){//wait for new task
             console.log(`  (Worker #${worker.id}): waiting`);
             setTimeout(()=>{

package/workers/asMaster.js

@@ -1,9 +1,9 @@
 var fs = require('fs');
 var pathUtil = require('path');
-const numCPUs = require('os').cpus().length;
 
 const TaskMgr = require('./TaskMgr');
 const randomsUtil = require('../utils/randoms');
+const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
 
 const resume = (config)=>{
     config.__resume = true;
@@ -60,7 +60,7 @@ const init = (config)=>{
     if(!config.initialTasks){ throw new Error('Please provide initialTasks data!');  };
     if(!config.processTask){ throw new Error('Please provide processTask function!'); };
 
-    if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
+    config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串, 非法值在此抛错(fail fast, 不建任务目录)
 
     let {taskRootFolder, taskName} = config;
 

package/workers/helper.js

@@ -1,22 +1,61 @@
-let fs = require('fs');
-let pathutil = require('path');
-
-let TaskMgr = require('./TaskMgr');
-
-let myconfig;
-const load = (config)=>{
-    myconfig = config;
-};
-const createNewTasks = (tasks)=>{
-    if (!myconfig) throw new Error('Please call helper.load(config) first!');
-    if (!Array.isArray(tasks)) tasks = [tasks];
-    TaskMgr.load(myconfig);
-    tasks.forEach((task)=>{
-        console.log('Creating new task', task)
-        TaskMgr.createNewTask(task);
-    });
-};
-module.exports = {
-    load,
-    createNewTasks
-};
+let fs = require('fs');
+let pathutil = require('path');
+
+let TaskMgr = require('./TaskMgr');
+
+let myconfig;
+let listeners = {};//event -> handler, worker 进程内模块级广播监听 registry(每个 event 一个 handler)
+
+const load = (config)=>{
+    myconfig = config;
+};
+const createNewTasks = (tasks)=>{
+    if (!myconfig) throw new Error('Please call helper.load(config) first!');
+    if (!Array.isArray(tasks)) tasks = [tasks];
+    TaskMgr.load(myconfig);
+    tasks.forEach((task)=>{
+        console.log('Creating new task', task)
+        TaskMgr.createNewTask(task);
+    });
+};
+
+//广播事件给所有 worker(经 master 中转); option.includingMe 默认 true(也发给自己)
+const emit = (event, payload, option)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emit: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emit: broadcast is only available in worker processes');
+    let includingMe = !option || option.includingMe !== false;
+    process.send({__broadcast: true, event, payload, includingMe});
+};
+//发送系统事件给 master(master 自己消费, 不转发); 内建事件 TERMINATE_ALL_WORKERS,
+//其余交给 config.setSysListener
+const emitSys = (event, payload)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.emitSys: event must be a non-empty string');
+    if (typeof process.send !== 'function') throw new Error('helper.emitSys: system events are only available in worker processes');
+    process.send({__sysEvent: true, event, payload});
+};
+//监听广播事件, 每个 event 每个 worker 进程只保留一个 handler(重复注册覆盖),
+//handler 签名 (payload, meta), meta 含 fromWorkerId/fromWorkerPid
+const setListener = (event, handler)=>{
+    if (typeof event !== 'string' || !event) throw new Error('helper.setListener: event must be a non-empty string');
+    if (typeof handler !== 'function') throw new Error('helper.setListener: handler must be a function');
+    listeners[event] = handler;
+};
+//内部方法: 供 subWorker 收到 master 中转的广播时调用, 按 event 分发
+const __dispatchBroadcast = (message)=>{
+    let {event, payload, meta} = message;
+    let handler = listeners[event];
+    if (!handler) return;
+    try{
+        handler(payload, meta);
+    }catch(e){
+        console.error(`[Broadcast] handler for event "${event}" threw:`, e);
+    }
+};
+module.exports = {
+    load,
+    createNewTasks,
+    emit,
+    emitSys,
+    setListener,
+    __dispatchBroadcast
+};