multi-tasks 3.0.2 → 3.0.3

package/README.md

@@ -1,164 +1,201 @@
-# multi-tasks
-
-Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. It manages progress and tasks based on the file system, which means that even if the host crashes, tasks can be resumed based on file records.
-
-**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
-
-### Install:
-
-```javascript
-npm install multi-tasks
-```
-
-### How to use:
-
-```javascript
-//see examples/example0
-let multiTasks = require('multi-tasks').multiTasks;
-
-//Step1, create your tasks that need to be executed simultaneously as an array.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    alltasks.push({
-        name: `task-${i}`,
-        data: `This prop is for a subtask`
-    });
-};
-
-//Step2, provide a function to process a certain sub-task and return the result data
-let processTask = (task, helper)=>{
-    let {taskCount} = task;//get your task data
-
-    //Run your processing logic here...
-
-    //then return the result as a plain javascript object or an async promise
-    return 'result data'; 
-};
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //how many workers are working in parallel
-    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
-    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
-    shouldTerminate:(info)=>{
-        //return true if you need to terminate the whole process
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});
-
-```
-
-### More about the task processing function
-
-```javascript
-//Example1, return a promise for async processes,
-//          you can return a promise or a non-promise result, 
-//          all result data can be found in the results/succ folder
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-
-//Example2, dynamically create a new task while processing
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-    
-    if(taskCount % 2 === 0){
-        //create a new task if needed
-        helper.createNewTasks({
-            msg:'a new task'
-        });
-        return;
-    }
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-
-//Example3, generate/throw exceptions in a task method
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
-    if(taskCount===5) aaa =  bbb;//this undefined exception will be captured by multi-tasks
-
-    return {data:'succ'};
-}
-
-
-```
-
-### Resuming
-
-Sometimes the task execution is interrupted due to some reasons (such as power outage), you can resume the execution like this
-
-```javascript
-
-multiTasks({
-    initialTasks: `/myworks/my_scan_tasks/`, //Point 'initialTasks' to the interrupted task directory, multi-tasks will read the tasks in the 'new' folder and initialize them to 'initialTasks' and then continue execution
-    ...
-    ...//Other configurations remain unchanged
-    ...
-});
-
-```
-
-### Testing:
-
-The usage patterns documented above are covered by automated tests (unit tests in `test/`, end-to-end tests in `teste2e/`). The e2e suite runs the API in real child processes with a mixed workload — successful tasks (both promise and non-promise results), planned failures (rejected promises and thrown exceptions), dynamically created tasks via `helper.createNewTasks`, and workers that crash randomly — then resumes repeatedly and verifies that every task lands exactly one result in `results/succ` or `results/errors`.
-
-### Changelog:
-
-- 3.0.2 Support taskTimeout: a task that does not finish in time is treated as failed with a timeout error
-- 3.0.1 Versions 3.0.0 and above are maintained by AI
-- 2.2.0 Support Resuming from an interrupted task
-- 2.1.2 Update readme
-- 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
-- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
-- 2.0.9 Update readme examples
-- 2.0.8 Update readme examples
-- 2.0.7 Update changelog
-- 2.0.6 Update readme, remove failed examples
-- 2.0.5 Update readme examples
-- 2.0.4 Support resume from a failed task
-- 2.0.3 Fix: mkdir bug on windows
-- 2.0.2 new feature: support shouldTerminate
-- 2.0.1 Avoid possible I/O conflicts.
-- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
-- 1.2.8 Fix: create task folder failed on MacOS
-- 1.2.7 Small updates
-- 1.2.6 Support onFinish event
-- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
-- 1.2.4 Fix: opt.numberOfWorkers not work
-- 1.2.3 Update README
-- 1.2.2 Update README and examples
-- 1.2.1 Handle exceptions and errors in subtasks
-- 1.2.0 Simplified usage by providing the function way and support return Promise
-- 1.1.4 Remove make-dir
-- 1.1.3 Simplified usage, see example0
-- 1.1.2 
-- 1.1.1 Rename files, updated changelog
-- 1.1.0 Simplified the usage of a customized Consumer, see example0
-- 1.0.8 Fix examples
-- 1.0.7 Remove moment
-- 1.0.6 Performance optimization
-
-### Gitee:
-
-[https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)
+# multi-tasks
+
+Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. It manages progress and tasks based on the file system, which means that even if the host crashes, tasks can be resumed based on file records.
+
+**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
+
+### Install:
+
+```javascript
+npm install multi-tasks
+```
+
+### API:
+
+- `multiTasks(config)` — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
+- `multiTasks.start(config)` — always start a fresh run.
+- `multiTasks.resume(config)` — resume an interrupted run.
+- `multiTasks.restart(config)` — resume and also retry failed tasks (needs `config.taskFolder`).
+- `helper.createNewTasks(tasks)` — create new tasks dynamically while processing.
+- Config options:
+  - `initialTasks` — array of task objects, or a folder path string to resume from.
+  - `processTask(task, helper)` — required; return a value or a Promise.
+  - `taskRootFolder` — root directory where progress and result files are stored.
+  - `taskId` — task folder name under `taskRootFolder`.
+  - `taskFolder` — full task folder path, required by `restart`.
+  - `numberOfWorkers` — how many worker processes run in parallel.
+  - `taskTimeout` — optional, milliseconds; an overdue task fails with a timeout error.
+  - `maxTaskRetries` — optional, max times a failed task is auto-retried (sent back to the queue immediately); covers worker crashes, exceptions/rejections from `processTask`, and `taskTimeout` timeouts; the retry count is kept in the task file as `__retryCount` — it survives `resume` and is reset by `restart`.
+  - `autoCloseAfterCompletion` — set `false` if you create new tasks dynamically.
+  - `shouldTerminate(info)` — return `true` to terminate the whole process.
+  - `onFinish(report)` — called once after all workers are done.
+
+### How to use:
+
+```javascript
+//see examples/example0
+let multiTasks = require('multi-tasks').multiTasks;
+
+//Step1, create your tasks that need to be executed simultaneously as an array.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
+    });
+};
+
+//Step2, provide a function to process a certain sub-task and return the result data
+let processTask = (task, helper)=>{
+    let {taskCount} = task;//get your task data
+
+    //Run your processing logic here...
+
+    //then return the result as a plain javascript object or an async promise
+    return 'result data'; 
+};
+
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //how many workers are working in parallel
+    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
+    //maxTaskRetries: 2, //optional, auto-retry a failed task (worker crash, processTask error, or timeout); retried tasks go back to the queue
+    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
+    shouldTerminate:(info)=>{
+        //return true if you need to terminate the whole process
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});
+
+```
+
+### More about the task processing function
+
+```javascript
+//Example1, return a promise for async processes,
+//          you can return a promise or a non-promise result, 
+//          all result data can be found in the results/succ folder
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Example2, dynamically create a new task while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+    
+    if(taskCount % 2 === 0){
+        //create a new task if needed
+        helper.createNewTasks({
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Example3, generate/throw exceptions in a task method
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
+    if(taskCount===5) aaa =  bbb;//this undefined exception will be captured by multi-tasks
+
+    return {data:'succ'};
+}
+
+//Example5, auto-retry a failed task:
+//          when a task fails - its worker process crashes, processTask throws
+//          or rejects, or it exceeds taskTimeout - it is sent back to the
+//          queue and retried, at most maxTaskRetries times; when the limit
+//          is reached it lands in finished_with_errors (crash:
+//          {type:'worker_crash'}, timeout: {type:'timeout'}, processTask
+//          error: the original error data)
+multiTasks({
+    initialTasks: alltasks,
+    processTask,
+    taskRootFolder: `../examples-tmp-data/example-retry`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    maxTaskRetries: 2,
+});
+
+```
+
+### Resuming
+
+Sometimes the task execution is interrupted due to some reasons (such as power outage), you can resume the execution like this
+
+```javascript
+
+multiTasks({
+    initialTasks: `/myworks/my_scan_tasks/`, //Point 'initialTasks' to the interrupted task directory, multi-tasks will read the tasks in the 'new' folder and initialize them to 'initialTasks' and then continue execution
+    ...
+    ...//Other configurations remain unchanged
+    ...
+});
+
+```
+
+### Testing:
+
+The usage patterns documented above are covered by automated tests (unit tests in `test/`, end-to-end tests in `teste2e/`). The e2e suite runs the API in real child processes with a mixed workload — successful tasks (both promise and non-promise results), planned failures (rejected promises and thrown exceptions), dynamically created tasks via `helper.createNewTasks`, and workers that crash randomly — then resumes repeatedly and verifies that every task lands exactly one result in `results/succ` or `results/errors`.
+
+### Changelog:
+
+- 3.0.3 Support 'maxTaskRetries'
+- 3.0.2 Support 'taskTimeout'
+- 3.0.1 Versions 3.0.0 and above are maintained by AI
+- 2.2.0 Support Resuming from an interrupted task
+- 2.1.2 Update readme
+- 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
+- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
+- 2.0.9 Update readme examples
+- 2.0.8 Update readme examples
+- 2.0.7 Update changelog
+- 2.0.6 Update readme, remove failed examples
+- 2.0.5 Update readme examples
+- 2.0.4 Support resume from a failed task
+- 2.0.3 Fix: mkdir bug on windows
+- 2.0.2 new feature: support shouldTerminate
+- 2.0.1 Avoid possible I/O conflicts.
+- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
+- 1.2.8 Fix: create task folder failed on MacOS
+- 1.2.7 Small updates
+- 1.2.6 Support onFinish event
+- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
+- 1.2.4 Fix: opt.numberOfWorkers not work
+- 1.2.3 Update README
+- 1.2.2 Update README and examples
+- 1.2.1 Handle exceptions and errors in subtasks
+- 1.2.0 Simplified usage by providing the function way and support return Promise
+- 1.1.4 Remove make-dir
+- 1.1.3 Simplified usage, see example0
+- 1.1.2 
+- 1.1.1 Rename files, updated changelog
+- 1.1.0 Simplified the usage of a customized Consumer, see example0
+- 1.0.8 Fix examples
+- 1.0.7 Remove moment
+- 1.0.6 Performance optimization
+
+### Gitee:
+
+[https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/workers/Multithread.js

@@ -74,7 +74,15 @@ const start = (config)=>{
                 }else if(resultType === 'error'){//超时边界: worker 已自行记错误
                     TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
                 }else{
-                    TaskMgr.saveTaskResult(subid, true, [{type: '超时', timeout: config.taskTimeout, cost}]);
+                    if(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
+                        let retryCount = TaskMgr.incrementTaskRetry(subid);
+                        if(retryCount <= config.maxTaskRetries){
+                            console.warn(`[Master]: task "${subid}" timeout, retry (${retryCount}/${config.maxTaskRetries})`);
+                            TaskMgr.mvTask(subid, 'running', 'new', ()=>{});
+                            return;
+                        };
+                    };
+                    TaskMgr.saveTaskResult(subid, true, [{type: 'timeout', timeout: config.taskTimeout, cost}]);
                     TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
                 }
             }catch(e){
@@ -91,6 +99,23 @@ const start = (config)=>{
             worker.kill();//不 disconnect: exit 事件会按崩溃补员
             finalizeTimeoutTask(subTask, cost);
         };
+        let handleCrashedTask = (subTask)=>{
+            if(!(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0)) return;
+            let subid = subTask.subid;
+            try{
+                let retryCount = TaskMgr.incrementTaskRetry(subid);
+                if(retryCount <= config.maxTaskRetries){
+                    console.warn(`[Master]: worker crashed, retry task "${subid}" (${retryCount}/${config.maxTaskRetries})`);
+                    TaskMgr.mvTask(subid, 'running', 'new', ()=>{});
+                }else{
+                    console.warn(`[Master]: task "${subid}" crashed ${retryCount} times, give up`);
+                    TaskMgr.saveTaskResult(subid, true, [{type: 'worker_crash', retryCount: config.maxTaskRetries}]);
+                    TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
+                };
+            }catch(e){
+                console.warn('[Master]: failed to handle crashed task', subid, e);
+            }
+        };
 
         let initWorker = (worker)=>{
             WorkerMgr.addWorker(worker);
@@ -128,9 +153,11 @@ const start = (config)=>{
                                 },()=>{}, {
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
-                                if(typeof config.taskTimeout === 'number' && config.taskTimeout > 0){
+                                let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
+                                let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
+                                if(hasTimeout || hasRetry){
                                     workerTaskTimers[worker.id] = {
-                                        timer: setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout),
+                                        timer: hasTimeout ? setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout) : null,
                                         subTask,
                                         startTimestamp
                                     };
@@ -152,10 +179,14 @@ const start = (config)=>{
                 initWorker(worker);
             }
         });
-        cluster.on('exit', (worker, code, signal) => {             
+        cluster.on('exit', (worker, code, signal) => {
+            let rec = workerTaskTimers[worker.id];//先取再清: 崩溃重试要用 rec.subTask
             clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
             if (code !== 0 && !worker.exitedAfterDisconnect) {
                 console.log('Worker died unexpectedly starting a new one...');
+                if(rec && rec.subTask){
+                    handleCrashedTask(rec.subTask);
+                };
                 let worker = cluster.fork();
                 initWorker(worker);
                 return;
@@ -252,7 +283,27 @@ function runTask(config, subTask, subTaskLog){
             };
             let hasError = errors.length > 0;
             if(hasError)console.warn('[Task Error]', subTask.subid, errors);
+            let retried = false;
+            if(hasError && typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
+                let retryCount = TaskMgr.incrementTaskRetry(subTask.subid);
+                if(retryCount <= config.maxTaskRetries){
+                    console.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
+                    TaskMgr.mvTask(subTask.subid, 'running', 'new', ()=>{});
+                    retried = true;
+                };
+            };
             TaskMgr.saveTaskLog(subTask.subid, subTaskLog);
+            if(retried){
+                resolve({
+                    startTimestamp,
+                    endTimestamp,
+                    cost
+                });
+                return;
+            };
+            if(!hasError){
+                TaskMgr.deleteSavedResult(subTask.subid);//清掉前序尝试可能残留的(超时竞争)错误结果
+            };
             TaskMgr.saveTaskResult(subTask.subid, hasError, hasError ? errors : resultData);
             TaskMgr.mvTask(subTask.subid, 'running',(hasError ? 'finished_with_errors':'finished'), (newPath)=>{
                 resolve({

package/workers/TaskMgr.js

@@ -225,6 +225,30 @@ const TaskMgr = {
             fcontent = JSON.stringify(log);
         };
         fs.writeFileSync(fpath, fcontent);
-    }
+    },
+    incrementTaskRetry:(subid)=>{
+        let {taskFolderRunning} = main_config;
+        let fpath = pathutil.resolve(taskFolderRunning, `./${subid}.task.json`);
+        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
+        task.__retryCount = (task.__retryCount || 0) + 1;
+        fs.writeFileSync(fpath, JSON.stringify(task));
+        return task.__retryCount;
+    },
+    deleteSavedResult:(subid)=>{
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        [taskFolderResult, taskFolderResultFailed].forEach((folder)=>{
+            let fpath = pathutil.resolve(folder, `${subid}.json`);
+            if(fs.existsSync(fpath)){
+                fs.unlinkSync(fpath);
+            }
+        });
+    },
+    resetTaskRetry:(subid)=>{
+        let {taskFolderNew} = main_config;
+        let fpath = pathutil.resolve(taskFolderNew, `./${subid}.task.json`);
+        let task = JSON.parse(fs.readFileSync(fpath, 'utf-8'));
+        delete task.__retryCount;
+        fs.writeFileSync(fpath, JSON.stringify(task));
+    },
 };
 module.exports = TaskMgr;

package/workers/asMaster.js

@@ -32,7 +32,9 @@ const resume = (config)=>{
                 if(config.__restart){
                     errfiles.forEach((fname)=>{
                         let subid = fname.replace(/\.task\.json$/, '');
-                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new'));
+                        promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
+                            TaskMgr.resetTaskRetry(subid);//restart 重跑失败任务时重置重试计数(resume 不重置)
+                        }));
                     });
                 };
                 Promise.all(promises).then(()=>{