multi-tasks 3.0.1 → 3.0.2

package/README.md

@@ -2,6 +2,8 @@
 
 Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. It manages progress and tasks based on the file system, which means that even if the host crashes, tasks can be resumed based on file records.
 
+**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
+
 ### Install:
 
 ```javascript
@@ -40,6 +42,7 @@ multiTasks({
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
     taskId: 'my-task',
     numberOfWorkers: 3, //how many workers are working in parallel
+    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
     //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
     shouldTerminate:(info)=>{
         //return true if you need to terminate the whole process
@@ -122,6 +125,7 @@ The usage patterns documented above are covered by automated tests (unit tests i
 
 ### Changelog:
 
+- 3.0.2 Support taskTimeout: a task that does not finish in time is treated as failed with a timeout error
 - 3.0.1 Versions 3.0.0 and above are maintained by AI
 - 2.2.0 Support Resuming from an interrupted task
 - 2.1.2 Update readme

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.0.1",
+  "version": "3.0.2",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/workers/Multithread.js

@@ -27,6 +27,8 @@ let USER_CONFIG;
 
 let taskCount = 0;
 
+let workerTaskTimers = {};//worker.id -> {timer, subTask, startTimestamp}, master 端任务超时倒计时
+
 let killAllWorkers = ()=>{  
     for (const id in cluster.workers) {  
         if (cluster.workers.hasOwnProperty(id)) {  
@@ -56,10 +58,45 @@ const start = (config)=>{
         const AsyncQueue = require(`../utils/asyncQueue`);
         const queue = new AsyncQueue();  
 
+        let clearTaskTimer = (workerId)=>{
+            let rec = workerTaskTimers[workerId];
+            if(rec){
+                clearTimeout(rec.timer);
+                delete workerTaskTimers[workerId];
+            }
+        };
+        let finalizeTimeoutTask = (subTask, cost)=>{
+            let subid = subTask.subid;
+            try{
+                let resultType = TaskMgr.getSavedResultType(subid);
+                if(resultType === 'succ'){//超时边界: worker 已成功写盘, 视为已完成
+                    TaskMgr.mvTask(subid, 'running', 'finished', ()=>{});
+                }else if(resultType === 'error'){//超时边界: worker 已自行记错误
+                    TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
+                }else{
+                    TaskMgr.saveTaskResult(subid, true, [{type: '超时', timeout: config.taskTimeout, cost}]);
+                    TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
+                }
+            }catch(e){
+                console.warn('[Master]: failed to finalize timeout task', subid, e);
+            }
+        };
+        let onTaskTimeout = (worker)=>{
+            let rec = workerTaskTimers[worker.id];
+            if(!rec) return;
+            delete workerTaskTimers[worker.id];
+            let {subTask, startTimestamp} = rec;
+            let cost = new Date()*1 - startTimestamp;
+            console.warn(`[Master]: task "${subTask.subid}" timeout after ${cost}ms, killing worker pid=${worker.process.pid}`);
+            worker.kill();//不 disconnect: exit 事件会按崩溃补员
+            finalizeTimeoutTask(subTask, cost);
+        };
+
         let initWorker = (worker)=>{
             WorkerMgr.addWorker(worker);
             worker.on('message', function(message) {
                 if(message === MSG_REQUEST_NEW_TASK){
+                    clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
                     queue.enqueue(async ()=>{
                         return TaskMgr.updateNewTasks();
                     });//use queue to make sure no I/O conflict
@@ -82,14 +119,22 @@ const start = (config)=>{
                                     return process.exit(0);
                                 }
                                 //send new tasks to the worker
+                                let startTimestamp = new Date()*1;
                                 worker.send({
                                     taskCount,
-                                    startTimestamp: new Date()*1,
+                                    startTimestamp,
                                     config,
                                     subTask
                                 },()=>{}, {
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
+                                if(typeof config.taskTimeout === 'number' && config.taskTimeout > 0){
+                                    workerTaskTimers[worker.id] = {
+                                        timer: setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout),
+                                        subTask,
+                                        startTimestamp
+                                    };
+                                }
                                 taskCount++;
                             }
                         });
@@ -108,6 +153,7 @@ const start = (config)=>{
             }
         });
         cluster.on('exit', (worker, code, signal) => {             
+            clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
             if (code !== 0 && !worker.exitedAfterDisconnect) {
                 console.log('Worker died unexpectedly starting a new one...');
                 let worker = cluster.fork();

package/workers/TaskMgr.js

@@ -205,6 +205,14 @@ const TaskMgr = {
         };
         fs.writeFileSync(fpath, fcontent);
     },
+    getSavedResultType:(subid)=>{
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let succPath = pathutil.resolve(taskFolderResult, `${subid}.json`);
+        let errPath = pathutil.resolve(taskFolderResultFailed, `${subid}.json`);
+        if(fs.existsSync(succPath)) return 'succ';
+        if(fs.existsSync(errPath)) return 'error';
+        return null;
+    },
     saveTaskLog:(subid, log)=>{
         let {logsRootFolder} = main_config;
         let folder = logsRootFolder;