multi-tasks 3.1.5 → 3.1.6

package/README.md

@@ -263,12 +263,13 @@ Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **
 
 ### Changelog:
 
+- 3.1.6 Fix silent task loss under heavy load with maxTaskRetries
 - 3.1.5 Update README
-- 3.1.4 Rewrite `multiTasks.resume`, `multiTasks.retry_fails` and `multiTasks.restart`
+- 3.1.4 Rewrite methods multiTasks.resume/retry_fails/restart
 - 3.1.3 Update README
 - 3.1.2 Update README
 - 3.1.1 Fix readme documentation
-- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
+- 3.1.0 Support worker broadcast and system events; default numberOfWorkers is now "50%" of CPU cores
 - 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
 - 3.0.3 Support 'maxTaskRetries'
 - 3.0.2 Support 'taskTimeout'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.1.5",
+  "version": "3.1.6",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/workers/Multithread.js

@@ -309,7 +309,7 @@ function runTask(config, subTask, subTaskLog){
             resultData = data;
         }).catch((ex)=>{
             errors.push({ex: serializeError(ex)});
-        }).finally(()=>{
+        }).finally(async ()=>{
             let {startTimestamp} = subTaskLog;
             let endTimestamp = new Date()*1;
             let cost = endTimestamp - startTimestamp;
@@ -325,8 +325,14 @@ function runTask(config, subTask, subTaskLog){
                 let retryCount = TaskMgr.incrementTaskRetry(subTask.subid);
                 if(retryCount <= config.maxTaskRetries){
                     console.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
-                    TaskMgr.mvTask(subTask.subid, 'running', 'new', ()=>{});
                     retried = true;
+                    //等 rename 落盘再 resolve/请求新任务: fire-and-forget 的异步 rename 会随 worker 被
+                    //autoClose 杀掉而消亡, 任务文件滞留 running 被静默丢弃(高负载偶发丢任务的根因, 勿回退)
+                    try{
+                        await TaskMgr.mvTaskPromise(subTask.subid, 'running', 'new');
+                    }catch(newPath){
+                        console.warn(`[Task Retry] "${subTask.subid}" failed to move back to new, left in running (recoverable via resume)`);
+                    };
                 };
             };
             TaskMgr.saveTaskLog(subTask.subid, subTaskLog);