multi-tasks 3.1.5 → 3.1.7

package/README.md

@@ -17,7 +17,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 **Entry functions:**
 
 - **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
-- **`multiTasks.resume(config)`** — resume an interrupted run; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
+- **`multiTasks.resume(config)`** — usually you don't need this: the simplest way to resume is to just run `multiTasks(config)` again, it auto-resumes when the task folder already exists. This method resumes an interrupted run from the task folder alone; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
 - **`multiTasks.retry_fails(config)`** — same as `resume`, and also retries the failed tasks.
 - **`multiTasks.restart(config)`** — wipe the task folder's progress and re-run from scratch; pass a config object with `taskFolder`. The config (including `processTask` and `initialTasks`) is restored from `task_config.json`; any field you pass overrides the snapshot, and overrides are written back into it. **All previous progress and results are deleted.**
 
@@ -33,7 +33,10 @@ The API has three parts: the entry functions, the config options, and the `helpe
 - **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
 - **`shouldTerminate(info)`** — return `true` to terminate the whole process.
 - **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
-- **`onFinish(report)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`); `report` is an empty object for now.
+- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` lets you walk every persisted task result:
+  - **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing).
+  - **`helper.foreachErrorResult((count, task, err) => {})`** — iterate all failed results (`results/errors`); `err` is the persisted error object as-is (`{ex}` / `{subTask, exception}` / `{type: 'timeout', ...}` / `{type: 'worker_crash', ...}`).
+  - Iteration follows filesystem order (not sorted). `onFinish` runs synchronously at the very end and the master process exits shortly after it returns — don't `await` inside it.
 
 **Other options:**
 
@@ -84,8 +87,14 @@ multiTasks({
     shouldTerminate:(info)=>{
         //return true if you need to terminate the whole process
     },
-    onFinish: (report)=>{
+    onFinish: (report, helper)=>{
         console.log('finish callback', report);
+        helper.foreachSuccResult((count, task, result)=>{
+            console.log(`#${count} succeeded:`, task.name, result);
+        });
+        helper.foreachErrorResult((count, task, err)=>{
+            console.log(`#${count} failed:`, task.name, err);
+        });
     }
 });
 
@@ -263,12 +272,14 @@ Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **
 
 ### Changelog:
 
+- 3.1.7 onFinish: iterate all succ/failed results via helper.foreachSuccResult/foreachErrorResult
+- 3.1.6 Fix silent task loss under heavy load with maxTaskRetries
 - 3.1.5 Update README
-- 3.1.4 Rewrite `multiTasks.resume`, `multiTasks.retry_fails` and `multiTasks.restart`
+- 3.1.4 Rewrite methods multiTasks.resume/retry_fails/restart
 - 3.1.3 Update README
 - 3.1.2 Update README
 - 3.1.1 Fix readme documentation
-- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
+- 3.1.0 Support worker broadcast and system events; default numberOfWorkers is now "50%" of CPU cores
 - 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
 - 3.0.3 Support 'maxTaskRetries'
 - 3.0.2 Support 'taskTimeout'

package/package.json

@@ -1,36 +1,36 @@
-{
-  "name": "multi-tasks",
-  "version": "3.1.5",
-  "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
-  "main": "index.js",
-  "files": [
-    "index.js",
-    "workers",
-    "utils/*.js"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "https://gitee.com/zhanglei923/multi-tasks.git"
-  },
-  "directories": {
-    "example": "examples"
-  },
-  "engines": {
-    "node": ">=14.14"
-  },
-  "scripts": {
-    "test": "jest",
-    "prepublishOnly": "npm test"
-  },
-  "keywords": [
-    "multitask",
-    "worker",
-    "multithread"
-  ],
-  "author": "zhanglei923@gmail.com",
-  "license": "MIT",
-  "devDependencies": {
-    "cli-progress": "^3.12.0",
-    "jest": "^29.7.0"
-  }
-}
+{
+  "name": "multi-tasks",
+  "version": "3.1.7",
+  "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "workers",
+    "utils/*.js"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://gitee.com/zhanglei923/multi-tasks.git"
+  },
+  "directories": {
+    "example": "examples"
+  },
+  "engines": {
+    "node": ">=14.14"
+  },
+  "scripts": {
+    "test": "jest",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT",
+  "devDependencies": {
+    "cli-progress": "^3.12.0",
+    "jest": "^29.7.0"
+  }
+}

package/utils/foreachResults.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+const path = require('path');
+
+//同步遍历某个结果目录(results/succ 或 results/errors)下的全部 .json 结果文件,
+//逐个回调 cb(count, task, result):
+//  count  —— 0 起始的遍历序号;
+//  task   —— 按 subid 从 taskProgressFolder 读回的任务文件内容(缺失/解析失败为 null);
+//  result —— 结果文件的 JSON 内容(解析失败, 如字面量 'undefined', 传 undefined, 不跳过)。
+//仅在 onFinish 时机使用: 此时文件已全部落盘, 无并发写。
+const forEachResult = (resultFolder, taskProgressFolder, cb) => {
+  const files = fs.readdirSync(resultFolder).filter((f) => f.endsWith('.json'));
+  files.forEach((fname, count) => {
+    const subid = fname.replace(/\.json$/, '');
+    let task = null;
+    try {
+      task = JSON.parse(fs.readFileSync(path.join(taskProgressFolder, `${subid}.task.json`), 'utf-8'));
+    } catch (e) {
+      task = null;
+    };
+    let result;
+    try {
+      result = JSON.parse(fs.readFileSync(path.join(resultFolder, fname), 'utf-8'));
+    } catch (e) {
+      result = undefined;
+    };
+    cb(count, task, result);
+  });
+};
+
+module.exports = { forEachResult };

package/workers/Multithread.js

@@ -58,7 +58,10 @@ const start = (config)=>{
         console.log(`  PID=${PID}`);
 
         const AsyncQueue = require(`../utils/asyncQueue`);
-        const queue = new AsyncQueue();  
+        const queue = new AsyncQueue();
+
+        const masterStartTimestamp = new Date()*1;//本轮 master 启动时间(resume 不追溯首轮)
+        let workerCrashCount = 0;//累计崩溃补员次数, 进 onFinish report
 
         let clearTaskTimer = (workerId)=>{
             let rec = workerTaskTimers[workerId];
@@ -217,6 +220,7 @@ const start = (config)=>{
             clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
             if (code !== 0 && !worker.exitedAfterDisconnect) {
                 console.log('Worker died unexpectedly starting a new one...');
+                workerCrashCount++;
                 if(rec && rec.subTask){
                     handleCrashedTask(rec.subTask);
                 };
@@ -230,7 +234,25 @@ const start = (config)=>{
             }
             if(allkilled){
                 if(USER_CONFIG.onFinish){
-                    USER_CONFIG.onFinish({});
+                    let endTimestamp = new Date()*1;
+                    let stats = TaskMgr.getFinalStats();
+                    let report = {
+                        taskId: config.masterTaskId,
+                        taskFolder: stats.taskFolder,
+                        startTimestamp: masterStartTimestamp,
+                        endTimestamp,
+                        cost: endTimestamp - masterStartTimestamp,
+                        total: stats.succeeded + stats.failed,
+                        succeeded: stats.succeeded,
+                        failed: stats.failed,
+                        failedTasks: stats.failedTasks,
+                        workers: {count: numOfWorkers, crashed: workerCrashCount},
+                    };
+                    let onFinishHelper = {
+                        foreachSuccResult: (cb) => TaskMgr.forEachSuccResult(cb),
+                        foreachErrorResult: (cb) => TaskMgr.forEachErrorResult(cb),
+                    };
+                    USER_CONFIG.onFinish(report, onFinishHelper);
                     delete USER_CONFIG.onFinish;
                 }
                 setTimeout(()=>{
@@ -309,7 +331,7 @@ function runTask(config, subTask, subTaskLog){
             resultData = data;
         }).catch((ex)=>{
             errors.push({ex: serializeError(ex)});
-        }).finally(()=>{
+        }).finally(async ()=>{
             let {startTimestamp} = subTaskLog;
             let endTimestamp = new Date()*1;
             let cost = endTimestamp - startTimestamp;
@@ -325,8 +347,14 @@ function runTask(config, subTask, subTaskLog){
                 let retryCount = TaskMgr.incrementTaskRetry(subTask.subid);
                 if(retryCount <= config.maxTaskRetries){
                     console.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
-                    TaskMgr.mvTask(subTask.subid, 'running', 'new', ()=>{});
                     retried = true;
+                    //等 rename 落盘再 resolve/请求新任务: fire-and-forget 的异步 rename 会随 worker 被
+                    //autoClose 杀掉而消亡, 任务文件滞留 running 被静默丢弃(高负载偶发丢任务的根因, 勿回退)
+                    try{
+                        await TaskMgr.mvTaskPromise(subTask.subid, 'running', 'new');
+                    }catch(newPath){
+                        console.warn(`[Task Retry] "${subTask.subid}" failed to move back to new, left in running (recoverable via resume)`);
+                    };
                 };
             };
             TaskMgr.saveTaskLog(subTask.subid, subTaskLog);

package/workers/TaskMgr.js

@@ -3,6 +3,7 @@ const pathutil = require('path');
 const makeDir = require('../utils/makedir');
 const randomsUtil = require('../utils/randoms');
 const { serializeConfig } = require('../utils/serializeFns');
+const { forEachResult } = require('../utils/foreachResults');
 
 let allSubTasks = {
     new:[],
@@ -175,6 +176,21 @@ const TaskMgr = {
             callback(files);
         });
     },
+    getFinalStats:()=>{//master 收尾用: 以盘上终态为准统计成功/失败(resume 跨轮累计也是对的)
+        const {taskFolder, taskFolderFinished, taskFolderFailed} = main_config;
+        const listSubids = (folder)=>{
+            return fs.readdirSync(folder)
+                .filter((fname)=>fname.endsWith('.task.json'))
+                .map((fname)=>fname.replace(/\.task\.json$/, ''));
+        };
+        const failedTasks = listSubids(taskFolderFailed);
+        return {
+            taskFolder,
+            succeeded: listSubids(taskFolderFinished).length,
+            failed: failedTasks.length,
+            failedTasks,
+        };
+    },
     mvTaskPromise: (subid, from, to)=> {
         return new Promise((resolve, reject)=>{
             TaskMgr.mvTask(subid, from, to, (newPath, succ)=>{
@@ -257,6 +273,14 @@ const TaskMgr = {
         delete task.__retryCount;
         fs.writeFileSync(fpath, JSON.stringify(task));
     },
+    forEachSuccResult:(cb)=>{
+        let {taskFolderResult, taskFolderFinished} = main_config;
+        forEachResult(taskFolderResult, taskFolderFinished, cb);
+    },
+    forEachErrorResult:(cb)=>{
+        let {taskFolderResultFailed, taskFolderFailed} = main_config;
+        forEachResult(taskFolderResultFailed, taskFolderFailed, cb);
+    },
     resetTaskFolders:(taskFolder)=>{
         fs.rmSync(taskFolder, {recursive: true, force: true});
     },