multi-tasks 3.1.6 → 3.1.8

package/README.md

@@ -17,7 +17,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 **Entry functions:**
 
 - **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
-- **`multiTasks.resume(config)`** — resume an interrupted run; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
+- **`multiTasks.resume(config)`** — usually you don't need this: the simplest way to resume is to just run `multiTasks(config)` again, it auto-resumes when the task folder already exists. This method resumes an interrupted run from the task folder alone; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
 - **`multiTasks.retry_fails(config)`** — same as `resume`, and also retries the failed tasks.
 - **`multiTasks.restart(config)`** — wipe the task folder's progress and re-run from scratch; pass a config object with `taskFolder`. The config (including `processTask` and `initialTasks`) is restored from `task_config.json`; any field you pass overrides the snapshot, and overrides are written back into it. **All previous progress and results are deleted.**
 
@@ -33,11 +33,14 @@ The API has three parts: the entry functions, the config options, and the `helpe
 - **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
 - **`shouldTerminate(info)`** — return `true` to terminate the whole process.
 - **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
-- **`onFinish(report)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`); `report` is an empty object for now.
+- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` lets you walk every persisted task result:
+  - **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing).
+  - **`helper.foreachErrorResult((count, task, err) => {})`** — iterate all failed results (`results/errors`); `err` is the persisted error object as-is (`{ex}` / `{subTask, exception}` / `{type: 'timeout', ...}` / `{type: 'worker_crash', ...}`).
+  - Iteration follows filesystem order (not sorted). `onFinish` runs synchronously at the very end and the master process exits shortly after it returns — don't `await` inside it.
 
 **Other options:**
 
-- **`taskFolder`** — full task folder path; only used by `resume`/`retry_fails`.
+- **`taskFolder`** — full task folder path; only used by `resume`/`retry_fails`/`restart`.
 
 **Task helper** (the `helper` object passed as the second argument of `processTask`):
 
@@ -84,8 +87,14 @@ multiTasks({
     shouldTerminate:(info)=>{
         //return true if you need to terminate the whole process
     },
-    onFinish: (report)=>{
+    onFinish: (report, helper)=>{
         console.log('finish callback', report);
+        helper.foreachSuccResult((count, task, result)=>{
+            console.log(`#${count} succeeded:`, task.name, result);
+        });
+        helper.foreachErrorResult((count, task, err)=>{
+            console.log(`#${count} failed:`, task.name, err);
+        });
     }
 });
 
@@ -263,6 +272,8 @@ Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **
 
 ### Changelog:
 
+- 3.1.8 Small refinements
+- 3.1.7 onFinish: iterate all succ/failed results via helper.foreachSuccResult/foreachErrorResult
 - 3.1.6 Fix silent task loss under heavy load with maxTaskRetries
 - 3.1.5 Update README
 - 3.1.4 Rewrite methods multiTasks.resume/retry_fails/restart

package/package.json

@@ -1,36 +1,36 @@
-{
-  "name": "multi-tasks",
-  "version": "3.1.6",
-  "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
-  "main": "index.js",
-  "files": [
-    "index.js",
-    "workers",
-    "utils/*.js"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "https://gitee.com/zhanglei923/multi-tasks.git"
-  },
-  "directories": {
-    "example": "examples"
-  },
-  "engines": {
-    "node": ">=14.14"
-  },
-  "scripts": {
-    "test": "jest",
-    "prepublishOnly": "npm test"
-  },
-  "keywords": [
-    "multitask",
-    "worker",
-    "multithread"
-  ],
-  "author": "zhanglei923@gmail.com",
-  "license": "MIT",
-  "devDependencies": {
-    "cli-progress": "^3.12.0",
-    "jest": "^29.7.0"
-  }
-}
+{
+  "name": "multi-tasks",
+  "version": "3.1.8",
+  "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "workers",
+    "utils/*.js"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://gitee.com/zhanglei923/multi-tasks.git"
+  },
+  "directories": {
+    "example": "examples"
+  },
+  "engines": {
+    "node": ">=14.14"
+  },
+  "scripts": {
+    "test": "jest",
+    "prepublishOnly": "npm test"
+  },
+  "keywords": [
+    "multitask",
+    "worker",
+    "multithread"
+  ],
+  "author": "zhanglei923@gmail.com",
+  "license": "MIT",
+  "devDependencies": {
+    "cli-progress": "^3.12.0",
+    "jest": "^29.7.0"
+  }
+}

package/utils/foreachResults.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+const path = require('path');
+
+//同步遍历某个结果目录(results/succ 或 results/errors)下的全部 .json 结果文件,
+//逐个回调 cb(count, task, result):
+//  count  —— 0 起始的遍历序号;
+//  task   —— 按 subid 从 taskProgressFolder 读回的任务文件内容(缺失/解析失败为 null);
+//  result —— 结果文件的 JSON 内容(解析失败, 如字面量 'undefined', 传 undefined, 不跳过)。
+//仅在 onFinish 时机使用: 此时文件已全部落盘, 无并发写。
+const forEachResult = (resultFolder, taskProgressFolder, cb) => {
+  const files = fs.readdirSync(resultFolder).filter((f) => f.endsWith('.json'));
+  files.forEach((fname, count) => {
+    const subid = fname.replace(/\.json$/, '');
+    let task = null;
+    try {
+      task = JSON.parse(fs.readFileSync(path.join(taskProgressFolder, `${subid}.task.json`), 'utf-8'));
+    } catch (e) {
+      task = null;
+    };
+    let result;
+    try {
+      result = JSON.parse(fs.readFileSync(path.join(resultFolder, fname), 'utf-8'));
+    } catch (e) {
+      result = undefined;
+    };
+    cb(count, task, result);
+  });
+};
+
+module.exports = { forEachResult };

package/workers/Multithread.js

@@ -58,7 +58,10 @@ const start = (config)=>{
         console.log(`  PID=${PID}`);
 
         const AsyncQueue = require(`../utils/asyncQueue`);
-        const queue = new AsyncQueue();  
+        const queue = new AsyncQueue();
+
+        const masterStartTimestamp = new Date()*1;//本轮 master 启动时间(resume 不追溯首轮)
+        let workerCrashCount = 0;//累计崩溃补员次数, 进 onFinish report
 
         let clearTaskTimer = (workerId)=>{
             let rec = workerTaskTimers[workerId];
@@ -155,49 +158,64 @@ const start = (config)=>{
                 }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
-                    queue.enqueue(async ()=>{
-                        return TaskMgr.updateNewTasks();
-                    });//use queue to make sure no I/O conflict
                     queue.enqueue(async () => {
-                        return TaskMgr.popNewTask().then((subTask)=>{
-                            if(!subTask){
-                                //no tasks left
-                                console.log('[Master]: no tasks left!');
-                                if(config.autoCloseAfterCompletion){
-                                    killWorker(worker);
-                                }else{//waiting for new tasks
-                                    worker.send(MSG_NO_TASK_FOUND,()=>{}, {
-                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                    });
-                                }
-                            }else{
-                                let terminateData = {taskCount,subTask,config, id: worker.id};
-                                if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
-                                    killAllWorkers();
-                                    return process.exit(0);
-                                }
-                                //send new tasks to the worker
-                                let startTimestamp = new Date()*1;
-                                worker.send({
-                                    taskCount,
-                                    startTimestamp,
-                                    config,
-                                    subTask
-                                },()=>{}, {
+                        //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
+                        await TaskMgr.updateNewTasks();
+                        let subTask = await TaskMgr.popNewTask();
+                        if(!subTask){
+                            //判空前强制绕开缓存重读一次磁盘再 pop: 防其他 worker 动态建任务(createNewTasks)
+                            //落在 update 与判空之间被漏判, 导致 autoClose 提前杀光 worker / 丢新任务
+                            await TaskMgr.loadNewTasks();
+                            subTask = await TaskMgr.popNewTask();
+                        }
+                        if(!subTask && config.autoCloseAfterCompletion){
+                            //杀 worker 前延时 1s 再兜底确认一次: 等其他 worker 可能正在落盘的动态新任务
+                            //(best-effort 再压小竞态窗口, 只限 autoClose 路径, 普通派发零开销)
+                            await new Promise((resolve)=>{ setTimeout(resolve, 1000); });
+                            await TaskMgr.loadNewTasks();
+                            subTask = await TaskMgr.popNewTask();
+                        }
+                        if(!subTask){
+                            //no tasks left
+                            console.log('[Master]: no tasks left!');
+                            if(config.autoCloseAfterCompletion){
+                                killWorker(worker);
+                            }else{//waiting for new tasks
+                                worker.send(MSG_NO_TASK_FOUND,()=>{}, {
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
-                                let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
-                                let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
-                                if(hasTimeout || hasRetry){
-                                    workerTaskTimers[worker.id] = {
-                                        timer: hasTimeout ? setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout) : null,
-                                        subTask,
-                                        startTimestamp
-                                    };
-                                }
-                                taskCount++;
                             }
+                            return;
+                        }
+                        let terminateData = {taskCount,subTask,config, id: worker.id};
+                        if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
+                            killAllWorkers();
+                            return process.exit(0);
+                        }
+                        //send new tasks to the worker
+                        let startTimestamp = new Date()*1;
+                        //IPC 用裁剪副本: worker 端只取目录路径, 不需要 initialTasks 全量数组;
+                        //带着它每次派发都要 O(N) 序列化(总量 O(N²))。config 本体保留: resume/restart 重写快照要用
+                        let ipcConfig = { ...config };
+                        delete ipcConfig.initialTasks;
+                        worker.send({
+                            taskCount,
+                            startTimestamp,
+                            config: ipcConfig,
+                            subTask
+                        },()=>{}, {
+                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                         });
+                        let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
+                        let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
+                        if(hasTimeout || hasRetry){
+                            workerTaskTimers[worker.id] = {
+                                timer: hasTimeout ? setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout) : null,
+                                subTask,
+                                startTimestamp
+                            };
+                        }
+                        taskCount++;
                     });
                 }
             });
@@ -211,12 +229,18 @@ const start = (config)=>{
                 let worker = cluster.fork();
                 initWorker(worker);
             }
+        }).catch((e)=>{
+            //init/resume 异步失败(rename/读盘错误等): 明确 FATAL + 非 0 退出,
+            //否则 unhandled rejection 在 warn 模式下会不 fork 不退出、静默挂死
+            console.error('[FATAL] failed to init tasks:', e);
+            process.exit(1);
         });
         cluster.on('exit', (worker, code, signal) => {
             let rec = workerTaskTimers[worker.id];//先取再清: 崩溃重试要用 rec.subTask
             clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
             if (code !== 0 && !worker.exitedAfterDisconnect) {
                 console.log('Worker died unexpectedly starting a new one...');
+                workerCrashCount++;
                 if(rec && rec.subTask){
                     handleCrashedTask(rec.subTask);
                 };
@@ -230,7 +254,25 @@ const start = (config)=>{
             }
             if(allkilled){
                 if(USER_CONFIG.onFinish){
-                    USER_CONFIG.onFinish({});
+                    let endTimestamp = new Date()*1;
+                    let stats = TaskMgr.getFinalStats();
+                    let report = {
+                        taskId: config.masterTaskId,
+                        taskFolder: stats.taskFolder,
+                        startTimestamp: masterStartTimestamp,
+                        endTimestamp,
+                        cost: endTimestamp - masterStartTimestamp,
+                        total: stats.succeeded + stats.failed,
+                        succeeded: stats.succeeded,
+                        failed: stats.failed,
+                        failedTasks: stats.failedTasks,
+                        workers: {count: numOfWorkers, crashed: workerCrashCount},
+                    };
+                    let onFinishHelper = {
+                        foreachSuccResult: (cb) => TaskMgr.forEachSuccResult(cb),
+                        foreachErrorResult: (cb) => TaskMgr.forEachErrorResult(cb),
+                    };
+                    USER_CONFIG.onFinish(report, onFinishHelper);
                     delete USER_CONFIG.onFinish;
                 }
                 setTimeout(()=>{

package/workers/TaskMgr.js

@@ -3,6 +3,7 @@ const pathutil = require('path');
 const makeDir = require('../utils/makedir');
 const randomsUtil = require('../utils/randoms');
 const { serializeConfig } = require('../utils/serializeFns');
+const { forEachResult } = require('../utils/foreachResults');
 
 let allSubTasks = {
     new:[],
@@ -175,6 +176,21 @@ const TaskMgr = {
             callback(files);
         });
     },
+    getFinalStats:()=>{//master 收尾用: 以盘上终态为准统计成功/失败(resume 跨轮累计也是对的)
+        const {taskFolder, taskFolderFinished, taskFolderFailed} = main_config;
+        const listSubids = (folder)=>{
+            return fs.readdirSync(folder)
+                .filter((fname)=>fname.endsWith('.task.json'))
+                .map((fname)=>fname.replace(/\.task\.json$/, ''));
+        };
+        const failedTasks = listSubids(taskFolderFailed);
+        return {
+            taskFolder,
+            succeeded: listSubids(taskFolderFinished).length,
+            failed: failedTasks.length,
+            failedTasks,
+        };
+    },
     mvTaskPromise: (subid, from, to)=> {
         return new Promise((resolve, reject)=>{
             TaskMgr.mvTask(subid, from, to, (newPath, succ)=>{
@@ -257,6 +273,14 @@ const TaskMgr = {
         delete task.__retryCount;
         fs.writeFileSync(fpath, JSON.stringify(task));
     },
+    forEachSuccResult:(cb)=>{
+        let {taskFolderResult, taskFolderFinished} = main_config;
+        forEachResult(taskFolderResult, taskFolderFinished, cb);
+    },
+    forEachErrorResult:(cb)=>{
+        let {taskFolderResultFailed, taskFolderFailed} = main_config;
+        forEachResult(taskFolderResultFailed, taskFolderFailed, cb);
+    },
     resetTaskFolders:(taskFolder)=>{
         fs.rmSync(taskFolder, {recursive: true, force: true});
     },

package/workers/index.js

@@ -59,8 +59,7 @@ const resume = (config)=>{
     config.taskFolder = _toAbsPath(config.taskFolder);
     if(!config.taskFolder){
         if(!config.taskRootFolder || !config.taskId){
-            console.log('[ERROR]', 'method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
-            return;
+            throw new Error('method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
         };
         config.taskFolder = pathUtil.resolve(config.taskRootFolder, config.taskId);
     };
@@ -74,8 +73,7 @@ const retry_fails = (config)=>{
     config = _normalizeResumeArg('retry_fails', config);
     let taskFolder = _toAbsPath(config.taskFolder);
     if(!taskFolder){
-        console.log('[ERROR]', 'method retry_fails need param "taskFolder"');
-        return;
+        throw new Error('method retry_fails need param "taskFolder"');
     }
     config.taskFolder = taskFolder;
     config.taskRootFolder = pathUtil.dirname(taskFolder);
@@ -89,8 +87,7 @@ const restart = (config)=>{
     config = _normalizeResumeArg('restart', config);
     let taskFolder = _toAbsPath(config.taskFolder);
     if(!taskFolder){
-        console.log('[ERROR]', 'method restart need param "taskFolder"');
-        return;
+        throw new Error('method restart need param "taskFolder"');
     }
     config.taskFolder = taskFolder;
     config.taskRootFolder = pathUtil.dirname(taskFolder);