multi-tasks 3.1.2 → 3.1.4

package/README.md

@@ -1,257 +1,292 @@
-# multi-tasks
-
-Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. Progress and tasks are stored on the file system, so tasks can be resumed even if the host crashes.
-
-**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
-
-### Install:
-
-```javascript
-npm install multi-tasks
-```
-
-### API:
-
-The API has three parts: the entry functions, the config options, and the `helper` object injected into `processTask`.
-
-**Entry functions:**
-
-- **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
-- **`multiTasks.start(config)`** — always start a fresh run.
-- **`multiTasks.resume(config)`** — resume an interrupted run.
-- **`multiTasks.restart(config)`** — resume and also retry failed tasks (needs `config.taskFolder`).
-
-**Config options:**
-
-- **`initialTasks`** — array of task objects, or a folder path string to resume from.
-- **`processTask(task, helper)`** — required; return a value or a Promise.
-- **`taskRootFolder`** — root directory where progress and result files are stored.
-- **`taskId`** — task folder name under `taskRootFolder`.
-- **`taskFolder`** — full task folder path, required by `restart`.
-- **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
-- **`taskTimeout`** — optional, milliseconds; an overdue task fails with a timeout error.
-- **`maxTaskRetries`** — optional, max times a failed task is auto-retried.
-- **`autoCloseAfterCompletion`** — set `false` if you create new tasks dynamically.
-- **`shouldTerminate(info)`** — return `true` to terminate the whole process.
-- **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
-- **`onFinish(report)`** — called once after all workers are done.
-
-**Task helper** (the `helper` object passed as the second argument of `processTask`):
-
-- **`helper.createNewTasks(tasks)`** — create new tasks dynamically while processing.
-- **`helper.emit(event, payload, option?)`** — broadcast an event to all workers (relayed by the master, best-effort); `option.includingMe` defaults to `true` — pass `{includingMe: false}` to exclude the sender. Payload must be JSON-serializable.
-- **`helper.setListener(event, handler)`** — listen for broadcast events; one handler per event per worker process (re-registering replaces it), so calling it inside `processTask` is always safe; `handler(payload, meta)` gets `meta.fromWorkerId` and `meta.fromWorkerPid`. Broadcasts are runtime-only messages — not persisted, not replayed on resume.
-- **`helper.emitSys(event, payload?)`** — send a system event to the master (consumed by the master itself, not relayed). Built-in event: `'TERMINATE_ALL_WORKERS'` force-kills all workers and exits the master; any other event goes to `config.setSysListener`.
-
-### How to use:
-
-```javascript
-//see examples/example0
-let multiTasks = require('multi-tasks').multiTasks;
-
-//Step1, create the tasks to run in parallel as an array.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    alltasks.push({
-        name: `task-${i}`,
-        data: `This prop is for a subtask`
-    });
-};
-
-//Step2, provide a function that processes each sub-task and returns the result
-let processTask = (task, helper)=>{
-    let {taskCount} = task;//get your task data
-
-    //Run your processing logic here...
-
-    //then return the result as a plain javascript object or an async promise
-    return 'result data'; 
-};
-
-//Step3, run!
-multiTasks({
-    initialTasks: alltasks, 
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and result files; you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //how many workers are working in parallel
-    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
-    //maxTaskRetries: 2, //optional, auto-retry a failed task (worker crash, processTask error, or timeout); retried tasks go back to the queue
-    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, set this to false
-    shouldTerminate:(info)=>{
-        //return true if you need to terminate the whole process
-    },
-    onFinish: (report)=>{
-        console.log('finish callback', report);
-    }
-});
-
-```
-
-### More about the task processing function
-
-```javascript
-//Example1, return a promise for async processes,
-//          you can return a promise or a non-promise result, 
-//          all result data can be found in the results/succ folder
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-
-//Example2, dynamically create a new task while processing
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-    
-    if(taskCount % 2 === 0){
-        //create a new task if needed
-        helper.createNewTasks({
-            msg:'a new task'
-        });
-        return;
-    }
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
-
-//Example3, generate/throw exceptions in a task method
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-
-    //Demo of exceptions/errors: they are captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
-    if(taskCount===5) aaa =  bbb;//this ReferenceError will be captured by multi-tasks
-
-    return {data:'succ'};
-}
-
-//Example4, timeout: a task that does not finish within taskTimeout is
-//          treated as failed with a timeout error
-multiTasks({
-    initialTasks: alltasks,
-    processTask,
-    taskRootFolder: `../examples-tmp-data/example-timeout`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    taskTimeout: 30000,
-});
-
-//Example5, auto-retry a failed task with maxTaskRetries
-multiTasks({
-    initialTasks: alltasks,
-    processTask,
-    taskRootFolder: `../examples-tmp-data/example-retry`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    maxTaskRetries: 2,
-});
-
-//Example6, broadcast events between workers with helper.emit/helper.setListener
-multiTasks({
-    initialTasks: alltasks,
-    taskRootFolder: `../examples-tmp-data/example-broadcast`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    processTask: (task, helper) => {
-        helper.setListener('task-done', (payload, meta) => {
-            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
-        });
-        helper.emit('task-done', {seq: task.seq});
-        return {data:'succ'};
-    },
-});
-
-//Example7, system events from a worker to the master with helper.emitSys
-multiTasks({
-    initialTasks: alltasks,
-    taskRootFolder: `../examples-tmp-data/example-sysevent`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    setSysListener: (event, payload, meta) => {
-        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
-    },
-    processTask: (task, helper) => {
-        helper.emitSys('task-started', {seq: task.seq});
-        if(task.fatal){
-            helper.emitSys('TERMINATE_ALL_WORKERS');//stop everything
-            return;
-        };
-        return {data:'succ'};
-    },
-});
-
-```
-
-### Resuming
-
-If the execution is interrupted (e.g. a power outage), resume it like this:
-
-```javascript
-
-multiTasks({
-    initialTasks: `/myworks/my_scan_tasks/`, //point 'initialTasks' to the interrupted task directory; multi-tasks reads the tasks in its 'new' folder and continues execution
-    ...
-    ...//Other configurations remain unchanged
-    ...
-});
-
-```
-
-### Changelog:
-
-- 3.1.2 Update README
-- 3.1.1 Fix readme documentation
-- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
-- 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
-- 3.0.3 Support 'maxTaskRetries'
-- 3.0.2 Support 'taskTimeout'
-- 3.0.1 Versions 3.0.0 and above are maintained by AI
-- 2.2.0 Support Resuming from an interrupted task
-- 2.1.2 Update readme
-- 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
-- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
-- 2.0.9 Update readme examples
-- 2.0.8 Update readme examples
-- 2.0.7 Update changelog
-- 2.0.6 Update readme, remove failed examples
-- 2.0.5 Update readme examples
-- 2.0.4 Support resume from a failed task
-- 2.0.3 Fix: mkdir bug on windows
-- 2.0.2 new feature: support shouldTerminate
-- 2.0.1 Avoid possible I/O conflicts.
-- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
-- 1.2.8 Fix: create task folder failed on MacOS
-- 1.2.7 Small updates
-- 1.2.6 Support onFinish event
-- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
-- 1.2.4 Fix: opt.numberOfWorkers not work
-- 1.2.3 Update README
-- 1.2.2 Update README and examples
-- 1.2.1 Handle exceptions and errors in subtasks
-- 1.2.0 Simplified usage by providing the function way and support return Promise
-- 1.1.4 Remove make-dir
-- 1.1.3 Simplified usage, see example0
-- 1.1.2 
-- 1.1.1 Rename files, updated changelog
-- 1.1.0 Simplified the usage of a customized Consumer, see example0
-- 1.0.8 Fix examples
-- 1.0.7 Remove moment
-- 1.0.6 Performance optimization
-
-### Gitee:
-
-[https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)
-
-### License:
-
-[MIT](https://opensource.org/license/MIT) 
+# multi-tasks
+
+Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. Progress and tasks are stored on the file system, so tasks can be resumed even if the host crashes.
+
+**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
+
+### Install:
+
+```javascript
+npm install multi-tasks
+```
+
+### API:
+
+The API has three parts: the entry functions, the config options, and the `helper` object injected into `processTask`.
+
+**Entry functions:**
+
+- **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
+- **`multiTasks.resume(config)`** — resume an interrupted run; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
+- **`multiTasks.retry_fails(config)`** — same as `resume`, and also retries the failed tasks.
+- **`multiTasks.restart(config)`** — wipe the task folder's progress and re-run from scratch; pass a config object with `taskFolder`. The config (including `processTask` and `initialTasks`) is restored from `task_config.json`; any field you pass overrides the snapshot, and overrides are written back into it. **All previous progress and results are deleted.**
+
+**Config options:**
+
+- **`initialTasks`** — array of task objects, or a folder path string to resume from.
+- **`processTask(task, helper)`** — required; return a value or a Promise.
+- **`taskRootFolder`** — the task run folder (named by `taskId`) is created under this directory.
+- **`taskId`** — task folder name under `taskRootFolder`.
+- **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
+- **`taskTimeout`** — optional, milliseconds; an overdue task fails with a timeout error.
+- **`maxTaskRetries`** — optional, max times a failed task is auto-retried.
+- **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
+- **`shouldTerminate(info)`** — return `true` to terminate the whole process.
+- **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
+- **`onFinish(report)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`); `report` is an empty object for now.
+
+**Other options:**
+
+- **`taskFolder`** — full task folder path; only used by `resume`/`retry_fails`.
+
+**Task helper** (the `helper` object passed as the second argument of `processTask`):
+
+- **`helper.createNewTasks(tasks)`** — create new tasks dynamically while processing.
+- **`helper.emit(event, payload, option?)`** — broadcast an event to all workers (relayed by the master, best-effort); `option.includingMe` defaults to `true` — pass `{includingMe: false}` to exclude the sender. Payload must be JSON-serializable.
+- **`helper.setListener(event, handler)`** — listen for broadcast events; one handler per event per worker process (re-registering replaces it), so calling it inside `processTask` is always safe; `handler(payload, meta)` gets `meta.fromWorkerId` and `meta.fromWorkerPid`. Broadcasts are runtime-only messages — not persisted, not replayed on resume.
+- **`helper.emitSys(event, payload?)`** — send a system event to the master (consumed by the master itself, not relayed). Built-in event: `'TERMINATE_ALL_WORKERS'` force-kills all workers and exits the master; any other event goes to `config.setSysListener`.
+
+### How to use:
+
+```javascript
+//see examples/example0
+let multiTasks = require('multi-tasks').multiTasks;
+
+//Step1, create the tasks to run in parallel as an array.
+let alltasks = [];
+for(let i=0;i<50;i++){
+    alltasks.push({
+        name: `task-${i}`,
+        data: `This prop is for a subtask`
+    });
+};
+
+//Step2, provide a function that processes each sub-task and returns the result
+let processTask = (task, helper)=>{
+    let {taskCount} = task;//get your task data
+
+    //Run your processing logic here...
+
+    //then return the result as a plain javascript object or an async promise
+    return 'result data'; 
+};
+
+//Step3, run!
+multiTasks({
+    initialTasks: alltasks, 
+    processTask, 
+    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and result files; you can check the progress here
+    taskId: 'my-task',
+    numberOfWorkers: 3, //how many workers are working in parallel
+    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
+    //maxTaskRetries: 2, //optional, auto-retry a failed task (worker crash, processTask error, or timeout); retried tasks go back to the queue
+    //autoCloseAfterCompletion: true, //set true to exit the process when all tasks are done; keep it false (default) if you create new tasks dynamically
+    shouldTerminate:(info)=>{
+        //return true if you need to terminate the whole process
+    },
+    onFinish: (report)=>{
+        console.log('finish callback', report);
+    }
+});
+
+```
+
+### More about the task processing function
+
+```javascript
+//Examples 1-3 below are independent variants of processTask, use one at a time.
+
+//Example1, return a promise for async processes,
+//          you can return a promise or a non-promise result, 
+//          all result data can be found in the results/succ folder
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Example2, dynamically create a new task while processing
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+    
+    if(taskCount % 2 === 0){
+        //create a new task if needed
+        helper.createNewTasks({
+            msg:'a new task'
+        });
+        return;
+    }
+
+    return new Promise((resolve, reject)=>{
+        resolve({
+            data:`task${taskCount} complete`
+        })
+    })
+};
+
+//Example3, generate/throw exceptions in a task method
+let processTask = (task, helper)=>{
+    let {taskCount} = task;
+
+    //Demo of exceptions/errors: they are captured and saved in the results/errors folder
+    if(taskCount===3) throw 'exception';
+    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
+    if(taskCount===5) aaa =  bbb;//this ReferenceError will be captured by multi-tasks
+
+    return {data:'succ'};
+}
+
+//Example4, timeout: a task that does not finish within taskTimeout is
+//          treated as failed with a timeout error
+multiTasks({
+    initialTasks: alltasks,
+    processTask,
+    taskRootFolder: `../examples-tmp-data/example-timeout`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    taskTimeout: 30000,
+});
+
+//Example5, auto-retry a failed task with maxTaskRetries
+multiTasks({
+    initialTasks: alltasks,
+    processTask,
+    taskRootFolder: `../examples-tmp-data/example-retry`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    maxTaskRetries: 2,
+});
+
+//Example6, broadcast events between workers with helper.emit/helper.setListener
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-broadcast`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    processTask: (task, helper) => {
+        helper.setListener('task-done', (payload, meta) => {
+            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
+        });
+        helper.emit('task-done', {seq: task.seq});
+        return {data:'succ'};
+    },
+});
+
+//Example7, system events from a worker to the master with helper.emitSys
+multiTasks({
+    initialTasks: alltasks,
+    taskRootFolder: `../examples-tmp-data/example-sysevent`,
+    taskId: 'my-task',
+    numberOfWorkers: 3,
+    setSysListener: (event, payload, meta) => {
+        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
+    },
+    processTask: (task, helper) => {
+        helper.emitSys('task-started', {seq: task.seq});
+        if(task.fatal){
+            helper.emitSys('TERMINATE_ALL_WORKERS');//stop everything
+            return;
+        };
+        return {data:'succ'};
+    },
+});
+
+```
+
+### Resuming
+
+If the execution is interrupted (e.g. a power outage), resume it by pointing to the task folder — nothing else is needed, the config (including `processTask`) is restored from `task_config.json`:
+
+```javascript
+
+multiTasks.resume({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //taskFolder: the full path of the existing task folder
+
+```
+
+Re-running `multiTasks(config)` (or pointing `initialTasks` at the folder) auto-resumes too; passed config wins over the snapshot.
+
+**Note:** revived functions must be **self-contained** (no outer-scope variables; `require(...)` inside the body) — otherwise pass them again, they always win:
+
+```javascript
+multiTasks.resume({
+    taskFolder: `/myworks/my_scan_tasks/my-task`,
+    processTask,
+});
+```
+
+### Retrying failed tasks
+
+To resume an interrupted run and also retry the tasks that failed (in task folder `progress/finished_with_errors`):
+
+```javascript
+
+multiTasks.retry_fails({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //taskFolder: the full path of the existing task folder
+
+```
+
+### Restarting from scratch
+
+To throw away all progress and results in a task folder and re-run all the initial tasks from the beginning (the config, including `processTask` and `initialTasks`, is restored from `task_config.json`):
+
+```javascript
+
+multiTasks.restart({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //deletes all previous progress and results, then re-runs all the initial tasks
+
+```
+
+Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass (e.g. `processTask`) override the snapshot and are written back into it; the same self-contained-function note as in Resuming applies. (**Breaking change:** the name `restart` was the old name of `retry_fails` in v2 and in the published v3.0–v3.1.x releases; this is a different API — calling it no longer retries failed tasks, it wipes the folder and re-runs everything.)
+
+### Changelog:
+
+- 3.1.4 Rewrite `multiTasks.resume`, `multiTasks.retry_fails` and `multiTasks.restart`
+- 3.1.3 Update README
+- 3.1.2 Update README
+- 3.1.1 Fix readme documentation
+- 3.1.0 Support worker broadcast ('helper.emit' and 'helper.setListener') and system events ('helper.emitSys' and 'setSysListener', with built-in 'TERMINATE_ALL_WORKERS'); default numberOfWorkers is now "50%" of CPU cores (was core count minus 1)
+- 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
+- 3.0.3 Support 'maxTaskRetries'
+- 3.0.2 Support 'taskTimeout'
+- 3.0.1 Versions 3.0.0 and above are maintained by AI
+- 2.2.0 Support Resuming from an interrupted task
+- 2.1.2 Update readme
+- 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
+- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
+- 2.0.9 Update readme examples
+- 2.0.8 Update readme examples
+- 2.0.7 Update changelog
+- 2.0.6 Update readme, remove failed examples
+- 2.0.5 Update readme examples
+- 2.0.4 Support resume from a failed task
+- 2.0.3 Fix: mkdir bug on windows
+- 2.0.2 new feature: support shouldTerminate
+- 2.0.1 Avoid possible I/O conflicts.
+- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
+- 1.2.8 Fix: create task folder failed on MacOS
+- 1.2.7 Small updates
+- 1.2.6 Support onFinish event
+- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
+- 1.2.4 Fix: opt.numberOfWorkers not work
+- 1.2.3 Update README
+- 1.2.2 Update README and examples
+- 1.2.1 Handle exceptions and errors in subtasks
+- 1.2.0 Simplified usage by providing the function way and support return Promise
+- 1.1.4 Remove make-dir
+- 1.1.3 Simplified usage, see example0
+- 1.1.2 
+- 1.1.1 Rename files, updated changelog
+- 1.1.0 Simplified the usage of a customized Consumer, see example0
+- 1.0.8 Fix examples
+- 1.0.7 Remove moment
+- 1.0.6 Performance optimization
+
+### Gitee:
+
+[https://gitee.com/zhanglei923/multi-tasks](https://gitee.com/zhanglei923/multi-tasks)
+
+### License:
+
+[MIT](https://opensource.org/license/MIT) 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.1.2",
+  "version": "3.1.4",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [
@@ -16,7 +16,7 @@
     "example": "examples"
   },
   "engines": {
-    "node": ">=14"
+    "node": ">=14.14"
   },
   "scripts": {
     "test": "jest",

package/utils/serializeFns.js

@@ -0,0 +1,46 @@
+'use strict';
+
+// config 里的函数型配置键: 写 task_config.json 快照时序列化为源码字符串,
+// resume 时复活为函数。未来新增函数型配置键在此登记。
+const FUNCTION_CONFIG_KEYS = ['processTask', 'onFinish', 'shouldTerminate', 'setSysListener'];
+
+//返回浅副本: 函数型键的函数值替换为源码字符串, 其余值原样保留。
+const serializeConfig = (config) => {
+  const out = { ...config };
+  FUNCTION_CONFIG_KEYS.forEach((key) => {
+    if (typeof out[key] === 'function') {
+      out[key] = out[key].toString();
+    };
+  });
+  return out;
+};
+
+//返回浅副本: 函数型键的字符串值复活为函数。复活环境注入 require/module/exports,
+//函数体内可直接 require(...)。eval 失败或复活结果非函数时抛 Error(消息带键名)。
+const reviveConfig = (config) => {
+  const out = { ...config };
+  FUNCTION_CONFIG_KEYS.forEach((key) => {
+    if (typeof out[key] === 'string') {
+      const src = out[key];
+      let revived;
+      try {
+        revived = new Function('require', 'module', 'exports', `return (${src}\n);`)(require, module, exports);
+      } catch (e) {
+        //方法简写(如 "processTask(task) {...}" 或 "async processTask(task) {...}")需包成函数声明再复活
+        let decl = src.startsWith('async ') ? `async function ${src.slice('async '.length)}` : `function ${src}`;
+        try {
+          revived = new Function('require', 'module', 'exports', `return (${decl}\n);`)(require, module, exports);
+        } catch (e2) {
+          throw new Error(`Failed to revive config function "${key}" from task_config.json: ${e.message}`);
+        };
+      };
+      if (typeof revived !== 'function') {
+        throw new Error(`Failed to revive config function "${key}" from task_config.json: source is not a function`);
+      };
+      out[key] = revived;
+    };
+  });
+  return out;
+};
+
+module.exports = { FUNCTION_CONFIG_KEYS, serializeConfig, reviveConfig };

package/workers/TaskMgr.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const pathutil = require('path');
 const makeDir = require('../utils/makedir');
 const randomsUtil = require('../utils/randoms');
+const { serializeConfig } = require('../utils/serializeFns');
 
 let allSubTasks = {
     new:[],
@@ -20,13 +21,16 @@ const TaskMgr = {
     initFolders:()=>{
         let {taskRootFolder, masterTaskId} = main_config;
         const taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
-        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__restart)){
+        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__retryFails || main_config.__restart)){
             let files = fs.readdirSync(taskFolder);
             if(files.length > 0){
                 console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
                 return process.exit(0);
             }
         };
+        if(main_config.__restart && fs.existsSync(taskFolder)){
+            TaskMgr.resetTaskFolders(taskFolder);//restart: 整目录重置, 随后由下方 makeDir 重建
+        };
         const progressRootFolder = pathutil.resolve(taskFolder, './progress');
         const resultsRootFolder = pathutil.resolve(taskFolder, './results');
         const logsRootFolder = pathutil.resolve(taskFolder, './logs');
@@ -61,9 +65,12 @@ const TaskMgr = {
             taskFolderResultFailed,
         };
 
-        //for restart a failed task
-        let main_config2 = JSON.parse(JSON.stringify(main_config));
+        //for resuming an interrupted task
+        let main_config2 = JSON.parse(JSON.stringify(serializeConfig(main_config)));//函数型配置键先序列化为源码字符串, 供 resume 从快照复活
         delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
+        Object.keys(main_config2).forEach((key)=>{
+            if(key.startsWith('__')) delete main_config2[key];//__-prefixed keys are one-shot runtime markers, never persisted
+        });
         fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
         return main_config;
     },
@@ -250,5 +257,8 @@ const TaskMgr = {
         delete task.__retryCount;
         fs.writeFileSync(fpath, JSON.stringify(task));
     },
+    resetTaskFolders:(taskFolder)=>{
+        fs.rmSync(taskFolder, {recursive: true, force: true});
+    },
 };
 module.exports = TaskMgr;

package/workers/asMaster.js

@@ -10,12 +10,17 @@ const resume = (config)=>{
     let {taskRootFolder, taskId} = config;
     let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
 
-    //load from config file
+    //load from config file: 旧快照作底(兼作任务目录合法性校验, 不存在抛 ENOENT),
+    //本次调用传入的配置覆盖同名字段——快照只补未传入的字段
     let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
+    Object.keys(task_config).forEach((key)=>{
+        if(key.startsWith('__')) delete task_config[key];//读侧剔除一次性标记, 兼容 <=3.1.3 旧快照
+    });
     config = {
-        ...config,
-        ...task_config
-    }
+        ...task_config,
+        ...config
+    };
+    config.masterTaskId = taskId;
 
     TaskMgr.load(config);
     config = TaskMgr.initFolders(config);
@@ -29,11 +34,11 @@ const resume = (config)=>{
                     let subid = fname.replace(/\.task\.json$/, '');
                     promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
                 });
-                if(config.__restart){
+                if(config.__retryFails){
                     errfiles.forEach((fname)=>{
                         let subid = fname.replace(/\.task\.json$/, '');
                         promises.push(TaskMgr.mvTaskPromise(subid, 'finished_with_errors', 'new').then(()=>{
-                            TaskMgr.resetTaskRetry(subid);//restart 重跑失败任务时重置重试计数(resume 不重置)
+                            TaskMgr.resetTaskRetry(subid);//retry_fails 重跑失败任务时重置重试计数(resume 不重置)
                         }));
                     });
                 };

package/workers/index.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const pathUtil = require('path');
 const Multithread = require('./Multithread');
 const backwardv1 = require('../utils/backward-v1');
+const { FUNCTION_CONFIG_KEYS, reviveConfig } = require('../utils/serializeFns');
 
 const _toAbsPath = (path)=>{
     if(!path) return path;
@@ -10,6 +11,38 @@ const _toAbsPath = (path)=>{
     return abspath;
 };
 
+const _normalizeResumeArg = (method, config)=>{
+    if(typeof config === 'string'){
+        throw new Error(`method ${method} need a config object with param "taskFolder" (string path form is no longer supported)`);
+    };
+    return backwardv1.v1tov2(config);
+};
+
+//读取任务目录里的 task_config.json 快照, 复活其中的函数字符串后作底;
+//用户传入的同名字段优先(合并结果会随 resume 被 initFolders 重写回快照, 成为新基线)。
+const _mergeSnapshotConfig = (config)=>{
+    let snapshot = JSON.parse(fs.readFileSync(pathUtil.resolve(config.taskFolder, 'task_config.json')));
+    Object.keys(snapshot).forEach((key)=>{
+        if(key.startsWith('__')) delete snapshot[key];//读侧同样剔除一次性标记: 兼容 <=3.1.3 旧版本写出的带标记快照(防 stale 标记误触发 wipe/错走链路)
+    });
+    let userConfig = { ...config };
+    if(typeof userConfig.initialTasks === 'string'){
+        delete userConfig.initialTasks;//目录指针用法(auto-resume), 不应覆盖/污染快照里的任务数组
+    };
+    //用户已传入同名函数时, 快照里的源码字符串无需复活(反正会被覆盖)——容忍不可复活的源码(如绑定函数)
+    FUNCTION_CONFIG_KEYS.forEach((key)=>{
+        if(typeof userConfig[key] === 'function' && typeof snapshot[key] === 'string'){
+            delete snapshot[key];
+        };
+    });
+    let revived = reviveConfig(snapshot);
+    let merged = { ...revived, ...userConfig };
+    if(typeof merged.processTask !== 'function'){
+        throw new Error('FATAL: no processTask available: task_config.json has none and none was passed in config.');
+    };
+    return merged;
+};
+
 const start = (config)=>{
     config = backwardv1.v1tov2(config);
 
@@ -19,28 +52,51 @@ const start = (config)=>{
     Multithread.start(config);
 };
 const resume = (config)=>{
-    config = backwardv1.v1tov2(config);
+    config = _normalizeResumeArg('resume', config);
     config.__resume = true;
 
-    config.initialTasks = [];
-
     config.taskRootFolder = _toAbsPath(config.taskRootFolder);
     config.taskFolder = _toAbsPath(config.taskFolder);
+    if(!config.taskFolder){
+        if(!config.taskRootFolder || !config.taskId){
+            console.log('[ERROR]', 'method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
+            return;
+        };
+        config.taskFolder = pathUtil.resolve(config.taskRootFolder, config.taskId);
+    };
+    config.taskRootFolder = pathUtil.dirname(config.taskFolder);
+    config.taskId = pathUtil.basename(config.taskFolder);
 
+    config = _mergeSnapshotConfig(config);
+    Multithread.start(config);
+};
+const retry_fails = (config)=>{
+    config = _normalizeResumeArg('retry_fails', config);
+    let taskFolder = _toAbsPath(config.taskFolder);
+    if(!taskFolder){
+        console.log('[ERROR]', 'method retry_fails need param "taskFolder"');
+        return;
+    }
+    config.taskFolder = taskFolder;
+    config.taskRootFolder = pathUtil.dirname(taskFolder);
+    config.taskId = pathUtil.basename(taskFolder);
+    config.__resume = true;
+    config.__retryFails = true;
+    config = _mergeSnapshotConfig(config);
     Multithread.start(config);
 };
 const restart = (config)=>{
-    config = backwardv1.v1tov2(config);
-    let {taskFolder} = config;
+    config = _normalizeResumeArg('restart', config);
+    let taskFolder = _toAbsPath(config.taskFolder);
     if(!taskFolder){
         console.log('[ERROR]', 'method restart need param "taskFolder"');
         return;
     }
+    config.taskFolder = taskFolder;
     config.taskRootFolder = pathUtil.dirname(taskFolder);
     config.taskId = pathUtil.basename(taskFolder);
-    config.__resume = true;
     config.__restart = true;
-    config.initialTasks = [];
+    config = _mergeSnapshotConfig(config);
     Multithread.start(config);
 };
 
@@ -77,7 +133,8 @@ const multiTasks = (config)=>{
 };
 
 multiTasks.start = start;
-multiTasks.restart = restart;
+multiTasks.retry_fails = retry_fails;
 multiTasks.resume = resume;
+multiTasks.restart = restart;
 
 module.exports = multiTasks;