multi-tasks 3.4.0 → 3.5.0

package/README.md

@@ -1,261 +1,245 @@
 # multi-tasks
 
-Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks, with zero dependencies. Progress and tasks are stored on the file system, so tasks can be resumed even if the host crashes.
+Run large batches of tasks in parallel across multiple Node.js processes, with zero dependencies. Progress and results are stored on the file system, so an interrupted run (crash, power outage) can resume where it left off.
+
+**New in 3.5.0 — `task_storage: 'memory'`**: keep all task state in the master process's memory instead of on disk. Same API, one config field — and ~17× faster on huge batches of small tasks. The trade-off: no resume. See [Task storage modes](#task-storage-modes).
+
+![task_storage benchmark: file mode 195.1s vs memory mode 11.2s — 17.4x faster](bench-storage.svg)
 
 ### Install:
 
-```javascript
+```text
 npm install multi-tasks
 ```
 
-Tip: monitor running tasks in real time — see [Real-Time Monitoring](#real-time-monitoring).
-
-### How to use:
+### Quick start:
 
 ```javascript
-//see examples/example0
 let multiTasks = require('multi-tasks').multiTasks;
 
-//Step1, provide the tasks as a function returning them (or a Promise of them);
-//it runs only once, in the master process.
-let initialTasks = async ()=>{
-    let alltasks = [];
-    for(let i=0;i<50;i++){
-        alltasks.push({
-            name: `task-${i}`,
-            data: `This prop is for a subtask`
-        });
-    };
-    return alltasks;
-};
-
-//Step2, provide a function that processes each sub-task and returns the result
-let processTask = (task, helper)=>{
-    let {taskCount} = task;//get your task data
-
-    //Run your processing logic here...
-
-    //then return the result as a plain javascript object or an async promise
-    return 'result data'; 
+//your tasks, as a plain array
+let tasks = [];
+for(let i = 0; i < 50; i++){
+    tasks.push({name: `task-${i}`});
 };
 
-//Step3, run!
 multiTasks({
-    initialTasks, //accepts array, function, or function returning a promise
-    processTask, 
-    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and result files; you can check the progress here
-    taskId: 'my-task',
-    numberOfWorkers: 3, //how many workers are working in parallel
-    //taskTimeout: 30000, //optional, in milliseconds, an overdue task is treated as failed with a timeout error
-    //maxTaskRetries: 2, //optional, auto-retry a failed task (worker crash, processTask error, or timeout); retried tasks go back to the queue
-    //workerPriority: 'low', //optional, one of 'low' | 'below_normal' | 'normal' | 'above_normal' | 'high' (or a nice number -20~19); lower the OS scheduling priority of workers to keep the machine responsive during long batches
-    //autoCloseAfterCompletion: true, //set true to exit the process when all tasks are done; keep it false (default) if you create new tasks dynamically
-    shouldTerminate:(info)=>{
-        //return true if you need to terminate the whole process
+    initialTasks: tasks,
+    //process one task and return its result (a plain value or a Promise)
+    processTask: (task, helper) => {
+        return `done: ${task.name}`;
+    },
+    taskRootFolder: './my-tasks', //progress and results live in ./my-tasks/demo
+    taskId: 'demo',
+    numberOfWorkers: 3, //worker processes running in parallel
+    autoCloseAfterCompletion: true, //exit the process when all tasks are done
+    onFinish: (report) => {
+        console.log(`${report.succeeded} succeeded, ${report.failed} failed`);
     },
-    onFinish: (report, helper)=>{
-        console.log('finish callback', report);
-        helper.foreachSuccResult((count, task, result)=>{
-            console.log(`#${count} succeeded:`, task.name, result);
-        });
-        helper.foreachErrorResult((count, task, err)=>{
-            console.log(`#${count} failed:`, task.name, err);
-        });
-    }
 });
-
 ```
 
-### More about the task processing function
+Results are written as JSON files under `<taskRootFolder>/<taskId>/results/succ`, failures under `results/errors`. While running, a live progress bar shows in the terminal — to watch the run from a browser, see [Real-Time Monitoring](#real-time-monitoring). A runnable copy of this example lives in `examples/example0`.
 
-```javascript
-//Examples 1-3 below are independent variants of processTask, use one at a time.
+### Results and errors:
 
-//Example1, return a promise for async processes,
-//          you can return a promise or a non-promise result, 
-//          all result data can be found in the results/succ folder
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
+`processTask` may return a plain value or a Promise:
 
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
+```javascript
+let processTask = (task, helper) => {
+    let {taskCount} = task; //0-based index of this task
+
+    return new Promise((resolve, reject) => {
+        setTimeout(() => {
+            resolve({data: `task${taskCount} complete`});
+        }, 100);
+    });
 };
+```
 
-//Example2, dynamically create a new task while processing
-let processTask = (task, helper)=>{
-    let {taskCount} = task;
-    
-    if(taskCount % 2 === 0){
-        //create a new task if needed
-        helper.createNewTasks({
-            msg:'a new task'
-        });
-        return;
-    }
-
-    return new Promise((resolve, reject)=>{
-        resolve({
-            data:`task${taskCount} complete`
-        })
-    })
-};
+A failing task never stops the batch — its error is captured and saved to `results/errors`, the rest keep going:
 
-//Example3, generate/throw exceptions in a task method
-let processTask = (task, helper)=>{
+```javascript
+let processTask = (task, helper) => {
     let {taskCount} = task;
 
-    //Demo of exceptions/errors: they are captured and saved in the results/errors folder
-    if(taskCount===3) throw 'exception';
-    if(taskCount===4) return Promise.reject({err:'a test error'});//use Promise.reject method
-    if(taskCount===5) aaa =  bbb;//this ReferenceError will be captured by multi-tasks
+    if(taskCount === 3) throw 'exception'; //a thrown value is persisted as-is
+    if(taskCount === 4) return Promise.reject({err: 'a test error'}); //a rejection value is persisted as-is
+    if(taskCount === 5) aaa = bbb; //a ReferenceError is captured too
 
-    return {data:'succ'};
-}
+    return {data: 'succ'};
+};
+```
 
-//Example4, timeout: a task that does not finish within taskTimeout is
-//          treated as failed with a timeout error
-multiTasks({
-    initialTasks: alltasks,
-    processTask,
-    taskRootFolder: `../examples-tmp-data/example-timeout`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    taskTimeout: 30000,
-});
+### Creating new tasks dynamically:
+
+```javascript
+//scenario: crawl a paginated API — each page tells you if there is a next one
+let processTask = async (task, helper) => {
+    let page = await fetchPage(task.url); //your logic here
+
+    if(page.nextUrl){
+        //a discovered next page becomes a new task;
+        //the pages are finite, so the queue always drains on its own
+        helper.createNewTasks({url: page.nextUrl});
+    };
+
+    return {url: task.url, items: page.items};
+};
 
-//Example5, auto-retry a failed task with maxTaskRetries
 multiTasks({
-    initialTasks: alltasks,
+    initialTasks: [{url: 'https://api.example.com/items?page=1'}], //start from the first page
     processTask,
-    taskRootFolder: `../examples-tmp-data/example-retry`,
-    taskId: 'my-task',
+    taskRootFolder: './my-tasks',
+    taskId: 'api-crawl',
     numberOfWorkers: 3,
-    maxTaskRetries: 2,
+    //keep autoCloseAfterCompletion false (the default) while tasks may still be created
 });
-
 ```
 
-### Event broadcast
+The point: new tasks come from work discovered while processing, and discovery stops when the data runs out — no artificial bound needed. Keep `autoCloseAfterCompletion` at its default (`false`) so workers stay alive for late-arriving tasks.
 
-```javascript
+### Timeouts and retries:
 
-//Example6, broadcast events between workers with helper.emit/helper.setListener
+```javascript
 multiTasks({
-    initialTasks: alltasks,
-    taskRootFolder: `../examples-tmp-data/example-broadcast`,
-    taskId: 'my-task',
+    initialTasks: tasks,
+    processTask,
+    taskRootFolder: './my-tasks',
+    taskId: 'demo',
     numberOfWorkers: 3,
-    processTask: (task, helper) => {
-        helper.setListener('task-done', (payload, meta) => {
-            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
-        });
-        helper.emit('task-done', {seq: task.seq});
-        return {data:'succ'};
-    },
+    taskTimeout: 30000, //a task that overruns 30s fails with a timeout error
+    maxTaskRetries: 2, //auto-retry a failed task (worker crash, processTask error, or timeout) up to 2 times
 });
-
-//Example7, system events from a worker to the master with helper.emitSys
-multiTasks({
-    initialTasks: alltasks,
-    taskRootFolder: `../examples-tmp-data/example-sysevent`,
-    taskId: 'my-task',
-    numberOfWorkers: 3,
-    setSysListener: (event, payload, meta) => {
-        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
-    },
-    processTask: (task, helper) => {
-        helper.emitSys('task-started', {seq: task.seq});
-        if(task.fatal){
-            helper.emitSys('TERMINATE_ALL_WORKERS');//stop everything
-            return;
-        };
-        return {data:'succ'};
-    },
-});
-
 ```
 
-### Performance: runs only once, in the master process
+### Resuming an interrupted run:
 
-Every worker process re-runs your entire entry script — so top-level task-generation code (building a huge array, scanning folders, querying a database, ...) runs once per worker. Wrap it in a function for performance: it is called **only once, in the master process**, and workers never call it. `initialTasks` accepts three forms — pick one:
+Just re-run the same code — `multiTasks(config)` detects the existing task folder and auto-resumes, re-running only the stuck tasks:
 
 ```javascript
-
-//Example8, the three forms of initialTasks
-
-//form 1: a plain array — assign it directly
-multiTasks({ initialTasks: alltasks, processTask, taskRootFolder, taskId });
-
-//form 2: a function returning the array
-multiTasks({ initialTasks: () => fetchTasksFromSomewhere(), processTask, taskRootFolder, taskId });
-
-//form 3: a function returning a Promise of the array
-multiTasks({ initialTasks: async () => await fetchTasksFromDb(), processTask, taskRootFolder, taskId });
-
+multiTasks(config); //same command, same config as the interrupted run
 ```
 
-The (resolved) value must be an array, otherwise the master exits with an error. The resulting array — not the function — is stored in `task_config.json`, so the self-contained-function note under Resuming does not apply to it.
+Or point `initialTasks` at the task folder — same effect:
 
-### Resuming
+```javascript
+multiTasks({...config, initialTasks: '/myworks/my_scan_tasks/my-task'});
+```
 
-If the execution is interrupted (e.g. a power outage), the simplest way to resume is to just re-run the same code — `multiTasks(config)` detects the existing task folder and auto-resumes, re-running only the stuck tasks:
+Or resume from the task folder alone — the config (including `processTask`) is restored from `task_config.json`:
 
 ```javascript
+multiTasks.resume({taskFolder: '/myworks/my_scan_tasks/my-task'}); //taskFolder: the full path of the existing task folder
+```
 
-//same command, same config as the interrupted run — that's all
-multiTasks(config);
+**Note:** functions revived from `task_config.json` must be **self-contained** (no outer-scope variables; `require(...)` inside the body) — otherwise pass them again; fields you pass always win:
 
+```javascript
+multiTasks.resume({taskFolder: '/myworks/my_scan_tasks/my-task', processTask});
 ```
 
-Or point `initialTasks` at the task folder — same effect:
+Note: a folder created with `task_storage: 'memory'` holds no recoverable state — resuming it throws.
 
-```javascript
+### Retrying failed tasks:
 
-multiTasks({ ...config, initialTasks: `/myworks/my_scan_tasks/my-task` });
+Resume an interrupted run and also re-run the tasks that failed (in `progress/finished_with_errors`):
 
+```javascript
+multiTasks.retry_fails({taskFolder: '/myworks/my_scan_tasks/my-task'});
 ```
 
-Or resume from the task folder alone — the config (including `processTask`) is restored from `task_config.json`:
+### Restarting from scratch:
+
+Throw away all progress and results in a task folder and re-run all the initial tasks (the config, including `processTask` and `initialTasks`, is restored from `task_config.json`):
 
 ```javascript
+multiTasks.restart({taskFolder: '/myworks/my_scan_tasks/my-task'});
+```
 
-multiTasks.resume({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //taskFolder: the full path of the existing task folder
+Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass override the saved config and are written back into it; the same self-contained-function note as in Resuming applies.
 
-```
+### Task storage modes:
 
-**Note:** revived functions must be **self-contained** (no outer-scope variables; `require(...)` inside the body) — otherwise pass them again, they always win:
+By default every task state change is written to disk immediately (`file` mode), which is what makes resume/retry/restart possible. For very large or very fast batches you can trade durability for speed:
 
 ```javascript
-multiTasks.resume({
-    taskFolder: `/myworks/my_scan_tasks/my-task`,
+multiTasks({
+    initialTasks: tasks,
     processTask,
+    taskRootFolder: './my-tasks',
+    taskId: 'demo',
+    task_storage: 'memory', //'file' (default) | 'memory'
 });
 ```
 
-### Retrying failed tasks
+- **`file`** (default) — every state change hits disk immediately; resume anytime.
+- **`memory`** — task states, results, logs and user messages live in the master process's memory only; no per-task files are ever written. The task folder is still created with the usual subfolders (`progress`/`results`/`logs` stay empty), holding only `task_config.json` and `.sys/run.log`. Dramatically faster for large batches of small tasks — about 20x faster in benchmarks. Results are still available through `onFinish(report, helper)`. `resume`/`retry_fails`/`restart` are not supported for this mode and throw when called — a killed run is simply gone.
 
-To resume an interrupted run and also retry the tasks that failed (in task folder `progress/finished_with_errors`):
+Benchmark — 30,000 tiny tasks × 28 workers, average of 3 runs on the same machine (17–27× across repeated benchmark sessions):
 
-```javascript
+![task_storage benchmark: file mode 195.1s vs memory mode 11.2s — 17.4x faster](bench-storage.svg)
 
-multiTasks.retry_fails({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //taskFolder: the full path of the existing task folder
+**Getting results out in `memory` mode**: since nothing per-task hits disk, `onFinish(report, helper)` is the one and only exit for your data — iterate everything with `helper.foreachResult` (or `foreachSuccResult`/`foreachErrorResult`) and persist it yourself in one batch if you need it kept:
 
+```javascript
+onFinish: (report, helper) => {
+    helper.foreachResult((count, task, result, succeeded) => {
+        //collect / batch-insert / write out — all results are here in memory
+    });
+},
 ```
 
-### Restarting from scratch
+If you need per-task files anyway, write them in bulk from `onFinish` — do **not** `writeFileSync` inside `processTask`: in benchmarks (30,000 tasks × 28 workers) per-task self-written files made the run ~3.3× slower than pure `memory` (27.7s vs 8.4s), because disk writes still dominate even without the library's bookkeeping. (That said, even the slow do-it-yourself variant beats `file` mode's ~195s — its cost is the resumable state machine, not the writes alone.)
 
-To throw away all progress and results in a task folder and re-run all the initial tasks from the beginning (the config, including `processTask` and `initialTasks`, is restored from `task_config.json`):
+Caveats of `memory` mode:
 
-```javascript
+- **Monitoring**: [multi-tasks-monitor](#real-time-monitoring) reads the task folder, so it has nothing to show for a `memory` run (no task states/results on disk).
+- **Worker crashes**: without `maxTaskRetries`, a crashed worker's in-flight task is not retried — with `file` it stays in `running` until a later `resume` picks it up; with `memory` it is lost for good.
+- **Dynamic tasks**: with `memory`, `helper.createNewTasks` reaches the master over IPC; if the master is killed before applying them, tasks created but not yet applied are lost. (`file` mode writes them to disk from the worker directly.)
+- **Dynamic tasks with `autoCloseAfterCompletion`**: in `memory` mode the master pops straight from the in-memory queue — there is no on-disk re-read fallback like in `file` mode, and no settle delay before killing an idle worker. A task another worker just sent via `helper.createNewTasks` may still be in IPC transit when the master decides the queue is empty, so it can miss the dispatch window and the workers get closed. If your workers create tasks dynamically, keep `autoCloseAfterCompletion` at its default `false`.
 
-multiTasks.restart({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //deletes all previous progress and results, then re-runs all the initial tasks
+### Broadcasting events between workers:
 
+```javascript
+multiTasks({
+    initialTasks: tasks,
+    processTask: (task, helper) => {
+        helper.setListener('task-done', (payload, meta) => {
+            console.log(`worker ${meta.fromWorkerPid} says: task ${payload.seq} done`);
+        });
+        helper.emit('task-done', {seq: task.seq}); //broadcast to all workers
+        return {data: 'succ'};
+    },
+    taskRootFolder: './my-tasks',
+    taskId: 'demo',
+    numberOfWorkers: 3,
+});
 ```
 
-Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass (e.g. `processTask`) override the snapshot and are written back into it; the same self-contained-function note as in Resuming applies.
+### System events:
+
+Send events from a worker to the master with `helper.emitSys`:
+
+```javascript
+multiTasks({
+    initialTasks: tasks,
+    setSysListener: (event, payload, meta) => {
+        console.log(`sys event "${event}" from worker ${meta.fromWorkerPid}:`, payload);
+    },
+    processTask: (task, helper) => {
+        helper.emitSys('task-started', {seq: task.seq});
+        if(task.fatal){
+            helper.emitSys('TERMINATE_ALL_WORKERS'); //built-in event: stop everything
+            return;
+        };
+        return {data: 'succ'};
+    },
+    taskRootFolder: './my-tasks',
+    taskId: 'demo',
+    numberOfWorkers: 3,
+});
+```
 
 ### Real-Time Monitoring
 
@@ -264,27 +248,24 @@ Monitor your running tasks with [multi-tasks-monitor](https://www.npmjs.com/pack
 Quick start (requires multi-tasks >=3.2.0): once your multi-tasks run has started, the startup log prints the monitor command with the task folder path already filled in — just copy-paste it into another terminal:
 
 ```text
-monitor install cmd : npm install multi-tasks-monitor
-monitor start cmd   : npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
+npm install multi-tasks-monitor
+npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
 ```
 
-For command-line options and more, see the [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) page.
-
 ### API:
 
-The API has three parts: the entry functions, the config options, and the `helper` object injected into `processTask`.
-
 **Entry functions:**
 
 - **`multiTasks(config)`** — run tasks; auto-resumes if the task folder already exists or `initialTasks` points to one.
-- **`multiTasks.resume(config)`** — usually you don't need this: the simplest way to resume is to just run `multiTasks(config)` again, it auto-resumes when the task folder already exists. This method resumes an interrupted run from the task folder alone; pass a config object with `taskFolder` (the full task folder path). The whole config (including `processTask`) is restored from `task_config.json` in that folder; any field you pass overrides the snapshot, and overrides are written back into it.
+- **`multiTasks.resume(config)`** — resume from the task folder alone (`{taskFolder}`); the config is restored from `task_config.json`, fields you pass override it and are written back. Usually you don't need this: re-running `multiTasks(config)` auto-resumes.
 - **`multiTasks.retry_fails(config)`** — same as `resume`, and also retries the failed tasks.
-- **`multiTasks.restart(config)`** — wipe the task folder's progress and re-run from scratch; pass a config object with `taskFolder`. The config (including `processTask` and `initialTasks`) is restored from `task_config.json`; any field you pass overrides the snapshot, and overrides are written back into it. **All previous progress and results are deleted.**
+- **`multiTasks.restart(config)`** — wipe the task folder's progress and results and re-run from scratch (`{taskFolder}`). **All previous progress and results are deleted.**
 
 **Config options:**
 
-- **`initialTasks`** — array of task objects, a function returning the array (or a Promise of it — called only once, in the master process; worker processes never call it; see [Performance: runs only once, in the master process](#performance-runs-only-once-in-the-master-process)), or a folder path string to resume from.
-- **`processTask(task, helper)`** — required; return a result or a Promise with result. Results are stored as JSON in `results/succ/<subid>.json`; if a result cannot be saved because it is not JSON-serializable, the task still counts as succeeded, and a warning shows up in the command line and in monitor tools.
+- **`initialTasks`** — an array of task objects; a function returning the array (or a Promise of it); or a task folder path string to resume from.
+  - Function form, for performance: every worker process re-runs your entire entry script, so top-level task-generation code (scanning folders, querying a database, ...) runs once per worker. Wrapped in a function it is called **only once, in the master process**, and workers never call it. The (resolved) value must be an array, otherwise the master exits with an error. The resulting array — not the function — is stored in `task_config.json`, so the self-contained-function note under Resuming does not apply to it.
+- **`processTask(task, helper)`** — required; return a result or a Promise with result. Results are stored as JSON in `results/succ/<subid>.json`; if a result is not JSON-serializable, the task still counts as succeeded (a placeholder is stored) and a warning shows up in the command line and in monitor tools.
 - **`taskRootFolder`** — the task run folder (named by `taskId`) is created under this directory.
 - **`taskId`** — task folder name under `taskRootFolder`.
 - **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
@@ -293,14 +274,15 @@ The API has three parts: the entry functions, the config options, and the `helpe
 - **`workerPriority`** — optional, OS scheduling priority for worker processes: `'low' | 'below_normal' | 'normal' | 'above_normal' | 'high'`, or a nice number (integer -20~19). Lower priority keeps the machine responsive while a long batch occupies all cores; invalid values throw before any task folder is created.
 - **`progressBar`** — `true` by default: shows a live progress bar in the terminal; set `false` to fall back to plain log output.
 - **`worker_status_poll_ms`** — worker status poll interval in ms; default `2000`; `0` (or `false`) disables the worker status table; `true` means the default rate.
+- **`task_storage`** — `'file'` (default) | `'memory'`: where task states and results are kept; see "Task storage modes".
 - **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
 - **`shouldTerminate(info)`** — return `true` to terminate the whole process. If it throws, the error is logged and the task is dispatched anyway (treated as "don't terminate").
 - **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
-- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, succButSaveFailed, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `succButSaveFailed` counts tasks that ran fine but whose results could not be JSON-serialized (details in `<taskFolder>/.sys/user_messages/`); `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` lets you walk every persisted task result:
-  - **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing; a `{type: 'result_serialize_error', message}` placeholder when the returned value could not be JSON-serialized).
+- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, succButSaveFailed, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `succButSaveFailed` counts tasks that ran fine but whose results could not be JSON-serialized (details in `<taskFolder>/.sys/user_messages/`); `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` walks every persisted task result:
+  - **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing; a `{type: 'result_serialize_error', message}` placeholder when the value could not be JSON-serialized).
   - **`helper.foreachErrorResult((count, task, err) => {})`** — iterate all failed results (`results/errors`); `err` is the persisted error object as-is (`{ex}` / `{subTask, exception}` / `{type: 'timeout', ...}` / `{type: 'worker_crash', ...}`). A non-`Error` rejection value that is not JSON-serializable is persisted as `{ex: {type: 'unserializable_error_value', ...}}`.
   - **`helper.foreachResult((count, task, result, succeeded) => {})`** — iterate all results regardless of outcome (succeeded first, then failed; `count` runs continuously across both); `succeeded` is `true` for results from `results/succ` and `false` for those from `results/errors` (where `result` is the persisted error object).
-  - Iteration follows filesystem order (not sorted). `onFinish` runs synchronously at the very end and the master process exits shortly after it returns — don't `await` inside it.
+  - Iteration follows filesystem order (not sorted). `onFinish` runs synchronously at the very end and the master process exits shortly after it returns — don't `await` inside it. In `memory` mode the helpers iterate the in-memory results the same way (in completion order).
 
 **Other options:**
 
@@ -315,6 +297,8 @@ The API has three parts: the entry functions, the config options, and the `helpe
 
 ### Changelog:
 
+- 3.5.0 Add `task_storage: 'memory'` mode (~17x faster, no resume)
+- 3.4.1 Rewrite README: simpler structure, all examples verified by test cases
 - 3.4.0 Show per-worker load (ELU/CPU/memory) under progressBar; stability fixes
 - 3.3.0 Unserializable results still succeed; shouldTerminate throwing is safe; progressBar counters added
 - 3.2.3 Support workerPriority: set OS scheduling priority for worker processes
@@ -325,47 +309,12 @@ The API has three parts: the entry functions, the config options, and the `helpe
 - 3.1.8 Small refinements
 - 3.1.7 onFinish: iterate all succ/failed results via helper.foreachSuccResult/foreachErrorResult
 - 3.1.6 Fix silent task loss under heavy load with maxTaskRetries
-- 3.1.5 Update README
 - 3.1.4 Rewrite methods multiTasks.resume/retry_fails/restart
-- 3.1.3 Update README
-- 3.1.2 Update README
-- 3.1.1 Fix readme documentation
 - 3.1.0 Support worker broadcast and system events; default numberOfWorkers is now "50%" of CPU cores
-- 3.0.4 numberOfWorkers accepts a percentage string of CPU cores, e.g. "50%"
 - 3.0.3 Support 'maxTaskRetries'
 - 3.0.2 Support 'taskTimeout'
 - 3.0.1 Versions 3.0.0 and above are maintained by AI
-- 2.2.0 Support Resuming from an interrupted task
-- 2.1.2 Update readme
-- 2.1.1 Fix: recreate a new one when a worker collapsed unexpectedly.
-- 2.1.0 Add a new output directory "log" where you can view the process time of each subtask
-- 2.0.9 Update readme examples
-- 2.0.8 Update readme examples
-- 2.0.7 Update changelog
-- 2.0.6 Update readme, remove failed examples
-- 2.0.5 Update readme examples
-- 2.0.4 Support resume from a failed task
-- 2.0.3 Fix: mkdir bug on windows
-- 2.0.2 new feature: support shouldTerminate
-- 2.0.1 Avoid possible I/O conflicts.
-- 2.0.0 Rewritten with a new architecture to support dynamic tasks.
-- 1.2.8 Fix: create task folder failed on MacOS
-- 1.2.7 Small updates
-- 1.2.6 Support onFinish event
-- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
-- 1.2.4 Fix: opt.numberOfWorkers not work
-- 1.2.3 Update README
-- 1.2.2 Update README and examples
-- 1.2.1 Handle exceptions and errors in subtasks
-- 1.2.0 Simplified usage by providing the function way and support return Promise
-- 1.1.4 Remove make-dir
-- 1.1.3 Simplified usage, see example0
-- 1.1.2 
-- 1.1.1 Rename files, updated changelog
-- 1.1.0 Simplified the usage of a customized Consumer, see example0
-- 1.0.8 Fix examples
-- 1.0.7 Remove moment
-- 1.0.6 Performance optimization
+- 3.0.0 and earlier: see the git history
 
 ### Gitee:
 
@@ -373,4 +322,4 @@ The API has three parts: the entry functions, the config options, and the `helpe
 
 ### License:
 
-[MIT](https://opensource.org/license/MIT) 
+[MIT](https://opensource.org/license/MIT)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/utils/backward-v1.js

@@ -1,4 +1,5 @@
 function v1tov2(config) {      
+    if(!config) throw new Error('config is required');//缺参时给有语义的报错, 而不是裸 TypeError(兼容入口 multiTasks/resume/retry_fails/restart 都汇到这里)
     if(config.multi_task_parent_folder && !config.taskRootFolder) config.taskRootFolder = config.multi_task_parent_folder;//backward compatible
     delete config.multi_task_parent_folder;
     

package/utils/randoms.js

@@ -2,10 +2,10 @@
 const crypto = require('crypto');
 const fmtDigit = (n)=>{
     return n > 9 ? "" + n: "0" + n;
-}
+};
 const getDateTimeTxt = ()=>{
     let d=new Date();
-    let YYYY=d.getFullYear()
+    let YYYY=d.getFullYear();
     let MM=fmtDigit(d.getMonth()+1);
     let DD=fmtDigit(d.getDate());
     let HH=fmtDigit(d.getHours());

package/utils/resolveWorkers.js

@@ -6,7 +6,7 @@ const PERCENT_PATTERN = /^(\d+(?:\.\d+)?)%$/;
 
 // 解析 numberOfWorkers 配置:
 // - undefined -> 默认 "50%", 按核数四舍五入折算, 最小为 1;
-// - number -> 原样返回(现状行为不变);
+// - number -> 必须是 >=1 的整数(0 会静默起不来 worker、小数/负数行为隐晦, fail-fast);
 // - string -> 必须是 "数字%" 形式且百分比在 1-100 之间, 按核数四舍五入折算, 最小为 1;
 // - 其余 -> 抛 Error。
 const percentToWorkers = (percent) => {
@@ -18,7 +18,9 @@ const resolveNumberOfWorkers = (value) => {
     return percentToWorkers(50);
   };
   if (typeof value === 'number') {
-    return value;
+    if (Number.isInteger(value) && value >= 1) {
+      return value;
+    };
   };
   if (typeof value === 'string') {
     const match = value.trim().match(PERCENT_PATTERN);