multi-tasks 3.2.1 → 3.2.2

package/README.md

@@ -1,8 +1,6 @@
 # multi-tasks
 
-Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks. Progress and tasks are stored on the file system, so tasks can be resumed even if the host crashes.
-
-**Zero dependencies:** multi-tasks has no runtime dependencies at all — it is built entirely on Node.js built-in modules (`cluster`, `fs`, `path`, `os`), so installing it adds nothing extra to your `node_modules`.
+Multi-tasks is a toolkit to manage long-term and large-scale parallel computing tasks, with zero dependencies. Progress and tasks are stored on the file system, so tasks can be resumed even if the host crashes.
 
 ### Install:
 
@@ -18,13 +16,17 @@ Tip: monitor running tasks in real time — see [Real-Time Monitoring](#real-tim
 //see examples/example0
 let multiTasks = require('multi-tasks').multiTasks;
 
-//Step1, create the tasks to run in parallel as an array.
-let alltasks = [];
-for(let i=0;i<50;i++){
-    alltasks.push({
-        name: `task-${i}`,
-        data: `This prop is for a subtask`
-    });
+//Step1, provide the tasks as a function returning them (or a Promise of them);
+//it runs only once, in the master process.
+let initialTasks = async ()=>{
+    let alltasks = [];
+    for(let i=0;i<50;i++){
+        alltasks.push({
+            name: `task-${i}`,
+            data: `This prop is for a subtask`
+        });
+    };
+    return alltasks;
 };
 
 //Step2, provide a function that processes each sub-task and returns the result
@@ -39,7 +41,7 @@ let processTask = (task, helper)=>{
 
 //Step3, run!
 multiTasks({
-    initialTasks: alltasks, 
+    initialTasks, //accepts array, function, or function returning a promise
     processTask, 
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and result files; you can check the progress here
     taskId: 'my-task',
@@ -175,6 +177,27 @@ multiTasks({
 
 ```
 
+### Performance: runs only once, in the master process
+
+Every worker process re-runs your entire entry script — so top-level task-generation code (building a huge array, scanning folders, querying a database, ...) runs once per worker. Wrap it in a function for performance: it is called **only once, in the master process**, and workers never call it. `initialTasks` accepts three forms — pick one:
+
+```javascript
+
+//Example8, the three forms of initialTasks
+
+//form 1: a plain array — assign it directly
+multiTasks({ initialTasks: alltasks, processTask, taskRootFolder, taskId });
+
+//form 2: a function returning the array
+multiTasks({ initialTasks: () => fetchTasksFromSomewhere(), processTask, taskRootFolder, taskId });
+
+//form 3: a function returning a Promise of the array
+multiTasks({ initialTasks: async () => await fetchTasksFromDb(), processTask, taskRootFolder, taskId });
+
+```
+
+The (resolved) value must be an array, otherwise the master exits with an error. The resulting array — not the function — is stored in `task_config.json`, so the self-contained-function note under Resuming does not apply to it.
+
 ### Resuming
 
 If the execution is interrupted (e.g. a power outage), the simplest way to resume is to just re-run the same code — `multiTasks(config)` detects the existing task folder and auto-resumes, re-running only the stuck tasks:
@@ -261,7 +284,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 
 **Config options:**
 
-- **`initialTasks`** — array of task objects, or a folder path string to resume from.
+- **`initialTasks`** — array of task objects, a function returning the array (or a Promise of it — called only once, in the master process; worker processes never call it; see [Performance: runs only once, in the master process](#performance-runs-only-once-in-the-master-process)), or a folder path string to resume from.
 - **`processTask(task, helper)`** — required; return a value or a Promise.
 - **`taskRootFolder`** — the task run folder (named by `taskId`) is created under this directory.
 - **`taskId`** — task folder name under `taskRootFolder`.
@@ -290,6 +313,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 
 ### Changelog:
 
+- 3.2.2 initialTasks accepts a function (sync or async) that is called only once, in the master process
 - 3.2.1 Fix retry bug: user-defined taskCount/index no longer overwritten on task retries
 - 3.2.0 Add progressBar (on by default); console output diverted to run.log; new startup banner
 - 3.1.9 Print the multi-tasks-monitor command on startup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/workers/index.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const pathUtil = require('path');
+const cluster = require('cluster');
 const Multithread = require('./Multithread');
 const backwardv1 = require('../utils/backward-v1');
 const { FUNCTION_CONFIG_KEYS, reviveConfig } = require('../utils/serializeFns');
@@ -52,19 +53,58 @@ const _computeSysLogFile = (config)=>{
     };
 };
 
+//initialTasks 支持函数形式(惰性生成): 只在 master 进程求值一次, worker 进程重跑入口脚本时跳过,
+//防生成逻辑(扫目录/查库等初始化脚本)在每个 worker 里各执行一遍。
+//函数可同步返回数组, 也可返回 Promise(resolve 值同样必须是数组)——同步路径保持同步 throw,
+//Promise 路径返回 then 链交给 _proceedAfterEval 等待。求值在入口统一完成(先于 _mergeSnapshotConfig
+//与 Multithread.start): 求值后的数组才会参与合并并写入 task_config.json(函数本身会被
+//JSON.stringify 丢弃, 不进存档、不参与 revive)。
+const _evalInitialTasks = (config)=>{
+    if(typeof config.initialTasks !== 'function') return null;
+    if(!(cluster.isPrimary || cluster.isMaster)) return null;
+    let tasks = config.initialTasks();
+    let applyTasks = (resolved)=>{
+        if(!Array.isArray(resolved)){
+            throw new Error('FATAL: initialTasks function should return an array!');
+        };
+        config.initialTasks = resolved;
+    };
+    if(tasks && typeof tasks.then === 'function'){//Promise 形式: 异步求值, resolve 后同样校验数组(假设真 Promise, 即 .then 返回 Promise; 非标准 thenable 不支持)
+        return tasks.then(applyTasks);
+    };
+    applyTasks(tasks);
+    return null;
+};
+
+//同步路径(pending 为 null)直接执行 proceed, 保持既有同步 throw 语义;
+//Promise 路径等 resolve 后执行, resolve 非数组/reject/proceed 抛错时打 FATAL 并 exit(1)
+//(与 init 异步失败一致; Promise 路径只发生在 master, 无同步 throw 对象)。
+const _proceedAfterEval = (pending, proceed)=>{
+    if(!pending){
+        proceed();
+        return;
+    };
+    return pending.then(proceed).catch((e)=>{
+        console.error('[FATAL] failed to evaluate initialTasks:', e);
+        process.exit(1);
+    });
+};
+
 const start = (config)=>{
     config = backwardv1.v1tov2(config);
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
 
-    config.taskRootFolder = _toAbsPath(config.taskRootFolder);
-    config.taskFolder = _toAbsPath(config.taskFolder);
-    _computeSysLogFile(config);
+        config.taskRootFolder = _toAbsPath(config.taskRootFolder);
+        config.taskFolder = _toAbsPath(config.taskFolder);
+        _computeSysLogFile(config);
 
-    Multithread.start(config);
+        Multithread.start(config);
+    });
 };
 const resume = (config)=>{
     config = _normalizeResumeArg('resume', config);
-    config.__resume = true;
-
+    //先校验/推导 taskFolder 再求值 initialTasks(与 retry_fails/restart 一致):
+    //缺目录时同步 throw fail fast, 不让可能昂贵的生成逻辑白跑一遍
     config.taskRootFolder = _toAbsPath(config.taskRootFolder);
     config.taskFolder = _toAbsPath(config.taskFolder);
     if(!config.taskFolder){
@@ -75,10 +115,12 @@ const resume = (config)=>{
     };
     config.taskRootFolder = pathUtil.dirname(config.taskFolder);
     config.taskId = pathUtil.basename(config.taskFolder);
-    _computeSysLogFile(config);
-
-    config = _mergeSnapshotConfig(config);
-    Multithread.start(config);
+    config.__resume = true;
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        config = _mergeSnapshotConfig(config);
+        Multithread.start(config);
+    });
 };
 const retry_fails = (config)=>{
     config = _normalizeResumeArg('retry_fails', config);
@@ -91,9 +133,11 @@ const retry_fails = (config)=>{
     config.taskId = pathUtil.basename(taskFolder);
     config.__resume = true;
     config.__retryFails = true;
-    _computeSysLogFile(config);
-    config = _mergeSnapshotConfig(config);
-    Multithread.start(config);
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        config = _mergeSnapshotConfig(config);
+        Multithread.start(config);
+    });
 };
 const restart = (config)=>{
     config = _normalizeResumeArg('restart', config);
@@ -105,9 +149,11 @@ const restart = (config)=>{
     config.taskRootFolder = pathUtil.dirname(taskFolder);
     config.taskId = pathUtil.basename(taskFolder);
     config.__restart = true;
-    _computeSysLogFile(config);
-    config = _mergeSnapshotConfig(config);
-    Multithread.start(config);
+    return _proceedAfterEval(_evalInitialTasks(config), ()=>{
+        _computeSysLogFile(config);
+        config = _mergeSnapshotConfig(config);
+        Multithread.start(config);
+    });
 };
 
 const multiTasks = (config)=>{
@@ -126,7 +172,7 @@ const multiTasks = (config)=>{
                 config.taskRootFolder = pathUtil.dirname(folder);
             }
         }
-        if(!isResume) throw new Error('FATAL: initialTasks should be an array or a task root folder!');
+        if(!isResume) throw new Error('FATAL: initialTasks should be an array, a function or a task root folder!');
     }else{
         let taskFolder = pathUtil.resolve(taskRootFolder, taskId);
         if(fs.existsSync(taskFolder)){
@@ -135,9 +181,9 @@ const multiTasks = (config)=>{
     }
 
     if(isResume){
-        resume(config);
+        return resume(config);
     }else{
-        start(config);
+        return start(config);
     }
 
 };