multi-tasks 3.1.7 → 3.1.9

package/README.md

@@ -10,6 +10,31 @@ Multi-tasks is a toolkit to manage long-term and large-scale parallel computing
 npm install multi-tasks
 ```
 
+## 🎉 What's New — Real-Time Monitoring Has Arrived! 🎉
+
+**1. Install** the monitor:
+
+```bash
+npm install multi-tasks-monitor
+```
+
+**2. Start your multi-tasks run first** — the monitor reads the task folder, which only exists once your run has started. Then either:
+
+- Easiest (multi-tasks 3.1.9+): your startup log prints the complete command — full task folder path included. Copy it into another terminal and run:
+
+```text
+****************************************************************
+[Master]: monitor this task with:
+    npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
+****************************************************************
+```
+
+- Or just run `npx multi-tasks-monitor` and pick the task folder on the page with "Select Directory".
+
+**3. Watch the monitor page.** The live dashboard opens in your browser automatically — and you can revisit http://localhost:3777 anytime. 🚀
+
+Running multi-tasks older than 3.1.9, or want the full command-line options? See the [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) page.
+
 ### API:
 
 The API has three parts: the entry functions, the config options, and the `helper` object injected into `processTask`.
@@ -40,7 +65,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 
 **Other options:**
 
-- **`taskFolder`** — full task folder path; only used by `resume`/`retry_fails`.
+- **`taskFolder`** — full task folder path; only used by `resume`/`retry_fails`/`restart`.
 
 **Task helper** (the `helper` object passed as the second argument of `processTask`):
 
@@ -272,6 +297,8 @@ Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **
 
 ### Changelog:
 
+- 3.1.9 Print the multi-tasks-monitor command on startup
+- 3.1.8 Small refinements
 - 3.1.7 onFinish: iterate all succ/failed results via helper.foreachSuccResult/foreachErrorResult
 - 3.1.6 Fix silent task loss under heavy load with maxTaskRetries
 - 3.1.5 Update README

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.1.7",
+  "version": "3.1.9",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/workers/Multithread.js

@@ -158,49 +158,64 @@ const start = (config)=>{
                 }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
-                    queue.enqueue(async ()=>{
-                        return TaskMgr.updateNewTasks();
-                    });//use queue to make sure no I/O conflict
                     queue.enqueue(async () => {
-                        return TaskMgr.popNewTask().then((subTask)=>{
-                            if(!subTask){
-                                //no tasks left
-                                console.log('[Master]: no tasks left!');
-                                if(config.autoCloseAfterCompletion){
-                                    killWorker(worker);
-                                }else{//waiting for new tasks
-                                    worker.send(MSG_NO_TASK_FOUND,()=>{}, {
-                                        keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
-                                    });
-                                }
-                            }else{
-                                let terminateData = {taskCount,subTask,config, id: worker.id};
-                                if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
-                                    killAllWorkers();
-                                    return process.exit(0);
-                                }
-                                //send new tasks to the worker
-                                let startTimestamp = new Date()*1;
-                                worker.send({
-                                    taskCount,
-                                    startTimestamp,
-                                    config,
-                                    subTask
-                                },()=>{}, {
+                        //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
+                        await TaskMgr.updateNewTasks();
+                        let subTask = await TaskMgr.popNewTask();
+                        if(!subTask){
+                            //判空前强制绕开缓存重读一次磁盘再 pop: 防其他 worker 动态建任务(createNewTasks)
+                            //落在 update 与判空之间被漏判, 导致 autoClose 提前杀光 worker / 丢新任务
+                            await TaskMgr.loadNewTasks();
+                            subTask = await TaskMgr.popNewTask();
+                        }
+                        if(!subTask && config.autoCloseAfterCompletion){
+                            //杀 worker 前延时 1s 再兜底确认一次: 等其他 worker 可能正在落盘的动态新任务
+                            //(best-effort 再压小竞态窗口, 只限 autoClose 路径, 普通派发零开销)
+                            await new Promise((resolve)=>{ setTimeout(resolve, 1000); });
+                            await TaskMgr.loadNewTasks();
+                            subTask = await TaskMgr.popNewTask();
+                        }
+                        if(!subTask){
+                            //no tasks left
+                            console.log('[Master]: no tasks left!');
+                            if(config.autoCloseAfterCompletion){
+                                killWorker(worker);
+                            }else{//waiting for new tasks
+                                worker.send(MSG_NO_TASK_FOUND,()=>{}, {
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
-                                let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
-                                let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
-                                if(hasTimeout || hasRetry){
-                                    workerTaskTimers[worker.id] = {
-                                        timer: hasTimeout ? setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout) : null,
-                                        subTask,
-                                        startTimestamp
-                                    };
-                                }
-                                taskCount++;
                             }
+                            return;
+                        }
+                        let terminateData = {taskCount,subTask,config, id: worker.id};
+                        if(USER_CONFIG.shouldTerminate && USER_CONFIG.shouldTerminate(terminateData)===true) {
+                            killAllWorkers();
+                            return process.exit(0);
+                        }
+                        //send new tasks to the worker
+                        let startTimestamp = new Date()*1;
+                        //IPC 用裁剪副本: worker 端只取目录路径, 不需要 initialTasks 全量数组;
+                        //带着它每次派发都要 O(N) 序列化(总量 O(N²))。config 本体保留: resume/restart 重写快照要用
+                        let ipcConfig = { ...config };
+                        delete ipcConfig.initialTasks;
+                        worker.send({
+                            taskCount,
+                            startTimestamp,
+                            config: ipcConfig,
+                            subTask
+                        },()=>{}, {
+                            keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                         });
+                        let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
+                        let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
+                        if(hasTimeout || hasRetry){
+                            workerTaskTimers[worker.id] = {
+                                timer: hasTimeout ? setTimeout(()=>{ onTaskTimeout(worker); }, config.taskTimeout) : null,
+                                subTask,
+                                startTimestamp
+                            };
+                        }
+                        taskCount++;
                     });
                 }
             });
@@ -210,10 +225,20 @@ const start = (config)=>{
         initPromise.then((updated_config)=>{
             console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
             config = updated_config;
+            console.log('');//以下: 配套监控工程命令提示, 仅 master 打印一次, 星号边框突出显示
+            console.log('****************************************************************');
+            console.log('[Master]: monitor this task with:');
+            console.log(`    npx multi-tasks-monitor --task-dir "${config.taskFolder}" --port 3777`);
+            console.log('****************************************************************');
             for (let i = 0; i < numOfWorkers; i++) {
                 let worker = cluster.fork();
                 initWorker(worker);
             }
+        }).catch((e)=>{
+            //init/resume 异步失败(rename/读盘错误等): 明确 FATAL + 非 0 退出,
+            //否则 unhandled rejection 在 warn 模式下会不 fork 不退出、静默挂死
+            console.error('[FATAL] failed to init tasks:', e);
+            process.exit(1);
         });
         cluster.on('exit', (worker, code, signal) => {
             let rec = workerTaskTimers[worker.id];//先取再清: 崩溃重试要用 rec.subTask

package/workers/index.js

@@ -59,8 +59,7 @@ const resume = (config)=>{
     config.taskFolder = _toAbsPath(config.taskFolder);
     if(!config.taskFolder){
         if(!config.taskRootFolder || !config.taskId){
-            console.log('[ERROR]', 'method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
-            return;
+            throw new Error('method resume need param "taskFolder" (or "taskRootFolder" + "taskId")');
         };
         config.taskFolder = pathUtil.resolve(config.taskRootFolder, config.taskId);
     };
@@ -74,8 +73,7 @@ const retry_fails = (config)=>{
     config = _normalizeResumeArg('retry_fails', config);
     let taskFolder = _toAbsPath(config.taskFolder);
     if(!taskFolder){
-        console.log('[ERROR]', 'method retry_fails need param "taskFolder"');
-        return;
+        throw new Error('method retry_fails need param "taskFolder"');
     }
     config.taskFolder = taskFolder;
     config.taskRootFolder = pathUtil.dirname(taskFolder);
@@ -89,8 +87,7 @@ const restart = (config)=>{
     config = _normalizeResumeArg('restart', config);
     let taskFolder = _toAbsPath(config.taskFolder);
     if(!taskFolder){
-        console.log('[ERROR]', 'method restart need param "taskFolder"');
-        return;
+        throw new Error('method restart need param "taskFolder"');
     }
     config.taskFolder = taskFolder;
     config.taskRootFolder = pathUtil.dirname(taskFolder);