multi-tasks 3.1.9 → 3.2.0

package/README.md

@@ -10,30 +10,7 @@ Multi-tasks is a toolkit to manage long-term and large-scale parallel computing
 npm install multi-tasks
 ```
 
-## 🎉 What's New — Real-Time Monitoring Has Arrived! 🎉
-
-**1. Install** the monitor:
-
-```bash
-npm install multi-tasks-monitor
-```
-
-**2. Start your multi-tasks run first** — the monitor reads the task folder, which only exists once your run has started. Then either:
-
-- Easiest (multi-tasks 3.1.9+): your startup log prints the complete command — full task folder path included. Copy it into another terminal and run:
-
-```text
-****************************************************************
-[Master]: monitor this task with:
-    npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
-****************************************************************
-```
-
-- Or just run `npx multi-tasks-monitor` and pick the task folder on the page with "Select Directory".
-
-**3. Watch the monitor page.** The live dashboard opens in your browser automatically — and you can revisit http://localhost:3777 anytime. 🚀
-
-Running multi-tasks older than 3.1.9, or want the full command-line options? See the [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) page.
+Tip: monitor running tasks in real time — see [Real-Time Monitoring](#real-time-monitoring).
 
 ### API:
 
@@ -55,6 +32,7 @@ The API has three parts: the entry functions, the config options, and the `helpe
 - **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
 - **`taskTimeout`** — optional, milliseconds; an overdue task fails with a timeout error.
 - **`maxTaskRetries`** — optional, max times a failed task is auto-retried.
+- **`progressBar`** — `true` by default: shows a live single-line progress bar in the terminal (set `false` to disable); logs (including your own `console.*` calls) always go to `<taskFolder>/.sys/run.log`, and print to the console as well only when the bar is off or stdout is not a TTY.
 - **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
 - **`shouldTerminate(info)`** — return `true` to terminate the whole process.
 - **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
@@ -295,8 +273,24 @@ multiTasks.restart({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //deletes
 
 Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass (e.g. `processTask`) override the snapshot and are written back into it; the same self-contained-function note as in Resuming applies. (**Breaking change:** the name `restart` was the old name of `retry_fails` in v2 and in the published v3.0–v3.1.x releases; this is a different API — calling it no longer retries failed tasks, it wipes the folder and re-runs everything.)
 
+### Real-Time Monitoring
+
+Monitor your running tasks with [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) — see its npm page for full details.
+
+Quick start (requires multi-tasks >=3.2.0): once your multi-tasks run has started, the startup log prints the monitor command with the task folder path already filled in — just copy-paste it into another terminal:
+
+```text
+****************************************************************
+[Master]: monitor this task with:
+    npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
+****************************************************************
+```
+
+For installation, command-line options, and more, see the [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) page.
+
 ### Changelog:
 
+- 3.2.0 Add progressBar (on by default); console output diverted to run.log; new startup banner
 - 3.1.9 Print the multi-tasks-monitor command on startup
 - 3.1.8 Small refinements
 - 3.1.7 onFinish: iterate all succ/failed results via helper.foreachSuccResult/foreachErrorResult

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "3.1.9",
+  "version": "3.2.0",
   "description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
   "main": "index.js",
   "files": [

package/utils/progressCtl.js

@@ -0,0 +1,118 @@
+//progressBar master 端控制器(零依赖): 持有渲染器实例与忙碌表, 封装 TTY 判定与节流渲染,
+//Multithread 只调语义化方法(setup/workerStarted/taskAssigned/...), 不碰渲染细节。
+//纯 master 端运行时状态, 不进快照不进 IPC; 统计数据(getStats/getRemaining)由调用方注入, 本模块不依赖 TaskMgr。
+//console 劫持与 run.log 已与 progressBar 解耦(永久生效, 见 utils/runLog), 不再是本模块的职责。
+//单行进度条(progressView): worker 增删不体现在渲染上, 这里只维护 busyWorkers 计数 running。
+const progressView = require('./progressView');
+
+let RENDER_INTERVAL = 100;//渲染节流: 高频小任务场景避免每次事件都 readdirSync(getFinalStats) 造成 O(N²) 盘扫
+
+let view = null;//progressView 渲染器实例, 未开启为 null
+let progressOn = false;//progressBar 且 stdout 为 TTY 时为真(非 TTY 回退普通日志)
+let busyWorkers = {};//worker.id -> subid, master 端忙碌表
+let getStats = null;//() => TaskMgr.getFinalStats(), setup 注入
+let getRemaining = null;//() => new 目录任务数, setup 注入
+let lastRenderAt = 0;
+let renderTimer = null;
+
+const reset = () => {//master start 入口/同进程再跑一轮时调用: 清掉上一轮全部残留(含未发的节流帧)
+  if (renderTimer) {
+    clearTimeout(renderTimer);
+    renderTimer = null;
+  };
+  view = null;
+  progressOn = false;
+  busyWorkers = {};
+  lastRenderAt = 0;
+};
+
+//开启判定 + 初始化: 返回是否真正开启(显式 progressBar:false 关闭, 缺省默认开启;
+//非 TTY 一律回退普通日志, 仅显式 true 时打回退警告, 默认开启静默回退)
+const setup = ({ config, stream, getStats: statsFn, getRemaining: remainingFn }) => {
+  reset();
+  if (config.progressBar === false) {
+    return false;
+  };
+  if (!stream.isTTY) {//ANSI 控制码写进重定向输出是垃圾: 回退为现有控制台日志
+    if (config.progressBar === true) {
+      console.warn('[Master]: progressBar needs a TTY stdout, fallback to console logs');
+    };
+    return false;
+  };
+  progressOn = true;
+  getStats = statsFn;
+  getRemaining = remainingFn;
+  view = progressView.create({ stream, barWidth: 30 });
+  return true;
+};
+
+const isOn = () => {
+  return progressOn;
+};
+
+const doRender = () => {
+  //完成数按盘上结果目录统计: 重试/超时/崩溃各种路径天然正确, 无需在协议里额外计数
+  let stats = getStats();
+  let done = stats.succeeded + stats.failed;
+  let running = Object.keys(busyWorkers).length;
+  lastRenderAt = Date.now();
+  view.render({ done, running, remaining: getRemaining() });
+};
+
+const render = () => {
+  if (!view) {
+    return;
+  };
+  if (renderTimer) {
+    return;//已挂着一个尾随渲染, 合并掉中间帧
+  };
+  let wait = RENDER_INTERVAL - (Date.now() - lastRenderAt);
+  if (wait <= 0) {
+    doRender();
+    return;
+  };
+  renderTimer = setTimeout(() => {
+    renderTimer = null;
+    if (view) {
+      doRender();
+    };
+  }, wait);
+  renderTimer.unref();//不阻碍 master 退出(最后一帧可能因进程退出丢失, 可接受)
+};
+
+const workerStarted = (id, pid) => {//单行进度条不展示 worker 行: 仅保留语义入口, 无渲染动作
+};
+
+const taskAssigned = (id, subid) => {
+  busyWorkers[id] = subid;
+  render();
+};
+
+const taskCompleted = (id) => {
+  if (busyWorkers[id]) {
+    delete busyWorkers[id];
+  };
+};
+
+const workerExited = (id) => {
+  delete busyWorkers[id];
+  render();
+};
+
+const finish = () => {//收尾: 换行定格进度条行(master 退出前调用); 未开启为 no-op
+  if (view) {
+    view.end();
+  };
+};
+
+module.exports = {
+  setup,
+  reset,
+  isOn,
+  render,
+  workerStarted,
+  taskAssigned,
+  taskCompleted,
+  workerExited,
+  finish,
+};

package/utils/progressView.js

@@ -0,0 +1,35 @@
+//终端进度条渲染器(零依赖): 单行进度条, 每帧 \r 回行首 + 清行 + 重写, 不上移光标、不换行。
+//纯渲染, 不碰 cluster/fs。单行形态对"两帧之间有外部写入共享终端"鲁棒:
+//旧帧最多留一行滚动残留, 下一帧在当前行自愈(多行版靠 ESC[nA 数行, 外部写入挤偏光标会让旧帧永久滞留)。
+const ESC = '';
+
+const create = ({ stream, barWidth }) => {
+  if (!stream) { throw new Error('progressView needs a stream'); };
+  const width = barWidth || 30;
+
+  const buildBarLine = ({ done, running, remaining }) => {
+    let total = done + running + remaining;
+    let filled = total > 0 ? Math.floor(width * done / total) : width;
+    let bar = '█'.repeat(filled) + '░'.repeat(width - filled);
+    let pct = (total > 0 ? 100 * done / total : 100).toFixed(2).padStart(6);//两位小数, 定宽防行长抖动
+    return `[${bar}] ${pct}% ${done}/${total} done, ${running} running`;
+  };
+
+  const render = ({ done, running, remaining }) => {
+    //行比终端列宽还长会折行、\r 回不到帧首: 按列宽截断
+    let cols = stream.columns || 80;
+    let line = buildBarLine({ done, running, remaining });
+    if (line.length > cols) {
+      line = line.slice(0, cols);
+    };
+    stream.write(`\r${ESC}[2K${line}`);
+  };
+
+  const end = () => {//收尾换行: 进度条行定格, 后续输出(shell 提示符等)落在下一行
+    stream.write('\n');
+  };
+
+  return { render, end };
+};
+
+module.exports = { create };

package/utils/runLog.js

@@ -0,0 +1,136 @@
+//日志分流(永久性机制, 与 progressBar 无关): 库的运行日志与 hook 后的 console.* 一律经 O_APPEND
+//写流(异步, 不阻塞事件循环) append 到任务目录的 run.log; echo 模式(progressBar 未开/非 TTY 回退)
+//同时透传原 console 上屏, 非 echo 模式(progressBar 进度区接管终端)只写文件不上屏。
+//master 与 worker 各进程各持一个写流 append 同一文件。
+//best-effort: 写流出错(目录不存在/权限等)标记后整轮丢弃; 进程退出可能丢未 flush 的缓冲尾部。
+const fs = require('fs');
+const util = require('util');
+
+let state = { enabled: false, filePath: null, stream: null, broken: false, hooked: false, echo: false };
+let originalConsole = null;
+
+const init = (filePath) => {
+  state.enabled = true;
+  state.filePath = filePath;
+  state.stream = null;//惰性创建: 首个 write 时才 open, 避免空跑也建文件
+  state.broken = false;
+};
+
+const reset = () => {//同进程再跑一轮 multiTasks 时复位, 否则日志会继续写上一轮的 run.log
+  unhookConsole();
+  if (state.stream) {
+    state.stream.destroy();
+  };
+  state.enabled = false;
+  state.filePath = null;
+  state.stream = null;
+  state.broken = false;
+  state.echo = false;
+};
+
+const isEnabled = () => {
+  return state.enabled;
+};
+
+//拦截 console: 把 console.log/warn/error 替换为写 run.log 的实现,
+//用户代码(processTask/onFinish 等)里的 console 输出同样进文件。
+//echo 为真时同时调原实现上屏(progressBar 未开/非 TTY 回退); 为假时只写文件(进度区接管终端)。
+//reset/unhookConsole 恢复原实现(必须在 enabled 变 false 之前恢复, 否则 write 透传 console 会递归)。
+const hookConsole = ({ echo } = {}) => {
+  if (!state.enabled || state.hooked) {
+    return;
+  };
+  state.hooked = true;
+  state.echo = !!echo;
+  originalConsole = { log: console.log, warn: console.warn, error: console.error };
+  console.log = write;
+  console.warn = warn;
+  console.error = error;
+};
+
+const unhookConsole = () => {
+  if (!state.hooked) {
+    return;
+  };
+  console.log = originalConsole.log;
+  console.warn = originalConsole.warn;
+  console.error = originalConsole.error;
+  originalConsole = null;
+  state.hooked = false;
+  state.echo = false;
+};
+
+const formatArg = (arg) => {
+  if (arg instanceof Error) {
+    return arg.stack || String(arg);
+  };
+  if (typeof arg === 'string') {
+    return arg;
+  };
+  //对象/数组走 util.inspect, 避免写进 run.log 变成 [object Object] 丢信息
+  return typeof arg === 'object' && arg !== null ? util.inspect(arg) : String(arg);
+};
+
+const appendToFile = (args) => {
+  if (state.broken) {
+    return;
+  };
+  try {
+    if (!state.stream) {
+      state.stream = fs.createWriteStream(state.filePath, { flags: 'a' });
+      state.stream.on('error', () => {//写盘失败不影响任务运行, 标记后整轮丢弃(避免循环重建)
+        state.broken = true;
+        state.stream.destroy();
+        state.stream = null;
+      });
+    };
+    state.stream.write(args.map(formatArg).join(' ') + '\n');
+  } catch (e) {
+    //best-effort: 日志写盘失败不影响任务运行
+  };
+};
+
+const write = (...args) => {
+  if (!state.enabled) {
+    console.log(...args);
+    return;
+  };
+  appendToFile(args);
+  if (state.echo && originalConsole) {
+    originalConsole.log(...args);
+  };
+};
+
+const warn = (...args) => {
+  if (!state.enabled) {
+    console.warn(...args);
+    return;
+  };
+  appendToFile(args);
+  if (state.echo && originalConsole) {
+    originalConsole.warn(...args);
+  };
+};
+
+const error = (...args) => {
+  if (!state.enabled) {
+    console.error(...args);
+    return;
+  };
+  appendToFile(args);
+  if (state.echo && originalConsole) {
+    originalConsole.error(...args);
+  };
+};
+
+//master/worker 端入口调用: 有分流路径即 init 并 hook console(永久性机制, 不再判 TTY/progressBar);
+//echo 由调用方按 progressBar 是否接管终端决定。无路径时什么都不做(维持 console)。判断收敛在此, 调用点不写 if。
+const initAndHook = (filePath, { echo } = {}) => {
+  if (!filePath) {
+    return;
+  };
+  init(filePath);
+  hookConsole({ echo });
+};
+
+module.exports = { init, initAndHook, reset, isEnabled, write, warn, error, hookConsole, unhookConsole };

package/utils/startupBanner.js

@@ -0,0 +1,50 @@
+//启动信息横幅渲染器(纯函数, 不碰 console/fs/cluster): 仅标题行 ASCII 框装饰,
+//框宽只由标题行内容决定——长路径只出现在平铺字段行, 不会撑破框、不会折行。
+//renderBanner: 终端上屏用(劫持前 console 直出); renderLogHeader: run.log 文件头部用(含 MODE 行);
+//writeLogHeader: 判 sysLogFile 后写 runLog——判断收敛在方法内部, 调用点不写 if;
+//无路径时(内部 API multiTasks.start 自动生成目录名)no-op, 不透传 console。
+const runLog = require('./runLog');
+
+const VERSION = require('../package.json').version;
+
+const KEY_WIDTH = 17;//键列宽: 需容纳最长键 'monitor start cmd', 改动时同步调
+
+const renderTitleBox = (mode) => {
+  const left = `multi-tasks v${VERSION}`;
+  const right = `MODE: ${mode}`;
+  const innerWidth = left.length + right.length + 4;//左右内容间至少 4 空格
+  const middle = `| ${left}${' '.repeat(innerWidth - left.length - right.length)}${right} |`;
+  const border = `+${'-'.repeat(innerWidth + 2)}+`;
+  return [border, middle, border];
+};
+
+const renderFieldLines = (info) => {
+  const rows = [
+    ['taskId', info.taskId],
+    ['taskFolder', info.taskFolder],
+    ['workers', `${info.numOfWorkers} (numCPUs=${info.numCPUs})`],
+    ['master PID', `${info.pid}`],
+    ['new tasks', `${info.newTasks}`],
+    ['run.log', info.sysLogFile || '(not available)'],
+    ['started at', info.startedAt],
+    ['monitor start cmd', `npx multi-tasks-monitor --task-dir "${info.taskFolder}" --port 3777`],
+  ];
+  return rows.map(([key, value]) => `${key.padEnd(KEY_WIDTH)} : ${value}`);
+};
+
+const renderBanner = (info) => {
+  return [...renderTitleBox(info.mode), ...renderFieldLines(info)].join('\n');
+};
+
+const renderLogHeader = (info) => {
+  return [`${'MODE'.padEnd(KEY_WIDTH)} : ${info.mode}`, ...renderFieldLines(info)].join('\n');
+};
+
+const writeLogHeader = (info) => {
+  if (!info.sysLogFile) {
+    return;
+  };
+  runLog.write(renderLogHeader(info));
+};
+
+module.exports = { renderBanner, renderLogHeader, writeLogHeader };

package/workers/Multithread.js

@@ -9,6 +9,9 @@ const TaskMgr = require('./TaskMgr');
 const WorkerMgr = require('./WorkerMgr');
 const helper = require('./helper');
 const { resolveNumberOfWorkers } = require('../utils/resolveWorkers');
+const runLog = require('../utils/runLog');
+const progressCtl = require('../utils/progressCtl');
+const startupBanner = require('../utils/startupBanner');
 
 let fmtdigit = (n)=>{
     return n > 9 ? "" + n: "0" + n;
@@ -34,7 +37,7 @@ let killAllWorkers = ()=>{
     for (const id in cluster.workers) {  
         if (cluster.workers.hasOwnProperty(id)) {  
             const worker = cluster.workers[id];  
-            console.log(`Killing worker ${worker.process.pid}`);  
+            runLog.write(`Killing worker ${worker.process.pid}`);  
             killWorker(worker);
         }  
     }  
@@ -49,13 +52,12 @@ const start = (config)=>{
     let isMaster = (cluster.isPrimary || cluster.isMaster);
 
     if (isMaster) {//init
-        console.log('[Master]: Start!!');
-        let PID = process.pid;
+        //同进程再跑一轮 multiTasks 时复位 progressBar 控制器状态,
+        //否则残留会让新一轮(不带 progressBar)继续渲染旧进度区
+        progressCtl.reset();
+        runLog.reset();//runLog 已脱离 progressCtl: 不复位则新一轮日志继续写旧 run.log、console 仍被旧 hook 拦截
         config.numberOfWorkers = resolveNumberOfWorkers(config.numberOfWorkers);//支持百分比字符串; 须在 fork 循环读值前折算, 非法值在此抛错
         let numOfWorkers = config.numberOfWorkers;
-        console.log(`  numCPUs=${numCPUs}`);
-        console.log(`  numOfWorkers=${numOfWorkers}`);
-        console.log(`  PID=${PID}`);
 
         const AsyncQueue = require(`../utils/asyncQueue`);
         const queue = new AsyncQueue();
@@ -82,7 +84,7 @@ const start = (config)=>{
                     if(typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
                         let retryCount = TaskMgr.incrementTaskRetry(subid);
                         if(retryCount <= config.maxTaskRetries){
-                            console.warn(`[Master]: task "${subid}" timeout, retry (${retryCount}/${config.maxTaskRetries})`);
+                            runLog.warn(`[Master]: task "${subid}" timeout, retry (${retryCount}/${config.maxTaskRetries})`);
                             TaskMgr.mvTask(subid, 'running', 'new', ()=>{});
                             return;
                         };
@@ -91,7 +93,7 @@ const start = (config)=>{
                     TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
                 }
             }catch(e){
-                console.warn('[Master]: failed to finalize timeout task', subid, e);
+                runLog.warn('[Master]: failed to finalize timeout task', subid, e);
             }
         };
         let onTaskTimeout = (worker)=>{
@@ -100,7 +102,7 @@ const start = (config)=>{
             delete workerTaskTimers[worker.id];
             let {subTask, startTimestamp} = rec;
             let cost = new Date()*1 - startTimestamp;
-            console.warn(`[Master]: task "${subTask.subid}" timeout after ${cost}ms, killing worker pid=${worker.process.pid}`);
+            runLog.warn(`[Master]: task "${subTask.subid}" timeout after ${cost}ms, killing worker pid=${worker.process.pid}`);
             worker.kill();//不 disconnect: exit 事件会按崩溃补员
             finalizeTimeoutTask(subTask, cost);
         };
@@ -110,15 +112,15 @@ const start = (config)=>{
             try{
                 let retryCount = TaskMgr.incrementTaskRetry(subid);
                 if(retryCount <= config.maxTaskRetries){
-                    console.warn(`[Master]: worker crashed, retry task "${subid}" (${retryCount}/${config.maxTaskRetries})`);
+                    runLog.warn(`[Master]: worker crashed, retry task "${subid}" (${retryCount}/${config.maxTaskRetries})`);
                     TaskMgr.mvTask(subid, 'running', 'new', ()=>{});
                 }else{
-                    console.warn(`[Master]: task "${subid}" crashed ${retryCount} times, give up`);
+                    runLog.warn(`[Master]: task "${subid}" crashed ${retryCount} times, give up`);
                     TaskMgr.saveTaskResult(subid, true, [{type: 'worker_crash', retryCount: config.maxTaskRetries}]);
                     TaskMgr.mvTask(subid, 'running', 'finished_with_errors', ()=>{});
                 };
             }catch(e){
-                console.warn('[Master]: failed to handle crashed task', subid, e);
+                runLog.warn('[Master]: failed to handle crashed task', subid, e);
             }
         };
 
@@ -127,8 +129,9 @@ const start = (config)=>{
             worker.on('message', function(message) {
                 if(message && message.__sysEvent){//worker 系统事件: master 自己消费, 不转发
                     if(message.event === 'TERMINATE_ALL_WORKERS'){
-                        console.warn(`[Master]: TERMINATE_ALL_WORKERS from worker ${worker.id}, killing all workers`);
+                        runLog.warn(`[Master]: TERMINATE_ALL_WORKERS from worker ${worker.id}, killing all workers`);
                         killAllWorkers();
+                        progressCtl.finish();//单行进度条换行定格, 别让 shell 提示符盖住它
                         return process.exit(0);
                     }
                     let meta = {fromWorkerId: worker.id, fromWorkerPid: worker.process.pid};
@@ -136,10 +139,10 @@ const start = (config)=>{
                         try{
                             USER_CONFIG.setSysListener(message.event, message.payload, meta);
                         }catch(e){
-                            console.error(`[Master]: setSysListener for event "${message.event}" threw:`, e);
+                            runLog.error(`[Master]: setSysListener for event "${message.event}" threw:`, e);
                         }
                     }else{
-                        console.warn(`[Master]: unknown sys event "${message.event}" from worker ${worker.id} (no setSysListener configured)`);
+                        runLog.warn(`[Master]: unknown sys event "${message.event}" from worker ${worker.id} (no setSysListener configured)`);
                     }
                     return;
                 }
@@ -151,13 +154,14 @@ const start = (config)=>{
                         try{
                             w.send({__broadcast: true, event: message.event, payload: message.payload, meta});
                         }catch(e){
-                            console.warn(`[Master]: failed to relay broadcast to worker ${w.id}`, e);
+                            runLog.warn(`[Master]: failed to relay broadcast to worker ${w.id}`, e);
                         }
                     }
                     return;
                 }
                 if(message === MSG_REQUEST_NEW_TASK){
                     clearTaskTimer(worker.id);//上一个任务已结束(无论成败), 撤销超时倒计时
+                    progressCtl.taskCompleted(worker.id);//worker 行回到 idle; 完成数按盘上结果目录统计, 不在此计数
                     queue.enqueue(async () => {
                         //use queue to make sure no I/O conflict; update/pop/判空须在同一个串行任务里原子完成
                         await TaskMgr.updateNewTasks();
@@ -177,7 +181,7 @@ const start = (config)=>{
                         }
                         if(!subTask){
                             //no tasks left
-                            console.log('[Master]: no tasks left!');
+                            runLog.write('[Master]: no tasks left!');
                             if(config.autoCloseAfterCompletion){
                                 killWorker(worker);
                             }else{//waiting for new tasks
@@ -185,6 +189,7 @@ const start = (config)=>{
                                     keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                                 });
                             }
+                            progressCtl.render();
                             return;
                         }
                         let terminateData = {taskCount,subTask,config, id: worker.id};
@@ -206,6 +211,7 @@ const start = (config)=>{
                         },()=>{}, {
                             keepOpen: (typeof config.keepOpen === 'undefined') ? false : config.keepOpen
                         });
+                        progressCtl.taskAssigned(worker.id, subTask.subid);
                         let hasTimeout = typeof config.taskTimeout === 'number' && config.taskTimeout > 0;
                         let hasRetry = typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0;
                         if(hasTimeout || hasRetry){
@@ -223,17 +229,37 @@ const start = (config)=>{
 
         let initPromise = config.__resume ? asMaster.resume(config) : asMaster.init(config);
         initPromise.then((updated_config)=>{
-            console.log(`  new tasks=${TaskMgr.getTasks('new').length}`);
             config = updated_config;
-            console.log('');//以下: 配套监控工程命令提示, 仅 master 打印一次, 星号边框突出显示
-            console.log('****************************************************************');
-            console.log('[Master]: monitor this task with:');
-            console.log(`    npx multi-tasks-monitor --task-dir "${config.taskFolder}" --port 3777`);
-            console.log('****************************************************************');
+            //一次性启动块(走 console, 在劫持前上屏): 标题框横幅 + 平铺关键字段, 模式由 __ 标记推导;
+            //init 成功后所有信息齐备(new tasks 数/masterTaskId/__sysLogFile), 一次打完
+            let startupInfo = {
+                mode: config.__restart ? 'RESTART' : (config.__retryFails ? 'RETRY_FAILS' : (config.__resume ? 'RESUME' : 'START')),
+                taskId: config.masterTaskId,
+                taskFolder: config.taskFolder,
+                numOfWorkers,
+                numCPUs,
+                pid: process.pid,
+                newTasks: TaskMgr.getTasks('new').length,
+                sysLogFile: config.__sysLogFile,
+                startedAt: getDateTimeTxt(),
+            };
+            console.log(startupBanner.renderBanner(startupInfo));
+            progressCtl.setup({//TTY 判定 + 渲染器创建都在控制器内; 统计口径注入, 控制器不依赖 TaskMgr
+                config,
+                stream: process.stdout,
+                getStats: () => TaskMgr.getFinalStats(),
+                getRemaining: () => TaskMgr.getTasks('new').length,
+            });
+            //console 劫持写 run.log 是永久机制, 与 progressBar 无关: 进度区接管终端(开且 TTY)时只写文件,
+            //否则 echo 同时透传 console 上屏; 判定收敛在 initAndHook 内, 无 __sysLogFile 时维持 console
+            runLog.initAndHook(config.__sysLogFile, { echo: !progressCtl.isOn() });
+            startupBanner.writeLogHeader(startupInfo);//同一组关键信息写 run.log 头部; 无 __sysLogFile 时内部 no-op
             for (let i = 0; i < numOfWorkers; i++) {
                 let worker = cluster.fork();
                 initWorker(worker);
+                progressCtl.workerStarted(worker.id, worker.process.pid);
             }
+            progressCtl.render();
         }).catch((e)=>{
             //init/resume 异步失败(rename/读盘错误等): 明确 FATAL + 非 0 退出,
             //否则 unhandled rejection 在 warn 模式下会不 fork 不退出、静默挂死
@@ -243,14 +269,17 @@ const start = (config)=>{
         cluster.on('exit', (worker, code, signal) => {
             let rec = workerTaskTimers[worker.id];//先取再清: 崩溃重试要用 rec.subTask
             clearTaskTimer(worker.id);//worker 退出(含崩溃)时清掉其超时计时器
+            progressCtl.workerExited(worker.id);
             if (code !== 0 && !worker.exitedAfterDisconnect) {
-                console.log('Worker died unexpectedly starting a new one...');
+                runLog.write('Worker died unexpectedly starting a new one...');
                 workerCrashCount++;
                 if(rec && rec.subTask){
                     handleCrashedTask(rec.subTask);
                 };
                 let worker = cluster.fork();
                 initWorker(worker);
+                progressCtl.workerStarted(worker.id, worker.process.pid);
+                progressCtl.render();
                 return;
             }
             let allkilled = true; 
@@ -281,24 +310,30 @@ const start = (config)=>{
                     delete USER_CONFIG.onFinish;
                 }
                 setTimeout(()=>{
-                    console.log('[Master]: all workers are done, process exit.')
+                    runLog.write('[Master]: all workers are done, process exit.')
+                    progressCtl.finish();//单行进度条换行定格, 别让 shell 提示符盖住它
                     process.exit(0);
                 }, 10);
             }
         }); 
     }else{
-        subWorker();
+        subWorker(config);
     }
 };
-function subWorker(){
+function subWorker(config){
     let worker = cluster.worker;
+    //worker 进程独立重跑入口, config.__sysLogFile 已由 workers/index.js 入口算好(与 master 同一路径):
+    //console 劫持写 run.log 永久生效, 与 progressBar 无关; progressBar 未显式关闭(缺省默认开启)且
+    //共享 stdout 为 TTY 时只写文件(与 master 判定一致), 否则 echo 同时上屏。判定收敛在 initAndHook 内,
+    //无路径时维持 console。入口即生效, 领不到任务的富余 worker 的 waiting 日志同样进 run.log。
+    runLog.initAndHook(config.__sysLogFile, { echo: !(config.progressBar !== false && process.stdout.isTTY) });
     worker.on('message', function(message) {
         if(message && message.__broadcast){//master 中转来的广播: 分发给 helper.on 注册的 handler
             helper.__dispatchBroadcast(message);
             return;
         }
         if(message === MSG_NO_TASK_FOUND){//wait for new task
-            console.log(`  (Worker #${worker.id}): waiting`);
+            runLog.write(`  (Worker #${worker.id}): waiting`);
             setTimeout(()=>{
                 worker.send(MSG_REQUEST_NEW_TASK);//request new task
             }, 5000 + Math.random()*5000);
@@ -310,7 +345,7 @@ function subWorker(){
             subTask.taskCount = taskCount;
             subTask.workerId = worker.id;
             subTask.workerPid = process.pid;
-            console.log(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
+            runLog.write(`[Task Start]: "${subTask.subid}"`, `pid=${process.pid}`, getDateTimeTxt());
             let subTaskLog = {
                 workerId: worker.id,
                 workerPid: process.pid,
@@ -320,12 +355,12 @@ function subWorker(){
             runTask(config, subTask, subTaskLog).then((tasklog)=>{
                 let {cost, startTimestamp, endTimestamp} = tasklog;
 
-                console.log(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
+                runLog.write(`[Task Finished] "${config.masterTaskId}"."${subTask.subid}".`, getDateTimeTxt(), `cost=${cost}ms`)
             }).then(()=>{
                 worker.send(MSG_REQUEST_NEW_TASK);//request new task
             });
         }else{
-            console.log('Can not find: message.subTask');
+            runLog.write('Can not find: message.subTask');
             throw new Error('can not find subTask!')
         }
     });
@@ -366,19 +401,19 @@ function runTask(config, subTask, subTaskLog){
                 cost
             };
             let hasError = errors.length > 0;
-            if(hasError)console.warn('[Task Error]', subTask.subid, errors);
+            if(hasError)runLog.warn('[Task Error]', subTask.subid, errors);
             let retried = false;
             if(hasError && typeof config.maxTaskRetries === 'number' && config.maxTaskRetries > 0){
                 let retryCount = TaskMgr.incrementTaskRetry(subTask.subid);
                 if(retryCount <= config.maxTaskRetries){
-                    console.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
+                    runLog.warn(`[Task Retry] "${subTask.subid}" failed, retry (${retryCount}/${config.maxTaskRetries})`);
                     retried = true;
                     //等 rename 落盘再 resolve/请求新任务: fire-and-forget 的异步 rename 会随 worker 被
                     //autoClose 杀掉而消亡, 任务文件滞留 running 被静默丢弃(高负载偶发丢任务的根因, 勿回退)
                     try{
                         await TaskMgr.mvTaskPromise(subTask.subid, 'running', 'new');
                     }catch(newPath){
-                        console.warn(`[Task Retry] "${subTask.subid}" failed to move back to new, left in running (recoverable via resume)`);
+                        runLog.warn(`[Task Retry] "${subTask.subid}" failed to move back to new, left in running (recoverable via resume)`);
                     };
                 };
             };

package/workers/TaskMgr.js

@@ -4,6 +4,7 @@ const makeDir = require('../utils/makedir');
 const randomsUtil = require('../utils/randoms');
 const { serializeConfig } = require('../utils/serializeFns');
 const { forEachResult } = require('../utils/foreachResults');
+const runLog = require('../utils/runLog');
 
 let allSubTasks = {
     new:[],
@@ -35,6 +36,7 @@ const TaskMgr = {
         const progressRootFolder = pathutil.resolve(taskFolder, './progress');
         const resultsRootFolder = pathutil.resolve(taskFolder, './results');
         const logsRootFolder = pathutil.resolve(taskFolder, './logs');
+        const sysRootFolder = pathutil.resolve(taskFolder, './.sys');//库自身运行日志(run.log)等系统文件, 与任务数据/结果隔离
 
         const taskFolderNew = pathutil.resolve(progressRootFolder, './new');
         const taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
@@ -44,6 +46,7 @@ const TaskMgr = {
         const taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
         const taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
         makeDir.sync(logsRootFolder);
+        makeDir.sync(sysRootFolder);
         makeDir.sync(taskFolderNew);
         makeDir.sync(taskFolderRunning);
         makeDir.sync(taskFolderFinished);
@@ -57,6 +60,9 @@ const TaskMgr = {
             progressRootFolder,
             resultsRootFolder,
             logsRootFolder,
+            sysRootFolder,
+            //run.log 路径唯一兜底计算点: 入口(workers/index.js)没算过时在这里补(__ 前缀, 不进快照)
+            __sysLogFile: main_config.__sysLogFile || pathutil.resolve(sysRootFolder, 'run.log'),
 
             taskFolderNew,
             taskFolderRunning,
@@ -97,14 +103,14 @@ const TaskMgr = {
                             try{
                                 fcontent = fs.readFileSync(fpath, 'utf-8');
                             }catch(e){
-                                console.log('[WARN]', `file not found`, fpath)
+                                runLog.warn('[WARN]', `file not found`, fpath)
                             }
                             if(fcontent){
                                 try{
                                     let task = JSON.parse(fcontent);
                                     allSubTasks[type].push(task);
                                 }catch(e){
-                                    console.warn('[WARN]', 'failed to parse task file, skipped', fpath);
+                                    runLog.warn('[WARN]', 'failed to parse task file, skipped', fpath);
                                 }
                             }
                         });
@@ -199,7 +205,7 @@ const TaskMgr = {
         });
     },
     mvTask: (subid, from, to, callback)=> {
-        console.log('Move task:', subid, `${from}>${to}`);
+        runLog.write('Move task:', subid, `${from}>${to}`);
         let {progressRootFolder} = main_config;
         let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
         let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
@@ -212,7 +218,7 @@ const TaskMgr = {
                 }
             });
         }catch(e){
-            console.log(e);
+            runLog.warn(e);
             callback(newPath, false);
         }
     },

package/workers/WorkerMgr.js

@@ -1,6 +1,8 @@
 let workers = {};
 let status = {};
 
+const runLog = require('../utils/runLog');
+
 const mgr = {
     addWorker:(worker)=>{
         let id = `${worker.id}`;
@@ -8,7 +10,7 @@ const mgr = {
             throw new Error('FATAL: duplicated worker!');
         }
         status[id] = {isBusy: true};
-        console.log('add worker', worker.id);
+        runLog.write('add worker', worker.id);
         workers[id] = worker;
     },
     getWorker:(id)=>{

package/workers/asMaster.js

@@ -77,8 +77,6 @@ const init = (config)=>{
     let randomNum = round((random() * pow(10,8)),8).toString(36);
     let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
 
-    console.log(`  masterTaskId===${masterTaskId}`);
-
     config.taskRootFolder = taskRootFolder;
     config.masterTaskId = masterTaskId;
 
@@ -93,7 +91,6 @@ const init = (config)=>{
     //to reload is for to reStart a collapsed task
     return new Promise((resolve)=>{
         TaskMgr.loadNewTasks().then(()=>{
-            console.log('******start multi-thread')
             resolve(config);
         });
     });

package/workers/helper.js

@@ -2,6 +2,7 @@ let fs = require('fs');
 let pathutil = require('path');
 
 let TaskMgr = require('./TaskMgr');
+let runLog = require('../utils/runLog');
 
 let myconfig;
 let listeners = {};//event -> handler, worker 进程内模块级广播监听 registry(每个 event 一个 handler)
@@ -14,7 +15,7 @@ const createNewTasks = (tasks)=>{
     if (!Array.isArray(tasks)) tasks = [tasks];
     TaskMgr.load(myconfig);
     tasks.forEach((task)=>{
-        console.log('Creating new task', task)
+        runLog.write('Creating new task', task)
         TaskMgr.createNewTask(task);
     });
 };
@@ -40,6 +41,10 @@ const setListener = (event, handler)=>{
     if (typeof handler !== 'function') throw new Error('helper.setListener: handler must be a function');
     listeners[event] = handler;
 };
+//任务日志: 一律写 <taskFolder>/.sys/run.log; progressBar 未开/非 TTY 回退时 echo 上屏(效果等同 console.log)
+const log = (...args)=>{
+    runLog.write(...args);
+};
 //内部方法: 供 subWorker 收到 master 中转的广播时调用, 按 event 分发
 const __dispatchBroadcast = (message)=>{
     let {event, payload, meta} = message;
@@ -48,7 +53,7 @@ const __dispatchBroadcast = (message)=>{
     try{
         handler(payload, meta);
     }catch(e){
-        console.error(`[Broadcast] handler for event "${event}" threw:`, e);
+        runLog.error(`[Broadcast] handler for event "${event}" threw:`, e);
     }
 };
 module.exports = {
@@ -57,5 +62,6 @@ module.exports = {
     emit,
     emitSys,
     setListener,
+    log,
     __dispatchBroadcast
 };

package/workers/index.js

@@ -43,11 +43,21 @@ const _mergeSnapshotConfig = (config)=>{
     return merged;
 };
 
+//run.log 路径在入口统一算好挂到 config(__ 前缀: 一次性派生值, 不进快照), 与 progressBar 无关;
+//master/worker 各进程都重跑入口、算出同一路径, Multithread 直接使用, 不做任何路径推导。
+const _computeSysLogFile = (config)=>{
+    let taskFolder = config.taskFolder || ((config.taskId && config.taskRootFolder) ? pathUtil.resolve(config.taskRootFolder, config.taskId) : null);
+    if(taskFolder){
+        config.__sysLogFile = pathUtil.resolve(taskFolder, '.sys', 'run.log');
+    };
+};
+
 const start = (config)=>{
     config = backwardv1.v1tov2(config);
 
     config.taskRootFolder = _toAbsPath(config.taskRootFolder);
     config.taskFolder = _toAbsPath(config.taskFolder);
+    _computeSysLogFile(config);
 
     Multithread.start(config);
 };
@@ -65,6 +75,7 @@ const resume = (config)=>{
     };
     config.taskRootFolder = pathUtil.dirname(config.taskFolder);
     config.taskId = pathUtil.basename(config.taskFolder);
+    _computeSysLogFile(config);
 
     config = _mergeSnapshotConfig(config);
     Multithread.start(config);
@@ -80,6 +91,7 @@ const retry_fails = (config)=>{
     config.taskId = pathUtil.basename(taskFolder);
     config.__resume = true;
     config.__retryFails = true;
+    _computeSysLogFile(config);
     config = _mergeSnapshotConfig(config);
     Multithread.start(config);
 };
@@ -93,6 +105,7 @@ const restart = (config)=>{
     config.taskRootFolder = pathUtil.dirname(taskFolder);
     config.taskId = pathUtil.basename(taskFolder);
     config.__restart = true;
+    _computeSysLogFile(config);
     config = _mergeSnapshotConfig(config);
     Multithread.start(config);
 };