multi-tasks 3.2.3 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -12
- package/package.json +36 -36
- package/utils/makedir.js +12 -18
- package/utils/progressCtl.js +287 -118
- package/utils/progressView.js +110 -35
- package/utils/randoms.js +5 -2
- package/utils/startupBanner.js +2 -1
- package/utils/workerStatusProbe.js +38 -0
- package/workers/Multithread.js +83 -44
- package/workers/TaskMgr.js +406 -293
- package/workers/WorkerMgr.js +18 -27
- package/workers/asMaster.js +101 -100
package/README.md
CHANGED
|
@@ -255,7 +255,7 @@ multiTasks.restart({ taskFolder: `/myworks/my_scan_tasks/my-task` }); //deletes
|
|
|
255
255
|
|
|
256
256
|
```
|
|
257
257
|
|
|
258
|
-
Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass (e.g. `processTask`) override the snapshot and are written back into it; the same self-contained-function note as in Resuming applies.
|
|
258
|
+
Unlike `retry_fails` (which re-runs only the failed tasks), `restart` re-runs **all the initial tasks** — tasks created dynamically via `helper.createNewTasks` are not restored and will not be re-run. Fields you pass (e.g. `processTask`) override the snapshot and are written back into it; the same self-contained-function note as in Resuming applies.
|
|
259
259
|
|
|
260
260
|
### Real-Time Monitoring
|
|
261
261
|
|
|
@@ -264,13 +264,11 @@ Monitor your running tasks with [multi-tasks-monitor](https://www.npmjs.com/pack
|
|
|
264
264
|
Quick start (requires multi-tasks >=3.2.0): once your multi-tasks run has started, the startup log prints the monitor command with the task folder path already filled in — just copy-paste it into another terminal:
|
|
265
265
|
|
|
266
266
|
```text
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
|
|
270
|
-
****************************************************************
|
|
267
|
+
monitor install cmd : npm install multi-tasks-monitor
|
|
268
|
+
monitor start cmd : npx multi-tasks-monitor --task-dir "D:\tasks\multitasks20260806..." --port 3777
|
|
271
269
|
```
|
|
272
270
|
|
|
273
|
-
For
|
|
271
|
+
For command-line options and more, see the [multi-tasks-monitor](https://www.npmjs.com/package/multi-tasks-monitor) page.
|
|
274
272
|
|
|
275
273
|
### API:
|
|
276
274
|
|
|
@@ -286,20 +284,22 @@ The API has three parts: the entry functions, the config options, and the `helpe
|
|
|
286
284
|
**Config options:**
|
|
287
285
|
|
|
288
286
|
- **`initialTasks`** — array of task objects, a function returning the array (or a Promise of it — called only once, in the master process; worker processes never call it; see [Performance: runs only once, in the master process](#performance-runs-only-once-in-the-master-process)), or a folder path string to resume from.
|
|
289
|
-
- **`processTask(task, helper)`** — required; return a
|
|
287
|
+
- **`processTask(task, helper)`** — required; return a result or a Promise with result. Results are stored as JSON in `results/succ/<subid>.json`; if a result cannot be saved because it is not JSON-serializable, the task still counts as succeeded, and a warning shows up in the command line and in monitor tools.
|
|
290
288
|
- **`taskRootFolder`** — the task run folder (named by `taskId`) is created under this directory.
|
|
291
289
|
- **`taskId`** — task folder name under `taskRootFolder`.
|
|
292
290
|
- **`numberOfWorkers`** — how many worker processes run in parallel; a number, or a percentage string of CPU cores like `"50%"` (default).
|
|
293
291
|
- **`taskTimeout`** — optional, milliseconds; an overdue task fails with a timeout error.
|
|
294
292
|
- **`maxTaskRetries`** — optional, max times a failed task is auto-retried.
|
|
295
293
|
- **`workerPriority`** — optional, OS scheduling priority for worker processes: `'low' | 'below_normal' | 'normal' | 'above_normal' | 'high'`, or a nice number (integer -20~19). Lower priority keeps the machine responsive while a long batch occupies all cores; invalid values throw before any task folder is created.
|
|
296
|
-
- **`progressBar`** — `true` by default: shows a live
|
|
294
|
+
- **`progressBar`** — `true` by default: shows a live progress bar in the terminal; set `false` to fall back to plain log output.
|
|
295
|
+
- **`worker_status_poll_ms`** — worker status poll interval in ms; default `2000`; `0` (or `false`) disables the worker status table; `true` means the default rate.
|
|
297
296
|
- **`autoCloseAfterCompletion`** — set `true` to kill workers and exit the process when no tasks are left; default is `false`, i.e. workers stay alive waiting for dynamically created tasks.
|
|
298
|
-
- **`shouldTerminate(info)`** — return `true` to terminate the whole process.
|
|
297
|
+
- **`shouldTerminate(info)`** — return `true` to terminate the whole process. If it throws, the error is logged and the task is dispatched anyway (treated as "don't terminate").
|
|
299
298
|
- **`setSysListener(event, payload, meta)`** — optional, handle custom system events (sent via `helper.emitSys`) on the master; `meta` has `fromWorkerId` and `fromWorkerPid`.
|
|
300
|
-
- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` lets you walk every persisted task result:
|
|
301
|
-
- **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing).
|
|
302
|
-
- **`helper.foreachErrorResult((count, task, err) => {})`** — iterate all failed results (`results/errors`); `err` is the persisted error object as-is (`{ex}` / `{subTask, exception}` / `{type: 'timeout', ...}` / `{type: 'worker_crash', ...}`).
|
|
299
|
+
- **`onFinish(report, helper)`** — called once after all workers exit (only when `autoCloseAfterCompletion` is `true`). `report` summarizes the run: `{ taskId, taskFolder, startTimestamp, endTimestamp, cost, total, succeeded, failed, failedTasks, succButSaveFailed, workers }` — `cost` is the run duration in ms (a resumed run measures only the current round); `failedTasks` lists the failed tasks' subids; `succButSaveFailed` counts tasks that ran fine but whose results could not be JSON-serialized (details in `<taskFolder>/.sys/user_messages/`); `workers` is `{ count, crashed }` — the configured worker count and how many workers crashed and were replaced. `helper` lets you walk every persisted task result:
|
|
300
|
+
- **`helper.foreachSuccResult((count, task, result) => {})`** — iterate all succeeded results (`results/succ`); `count` is a 0-based index, `task` is the original task object (`null` if its task file is missing or unparseable), `result` is the value returned by `processTask` (`undefined` when the task returned nothing; a `{type: 'result_serialize_error', message}` placeholder when the returned value could not be JSON-serialized).
|
|
301
|
+
- **`helper.foreachErrorResult((count, task, err) => {})`** — iterate all failed results (`results/errors`); `err` is the persisted error object as-is (`{ex}` / `{subTask, exception}` / `{type: 'timeout', ...}` / `{type: 'worker_crash', ...}`). A non-`Error` rejection value that is not JSON-serializable is persisted as `{ex: {type: 'unserializable_error_value', ...}}`.
|
|
302
|
+
- **`helper.foreachResult((count, task, result, succeeded) => {})`** — iterate all results regardless of outcome (succeeded first, then failed; `count` runs continuously across both); `succeeded` is `true` for results from `results/succ` and `false` for those from `results/errors` (where `result` is the persisted error object).
|
|
303
303
|
- Iteration follows filesystem order (not sorted). `onFinish` runs synchronously at the very end and the master process exits shortly after it returns — don't `await` inside it.
|
|
304
304
|
|
|
305
305
|
**Other options:**
|
|
@@ -315,6 +315,8 @@ The API has three parts: the entry functions, the config options, and the `helpe
|
|
|
315
315
|
|
|
316
316
|
### Changelog:
|
|
317
317
|
|
|
318
|
+
- 3.4.0 Show per-worker load (ELU/CPU/memory) under progressBar; stability fixes
|
|
319
|
+
- 3.3.0 Unserializable results still succeed; shouldTerminate throwing is safe; progressBar counters added
|
|
318
320
|
- 3.2.3 Support workerPriority: set OS scheduling priority for worker processes
|
|
319
321
|
- 3.2.2 initialTasks accepts a function (sync or async) that is called only once, in the master process
|
|
320
322
|
- 3.2.1 Fix retry bug: user-defined taskCount/index no longer overwritten on task retries
|
package/package.json
CHANGED
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "multi-tasks",
|
|
3
|
-
"version": "3.
|
|
4
|
-
"description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
|
|
5
|
-
"main": "index.js",
|
|
6
|
-
"files": [
|
|
7
|
-
"index.js",
|
|
8
|
-
"workers",
|
|
9
|
-
"utils/*.js"
|
|
10
|
-
],
|
|
11
|
-
"repository": {
|
|
12
|
-
"type": "git",
|
|
13
|
-
"url": "https://gitee.com/zhanglei923/multi-tasks.git"
|
|
14
|
-
},
|
|
15
|
-
"directories": {
|
|
16
|
-
"example": "examples"
|
|
17
|
-
},
|
|
18
|
-
"engines": {
|
|
19
|
-
"node": ">=14.14"
|
|
20
|
-
},
|
|
21
|
-
"scripts": {
|
|
22
|
-
"test": "jest",
|
|
23
|
-
"prepublishOnly": "npm test"
|
|
24
|
-
},
|
|
25
|
-
"keywords": [
|
|
26
|
-
"multitask",
|
|
27
|
-
"worker",
|
|
28
|
-
"multithread"
|
|
29
|
-
],
|
|
30
|
-
"author": "zhanglei923@gmail.com",
|
|
31
|
-
"license": "MIT",
|
|
32
|
-
"devDependencies": {
|
|
33
|
-
"cli-progress": "^3.12.0",
|
|
34
|
-
"jest": "^29.7.0"
|
|
35
|
-
}
|
|
36
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "multi-tasks",
|
|
3
|
+
"version": "3.4.0",
|
|
4
|
+
"description": "Multi-process task scheduling based on Node.js cluster, with crash resume and failed-task restart support",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"files": [
|
|
7
|
+
"index.js",
|
|
8
|
+
"workers",
|
|
9
|
+
"utils/*.js"
|
|
10
|
+
],
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "https://gitee.com/zhanglei923/multi-tasks.git"
|
|
14
|
+
},
|
|
15
|
+
"directories": {
|
|
16
|
+
"example": "examples"
|
|
17
|
+
},
|
|
18
|
+
"engines": {
|
|
19
|
+
"node": ">=14.14"
|
|
20
|
+
},
|
|
21
|
+
"scripts": {
|
|
22
|
+
"test": "jest",
|
|
23
|
+
"prepublishOnly": "npm test"
|
|
24
|
+
},
|
|
25
|
+
"keywords": [
|
|
26
|
+
"multitask",
|
|
27
|
+
"worker",
|
|
28
|
+
"multithread"
|
|
29
|
+
],
|
|
30
|
+
"author": "zhanglei923@gmail.com",
|
|
31
|
+
"license": "MIT",
|
|
32
|
+
"devDependencies": {
|
|
33
|
+
"cli-progress": "^3.12.0",
|
|
34
|
+
"jest": "^29.7.0"
|
|
35
|
+
}
|
|
36
|
+
}
|
package/utils/makedir.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
|
-
const path = require('path');
|
|
3
2
|
|
|
4
3
|
function sync(directoryPath) {
|
|
5
4
|
directoryPath = directoryPath.replace(/\\/g, '/');
|
|
@@ -12,24 +11,19 @@ function getDirectFiles(dir, callback) {
|
|
|
12
11
|
console.log('FATAL: directory is missing:', dir);
|
|
13
12
|
return process.exit(1);
|
|
14
13
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
let entries;
|
|
15
|
+
try{
|
|
16
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });//一次拿齐 isFile, 省掉逐文件 statSync(大任务量启动/resume 显著)
|
|
17
|
+
}catch(err){
|
|
18
|
+
console.error("Could not list the directory.", err);
|
|
19
|
+
return callback([]);
|
|
20
|
+
}
|
|
21
|
+
entries.forEach((entry)=>{
|
|
22
|
+
if(entry.isFile()){
|
|
23
|
+
childrenfiles.push(entry.name);
|
|
19
24
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
try{
|
|
23
|
-
let stats = fs.statSync(filePath);
|
|
24
|
-
if (stats.isFile()) {
|
|
25
|
-
childrenfiles.push(file);
|
|
26
|
-
}
|
|
27
|
-
}catch(e){
|
|
28
|
-
//
|
|
29
|
-
}
|
|
30
|
-
});
|
|
31
|
-
callback(childrenfiles);
|
|
32
|
-
});
|
|
25
|
+
});
|
|
26
|
+
callback(childrenfiles);
|
|
33
27
|
}
|
|
34
28
|
|
|
35
29
|
module.exports = {sync, getDirectFiles};
|
package/utils/progressCtl.js
CHANGED
|
@@ -1,118 +1,287 @@
|
|
|
1
|
-
//progressBar master 端控制器(零依赖): 持有渲染器实例与忙碌表, 封装 TTY 判定与节流渲染,
|
|
2
|
-
//Multithread 只调语义化方法(setup/workerStarted/taskAssigned/...), 不碰渲染细节。
|
|
3
|
-
//纯 master 端运行时状态, 不进快照不进 IPC; 统计数据(getStats/getRemaining)由调用方注入, 本模块不依赖 TaskMgr。
|
|
4
|
-
//console 劫持与 run.log 已与 progressBar 解耦(永久生效, 见 utils/runLog), 不再是本模块的职责。
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
let
|
|
12
|
-
let
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
let
|
|
16
|
-
let
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
};
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
return;
|
|
73
|
-
};
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
};
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
1
|
+
//progressBar master 端控制器(零依赖): 持有渲染器实例与忙碌表, 封装 TTY 判定与节流渲染,
|
|
2
|
+
//Multithread 只调语义化方法(setup/workerStarted/taskAssigned/...), 不碰渲染细节。
|
|
3
|
+
//纯 master 端运行时状态, 不进快照不进 IPC; 统计数据(getStats/getRemaining)由调用方注入, 本模块不依赖 TaskMgr。
|
|
4
|
+
//console 劫持与 run.log 已与 progressBar 解耦(永久生效, 见 utils/runLog), 不再是本模块的职责。
|
|
5
|
+
//N 行帧进度区(progressView): 首行 bar, 末行计数器行, 状态显示开启(worker_status_poll_ms 缺省 2000, <=0 关)时中间整组插入
|
|
6
|
+
//ASCII 状态表(每组 6 行: id 表头/分隔横线/elu/cpu/rss/heap——去顶/底框、去列间隔符纯空格分栏(列间 1 空格), 列宽取各列内容最大宽,
|
|
7
|
+
//每组列数按终端宽度自适应, 无数据单列 - 占位); worker 增删不体现在 bar/计数器行上, 这里只维护 busyWorkers 计数 running 与
|
|
8
|
+
//workerStatuses 状态表(__workerStatus 消息更新, worker 退出删条目)。
|
|
9
|
+
const progressView = require('./progressView');
|
|
10
|
+
|
|
11
|
+
let RENDER_INTERVAL = 100;//渲染节流: 高频小任务场景避免每次事件都 readdirSync(getFinalStats) 造成 O(N²) 盘扫
|
|
12
|
+
let STATS_CACHE_MS = 1000;//盘上统计缓存 TTL: doRender 每帧 getStats(3 次 readdirSync), 高频小任务下 O(N) 盘扫放大;
|
|
13
|
+
//计数器行允许秒级滞后(完成数按盘上结果目录统计的口径不变), finish 收尾绕过缓存始终取最新
|
|
14
|
+
|
|
15
|
+
let view = null;//progressView 渲染器实例, 未开启为 null
|
|
16
|
+
let progressOn = false;//progressBar 且 stdout 为 TTY 时为真(非 TTY 回退普通日志)
|
|
17
|
+
let busyWorkers = {};//worker.id -> subid, master 端忙碌表
|
|
18
|
+
let workerStatuses = {};//worker.id -> {elu, cpu, rss, heapUsed}, worker 周期自报(__workerStatus 消息)的状态表
|
|
19
|
+
let statusEnabled = false;//worker_status_poll_ms 缺省(默认 2000)或 >0 时为真; <=0 整个特性关闭(帧不带状态行)
|
|
20
|
+
let getStats = null;//() => TaskMgr.getFinalStats(), setup 注入
|
|
21
|
+
let getRemaining = null;//() => new 目录任务数, setup 注入
|
|
22
|
+
let lastRenderAt = 0;
|
|
23
|
+
let renderTimer = null;
|
|
24
|
+
let resizeStream = null;//resize 监听挂载的 stream, reset 时摘除防累积
|
|
25
|
+
let resizeListener = null;
|
|
26
|
+
let outStream = null;//渲染目标 stream, 供列宽变化检测(resize 事件缺失的终端靠它轮询)
|
|
27
|
+
let lastCols = 0;//上一帧使用的列宽, 0 = 尚未渲染
|
|
28
|
+
let resizeSettleTimer = null;
|
|
29
|
+
let lastExtraLineCount = 0;//上一帧状态区行数: 组数变化(帧行数变)时先 reanchor 抹除旧帧再重写, 原地重绘必错位
|
|
30
|
+
let statsCache = null;//盘上统计缓存(TTL 见 STATS_CACHE_MS), reset 时清空
|
|
31
|
+
let statsCachedAt = 0;
|
|
32
|
+
|
|
33
|
+
const RESIZE_SETTLE = 200;//列宽变化防抖: 拖动窗口会连续变化, 等稳定再重锚定重绘一次(防连续重锚定刷屏)
|
|
34
|
+
const MAX_COLS_PER_GROUP = 7;//每组列数上限(用户拍板): 列再多表也读不过来, 超出的分多组堆叠
|
|
35
|
+
|
|
36
|
+
const currentCols = () => {
|
|
37
|
+
return (outStream && outStream.columns) || 80;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const reset = () => {//master start 入口/同进程再跑一轮时调用: 清掉上一轮全部残留(含未发的节流帧)
|
|
41
|
+
if (renderTimer) {
|
|
42
|
+
clearTimeout(renderTimer);
|
|
43
|
+
renderTimer = null;
|
|
44
|
+
};
|
|
45
|
+
if (resizeSettleTimer) {
|
|
46
|
+
clearTimeout(resizeSettleTimer);
|
|
47
|
+
resizeSettleTimer = null;
|
|
48
|
+
};
|
|
49
|
+
if (resizeStream && resizeListener) {
|
|
50
|
+
resizeStream.removeListener('resize', resizeListener);
|
|
51
|
+
};
|
|
52
|
+
resizeStream = null;
|
|
53
|
+
resizeListener = null;
|
|
54
|
+
outStream = null;
|
|
55
|
+
lastCols = 0;
|
|
56
|
+
lastExtraLineCount = 0;
|
|
57
|
+
view = null;
|
|
58
|
+
progressOn = false;
|
|
59
|
+
busyWorkers = {};
|
|
60
|
+
workerStatuses = {};
|
|
61
|
+
statusEnabled = false;
|
|
62
|
+
lastRenderAt = 0;
|
|
63
|
+
statsCache = null;
|
|
64
|
+
statsCachedAt = 0;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
//开启判定 + 初始化: 返回是否真正开启(显式 progressBar:false 关闭, 缺省默认开启;
|
|
68
|
+
//非 TTY 一律回退普通日志, 仅显式 true 时打回退警告, 默认开启静默回退)
|
|
69
|
+
const setup = ({ config, stream, getStats: statsFn, getRemaining: remainingFn }) => {
|
|
70
|
+
reset();
|
|
71
|
+
if (config.progressBar === false) {
|
|
72
|
+
return false;
|
|
73
|
+
};
|
|
74
|
+
if (!stream.isTTY) {//ANSI 控制码写进重定向输出是垃圾: 回退为现有控制台日志
|
|
75
|
+
if (config.progressBar === true) {
|
|
76
|
+
console.warn('[Master]: progressBar needs a TTY stdout, fallback to console logs');
|
|
77
|
+
};
|
|
78
|
+
return false;
|
|
79
|
+
};
|
|
80
|
+
progressOn = true;
|
|
81
|
+
getStats = statsFn;
|
|
82
|
+
getRemaining = remainingFn;
|
|
83
|
+
//状态行开关与 worker 侧探针同口径: 缺省即开(默认间隔 2000ms), 显式 <=0 或 false 关闭, true 等价缺省;
|
|
84
|
+
//开启期间每帧都带状态表(无数据用占位)
|
|
85
|
+
let statusInterval = config.worker_status_poll_ms;
|
|
86
|
+
statusEnabled = (typeof statusInterval === 'undefined' || statusInterval === true) ? true : statusInterval > 0;
|
|
87
|
+
view = progressView.create({ stream });//N 行帧进度区: bar 行固定 36 列(见 progressView)
|
|
88
|
+
outStream = stream;
|
|
89
|
+
if (typeof stream.on === 'function') {//resize 事件仅作加速路径: 真正的检测在 render 时比对列宽(事件缺失的终端也自愈)
|
|
90
|
+
resizeStream = stream;
|
|
91
|
+
resizeListener = () => {
|
|
92
|
+
if (view) {
|
|
93
|
+
scheduleResizeSettle();
|
|
94
|
+
};
|
|
95
|
+
};
|
|
96
|
+
stream.on('resize', resizeListener);
|
|
97
|
+
};
|
|
98
|
+
return true;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const isOn = () => {
|
|
102
|
+
return progressOn;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const fmtMem = (bytes) => {//内存自适应单位(用户拍板): <1G 整数 MB(256M), >=1G 一位小数 GB(1.5G)
|
|
106
|
+
const mb = bytes / 1048576;
|
|
107
|
+
return mb >= 1024 ? `${(mb / 1024).toFixed(1)}G` : `${Math.round(mb)}M`;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
const buildStatusLines = () => {//状态区是 ASCII 表, 每组 6 行(id 表头/分隔横线/elu/cpu/rss/heap):
|
|
111
|
+
//压缩拍板——去顶/底框、去列间隔符(纯空格分栏), 表头与数据间横线保留(连续虚线), 行尾空格 trim;
|
|
112
|
+
//每列宽取该列 id/elu/cpu/rss/heap 五格内容的最大宽, 最大数字(100%/1.5G)与长 worker id(#10/#100)都不会错位; 按数字 id 排序;
|
|
113
|
+
//每组列数按终端宽度自适应且封顶 MAX_COLS_PER_GROUP(用户拍板): 行预算 (cols-1) 减 label 段后每列占 列宽+1 字符(列间 1 空格, 用户拍板),
|
|
114
|
+
//能放几列放几列(保底 1 列)——宽终端少分组、帧行数大减(缓解帧行数超屏高), 窄终端少放列、不再把表截烂;
|
|
115
|
+
//超出分多组整组堆叠(组数变 = 帧行数变, doRender 检测后走 reanchor 抹除重绘, 不原地错位重绘);
|
|
116
|
+
//无数据时单列 '-' 占位表(行数恒定: 开即每帧至少一组 6 行)
|
|
117
|
+
let ids = Object.keys(workerStatuses).sort((a, b) => a - b);
|
|
118
|
+
let cells = ids.map((id) => ({
|
|
119
|
+
id: `#${id}`,
|
|
120
|
+
elu: `${Math.round(workerStatuses[id].elu * 100)}%`,
|
|
121
|
+
cpu: `${Math.round(workerStatuses[id].cpu)}%`,
|
|
122
|
+
rss: fmtMem(workerStatuses[id].rss),
|
|
123
|
+
heap: fmtMem(workerStatuses[id].heapUsed),
|
|
124
|
+
}));
|
|
125
|
+
if (cells.length === 0) {
|
|
126
|
+
cells = [{ id: '-', elu: '-', cpu: '-', rss: '-', heap: '-' }];
|
|
127
|
+
};
|
|
128
|
+
let labels = ['id', 'elu', 'cpu', 'rss', 'heap'];
|
|
129
|
+
let labelWidth = Math.max(...labels.map((s) => s.length));
|
|
130
|
+
let colWidths = cells.map((cell) => Math.max(cell.id.length, cell.elu.length, cell.cpu.length, cell.rss.length, cell.heap.length));
|
|
131
|
+
//按全局最大列宽定每组列数: 任一组行宽 = labelWidth+1 + Σ(w+1) <= labelWidth+1 + colsPerGroup*(maxCellWidth+1) <= cols-1, 恒不超宽
|
|
132
|
+
let maxCellWidth = Math.max(...colWidths);
|
|
133
|
+
//终端宽度自适应 + 封顶 MAX_COLS_PER_GROUP(用户拍板): 宽终端少分组, 窄终端少放列不截烂表, 保底 1 列
|
|
134
|
+
let colsPerGroup = Math.min(MAX_COLS_PER_GROUP, Math.max(1, Math.floor((currentCols() - 1 - (labelWidth + 1)) / (maxCellWidth + 1))));
|
|
135
|
+
let lines = [];
|
|
136
|
+
for (let g = 0; g < cells.length; g += colsPerGroup) {
|
|
137
|
+
let group = cells.slice(g, g + colsPerGroup);
|
|
138
|
+
let groupWidths = colWidths.slice(g, g + colsPerGroup);
|
|
139
|
+
let cellText = (content, w) => ` ${content.padEnd(w)}`;//列间 1 空格(前导), padEnd 保列对齐, 行尾由 join 后 trimEnd 收掉
|
|
140
|
+
let divider = [labelWidth, ...groupWidths].map((w) => '-'.repeat(w + 1)).join('');
|
|
141
|
+
let line = (label, key) => {
|
|
142
|
+
return (cellText(label, labelWidth) + group.map((cell, c) => cellText(cell[key], groupWidths[c])).join('')).trimEnd();
|
|
143
|
+
};
|
|
144
|
+
lines.push(line('id', 'id'), divider, line('elu', 'elu'), line('cpu', 'cpu'), line('rss', 'rss'), line('heap', 'heap'));
|
|
145
|
+
};
|
|
146
|
+
return lines;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const readStats = () => {//TTL 内命中缓存: 帧渲染不重复扫盘; 过期重新取(finish 不走这里, 直接 getStats)
|
|
150
|
+
if (statsCache && (Date.now() - statsCachedAt) < STATS_CACHE_MS) {
|
|
151
|
+
return statsCache;
|
|
152
|
+
};
|
|
153
|
+
statsCache = getStats();
|
|
154
|
+
statsCachedAt = Date.now();
|
|
155
|
+
return statsCache;
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
const doRender = () => {
|
|
159
|
+
//完成数按盘上结果目录统计: 重试/超时/崩溃各种路径天然正确, 无需在协议里额外计数;
|
|
160
|
+
//failed/succ-but-save-failed 同源实时挂在行尾, 全程可见
|
|
161
|
+
let stats = readStats();
|
|
162
|
+
let done = stats.succeeded + stats.failed;
|
|
163
|
+
let running = Object.keys(busyWorkers).length;
|
|
164
|
+
lastRenderAt = Date.now();
|
|
165
|
+
lastCols = currentCols();
|
|
166
|
+
let data = { done, succ: stats.succeeded, running, remaining: getRemaining(), failed: stats.failed, succButSaveFailed: stats.succButSaveFailed || 0 };
|
|
167
|
+
if (statusEnabled) {
|
|
168
|
+
data.extraLines = buildStatusLines();
|
|
169
|
+
if (lastExtraLineCount > 0 && data.extraLines.length !== lastExtraLineCount) {
|
|
170
|
+
view.reanchor();//状态表组数变化(第 7/13... 个 worker 上报)致帧行数变: ESC[nA 按新行数算必错位, 抹除旧帧从锚点重写(同 resize 路径)
|
|
171
|
+
};
|
|
172
|
+
lastExtraLineCount = data.extraLines.length;
|
|
173
|
+
};
|
|
174
|
+
view.render(data);
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const scheduleResizeSettle = () => {//列宽变化(事件或 render 时检测): 防抖等宽度稳定, 再重锚定 + 重绘一次
|
|
178
|
+
if (resizeSettleTimer) {
|
|
179
|
+
clearTimeout(resizeSettleTimer);
|
|
180
|
+
};
|
|
181
|
+
resizeSettleTimer = setTimeout(() => {
|
|
182
|
+
resizeSettleTimer = null;
|
|
183
|
+
if (view) {
|
|
184
|
+
view.reanchor();
|
|
185
|
+
doRender();
|
|
186
|
+
};
|
|
187
|
+
}, RESIZE_SETTLE);
|
|
188
|
+
resizeSettleTimer.unref();//不阻碍 master 退出
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
const colsChanged = () => {//列宽变了 = 旧帧已被终端重排, 占用行数不可信: 任何光标序列重绘必错位
|
|
192
|
+
return lastCols > 0 && currentCols() !== lastCols;
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
const render = () => {
|
|
196
|
+
if (!view) {
|
|
197
|
+
return;
|
|
198
|
+
};
|
|
199
|
+
if (colsChanged()) {//原地重绘必错位, 走防抖重锚定
|
|
200
|
+
scheduleResizeSettle();
|
|
201
|
+
return;
|
|
202
|
+
};
|
|
203
|
+
if (renderTimer) {
|
|
204
|
+
return;//已挂着一个尾随渲染, 合并掉中间帧
|
|
205
|
+
};
|
|
206
|
+
let wait = RENDER_INTERVAL - (Date.now() - lastRenderAt);
|
|
207
|
+
if (wait <= 0) {
|
|
208
|
+
doRender();
|
|
209
|
+
return;
|
|
210
|
+
};
|
|
211
|
+
renderTimer = setTimeout(() => {
|
|
212
|
+
renderTimer = null;
|
|
213
|
+
if (!view) {
|
|
214
|
+
return;
|
|
215
|
+
};
|
|
216
|
+
if (colsChanged()) {//拖动期间尾随帧触发: 用旧列宽光标序列重绘是满屏碎片根因, 转防抖重锚定
|
|
217
|
+
scheduleResizeSettle();
|
|
218
|
+
return;
|
|
219
|
+
};
|
|
220
|
+
doRender();
|
|
221
|
+
}, wait);
|
|
222
|
+
renderTimer.unref();//不阻碍 master 退出(最后一帧可能因进程退出丢失, 可接受)
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
const workerStarted = (id, pid) => {//两行进度区不展示 worker 行: 仅保留语义入口, 无渲染动作
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
const taskAssigned = (id, subid) => {
|
|
229
|
+
busyWorkers[id] = subid;
|
|
230
|
+
render();
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
const taskCompleted = (id) => {
|
|
234
|
+
if (busyWorkers[id]) {
|
|
235
|
+
delete busyWorkers[id];
|
|
236
|
+
};
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
const workerExited = (id) => {
|
|
240
|
+
delete busyWorkers[id];
|
|
241
|
+
delete workerStatuses[id];
|
|
242
|
+
render();
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
//worker 周期自报的状态(__workerStatus 消息): 畸形 payload 忽略不进表; 特性关闭时同样忽略(帧本就不带状态行)
|
|
246
|
+
const workerStatus = (id, payload) => {
|
|
247
|
+
if (!statusEnabled) {
|
|
248
|
+
return;
|
|
249
|
+
};
|
|
250
|
+
if (!payload || typeof payload !== 'object') {
|
|
251
|
+
return;
|
|
252
|
+
};
|
|
253
|
+
let { elu, cpu, rss, heapUsed } = payload;
|
|
254
|
+
let nums = [elu, cpu, rss, heapUsed];
|
|
255
|
+
if (nums.some((n) => typeof n !== 'number' || !isFinite(n))) {
|
|
256
|
+
return;
|
|
257
|
+
};
|
|
258
|
+
workerStatuses[`${id}`] = { elu, cpu, rss, heapUsed };
|
|
259
|
+
render();
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
const finish = () => {//收尾: 换行定格进度条行 + 输出计数器行(master 退出前调用); 未开启为 no-op
|
|
263
|
+
if (resizeSettleTimer) {//收尾后不再重锚定重绘: 防抖帧会落在计数器行之后
|
|
264
|
+
clearTimeout(resizeSettleTimer);
|
|
265
|
+
resizeSettleTimer = null;
|
|
266
|
+
};
|
|
267
|
+
if (view) {
|
|
268
|
+
view.end();
|
|
269
|
+
if (getStats) {
|
|
270
|
+
let stats = getStats();
|
|
271
|
+
view.writeLine(`[Tasks] succeeded=${stats.succeeded}, failed=${stats.failed}, succ-but-save-failed=${stats.succButSaveFailed || 0}`);
|
|
272
|
+
};
|
|
273
|
+
};
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
module.exports = {
|
|
277
|
+
setup,
|
|
278
|
+
reset,
|
|
279
|
+
isOn,
|
|
280
|
+
render,
|
|
281
|
+
workerStarted,
|
|
282
|
+
taskAssigned,
|
|
283
|
+
taskCompleted,
|
|
284
|
+
workerExited,
|
|
285
|
+
workerStatus,
|
|
286
|
+
finish,
|
|
287
|
+
};
|