@maiyunnet/kebab 9.15.6 → 9.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/doc/kebab-rag.md +161 -78
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/lib/core.d.ts +22 -3
- package/lib/core.js +23 -4
- package/lib/text.d.ts +6 -0
- package/lib/text.js +24 -1
- package/lib/undici/request.d.ts +6 -1
- package/lib/undici/request.js +8 -0
- package/lib/undici/response.d.ts +1 -1
- package/lib/undici/response.js +2 -1
- package/lib/undici.d.ts +7 -3
- package/lib/undici.js +48 -8
- package/package.json +1 -1
- package/sys/child.js +15 -12
- package/sys/ctr.js +3 -3
- package/sys/monitor/watchdog.js +9 -5
- package/sys/monitor.d.ts +8 -1
- package/sys/monitor.js +198 -81
- package/sys/route.d.ts +1 -1
- package/sys/route.js +14 -14
- package/www/example/ctr/test.d.ts +2 -0
- package/www/example/ctr/test.js +44 -3
package/sys/monitor.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Project: Kebab, User: JianSuoQiYue
|
|
3
3
|
* Date: 2026-02-07
|
|
4
|
-
* Last: 2026-
|
|
4
|
+
* Last: 2026-08-22
|
|
5
5
|
* --- 性能监控库,用于检测 CPU/内存骤升并记录可疑请求、堆栈、CPU Profile、堆快照 ---
|
|
6
6
|
* --- 包含 Worker 看门狗线程,用于在事件循环完全阻塞时实时检测并记录 ---
|
|
7
7
|
*/
|
|
@@ -16,12 +16,14 @@ import * as lText from '#kebab/lib/text.js';
|
|
|
16
16
|
import * as lTime from '#kebab/lib/time.js';
|
|
17
17
|
import * as lFs from '#kebab/lib/fs.js';
|
|
18
18
|
// --- 阈值配置 ---
|
|
19
|
-
/** --- CPU 使用率阈值,
|
|
19
|
+
/** --- CPU 使用率阈值,100 代表占满一个逻辑核心 --- */
|
|
20
20
|
let cpuThreshold = 80;
|
|
21
21
|
/** --- 内存使用率阈值,单位 MB --- */
|
|
22
22
|
let memThreshold = 0;
|
|
23
23
|
/** --- 事件循环延迟阈值,单位 ms --- */
|
|
24
24
|
let eloopThreshold = 500;
|
|
25
|
+
/** --- 是否在内存超阈值时自动采集堆快照 --- */
|
|
26
|
+
let heapSnapshotEnabled = false;
|
|
25
27
|
/** --- 监控间隔,单位 ms --- */
|
|
26
28
|
const INTERVAL = 5_000;
|
|
27
29
|
/** --- 连续超阈值次数达到此值才记录日志,防止瞬间波动 --- */
|
|
@@ -30,15 +32,23 @@ const SPIKE_COUNT = 2;
|
|
|
30
32
|
const PROFILE_DURATION = 5_000;
|
|
31
33
|
/** --- 两次诊断采集的最小间隔,防止频繁写磁盘,单位 ms --- */
|
|
32
34
|
const DIAGNOSTIC_COOLDOWN = 60_000;
|
|
35
|
+
/** --- 单次快照或日志最多输出的活跃请求详情数 --- */
|
|
36
|
+
const MAX_ACTIVE_REQUEST_DETAILS = 100;
|
|
33
37
|
// --- 内部状态 ---
|
|
34
38
|
/** --- 定时器 --- */
|
|
35
39
|
let timer = null;
|
|
36
|
-
/** ---
|
|
37
|
-
let
|
|
38
|
-
/** ---
|
|
39
|
-
let
|
|
40
|
-
/** ---
|
|
41
|
-
let
|
|
40
|
+
/** --- 周期检查的上次 CPU 累计用量 --- */
|
|
41
|
+
let lastCheckCpuUsage = null;
|
|
42
|
+
/** --- 周期检查的上次 CPU 采样时间戳 --- */
|
|
43
|
+
let lastCheckCpuTime = 0;
|
|
44
|
+
/** --- 周期检查的上次系统各核 CPU 时间快照 --- */
|
|
45
|
+
let lastCheckOsCpus = null;
|
|
46
|
+
/** --- 对外快照的上次 CPU 累计用量 --- */
|
|
47
|
+
let lastSnapshotCpuUsage = null;
|
|
48
|
+
/** --- 对外快照的上次 CPU 采样时间戳 --- */
|
|
49
|
+
let lastSnapshotCpuTime = 0;
|
|
50
|
+
/** --- 对外快照的上次系统各核 CPU 时间快照 --- */
|
|
51
|
+
let lastSnapshotOsCpus = null;
|
|
42
52
|
/** --- 事件循环延迟直方图 --- */
|
|
43
53
|
let eloopHistogram = null;
|
|
44
54
|
/** --- 连续超阈值计数 --- */
|
|
@@ -47,6 +57,8 @@ let spikeCounter = 0;
|
|
|
47
57
|
let requestCounter = 0;
|
|
48
58
|
/** --- 上次诊断采集时间 --- */
|
|
49
59
|
let lastDiagnosticTime = 0;
|
|
60
|
+
/** --- 是否正在采集诊断数据 --- */
|
|
61
|
+
let diagnosing = false;
|
|
50
62
|
/** --- 是否正在进行 CPU Profile 采集 --- */
|
|
51
63
|
let profiling = false;
|
|
52
64
|
/**
|
|
@@ -58,7 +70,7 @@ function isDebugMode() {
|
|
|
58
70
|
// --- 看门狗相关 ---
|
|
59
71
|
/** --- 看门狗 Worker 实例 --- */
|
|
60
72
|
let watchdog = null;
|
|
61
|
-
/** --- 心跳共享内存(
|
|
73
|
+
/** --- 心跳共享内存(Uint32: 秒级时间戳) --- */
|
|
62
74
|
let heartbeatBuffer = null;
|
|
63
75
|
/** --- 心跳共享内存视图 --- */
|
|
64
76
|
let heartbeatView = null;
|
|
@@ -68,6 +80,10 @@ const WATCHDOG_THRESHOLD = 15;
|
|
|
68
80
|
const WATCHDOG_INTERVAL = 5_000;
|
|
69
81
|
/** --- 看门狗告警冷却时间,持续阻塞时不会每次都写日志,单位秒 --- */
|
|
70
82
|
const WATCHDOG_COOLDOWN = 30;
|
|
83
|
+
/** --- 看门狗异常退出后的重启延迟,单位 ms --- */
|
|
84
|
+
const WATCHDOG_RESTART_DELAY = 5_000;
|
|
85
|
+
/** --- 看门狗重启定时器 --- */
|
|
86
|
+
let watchdogRestartTimer = null;
|
|
71
87
|
/** --- 活跃请求池 --- */
|
|
72
88
|
const activeRequests = new Map();
|
|
73
89
|
/**
|
|
@@ -78,21 +94,45 @@ export function start(opt) {
|
|
|
78
94
|
if (timer) {
|
|
79
95
|
return;
|
|
80
96
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
97
|
+
const nextCpuThreshold = opt?.cpu ?? 80;
|
|
98
|
+
const nextMemThreshold = opt?.mem ?? 0;
|
|
99
|
+
const nextEloopThreshold = opt?.eloop ?? 500;
|
|
100
|
+
if (!Number.isFinite(nextCpuThreshold) || nextCpuThreshold <= 0) {
|
|
101
|
+
throw new RangeError('Monitor CPU threshold must be greater than 0.');
|
|
102
|
+
}
|
|
103
|
+
if (!Number.isFinite(nextMemThreshold) || nextMemThreshold < 0) {
|
|
104
|
+
throw new RangeError('Monitor memory threshold cannot be negative.');
|
|
105
|
+
}
|
|
106
|
+
if (!Number.isFinite(nextEloopThreshold) || nextEloopThreshold <= 0) {
|
|
107
|
+
throw new RangeError('Monitor event loop threshold must be greater than 0.');
|
|
108
|
+
}
|
|
109
|
+
cpuThreshold = nextCpuThreshold;
|
|
110
|
+
memThreshold = nextMemThreshold;
|
|
111
|
+
eloopThreshold = nextEloopThreshold;
|
|
112
|
+
heapSnapshotEnabled = opt?.heapSnapshot ?? false;
|
|
84
113
|
if (memThreshold === 0) {
|
|
85
|
-
// ---
|
|
86
|
-
|
|
114
|
+
// --- 同时考虑容器/系统约束和 V8 堆上限,避免默认阈值高于进程实际可用范围 ---
|
|
115
|
+
const constrainedMemory = process.constrainedMemory();
|
|
116
|
+
const systemLimit = constrainedMemory > 0
|
|
117
|
+
? Math.min(constrainedMemory, os.totalmem())
|
|
118
|
+
: os.totalmem();
|
|
119
|
+
const heapLimit = v8.getHeapStatistics().heap_size_limit;
|
|
120
|
+
memThreshold = Math.floor(Math.min(systemLimit * 0.8, heapLimit * 0.9) / 1024 / 1024);
|
|
87
121
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
122
|
+
const cpuUsage = process.cpuUsage();
|
|
123
|
+
const cpuTime = Date.now();
|
|
124
|
+
const osCpus = os.cpus();
|
|
125
|
+
lastCheckCpuUsage = cpuUsage;
|
|
126
|
+
lastCheckCpuTime = cpuTime;
|
|
127
|
+
lastCheckOsCpus = osCpus;
|
|
128
|
+
lastSnapshotCpuUsage = cpuUsage;
|
|
129
|
+
lastSnapshotCpuTime = cpuTime;
|
|
130
|
+
lastSnapshotOsCpus = osCpus;
|
|
91
131
|
spikeCounter = 0;
|
|
92
132
|
lastDiagnosticTime = 0;
|
|
93
133
|
// --- 初始化心跳共享内存(索引 0: 秒级时间戳, 索引 1: 调试模式标志) ---
|
|
94
134
|
heartbeatBuffer = new SharedArrayBuffer(8);
|
|
95
|
-
heartbeatView = new
|
|
135
|
+
heartbeatView = new Uint32Array(heartbeatBuffer);
|
|
96
136
|
Atomics.store(heartbeatView, 0, Math.floor(Date.now() / 1000));
|
|
97
137
|
Atomics.store(heartbeatView, 1, isDebugMode() ? 1 : 0);
|
|
98
138
|
// --- 启用事件循环延迟直方图 ---
|
|
@@ -121,7 +161,18 @@ export function stop() {
|
|
|
121
161
|
});
|
|
122
162
|
watchdog = null;
|
|
123
163
|
}
|
|
164
|
+
if (watchdogRestartTimer) {
|
|
165
|
+
clearTimeout(watchdogRestartTimer);
|
|
166
|
+
watchdogRestartTimer = null;
|
|
167
|
+
}
|
|
124
168
|
activeRequests.clear();
|
|
169
|
+
lastCheckCpuUsage = null;
|
|
170
|
+
lastCheckCpuTime = 0;
|
|
171
|
+
lastCheckOsCpus = null;
|
|
172
|
+
lastSnapshotCpuUsage = null;
|
|
173
|
+
lastSnapshotCpuTime = 0;
|
|
174
|
+
lastSnapshotOsCpus = null;
|
|
175
|
+
spikeCounter = 0;
|
|
125
176
|
heartbeatBuffer = null;
|
|
126
177
|
heartbeatView = null;
|
|
127
178
|
}
|
|
@@ -160,13 +211,28 @@ function startWatchdog() {
|
|
|
160
211
|
if (watchdog === worker) {
|
|
161
212
|
watchdog = null;
|
|
162
213
|
}
|
|
214
|
+
scheduleWatchdogRestart();
|
|
163
215
|
});
|
|
164
216
|
lCore.debug(`[MONITOR] [THREAD] [${process.pid}] Watchdog started`);
|
|
165
217
|
}
|
|
166
218
|
catch (e) {
|
|
167
219
|
lCore.debug('[MONITOR] Failed to start watchdog', e);
|
|
220
|
+
scheduleWatchdogRestart();
|
|
168
221
|
}
|
|
169
222
|
}
|
|
223
|
+
/**
|
|
224
|
+
* --- 看门狗异常退出后延迟重启,避免失去阻塞检测能力 ---
|
|
225
|
+
*/
|
|
226
|
+
function scheduleWatchdogRestart() {
|
|
227
|
+
if (!timer || watchdog || watchdogRestartTimer) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
watchdogRestartTimer = setTimeout(() => {
|
|
231
|
+
watchdogRestartTimer = null;
|
|
232
|
+
startWatchdog();
|
|
233
|
+
}, WATCHDOG_RESTART_DELAY);
|
|
234
|
+
watchdogRestartTimer.unref();
|
|
235
|
+
}
|
|
170
236
|
/**
|
|
171
237
|
* --- 注册一个活跃请求,返回追踪 ID ---
|
|
172
238
|
* @param url 请求 URL
|
|
@@ -175,12 +241,14 @@ function startWatchdog() {
|
|
|
175
241
|
export function track(url, method) {
|
|
176
242
|
const now = Date.now();
|
|
177
243
|
const id = `${++requestCounter}-${now}`;
|
|
244
|
+
const queryIndex = url.search(/[?#]/u);
|
|
178
245
|
activeRequests.set(id, {
|
|
179
|
-
|
|
246
|
+
// --- 查询参数可能含凭据或个人信息,诊断中只保留请求路径 ---
|
|
247
|
+
'url': queryIndex === -1 ? url : url.slice(0, queryIndex),
|
|
180
248
|
'method': method,
|
|
181
249
|
'start': now,
|
|
182
250
|
'startCpu': process.cpuUsage(),
|
|
183
|
-
'startMem': process.memoryUsage()
|
|
251
|
+
'startMem': process.memoryUsage.rss(),
|
|
184
252
|
});
|
|
185
253
|
return id;
|
|
186
254
|
}
|
|
@@ -201,15 +269,21 @@ export function getSnapshot() {
|
|
|
201
269
|
const now = Date.now();
|
|
202
270
|
// --- 计算进程 CPU 使用率(单核基准) ---
|
|
203
271
|
let cpuProcess = 0;
|
|
204
|
-
if (
|
|
272
|
+
if (lastSnapshotCpuUsage && lastSnapshotCpuTime) {
|
|
205
273
|
/** --- 经过的时间(微秒) --- */
|
|
206
|
-
const elapsed = (now -
|
|
207
|
-
const userDiff = cpuUsage.user -
|
|
208
|
-
const sysDiff = cpuUsage.system -
|
|
209
|
-
|
|
274
|
+
const elapsed = (now - lastSnapshotCpuTime) * 1_000;
|
|
275
|
+
const userDiff = cpuUsage.user - lastSnapshotCpuUsage.user;
|
|
276
|
+
const sysDiff = cpuUsage.system - lastSnapshotCpuUsage.system;
|
|
277
|
+
if (elapsed > 0) {
|
|
278
|
+
cpuProcess = ((userDiff + sysDiff) / elapsed) * 100;
|
|
279
|
+
}
|
|
210
280
|
}
|
|
281
|
+
lastSnapshotCpuUsage = cpuUsage;
|
|
282
|
+
lastSnapshotCpuTime = now;
|
|
211
283
|
// --- 计算系统总 CPU 使用率 ---
|
|
212
|
-
const
|
|
284
|
+
const osCpus = os.cpus();
|
|
285
|
+
const cpuOs = getOsCpuPercent(lastSnapshotOsCpus, osCpus);
|
|
286
|
+
lastSnapshotOsCpus = osCpus;
|
|
213
287
|
// --- 计算事件循环延迟(P99,纳秒转毫秒) ---
|
|
214
288
|
let eloopLag = 0;
|
|
215
289
|
if (eloopHistogram) {
|
|
@@ -218,6 +292,9 @@ export function getSnapshot() {
|
|
|
218
292
|
// --- 收集活跃请求 ---
|
|
219
293
|
const requests = [];
|
|
220
294
|
for (const [, req] of activeRequests) {
|
|
295
|
+
if (requests.length >= MAX_ACTIVE_REQUEST_DETAILS) {
|
|
296
|
+
break;
|
|
297
|
+
}
|
|
221
298
|
const reqCpu = process.cpuUsage(req.startCpu);
|
|
222
299
|
requests.push({
|
|
223
300
|
'url': req.url,
|
|
@@ -228,8 +305,6 @@ export function getSnapshot() {
|
|
|
228
305
|
'memDelta': mem.rss - req.startMem,
|
|
229
306
|
});
|
|
230
307
|
}
|
|
231
|
-
// --- 按持续时间降序排序 ---
|
|
232
|
-
requests.sort((a, b) => b.duration - a.duration);
|
|
233
308
|
return {
|
|
234
309
|
'pid': process.pid,
|
|
235
310
|
'time': now,
|
|
@@ -272,18 +347,22 @@ function check() {
|
|
|
272
347
|
let cpuPercent = 0;
|
|
273
348
|
/** --- 实际经过的时间(ms),用于检测事件循环阻塞 --- */
|
|
274
349
|
let actualElapsed = INTERVAL;
|
|
275
|
-
if (
|
|
276
|
-
actualElapsed = now -
|
|
350
|
+
if (lastCheckCpuUsage && lastCheckCpuTime) {
|
|
351
|
+
actualElapsed = now - lastCheckCpuTime;
|
|
277
352
|
const elapsedUs = actualElapsed * 1_000;
|
|
278
|
-
const userDiff = cpuUsage.user -
|
|
279
|
-
const sysDiff = cpuUsage.system -
|
|
280
|
-
|
|
353
|
+
const userDiff = cpuUsage.user - lastCheckCpuUsage.user;
|
|
354
|
+
const sysDiff = cpuUsage.system - lastCheckCpuUsage.system;
|
|
355
|
+
if (elapsedUs > 0) {
|
|
356
|
+
cpuPercent = ((userDiff + sysDiff) / elapsedUs) * 100;
|
|
357
|
+
}
|
|
281
358
|
}
|
|
282
|
-
|
|
283
|
-
|
|
359
|
+
lastCheckCpuUsage = cpuUsage;
|
|
360
|
+
lastCheckCpuTime = now;
|
|
284
361
|
// --- 计算系统总 CPU 使用率 ---
|
|
285
|
-
const
|
|
286
|
-
const
|
|
362
|
+
const osCpus = os.cpus();
|
|
363
|
+
const cpuOs = getOsCpuPercent(lastCheckOsCpus, osCpus);
|
|
364
|
+
lastCheckOsCpus = osCpus;
|
|
365
|
+
const memMB = process.memoryUsage.rss() / 1024 / 1024;
|
|
287
366
|
let eloopLag = 0;
|
|
288
367
|
if (eloopHistogram) {
|
|
289
368
|
eloopLag = Math.round(eloopHistogram.percentile(99) / 1_000_000);
|
|
@@ -345,50 +424,34 @@ function logSpike(alerts, cpuPercent, cpuOs, eloopLag, blocked = false) {
|
|
|
345
424
|
const now = Date.now();
|
|
346
425
|
const requestDetails = [];
|
|
347
426
|
for (const [, req] of activeRequests) {
|
|
427
|
+
if (requestDetails.length >= MAX_ACTIVE_REQUEST_DETAILS) {
|
|
428
|
+
break;
|
|
429
|
+
}
|
|
348
430
|
const reqCpu = process.cpuUsage(req.startCpu);
|
|
349
431
|
const duration = now - req.start;
|
|
350
432
|
const cpuTotal = reqCpu.user + reqCpu.system;
|
|
351
433
|
const memDelta = mem.rss - req.startMem;
|
|
352
|
-
requestDetails.push(`[${req.method}] ${req.url} (${duration}ms,
|
|
434
|
+
requestDetails.push(`[${req.method}] ${req.url} (${duration}ms, PROC_CPU_DELTA: ${cpuTotal}us, PROC_RSS_DELTA: ${memDelta >= 0 ? '+' : '-'}${lText.sizeFormat(Math.abs(memDelta), '')})`);
|
|
435
|
+
}
|
|
436
|
+
const omittedRequests = activeRequests.size - requestDetails.length;
|
|
437
|
+
if (omittedRequests > 0) {
|
|
438
|
+
requestDetails.push(`... ${omittedRequests} more`);
|
|
353
439
|
}
|
|
354
440
|
// --- 诊断采集策略 ---
|
|
355
441
|
// --- blocked=true:阻塞期间 watchdog 已通过 connectToMainThread() 远程采集了精确堆栈和 Profile ---
|
|
356
442
|
// --- blocked=false:持续性骤升,此处由主线程自行采集 Report/Profile/HeapSnapshot ---
|
|
357
|
-
if (!blocked && now - lastDiagnosticTime >= DIAGNOSTIC_COOLDOWN) {
|
|
443
|
+
if (!blocked && !diagnosing && now - lastDiagnosticTime >= DIAGNOSTIC_COOLDOWN) {
|
|
358
444
|
lastDiagnosticTime = now;
|
|
445
|
+
diagnosing = true;
|
|
359
446
|
const hasCpuSpike = cpuPercent >= cpuThreshold;
|
|
360
447
|
const hasMemSpike = (mem.rss / 1024 / 1024) >= memThreshold;
|
|
361
|
-
const
|
|
362
|
-
const
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
process.report.writeReport(reportName);
|
|
369
|
-
lCore.display(`[MONITOR] Diagnostic report: ${diagDir}${reportName}`);
|
|
370
|
-
}
|
|
371
|
-
catch (e) {
|
|
372
|
-
lCore.debug('[MONITOR] Diagnostic report failed', e);
|
|
373
|
-
}
|
|
374
|
-
// --- 2. CPU Profile(精确到函数+行号的 CPU 消耗定位) ---
|
|
375
|
-
if (hasCpuSpike) {
|
|
376
|
-
collectCpuProfile(diagDir, ts).catch((e) => {
|
|
377
|
-
lCore.debug('[MONITOR] CPU profile failed', e);
|
|
378
|
-
});
|
|
379
|
-
}
|
|
380
|
-
// --- 3. Heap Snapshot(内存泄漏定位到对象分配源) ---
|
|
381
|
-
if (hasMemSpike) {
|
|
382
|
-
try {
|
|
383
|
-
const heapFile = v8.writeHeapSnapshot(`${diagDir}heap-${ts}.heapsnapshot`);
|
|
384
|
-
lCore.display(`[MONITOR] Heap snapshot: ${heapFile}`);
|
|
385
|
-
}
|
|
386
|
-
catch (e) {
|
|
387
|
-
lCore.debug('[MONITOR] Heap snapshot failed', e);
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
}).catch((e) => {
|
|
391
|
-
lCore.debug('[MONITOR] Failed to create diagnostic directory', e);
|
|
448
|
+
const diagnosticTime = new Date();
|
|
449
|
+
const ts = lTime.format(null, 'YmdHis', diagnosticTime);
|
|
450
|
+
const diagDir = `${kebab.LOG_CWD}monitor/${lTime.format(null, 'Y/m/d/His', diagnosticTime)}-pid-${process.pid}/`;
|
|
451
|
+
collectDiagnostics(diagDir, ts, hasCpuSpike, hasMemSpike && heapSnapshotEnabled).catch((e) => {
|
|
452
|
+
lCore.debug('[MONITOR] Diagnostic collection failed', e);
|
|
453
|
+
}).finally(() => {
|
|
454
|
+
diagnosing = false;
|
|
392
455
|
});
|
|
393
456
|
}
|
|
394
457
|
const msg = `SPIKE [${alerts.join(', ')}] ` +
|
|
@@ -407,19 +470,19 @@ function logSpike(alerts, cpuPercent, cpuOs, eloopLag, blocked = false) {
|
|
|
407
470
|
}
|
|
408
471
|
/**
|
|
409
472
|
* --- 通过 os.cpus() 两次采样的 Delta 计算系统总 CPU 使用率 ---
|
|
473
|
+
* @param previous 上次系统各核 CPU 时间快照
|
|
474
|
+
* @param current 当前系统各核 CPU 时间快照
|
|
410
475
|
* @returns 0-100 的百分比值,和任务管理器/top 命令一致
|
|
411
476
|
*/
|
|
412
|
-
function getOsCpuPercent() {
|
|
413
|
-
|
|
414
|
-
if (lastOsCpus?.length !== cpus.length) {
|
|
415
|
-
lastOsCpus = cpus;
|
|
477
|
+
function getOsCpuPercent(previous, current) {
|
|
478
|
+
if (previous?.length !== current.length) {
|
|
416
479
|
return 0;
|
|
417
480
|
}
|
|
418
481
|
let totalIdle = 0;
|
|
419
482
|
let totalTick = 0;
|
|
420
|
-
for (let i = 0; i <
|
|
421
|
-
const cur =
|
|
422
|
-
const prev =
|
|
483
|
+
for (let i = 0; i < current.length; ++i) {
|
|
484
|
+
const cur = current[i].times;
|
|
485
|
+
const prev = previous[i].times;
|
|
423
486
|
const idleDiff = cur.idle - prev.idle;
|
|
424
487
|
const totalDiff = (cur.user - prev.user) +
|
|
425
488
|
(cur.nice - prev.nice) +
|
|
@@ -429,11 +492,58 @@ function getOsCpuPercent() {
|
|
|
429
492
|
totalIdle += idleDiff;
|
|
430
493
|
totalTick += totalDiff;
|
|
431
494
|
}
|
|
432
|
-
|
|
433
|
-
if (totalTick === 0) {
|
|
495
|
+
if (totalTick <= 0) {
|
|
434
496
|
return 0;
|
|
435
497
|
}
|
|
436
|
-
|
|
498
|
+
const percent = (1 - totalIdle / totalTick) * 100;
|
|
499
|
+
return Math.round(Math.max(0, Math.min(100, percent)) * 100) / 100;
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* --- 采集诊断报告及按需 Profile,避免并行采集互相污染数据 ---
|
|
503
|
+
* @param dir 诊断文件输出目录
|
|
504
|
+
* @param ts 时间戳字符串
|
|
505
|
+
* @param hasCpuSpike 是否发生 CPU 骤升
|
|
506
|
+
* @param collectHeapSnapshot 是否采集堆快照
|
|
507
|
+
*/
|
|
508
|
+
async function collectDiagnostics(dir, ts, hasCpuSpike, collectHeapSnapshot) {
|
|
509
|
+
if (!await lFs.mkdir(dir, 0o700)) {
|
|
510
|
+
throw new Error(`Failed to create diagnostic directory: ${dir}`);
|
|
511
|
+
}
|
|
512
|
+
if (!await lFs.chmod(dir, 0o700)) {
|
|
513
|
+
throw new Error(`Failed to secure diagnostic directory: ${dir}`);
|
|
514
|
+
}
|
|
515
|
+
// --- Diagnostic Report 默认会包含命令行、环境变量和网络端点,落盘前显式脱敏 ---
|
|
516
|
+
const reportName = `report-${ts}.json`;
|
|
517
|
+
const report = process.report.getReport();
|
|
518
|
+
if (report.header) {
|
|
519
|
+
report.header.commandLine = [];
|
|
520
|
+
delete report.header.networkInterfaces;
|
|
521
|
+
}
|
|
522
|
+
delete report.environmentVariables;
|
|
523
|
+
for (const handle of report.libuv ?? []) {
|
|
524
|
+
delete handle.localEndpoint;
|
|
525
|
+
delete handle.remoteEndpoint;
|
|
526
|
+
}
|
|
527
|
+
const reportPath = `${dir}${reportName}`;
|
|
528
|
+
const written = await lFs.putContent(reportPath, lText.stringifyJson(report, 2), { 'encoding': 'utf8', 'mode': 0o600 });
|
|
529
|
+
if (!written) {
|
|
530
|
+
throw new Error(`Failed to write diagnostic report: ${reportPath}`);
|
|
531
|
+
}
|
|
532
|
+
if (!await lFs.chmod(reportPath, 0o600)) {
|
|
533
|
+
throw new Error(`Failed to secure diagnostic report: ${reportPath}`);
|
|
534
|
+
}
|
|
535
|
+
lCore.display(`[MONITOR] Diagnostic report: ${reportPath}`);
|
|
536
|
+
if (hasCpuSpike) {
|
|
537
|
+
await collectCpuProfile(dir, ts);
|
|
538
|
+
}
|
|
539
|
+
// --- 堆快照会同步阻塞并额外占用大量内存,仅在调用方明确开启时采集 ---
|
|
540
|
+
if (collectHeapSnapshot) {
|
|
541
|
+
const heapFile = v8.writeHeapSnapshot(`${dir}heap-${ts}.heapsnapshot`);
|
|
542
|
+
if (!await lFs.chmod(heapFile, 0o600)) {
|
|
543
|
+
throw new Error(`Failed to secure heap snapshot: ${heapFile}`);
|
|
544
|
+
}
|
|
545
|
+
lCore.display(`[MONITOR] Heap snapshot: ${heapFile}`);
|
|
546
|
+
}
|
|
437
547
|
}
|
|
438
548
|
/**
|
|
439
549
|
* --- 通过 Inspector 协议采集 CPU Profile ---
|
|
@@ -478,9 +588,16 @@ async function collectCpuProfile(dir, ts) {
|
|
|
478
588
|
});
|
|
479
589
|
});
|
|
480
590
|
const filePath = `${dir}cpu-${ts}.cpuprofile`;
|
|
481
|
-
await lFs.putContent(filePath, lText.stringifyJson(profile), {
|
|
591
|
+
const written = await lFs.putContent(filePath, lText.stringifyJson(profile), {
|
|
482
592
|
'encoding': 'utf8',
|
|
593
|
+
'mode': 0o600,
|
|
483
594
|
});
|
|
595
|
+
if (!written) {
|
|
596
|
+
throw new Error(`Failed to write CPU profile: ${filePath}`);
|
|
597
|
+
}
|
|
598
|
+
if (!await lFs.chmod(filePath, 0o600)) {
|
|
599
|
+
throw new Error(`Failed to secure CPU profile: ${filePath}`);
|
|
600
|
+
}
|
|
484
601
|
lCore.display(`[MONITOR] CPU profile: ${filePath}`);
|
|
485
602
|
}
|
|
486
603
|
finally {
|
package/sys/route.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Project: Kebab, User: JianSuoQiYue
|
|
3
3
|
* Date: 2019-4-15 13:40
|
|
4
|
-
* Last: 2020-4-14 13:52:00, 2022-09-07 01:43:31, 2023-12-29 17:24:03, 2024-2-7 00:28:50, 2024-6-6 15:15:54, 2025-6-13 19:23:53, 2025-9-22 15:48:53, 2025-9-23 11:26:50, 2025-10-30 17:44:41
|
|
4
|
+
* Last: 2020-4-14 13:52:00, 2022-09-07 01:43:31, 2023-12-29 17:24:03, 2024-2-7 00:28:50, 2024-6-6 15:15:54, 2025-6-13 19:23:53, 2025-9-22 15:48:53, 2025-9-23 11:26:50, 2025-10-30 17:44:41, 2026-8-22
|
|
5
5
|
*/
|
|
6
6
|
import * as http from 'http';
|
|
7
7
|
import * as http2 from 'http2';
|
package/sys/route.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Project: Kebab, User: JianSuoQiYue
|
|
3
3
|
* Date: 2019-4-15 13:40
|
|
4
|
-
* Last: 2020-4-14 13:52:00, 2022-09-07 01:43:31, 2023-12-29 17:24:03, 2024-2-7 00:28:50, 2024-6-6 15:15:54, 2025-6-13 19:23:53, 2025-9-22 15:48:53, 2025-9-23 11:26:50, 2025-10-30 17:44:41
|
|
4
|
+
* Last: 2020-4-14 13:52:00, 2022-09-07 01:43:31, 2023-12-29 17:24:03, 2024-2-7 00:28:50, 2024-6-6 15:15:54, 2025-6-13 19:23:53, 2025-9-22 15:48:53, 2025-9-23 11:26:50, 2025-10-30 17:44:41, 2026-8-22
|
|
5
5
|
*/
|
|
6
6
|
import * as http from 'http';
|
|
7
7
|
import * as stream from 'stream';
|
|
@@ -300,7 +300,7 @@ export async function run(data) {
|
|
|
300
300
|
cctr.setPrototype('_socket', wsSocket);
|
|
301
301
|
}
|
|
302
302
|
catch (e) {
|
|
303
|
-
lCore.log(cctr, lText.
|
|
303
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
304
304
|
data.socket?.destroy();
|
|
305
305
|
return true;
|
|
306
306
|
}
|
|
@@ -314,7 +314,7 @@ export async function run(data) {
|
|
|
314
314
|
rtn = await cctr.onLoad();
|
|
315
315
|
}
|
|
316
316
|
catch (e) {
|
|
317
|
-
lCore.log(cctr, lText.
|
|
317
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
318
318
|
data.socket?.destroy();
|
|
319
319
|
return true;
|
|
320
320
|
}
|
|
@@ -323,7 +323,7 @@ export async function run(data) {
|
|
|
323
323
|
rtn = await cctr.onReady();
|
|
324
324
|
}
|
|
325
325
|
catch (e) {
|
|
326
|
-
lCore.log(cctr, lText.
|
|
326
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
327
327
|
data.socket?.destroy();
|
|
328
328
|
return true;
|
|
329
329
|
}
|
|
@@ -380,7 +380,7 @@ export async function run(data) {
|
|
|
380
380
|
}
|
|
381
381
|
}
|
|
382
382
|
catch (e) {
|
|
383
|
-
lCore.log(cctr, lText.
|
|
383
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
384
384
|
}
|
|
385
385
|
break;
|
|
386
386
|
}
|
|
@@ -393,23 +393,23 @@ export async function run(data) {
|
|
|
393
393
|
await cctr['onDrain']();
|
|
394
394
|
}
|
|
395
395
|
catch (e) {
|
|
396
|
-
lCore.log(cctr, lText.
|
|
396
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
397
397
|
}
|
|
398
398
|
}).on('error', (e) => {
|
|
399
|
-
lCore.log(cctr, lText.
|
|
399
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
400
400
|
}).on('end', async () => {
|
|
401
401
|
try {
|
|
402
402
|
await cctr['onEnd']();
|
|
403
403
|
}
|
|
404
404
|
catch (e) {
|
|
405
|
-
lCore.log(cctr, lText.
|
|
405
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
406
406
|
}
|
|
407
407
|
}).on('close', async () => {
|
|
408
408
|
try {
|
|
409
409
|
await cctr['onClose']();
|
|
410
410
|
}
|
|
411
411
|
catch (e) {
|
|
412
|
-
lCore.log(cctr, lText.
|
|
412
|
+
lCore.log(cctr, lText.stringifyError(e), '-error');
|
|
413
413
|
}
|
|
414
414
|
resolve();
|
|
415
415
|
});
|
|
@@ -467,7 +467,7 @@ export async function run(data) {
|
|
|
467
467
|
rtn = await middle.onLoad();
|
|
468
468
|
}
|
|
469
469
|
catch (e) {
|
|
470
|
-
lCore.log(middle, '(E03)' + lText.
|
|
470
|
+
lCore.log(middle, '(E03)' + lText.stringifyError(e), '-error');
|
|
471
471
|
respond500(data.res);
|
|
472
472
|
return true;
|
|
473
473
|
}
|
|
@@ -479,7 +479,7 @@ export async function run(data) {
|
|
|
479
479
|
rtn = await middle.onReady();
|
|
480
480
|
}
|
|
481
481
|
catch (e) {
|
|
482
|
-
lCore.log(middle, '(E05)' + lText.
|
|
482
|
+
lCore.log(middle, '(E05)' + lText.stringifyError(e), '-error');
|
|
483
483
|
respond500(data.res);
|
|
484
484
|
return true;
|
|
485
485
|
}
|
|
@@ -556,7 +556,7 @@ export async function run(data) {
|
|
|
556
556
|
}
|
|
557
557
|
}
|
|
558
558
|
catch (e) {
|
|
559
|
-
lCore.log(cctr, '(E05)' + lText.
|
|
559
|
+
lCore.log(cctr, '(E05)' + lText.stringifyError(e), '-error');
|
|
560
560
|
respond500(data.res);
|
|
561
561
|
await waitCtr(cctr);
|
|
562
562
|
return true;
|
|
@@ -567,7 +567,7 @@ export async function run(data) {
|
|
|
567
567
|
httpCode = cctr.getPrototype('_httpCode');
|
|
568
568
|
}
|
|
569
569
|
catch (e) {
|
|
570
|
-
lCore.log(cctr, '(E04)' + lText.
|
|
570
|
+
lCore.log(cctr, '(E04)' + lText.stringifyError(e), '-error');
|
|
571
571
|
respond500(data.res);
|
|
572
572
|
await waitCtr(cctr);
|
|
573
573
|
return true;
|
|
@@ -1291,7 +1291,7 @@ export function getFormData(req, events = {}, limits = {}) {
|
|
|
1291
1291
|
clearTimer();
|
|
1292
1292
|
cleanupActiveFile();
|
|
1293
1293
|
lCore.debug('[ROUTE][GETFORMDATA] request error before getFormData: ' + e.message);
|
|
1294
|
-
lCore.log({}, '[ROUTE][GETFORMDATA] request error before getFormData: ' + (e
|
|
1294
|
+
lCore.log({}, '[ROUTE][GETFORMDATA] request error before getFormData: ' + lText.stringifyError(e), '-error');
|
|
1295
1295
|
cleanupFiles();
|
|
1296
1296
|
resolve(false);
|
|
1297
1297
|
});
|
|
@@ -87,6 +87,8 @@ export default class extends sCtr.Ctr {
|
|
|
87
87
|
undiciFollow2(): kebab.Json;
|
|
88
88
|
undiciReuse(): Promise<string>;
|
|
89
89
|
undiciError(): Promise<string>;
|
|
90
|
+
undiciRetry(): Promise<string>;
|
|
91
|
+
undiciRetrySource(): Promise<string | boolean>;
|
|
90
92
|
undiciHosts(): Promise<string>;
|
|
91
93
|
undiciMproxy(): Promise<string | boolean>;
|
|
92
94
|
undiciMproxy1(): Promise<string | boolean>;
|
package/www/example/ctr/test.js
CHANGED
|
@@ -186,6 +186,7 @@ export default class extends sCtr.Ctr {
|
|
|
186
186
|
`<br><a href="${this._config.const.urlBase}test/undici-follow">View "test/undici-follow"</a>`,
|
|
187
187
|
`<br><a href="${this._config.const.urlBase}test/undici-reuse">View "test/undici-reuse"</a>`,
|
|
188
188
|
`<br><a href="${this._config.const.urlBase}test/undici-error">View "test/undici-error"</a>`,
|
|
189
|
+
`<br><a href="${this._config.const.urlBase}test/undici-retry">View "test/undici-retry"</a>`,
|
|
189
190
|
`<br><a href="${this._config.const.urlBase}test/undici-hosts">View "test/undici-hosts"</a>`,
|
|
190
191
|
`<br><a href="${this._config.const.urlBase}test/undici-rproxy/dist/core.js">View "test/undici-rproxy/dist/core.js"</a> <a href="${this._config.const.urlBase}test/undici-rproxy/package.json">View "package.json"</a>`,
|
|
191
192
|
`<br><a href="${this._config.const.urlBase}test/undici-mproxy">View "test/undici-mproxy"</a>`,
|
|
@@ -2212,6 +2213,46 @@ content: <pre>${(await res.getContent())?.toString() ?? 'null'}</pre>
|
|
|
2212
2213
|
error: <pre>${JSON.stringify(res.error, null, 4)}</pre>`);
|
|
2213
2214
|
return echo.join('') + this._getEnd();
|
|
2214
2215
|
}
|
|
2216
|
+
async undiciRetry() {
|
|
2217
|
+
const url = `${this._internalUrl}test/undici-retry-source`;
|
|
2218
|
+
const time = Date.now();
|
|
2219
|
+
const res = await lUndici.get(url, {
|
|
2220
|
+
'retry': 1,
|
|
2221
|
+
'log': false,
|
|
2222
|
+
});
|
|
2223
|
+
const content = await res.getContent();
|
|
2224
|
+
return `<pre>const res = await lUndici.get('${url}', {
|
|
2225
|
+
'retry': 1,
|
|
2226
|
+
'log': false,
|
|
2227
|
+
});
|
|
2228
|
+
const content = await res.getContent();</pre>
|
|
2229
|
+
time: ${Date.now() - time}ms
|
|
2230
|
+
headers: <pre>${lText.htmlescape(lText.stringifyJson(res.headers))}</pre>
|
|
2231
|
+
content: <pre>${lText.htmlescape(content?.toString() ?? 'null')}</pre>
|
|
2232
|
+
error: <pre>${lText.htmlescape(res.error ? lText.stringifyError(res.error) : 'null')}</pre>` + this._getEnd();
|
|
2233
|
+
}
|
|
2234
|
+
async undiciRetrySource() {
|
|
2235
|
+
const etag = '"undici-retry"';
|
|
2236
|
+
if ((this._headers['range'] === 'bytes=1-1') && (this._headers['if-match'] === etag)) {
|
|
2237
|
+
this._httpCode = 206;
|
|
2238
|
+
this._res.setHeader('content-length', '1');
|
|
2239
|
+
this._res.setHeader('content-range', 'bytes 1-1/2');
|
|
2240
|
+
this._res.setHeader('etag', etag);
|
|
2241
|
+
return 'K';
|
|
2242
|
+
}
|
|
2243
|
+
lCore.writeHead(this._res, 200, {
|
|
2244
|
+
'content-length': '2',
|
|
2245
|
+
etag,
|
|
2246
|
+
});
|
|
2247
|
+
lCore.write(this._res, 'O');
|
|
2248
|
+
await new Promise(resolve => {
|
|
2249
|
+
setImmediate(() => {
|
|
2250
|
+
this._res.destroy();
|
|
2251
|
+
resolve();
|
|
2252
|
+
});
|
|
2253
|
+
});
|
|
2254
|
+
return false;
|
|
2255
|
+
}
|
|
2215
2256
|
async undiciHosts() {
|
|
2216
2257
|
const echo = [];
|
|
2217
2258
|
const res = await lUndici.get('http://nodejs.org:' + this._config.const.hostport.toString() + this._config.const.urlBase + 'test', {
|
|
@@ -4083,7 +4124,7 @@ send.addEventListener('click', async () => {
|
|
|
4083
4124
|
'<b>Monitor Snapshot</b>',
|
|
4084
4125
|
'<hr>',
|
|
4085
4126
|
`<b>PID:</b> ${snapshot.pid}`,
|
|
4086
|
-
`<br><b>Process CPU (
|
|
4127
|
+
`<br><b>Process CPU (100% per core):</b> ${snapshot.cpuProcess}%`,
|
|
4087
4128
|
`<br><b>System CPU (total):</b> ${snapshot.cpuOs}%`,
|
|
4088
4129
|
`<br><b>Event Loop Lag:</b> ${snapshot.eloopLag}ms`,
|
|
4089
4130
|
'<br><br><b>Memory:</b>',
|
|
@@ -4102,7 +4143,7 @@ send.addEventListener('click', async () => {
|
|
|
4102
4143
|
`<br><br><b>Active Requests:</b> ${snapshot.activeCount}`,
|
|
4103
4144
|
];
|
|
4104
4145
|
for (const req of snapshot.activeRequests) {
|
|
4105
|
-
echo.push(`<br>[${req.method}] ${lText.htmlescape(req.url)} - ${req.duration}ms,
|
|
4146
|
+
echo.push(`<br>[${req.method}] ${lText.htmlescape(req.url)} - ${req.duration}ms, PROCESS_CPU_DELTA: ${req.cpuUser + req.cpuSystem}us, PROCESS_MEM_DELTA: ${lText.sizeFormat(req.memDelta)}`);
|
|
4106
4147
|
}
|
|
4107
4148
|
echo.push('<br><br>' + this._getEnd());
|
|
4108
4149
|
return echo.join('');
|
|
@@ -4156,7 +4197,7 @@ send.addEventListener('click', async () => {
|
|
|
4156
4197
|
`System CPU ${after.cpuOs}%, ` +
|
|
4157
4198
|
`RSS ${lText.sizeFormat(after.mem.rss)}, ` +
|
|
4158
4199
|
`Iterations: ${count}`);
|
|
4159
|
-
echo.push('<br><br>Check <code>log/monitor/{pid}/</code> for' +
|
|
4200
|
+
echo.push('<br><br>Check <code>log/monitor/YYYY/MM/DD/HHmmss-pid-{pid}/</code> for' +
|
|
4160
4201
|
' diagnostic files.');
|
|
4161
4202
|
echo.push('<br><br>' + this._getEnd());
|
|
4162
4203
|
return echo.join('');
|