@maiyunnet/kebab 9.21.0 → 9.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/doc/kebab-rag.md +436 -10
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/lib/core.d.ts +59 -0
- package/lib/core.js +81 -0
- package/package.json +1 -1
- package/sys/master.js +54 -0
- package/sys/monitor/files.d.ts +34 -0
- package/sys/monitor/files.js +260 -0
- package/sys/monitor/watchdog.js +284 -215
- package/sys/monitor.d.ts +28 -3
- package/sys/monitor.js +527 -250
- package/www/example/ctr/test.d.ts +21 -0
- package/www/example/ctr/test.js +250 -2
package/sys/monitor/watchdog.js
CHANGED
|
@@ -1,270 +1,339 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Project: Kebab, User: JianSuoQiYue
|
|
3
3
|
* Date: 2026-02-08
|
|
4
|
-
* Last: 2026-
|
|
5
|
-
* ---
|
|
6
|
-
* ---
|
|
7
|
-
* --- 看门狗必须最小依赖、最大自治,不引入项目库,确保主线程异常时仍能可靠运行 ---
|
|
4
|
+
* Last: 2026-10-05
|
|
5
|
+
* --- 独立线程监测主线程心跳,阻塞期间保存资源采样、请求、调用栈和 CPU Profile ---
|
|
6
|
+
* --- 只依赖 Node.js;主线程无法运行时,日志与 Inspector 超时仍可独立处理 ---
|
|
8
7
|
*/
|
|
9
8
|
import * as workerThreads from 'worker_threads';
|
|
10
9
|
import * as fs from 'fs';
|
|
11
10
|
import * as path from 'path';
|
|
12
|
-
import * as
|
|
11
|
+
import * as os from 'os';
|
|
12
|
+
import * as inspector from 'inspector/promises';
|
|
13
13
|
const data = workerThreads.workerData;
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
/** --- 0: 心跳 ms, 1: 调试, 2: Inspector 所有者(0-空闲,1-常规 Profile,2-阻塞 Profile), 3: 同步诊断, 4: 已保存现场的心跳 --- */
|
|
15
|
+
const view = new BigInt64Array(data.buffer);
|
|
16
|
+
const activeRequests = new Map(data.requests);
|
|
17
|
+
const recentRequests = [];
|
|
18
|
+
const samples = [];
|
|
19
|
+
let snapshot = null;
|
|
20
|
+
let lastAlertHeartbeat = 0n;
|
|
16
21
|
let lastAlertTime = 0;
|
|
17
|
-
/** --- 是否正在诊断采集 --- */
|
|
18
22
|
let capturing = false;
|
|
23
|
+
let lastCpu = process.cpuUsage();
|
|
24
|
+
let lastTime = Number(process.hrtime.bigint() / 1000000n);
|
|
25
|
+
workerThreads.parentPort?.on('message', (message) => {
|
|
26
|
+
if (message.type === 'snapshot') {
|
|
27
|
+
snapshot = message.snapshot;
|
|
28
|
+
data.logFormat = message.logFormat;
|
|
29
|
+
}
|
|
30
|
+
else if (message.type === 'request') {
|
|
31
|
+
activeRequests.set(message.id, message.request);
|
|
32
|
+
}
|
|
33
|
+
else if (message.type === 'complete') {
|
|
34
|
+
activeRequests.delete(message.id);
|
|
35
|
+
recentRequests.push(message.request);
|
|
36
|
+
if (recentRequests.length > 100) {
|
|
37
|
+
recentRequests.shift();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
collectProfile(message.dir, message.ts).catch((error) => {
|
|
42
|
+
writeLog(`WATCHDOG: Unexpected profile failure: ${String(error)}`);
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
});
|
|
19
46
|
/**
|
|
20
|
-
* ---
|
|
47
|
+
* --- 格式化本地时间,用于日志目录和文件名 ---
|
|
48
|
+
* @param date 要格式化的时间
|
|
49
|
+
* @returns YmdHis 字符串
|
|
21
50
|
*/
|
|
22
|
-
function fmtTs() {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
String(
|
|
26
|
-
String(d.getDate()).padStart(2, '0') +
|
|
27
|
-
String(d.getHours()).padStart(2, '0') +
|
|
28
|
-
String(d.getMinutes()).padStart(2, '0') +
|
|
29
|
-
String(d.getSeconds()).padStart(2, '0');
|
|
51
|
+
function fmtTs(date) {
|
|
52
|
+
return String(date.getFullYear()) + String(date.getMonth() + 1).padStart(2, '0') +
|
|
53
|
+
String(date.getDate()).padStart(2, '0') + String(date.getHours()).padStart(2, '0') +
|
|
54
|
+
String(date.getMinutes()).padStart(2, '0') + String(date.getSeconds()).padStart(2, '0');
|
|
30
55
|
}
|
|
31
56
|
/**
|
|
32
|
-
* ---
|
|
57
|
+
* --- 独立写入看门狗日志,格式和字段与 Core.log 一致,失败时保留标准错误输出 ---
|
|
58
|
+
* --- 不向主线程转发,也不加载 Core 的控制器、网络等依赖 ---
|
|
59
|
+
* @param msg 日志内容
|
|
60
|
+
* @returns 无返回值
|
|
33
61
|
*/
|
|
34
|
-
function
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
62
|
+
function writeLog(msg) {
|
|
63
|
+
const now = new Date();
|
|
64
|
+
const ts = fmtTs(now);
|
|
65
|
+
const dir = path.join(data.logDir, 'system-monitor', ts.slice(0, 4), ts.slice(4, 6), ts.slice(6, 8));
|
|
66
|
+
try {
|
|
67
|
+
fs.mkdirSync(dir, { 'recursive': true, 'mode': 0o700 });
|
|
68
|
+
const format = data.logFormat ?? 'jsonl';
|
|
69
|
+
const file = path.join(dir, `${ts.slice(8, 10)}.${format}`);
|
|
70
|
+
// --- Worker 没有单个请求上下文;内存使用 MB,保持日志字段可直接展示 ---
|
|
71
|
+
const entry = {
|
|
72
|
+
'time': `${ts.slice(8, 10)}:${ts.slice(10, 12)}:${ts.slice(12, 14)}`,
|
|
73
|
+
'unix': Math.floor(now.getTime() / 1000),
|
|
74
|
+
'url': '', 'cookie': {}, 'session': {}, 'userAgent': '',
|
|
75
|
+
'realIp': '', 'cfIp': '', 'xIp': '',
|
|
76
|
+
'osMem': `${Math.round((os.totalmem() - os.freemem()) / 1024 / 1024 * 100) / 100}MB`,
|
|
77
|
+
'procMem': `${Math.round(process.memoryUsage.rss() / 1024 / 1024 * 100) / 100}MB`,
|
|
78
|
+
'message': msg,
|
|
79
|
+
};
|
|
80
|
+
let content;
|
|
81
|
+
if (format === 'jsonl') {
|
|
82
|
+
content = JSON.stringify(entry) + '\n';
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
if (!fs.existsSync(file)) {
|
|
86
|
+
fs.writeFileSync(file, 'TIME,UNIX,URL,COOKIE,SESSION,USER_AGENT,REALIP,CFIP,XIP,OS,PROCESS,MESSAGE\n', { 'mode': 0o600 });
|
|
87
|
+
}
|
|
88
|
+
const fields = [
|
|
89
|
+
entry.time, String(entry.unix), entry.url, '', '{}', 'No HTTP_USER_AGENT',
|
|
90
|
+
entry.realIp, entry.cfIp, entry.xIp, entry.osMem, entry.procMem, entry.message,
|
|
91
|
+
];
|
|
92
|
+
content = fields.map(field => `"${field.replace(/"/g, '""')}"`).join(',') + '\n';
|
|
93
|
+
}
|
|
94
|
+
fs.appendFileSync(file, content, { 'mode': 0o600 });
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
process.stderr.write(`[MONITOR] Watchdog log failed: ${String(error)}\n`);
|
|
98
|
+
}
|
|
39
99
|
}
|
|
40
100
|
/**
|
|
41
|
-
* ---
|
|
42
|
-
* @param
|
|
101
|
+
* --- 写入权限受限的诊断文件 ---
|
|
102
|
+
* @param dir 诊断目录
|
|
103
|
+
* @param name 文件名
|
|
104
|
+
* @param content 文件内容
|
|
105
|
+
* @returns 是否保存成功
|
|
43
106
|
*/
|
|
44
|
-
function
|
|
45
|
-
const d = new Date();
|
|
46
|
-
const dir = path.join(data.logDir, 'system-monitor', String(d.getFullYear()), String(d.getMonth() + 1).padStart(2, '0'), String(d.getDate()).padStart(2, '0'));
|
|
107
|
+
function writeFile(dir, name, content) {
|
|
47
108
|
try {
|
|
48
|
-
fs.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
109
|
+
fs.writeFileSync(path.join(dir, name), content, { 'mode': 0o600 });
|
|
110
|
+
fs.chmodSync(path.join(dir, name), 0o600);
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
process.stderr.write(`[MONITOR] Watchdog diagnostic write failed: ${String(error)}\n`);
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* --- 按主线程请求采集 Profile,开始、停止、超时和文件写入都不依赖主线程事件循环 ---
|
|
120
|
+
* @param dir 主线程已创建的诊断目录
|
|
121
|
+
* @param ts 唯一采集时间标识
|
|
122
|
+
* @returns 无返回值,独立保存结果并通知主线程
|
|
123
|
+
*/
|
|
124
|
+
async function collectProfile(dir, ts) {
|
|
125
|
+
const file = `cpu-${ts}.cpuprofile`;
|
|
126
|
+
const result = { 'file': file, 'saved': false, 'errors': [] };
|
|
127
|
+
if (Atomics.compareExchange(view, 2, 0n, 1n) !== 0n) {
|
|
128
|
+
result.errors.push('Inspector is already collecting a profile.');
|
|
129
|
+
const recorded = writeFile(dir, `profile-${ts}.json`, JSON.stringify(result, null, 2));
|
|
130
|
+
workerThreads.parentPort?.postMessage({ 'type': 'profile', ...result, recorded });
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
const session = new inspector.Session();
|
|
134
|
+
let timedOut = false;
|
|
135
|
+
const timeout = setTimeout(() => {
|
|
136
|
+
timedOut = true;
|
|
137
|
+
session.disconnect();
|
|
138
|
+
}, data.profileDuration + 10_000);
|
|
139
|
+
try {
|
|
140
|
+
session.connectToMainThread();
|
|
141
|
+
await session.post('Profiler.enable');
|
|
142
|
+
await session.post('Profiler.start');
|
|
143
|
+
await new Promise(resolve => setTimeout(resolve, data.profileDuration));
|
|
144
|
+
const { profile } = await session.post('Profiler.stop');
|
|
145
|
+
result.saved = writeFile(dir, file, JSON.stringify(profile));
|
|
146
|
+
if (!result.saved) {
|
|
147
|
+
result.errors.push('Failed to save CPU profile.');
|
|
53
148
|
}
|
|
54
|
-
const now = Math.floor(Date.now() / 1000);
|
|
55
|
-
fs.appendFileSync(file, `"${fmtTime()}","${now}","${msg}"\n`);
|
|
56
149
|
}
|
|
57
|
-
catch {
|
|
58
|
-
|
|
150
|
+
catch (error) {
|
|
151
|
+
result.errors.push(timedOut ? 'Inspector profile collection timed out.' : String(error));
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
clearTimeout(timeout);
|
|
155
|
+
session.disconnect();
|
|
156
|
+
Atomics.compareExchange(view, 2, 1n, 0n);
|
|
157
|
+
const recorded = writeFile(dir, `profile-${ts}.json`, JSON.stringify(result, null, 2));
|
|
158
|
+
if (!recorded) {
|
|
159
|
+
writeLog('WATCHDOG: Failed to save CPU profile result.');
|
|
160
|
+
}
|
|
161
|
+
workerThreads.parentPort?.postMessage({ 'type': 'profile', ...result, recorded });
|
|
59
162
|
}
|
|
60
163
|
}
|
|
61
164
|
/**
|
|
62
|
-
* ---
|
|
63
|
-
* @param
|
|
165
|
+
* --- 保存阻塞现场;暂停后立即恢复,再写磁盘,整个会话有统一超时和释放出口 ---
|
|
166
|
+
* @param blockMs 心跳中断时间 ms
|
|
167
|
+
* @param heartbeat 这次阻塞前的心跳,用于区分连续阻塞和新事件
|
|
168
|
+
* @returns 无返回值,采集结果保留在 blocked-event.json
|
|
64
169
|
*/
|
|
65
|
-
function captureDiag(
|
|
170
|
+
async function captureDiag(blockMs, heartbeat) {
|
|
66
171
|
if (capturing) {
|
|
67
172
|
return;
|
|
68
173
|
}
|
|
69
174
|
capturing = true;
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
const
|
|
175
|
+
const now = new Date();
|
|
176
|
+
const ts = fmtTs(now);
|
|
177
|
+
const id = `${ts}-${now.getTime()}`;
|
|
178
|
+
const dir = path.join(data.logDir, 'monitor', ts.slice(0, 4), ts.slice(4, 6), ts.slice(6, 8), `${ts.slice(8)}-pid-${data.pid}-${now.getTime()}-blocked`);
|
|
179
|
+
const cpu = process.cpuUsage();
|
|
180
|
+
const rss = process.memoryUsage.rss();
|
|
181
|
+
const requests = [];
|
|
182
|
+
for (const req of activeRequests.values()) {
|
|
183
|
+
if (requests.length >= 100) {
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
requests.push({
|
|
187
|
+
'url': req.url, 'method': req.method, 'duration': now.getTime() - req.start,
|
|
188
|
+
'cpuUser': cpu.user - req.startCpu.user, 'cpuSystem': cpu.system - req.startCpu.system,
|
|
189
|
+
'memDelta': rss - req.startMem,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
const record = {
|
|
193
|
+
'pid': data.pid, 'time': now.getTime(), 'heartbeat': Number(heartbeat), 'blockMs': blockMs,
|
|
194
|
+
'samples': [...samples], 'lastMainSnapshot': snapshot, 'activeRequests': requests, 'activeCount': activeRequests.size,
|
|
195
|
+
'recentRequests': recentRequests.filter(req => now.getTime() - (req.completedAt ?? 0) <= 60_000),
|
|
196
|
+
'status': 'pending', 'files': [], 'errors': [],
|
|
197
|
+
};
|
|
73
198
|
try {
|
|
74
|
-
|
|
199
|
+
fs.mkdirSync(dir, { 'recursive': true, 'mode': 0o700 });
|
|
200
|
+
fs.chmodSync(dir, 0o700);
|
|
75
201
|
}
|
|
76
|
-
catch {
|
|
202
|
+
catch (error) {
|
|
203
|
+
writeLog(`WATCHDOG: Failed to create diagnostic directory: ${String(error)}`);
|
|
77
204
|
capturing = false;
|
|
78
205
|
return;
|
|
79
206
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
catch {
|
|
92
|
-
// --- 忽略 ---
|
|
93
|
-
}
|
|
94
|
-
try {
|
|
95
|
-
session.post('Debugger.disable');
|
|
96
|
-
}
|
|
97
|
-
catch {
|
|
98
|
-
// --- 忽略 ---
|
|
99
|
-
}
|
|
100
|
-
try {
|
|
101
|
-
session.disconnect();
|
|
102
|
-
}
|
|
103
|
-
catch {
|
|
104
|
-
// --- 忽略 ---
|
|
105
|
-
}
|
|
106
|
-
capturing = false;
|
|
207
|
+
if (!writeFile(dir, 'blocked-event.json', JSON.stringify(record, null, 2))) {
|
|
208
|
+
capturing = false;
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
writeLog(`WATCHDOG: Main thread heartbeat delayed ${blockMs}ms, PID:${data.pid}, EVENT:${dir}`);
|
|
212
|
+
if (Atomics.compareExchange(view, 2, 0n, 2n) !== 0n) {
|
|
213
|
+
record.status = 'skipped-busy';
|
|
214
|
+
lastAlertTime = Number(process.hrtime.bigint() / 1000000n) - data.cooldown * 1_000 + 5_000;
|
|
215
|
+
record.errors.push('Inspector is already collecting a profile.');
|
|
216
|
+
if (!writeFile(dir, 'blocked-event.json', JSON.stringify(record, null, 2))) {
|
|
217
|
+
writeLog('WATCHDOG: Failed to save Inspector busy result.');
|
|
107
218
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
219
|
+
capturing = false;
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
const session = new inspector.Session();
|
|
223
|
+
const scriptUrls = {};
|
|
224
|
+
let timedOut = false;
|
|
225
|
+
let finishPause = null;
|
|
226
|
+
const timeout = setTimeout(() => {
|
|
227
|
+
timedOut = true;
|
|
228
|
+
finishPause?.(false);
|
|
229
|
+
// --- 断开会话会使挂起的 post 失败,并释放我们设置的暂停状态 ---
|
|
230
|
+
session.disconnect();
|
|
231
|
+
}, data.profileDuration + 10_000);
|
|
232
|
+
session.on('Debugger.scriptParsed', (message) => {
|
|
233
|
+
scriptUrls[message.params.scriptId] = message.params.url;
|
|
111
234
|
});
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
let stackLines = [];
|
|
235
|
+
try {
|
|
236
|
+
session.connectToMainThread();
|
|
237
|
+
// --- 先启动 Profile,避免等待调用栈时错过仍在执行的热点 ---
|
|
238
|
+
await session.post('Profiler.enable');
|
|
239
|
+
await session.post('Profiler.start');
|
|
240
|
+
const profileStarted = Number(process.hrtime.bigint() / 1000000n);
|
|
119
241
|
try {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
const line = f.location.lineNumber + 1;
|
|
126
|
-
const col = (f.location.columnNumber ?? 0) + 1;
|
|
127
|
-
const name = f.functionName || '(anonymous)';
|
|
128
|
-
return `#${i} ${name} (${url}:${line}:${col})`;
|
|
242
|
+
await session.post('Debugger.enable');
|
|
243
|
+
const frames = await new Promise((resolve) => {
|
|
244
|
+
finishPause = resolve;
|
|
245
|
+
session.once('Debugger.paused', (message) => { resolve(message.params.callFrames); });
|
|
246
|
+
void session.post('Debugger.pause').catch(() => { resolve(false); });
|
|
129
247
|
});
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// --- 忽略 ---
|
|
142
|
-
}
|
|
143
|
-
try {
|
|
144
|
-
session.disconnect();
|
|
248
|
+
if (frames !== false && !timedOut) {
|
|
249
|
+
const lines = frames.map((frame, i) => {
|
|
250
|
+
const url = frame.url || scriptUrls[frame.location.scriptId] || '';
|
|
251
|
+
return `#${i} ${frame.functionName || '(anonymous)'} ` +
|
|
252
|
+
`(${url}:${frame.location.lineNumber + 1}:${(frame.location.columnNumber ?? 0) + 1})`;
|
|
253
|
+
});
|
|
254
|
+
await session.post('Debugger.resume');
|
|
255
|
+
const name = `blocked-stack-${id}.txt`;
|
|
256
|
+
if (writeFile(dir, name, `Heartbeat delayed: ${blockMs}ms\nPID: ${data.pid}\nCaptured: ${now.toISOString()}\n\n${lines.join('\n')}\n`)) {
|
|
257
|
+
record.files.push(name);
|
|
258
|
+
Atomics.store(view, 4, heartbeat);
|
|
145
259
|
}
|
|
146
|
-
|
|
147
|
-
|
|
260
|
+
else {
|
|
261
|
+
record.errors.push('Failed to save blocking stack.');
|
|
148
262
|
}
|
|
149
|
-
capturing = false;
|
|
150
|
-
return;
|
|
151
263
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
fs.mkdirSync(diagDir, {
|
|
156
|
-
'recursive': true, 'mode': 0o700,
|
|
157
|
-
});
|
|
158
|
-
fs.chmodSync(diagDir, 0o700);
|
|
159
|
-
const content = `Event Loop Blocked: ${blockSec}s\n` +
|
|
160
|
-
`Captured: ${new Date().toISOString()}\n` +
|
|
161
|
-
`PID: ${data.pid}\n\n` +
|
|
162
|
-
`Call Stack:\n${stackLines.join('\n')}\n`;
|
|
163
|
-
fs.writeFileSync(path.join(diagDir, `blocked-stack-${ts}.txt`), content, { 'mode': 0o600 });
|
|
164
|
-
fs.chmodSync(path.join(diagDir, `blocked-stack-${ts}.txt`), 0o600);
|
|
165
|
-
}
|
|
166
|
-
catch {
|
|
167
|
-
// --- 忽略 ---
|
|
168
|
-
}
|
|
169
|
-
// --- 采集 CPU Profile ---
|
|
170
|
-
collectProfile(session, diagDir, ts);
|
|
171
|
-
});
|
|
172
|
-
});
|
|
173
|
-
});
|
|
174
|
-
session.post('Debugger.enable', (err) => {
|
|
175
|
-
if (err || timedOut) {
|
|
176
|
-
clearTimeout(pauseTimeout);
|
|
264
|
+
else {
|
|
265
|
+
record.errors.push('No blocking stack received before timeout.');
|
|
266
|
+
}
|
|
177
267
|
if (!timedOut) {
|
|
178
|
-
session.
|
|
179
|
-
capturing = false;
|
|
268
|
+
await session.post('Debugger.disable');
|
|
180
269
|
}
|
|
181
|
-
return;
|
|
182
270
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
session.post('Debugger.disable');
|
|
190
|
-
}
|
|
191
|
-
catch {
|
|
192
|
-
// --- 忽略 ---
|
|
193
|
-
}
|
|
194
|
-
session.disconnect();
|
|
195
|
-
capturing = false;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
catch {
|
|
201
|
-
clearTimeout(pauseTimeout);
|
|
202
|
-
try {
|
|
203
|
-
session.disconnect();
|
|
204
|
-
}
|
|
205
|
-
catch {
|
|
206
|
-
// --- 忽略 ---
|
|
271
|
+
catch (error) {
|
|
272
|
+
record.errors.push(`Stack: ${String(error)}`);
|
|
273
|
+
// --- 栈采集失败仍尝试恢复主线程,CPU Profile 可以独立完成 ---
|
|
274
|
+
if (!timedOut) {
|
|
275
|
+
await session.post('Debugger.resume').catch(() => undefined);
|
|
276
|
+
await session.post('Debugger.disable').catch(() => undefined);
|
|
207
277
|
}
|
|
208
|
-
capturing = false;
|
|
209
278
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
* --- 采集 CPU Profile ---
|
|
214
|
-
* @param session Inspector 会话
|
|
215
|
-
* @param diagDir 诊断输出目录
|
|
216
|
-
* @param ts 时间戳字符串
|
|
217
|
-
*/
|
|
218
|
-
function collectProfile(session, diagDir, ts) {
|
|
219
|
-
session.post('Profiler.enable', (err) => {
|
|
220
|
-
if (err) {
|
|
221
|
-
session.disconnect();
|
|
222
|
-
capturing = false;
|
|
223
|
-
return;
|
|
279
|
+
const remaining = data.profileDuration - (Number(process.hrtime.bigint() / 1000000n) - profileStarted);
|
|
280
|
+
if (remaining > 0) {
|
|
281
|
+
await new Promise(resolve => setTimeout(resolve, remaining));
|
|
224
282
|
}
|
|
225
|
-
session.post('Profiler.
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
}
|
|
249
|
-
|
|
283
|
+
const { profile } = await session.post('Profiler.stop');
|
|
284
|
+
const name = `blocked-cpu-${id}.cpuprofile`;
|
|
285
|
+
if (writeFile(dir, name, JSON.stringify(profile))) {
|
|
286
|
+
record.files.push(name);
|
|
287
|
+
Atomics.store(view, 4, heartbeat);
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
record.errors.push('Failed to save CPU profile.');
|
|
291
|
+
}
|
|
292
|
+
record.status = record.errors.length ? 'partial' : 'complete';
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
record.status = record.files.length ? 'partial' : 'failed';
|
|
296
|
+
record.errors.push(timedOut ? 'Inspector collection timed out.' : String(error));
|
|
297
|
+
}
|
|
298
|
+
finally {
|
|
299
|
+
clearTimeout(timeout);
|
|
300
|
+
session.disconnect();
|
|
301
|
+
Atomics.compareExchange(view, 2, 2n, 0n);
|
|
302
|
+
capturing = false;
|
|
303
|
+
if (!writeFile(dir, 'blocked-event.json', JSON.stringify(record, null, 2))) {
|
|
304
|
+
record.errors.push('Failed to update event summary.');
|
|
305
|
+
record.status = 'partial';
|
|
306
|
+
}
|
|
307
|
+
writeLog(`WATCHDOG: Diagnostic ${record.status}, PID:${data.pid}, EVENT:${dir}`);
|
|
308
|
+
workerThreads.parentPort?.postMessage({ 'type': 'diagnostic', 'heartbeat': Number(heartbeat), dir,
|
|
309
|
+
'files': record.files.map(name => path.join(dir, name)), 'errors': record.errors, 'status': record.status });
|
|
310
|
+
}
|
|
250
311
|
}
|
|
251
|
-
// ---
|
|
312
|
+
// --- 每秒独立采样;新心跳代表新事件,不让上一事件的冷却吞掉下一次阻塞 ---
|
|
252
313
|
setInterval(() => {
|
|
253
|
-
const
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
314
|
+
const time = Number(process.hrtime.bigint() / 1000000n);
|
|
315
|
+
const cpu = process.cpuUsage();
|
|
316
|
+
const elapsed = time - lastTime;
|
|
317
|
+
const heartbeat = Atomics.load(view, 0);
|
|
318
|
+
const blockMs = time - Number(heartbeat);
|
|
319
|
+
const cpuPercent = elapsed > 0
|
|
320
|
+
? (cpu.user - lastCpu.user + cpu.system - lastCpu.system) / (elapsed * 1_000) * 100 : 0;
|
|
321
|
+
lastCpu = cpu;
|
|
322
|
+
lastTime = time;
|
|
323
|
+
samples.push({ 'time': Date.now(), elapsed, 'cpu': Math.round(cpuPercent * 100) / 100,
|
|
324
|
+
'rss': process.memoryUsage.rss(), 'heartbeatLag': blockMs });
|
|
325
|
+
if (samples.length > 60) {
|
|
326
|
+
samples.shift();
|
|
257
327
|
}
|
|
258
|
-
|
|
259
|
-
if (Atomics.load(view, 1) === 1) {
|
|
328
|
+
if (heartbeat <= 0n || blockMs < data.threshold || Atomics.load(view, 1) === 1n || Atomics.load(view, 3) === 1n) {
|
|
260
329
|
return;
|
|
261
330
|
}
|
|
262
|
-
if (
|
|
331
|
+
if (capturing || heartbeat === lastAlertHeartbeat && time - lastAlertTime < data.cooldown * 1_000) {
|
|
263
332
|
return;
|
|
264
333
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
334
|
+
lastAlertHeartbeat = heartbeat;
|
|
335
|
+
lastAlertTime = time;
|
|
336
|
+
captureDiag(blockMs, heartbeat).catch((error) => {
|
|
337
|
+
writeLog(`WATCHDOG: Unexpected collection failure: ${String(error)}`);
|
|
338
|
+
});
|
|
270
339
|
}, data.interval);
|
package/sys/monitor.d.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
|
+
/** --- 活跃请求的描述 --- */
|
|
2
|
+
export interface IActiveRequest {
|
|
3
|
+
'url': string;
|
|
4
|
+
'method': string;
|
|
5
|
+
/** --- 请求开始的时间戳(ms) --- */
|
|
6
|
+
'start': number;
|
|
7
|
+
/** --- 请求开始时的 CPU 累计用量基准 --- */
|
|
8
|
+
'startCpu': NodeJS.CpuUsage;
|
|
9
|
+
/** --- 请求开始时的进程整体内存 RSS 基准(bytes) --- */
|
|
10
|
+
'startMem': number;
|
|
11
|
+
}
|
|
1
12
|
/** --- 单个请求快照 --- */
|
|
2
|
-
interface ISnapshotRequest {
|
|
13
|
+
export interface ISnapshotRequest {
|
|
3
14
|
'url': string;
|
|
4
15
|
'method': string;
|
|
5
16
|
'duration': number;
|
|
@@ -9,6 +20,8 @@ interface ISnapshotRequest {
|
|
|
9
20
|
'cpuSystem': number;
|
|
10
21
|
/** --- 请求存续期间的进程整体 RSS 增量(bytes),不代表请求独占 --- */
|
|
11
22
|
'memDelta': number;
|
|
23
|
+
/** --- 已完成请求的结束时间,活跃请求不含此字段 --- */
|
|
24
|
+
'completedAt'?: number;
|
|
12
25
|
}
|
|
13
26
|
/** --- 整体资源快照 --- */
|
|
14
27
|
export interface ISnapshot {
|
|
@@ -35,14 +48,19 @@ export interface ISnapshot {
|
|
|
35
48
|
'free': number;
|
|
36
49
|
};
|
|
37
50
|
'eloopLag': number;
|
|
51
|
+
/** --- 本采样窗口最大延迟,用于复盘单次卡顿;持续告警仍按 P99 判断 --- */
|
|
52
|
+
'eloopMax': number;
|
|
38
53
|
/** --- 最早开始的活跃请求,最多 100 条 --- */
|
|
39
54
|
'activeRequests': ISnapshotRequest[];
|
|
40
55
|
/** --- 全部活跃请求数量,可能大于 activeRequests.length --- */
|
|
41
56
|
'activeCount': number;
|
|
57
|
+
/** --- 最近一分钟完成的请求,最多 100 条;资源增量均为整个进程的增量 --- */
|
|
58
|
+
'recentRequests': ISnapshotRequest[];
|
|
42
59
|
}
|
|
43
60
|
/**
|
|
44
61
|
* --- 启动性能监控 ---
|
|
45
62
|
* @param opt 可选的阈值配置
|
|
63
|
+
* @returns 是否成功启动,配置无效时返回 false
|
|
46
64
|
*/
|
|
47
65
|
export declare function start(opt?: {
|
|
48
66
|
/** --- CPU 使用率阈值百分比,默认 80 --- */
|
|
@@ -53,24 +71,31 @@ export declare function start(opt?: {
|
|
|
53
71
|
'eloop'?: number;
|
|
54
72
|
/** --- 内存超阈值时是否采集堆快照,默认 false --- */
|
|
55
73
|
'heapSnapshot'?: boolean;
|
|
56
|
-
|
|
74
|
+
/** --- 同一指标持续超阈值的确认时间 ms,默认 10000 --- */
|
|
75
|
+
'duration'?: number;
|
|
76
|
+
/** --- 主线程心跳中断的确认时间 ms,默认 5000,最小 1000 --- */
|
|
77
|
+
'blocked'?: number;
|
|
78
|
+
}): boolean;
|
|
57
79
|
/**
|
|
58
80
|
* --- 停止性能监控 ---
|
|
81
|
+
* @returns 无返回值
|
|
59
82
|
*/
|
|
60
83
|
export declare function stop(): void;
|
|
61
84
|
/**
|
|
62
85
|
* --- 注册一个活跃请求,返回追踪 ID ---
|
|
63
86
|
* @param url 请求 URL
|
|
64
87
|
* @param method 请求方法
|
|
88
|
+
* @returns 追踪 ID
|
|
65
89
|
*/
|
|
66
90
|
export declare function track(url: string, method: string): string;
|
|
67
91
|
/**
|
|
68
92
|
* --- 移除已完成的请求追踪 ---
|
|
69
93
|
* @param id 追踪 ID
|
|
94
|
+
* @returns 无返回值
|
|
70
95
|
*/
|
|
71
96
|
export declare function untrack(id: string): void;
|
|
72
97
|
/**
|
|
73
98
|
* --- 获取当前资源快照 ---
|
|
99
|
+
* @returns 资源与请求快照
|
|
74
100
|
*/
|
|
75
101
|
export declare function getSnapshot(): ISnapshot;
|
|
76
|
-
export {};
|