@maiyunnet/kebab 9.21.0 → 9.21.2

package/sys/monitor.js

@@ -1,8 +1,8 @@
 /**
  * Project: Kebab, User: JianSuoQiYue
  * Date: 2026-02-07
- * Last: 2026-08-22
- * --- 性能监控库，用于检测 CPU/内存骤升并记录可疑请求、堆栈、CPU Profile、堆快照 ---
+ * Last: 2026-10-05
+ * --- 性能监控库，用于确认 CPU/内存持续超阈值，保存相关请求、资源采样和诊断文件 ---
  * --- 包含 Worker 看门狗线程，用于在事件循环完全阻塞时实时检测并记录 ---
  */
 import * as os from 'os';
@@ -24,13 +24,13 @@ let memThreshold = 0;
 let eloopThreshold = 500;
 /** --- 是否在内存超阈值时自动采集堆快照 --- */
 let heapSnapshotEnabled = false;
-/** --- 监控间隔，单位 ms --- */
-const INTERVAL = 5_000;
-/** --- 连续超阈值次数达到此值才记录日志，防止瞬间波动 --- */
-const SPIKE_COUNT = 2;
+/** --- 监控间隔，单位 ms；主线程被阻塞时由 Worker 独立采样 --- */
+const INTERVAL = 1_000;
+/** --- 同一指标持续超阈值的确认时间，单位 ms --- */
+let spikeDuration = 10_000;
 /** --- CPU Profile 采集时长，单位 ms --- */
 const PROFILE_DURATION = 5_000;
-/** --- 两次诊断采集的最小间隔，防止频繁写磁盘，单位 ms --- */
+/** --- 同一持续异常重复采集的间隔，单位 ms；新事件或指标复发不受此限制 --- */
 const DIAGNOSTIC_COOLDOWN = 60_000;
 /** --- 单次快照或日志最多输出的活跃请求详情数 --- */
 const MAX_ACTIVE_REQUEST_DETAILS = 100;
@@ -51,33 +51,43 @@ let lastSnapshotCpuTime = 0;
 let lastSnapshotOsCpus = null;
 /** --- 事件循环延迟直方图 --- */
 let eloopHistogram = null;
-/** --- 连续超阈值计数 --- */
-let spikeCounter = 0;
+/** --- 各指标首次超阈值的单调时钟时间，不能跨指标累计 --- */
+let spikeSince = {
+    'cpu': null, 'mem': null, 'eloop': null,
+};
 /** --- 请求自增计数器 --- */
 let requestCounter = 0;
-/** --- 上次诊断采集时间 --- */
-let lastDiagnosticTime = 0;
 /** --- 是否正在采集诊断数据 --- */
 let diagnosing = false;
 /** --- 是否正在进行 CPU Profile 采集 --- */
 let profiling = false;
+/** --- 同步 Report/HeapSnapshot 会影响本采样窗口，完成后重新建立检测基准 --- */
+let diagnosticOverhead = false;
 /**
  * --- 检测当前是否处于调试模式（以 --inspect 启动或 IDE 调试器已连接） ---
+ * @returns 是否处于调试模式
  */
 function isDebugMode() {
     return !!inspector.url();
 }
+/**
+ * --- 获取单调时钟毫秒数，避免系统校时影响 CPU 比例和心跳 ---
+ * @returns 单调时钟毫秒数
+ */
+function clockMs() {
+    return Number(process.hrtime.bigint() / 1000000n);
+}
 // --- 看门狗相关 ---
 /** --- 看门狗 Worker 实例 --- */
 let watchdog = null;
-/** --- 心跳共享内存（Uint32: 秒级时间戳） --- */
+/** --- 心跳共享内存，协议见 monitor/watchdog.ts --- */
 let heartbeatBuffer = null;
 /** --- 心跳共享内存视图 --- */
 let heartbeatView = null;
-/** --- 看门狗检测阈值，主线程超过此秒数未更新心跳则认为阻塞，单位秒 --- */
-const WATCHDOG_THRESHOLD = 15;
+/** --- 主线程持续未更新心跳的确认时间，单位 ms --- */
+let watchdogThreshold = 5_000;
 /** --- 看门狗检测间隔，单位 ms --- */
-const WATCHDOG_INTERVAL = 5_000;
+const WATCHDOG_INTERVAL = 1_000;
 /** --- 看门狗告警冷却时间，持续阻塞时不会每次都写日志，单位秒 --- */
 const WATCHDOG_COOLDOWN = 30;
 /** --- 看门狗异常退出后的重启延迟，单位 ms --- */
@@ -86,29 +96,52 @@ const WATCHDOG_RESTART_DELAY = 5_000;
 let watchdogRestartTimer = null;
 /** --- 活跃请求池 --- */
 const activeRequests = new Map();
+/** --- 有界保存最近完成的请求，避免采集时请求已经移出活跃池 --- */
+const recentRequests = [];
+/** --- 最近一分钟的资源采样，不包含请求参数或请求体 --- */
+const sampleHistory = [];
+let lastWatchdogDiagnostic = null;
+const watchdogIncidents = new Map();
+/** --- 当前持续异常及其上次重型诊断时间 --- */
+let incident = null;
+let lastDiagnosticTime = 0;
+let lastEventSaveTime = 0;
+/** --- 串行写入事件摘要，避免初次保存覆盖恢复或采集结果 --- */
+let eventWrites = Promise.resolve();
 /**
  * --- 启动性能监控 ---
  * @param opt 可选的阈值配置
+ * @returns 是否成功启动，配置无效时返回 false
  */
 export function start(opt) {
     if (timer) {
-        return;
+        return true;
     }
     const nextCpuThreshold = opt?.cpu ?? 80;
     const nextMemThreshold = opt?.mem ?? 0;
     const nextEloopThreshold = opt?.eloop ?? 500;
+    const nextDuration = opt?.duration ?? 10_000;
+    const nextBlocked = opt?.blocked ?? 5_000;
     if (!Number.isFinite(nextCpuThreshold) || nextCpuThreshold <= 0) {
-        throw new RangeError('Monitor CPU threshold must be greater than 0.');
+        return false;
     }
     if (!Number.isFinite(nextMemThreshold) || nextMemThreshold < 0) {
-        throw new RangeError('Monitor memory threshold cannot be negative.');
+        return false;
     }
     if (!Number.isFinite(nextEloopThreshold) || nextEloopThreshold <= 0) {
-        throw new RangeError('Monitor event loop threshold must be greater than 0.');
+        return false;
+    }
+    if (!Number.isFinite(nextDuration) || nextDuration < INTERVAL) {
+        return false;
+    }
+    if (!Number.isFinite(nextBlocked) || nextBlocked < INTERVAL) {
+        return false;
     }
     cpuThreshold = nextCpuThreshold;
     memThreshold = nextMemThreshold;
     eloopThreshold = nextEloopThreshold;
+    spikeDuration = nextDuration;
+    watchdogThreshold = nextBlocked;
     heapSnapshotEnabled = opt?.heapSnapshot ?? false;
     if (memThreshold === 0) {
         // --- 同时考虑容器/系统约束和 V8 堆上限，避免默认阈值高于进程实际可用范围 ---
@@ -120,7 +153,7 @@ export function start(opt) {
         memThreshold = Math.floor(Math.min(systemLimit * 0.8, heapLimit * 0.9) / 1024 / 1024);
     }
     const cpuUsage = process.cpuUsage();
-    const cpuTime = Date.now();
+    const cpuTime = clockMs();
     const osCpus = os.cpus();
     lastCheckCpuUsage = cpuUsage;
     lastCheckCpuTime = cpuTime;
@@ -128,13 +161,19 @@ export function start(opt) {
     lastSnapshotCpuUsage = cpuUsage;
     lastSnapshotCpuTime = cpuTime;
     lastSnapshotOsCpus = osCpus;
-    spikeCounter = 0;
+    spikeSince = { 'cpu': null, 'mem': null, 'eloop': null };
+    sampleHistory.length = 0;
+    recentRequests.length = 0;
+    incident = null;
+    lastWatchdogDiagnostic = null;
+    watchdogIncidents.clear();
+    diagnosticOverhead = false;
     lastDiagnosticTime = 0;
-    // --- 初始化心跳共享内存（索引 0: 秒级时间戳, 索引 1: 调试模式标志） ---
-    heartbeatBuffer = new SharedArrayBuffer(8);
-    heartbeatView = new Uint32Array(heartbeatBuffer);
-    Atomics.store(heartbeatView, 0, Math.floor(Date.now() / 1000));
-    Atomics.store(heartbeatView, 1, isDebugMode() ? 1 : 0);
+    // --- 0: 单调时钟心跳, 1: 调试标志, 2: Inspector 占用, 3: 主线程诊断开销, 4: 已采集阻塞的心跳 ---
+    heartbeatBuffer = new SharedArrayBuffer(40);
+    heartbeatView = new BigInt64Array(heartbeatBuffer);
+    Atomics.store(heartbeatView, 0, BigInt(cpuTime));
+    Atomics.store(heartbeatView, 1, isDebugMode() ? 1n : 0n);
     // --- 启用事件循环延迟直方图 ---
     eloopHistogram = perfHooks.monitorEventLoopDelay({ 'resolution': 20 });
     eloopHistogram.enable();
@@ -142,11 +181,20 @@ export function start(opt) {
     // --- 启动看门狗线程 ---
     startWatchdog();
     lCore.debug(`[MONITOR] [PARENT] [${process.pid}] Started, CPU Threshold: ${cpuThreshold}%, MEM Threshold: ${memThreshold}MB, ELOOP Threshold: ${eloopThreshold}ms`);
+    return true;
 }
 /**
  * --- 停止性能监控 ---
+ * @returns 无返回值
  */
 export function stop() {
+    watchdogIncidents.clear();
+    if (incident) {
+        incident.endedAt = Date.now();
+        incident.endReason = 'stopped';
+        saveIncident(incident);
+        incident = null;
+    }
     if (timer) {
         clearInterval(timer);
         timer = null;
@@ -166,19 +214,22 @@ export function stop() {
         watchdogRestartTimer = null;
     }
     activeRequests.clear();
+    recentRequests.length = 0;
+    sampleHistory.length = 0;
     lastCheckCpuUsage = null;
     lastCheckCpuTime = 0;
     lastCheckOsCpus = null;
     lastSnapshotCpuUsage = null;
     lastSnapshotCpuTime = 0;
     lastSnapshotOsCpus = null;
-    spikeCounter = 0;
+    spikeSince = { 'cpu': null, 'mem': null, 'eloop': null };
     heartbeatBuffer = null;
     heartbeatView = null;
 }
 /**
  * --- 启动看门狗 Worker 线程，独立事件循环监控主线程心跳 ---
  * --- 实现代码见 monitor/watchdog.ts ---
+ * @returns 无返回值
  */
 function startWatchdog() {
     if (!heartbeatBuffer) {
@@ -190,18 +241,35 @@ function startWatchdog() {
             'workerData': {
                 'buffer': heartbeatBuffer,
                 'logDir': kebab.LOG_CWD,
+                'logFormat': lCore.globalConfig.logFormat ?? 'jsonl',
                 'pid': process.pid,
-                'threshold': WATCHDOG_THRESHOLD,
+                'threshold': watchdogThreshold,
                 'cooldown': WATCHDOG_COOLDOWN,
                 'interval': WATCHDOG_INTERVAL,
                 'profileDuration': PROFILE_DURATION,
+                'requests': Array.from(activeRequests.entries()),
             },
         });
         watchdog = worker;
         // --- 不阻止进程退出 ---
         worker.unref();
+        worker.on('message', (message) => {
+            if (watchdog !== worker || message.type !== 'diagnostic') {
+                return;
+            }
+            lastWatchdogDiagnostic = message;
+            const pending = watchdogIncidents.get(message.heartbeat);
+            if (pending) {
+                pending.result.status = `watchdog-${message.status}`;
+                pending.result.files = [`${message.dir}/blocked-event.json`, ...message.files];
+                pending.result.errors.push(...message.errors);
+                watchdogIncidents.delete(message.heartbeat);
+                saveIncident(pending.event);
+            }
+        });
         worker.on('error', (err) => {
-            lCore.debug('[MONITOR] Watchdog error', err);
+            lCore.log({}, `WATCHDOG_ERROR ${lText.stringifyError(err)}`, '-monitor');
+            lCore.display('[MONITOR] Watchdog error', err);
         });
         worker.on('exit', (exitCode) => {
             if (exitCode !== 0 && timer) {
@@ -216,12 +284,14 @@ function startWatchdog() {
         lCore.debug(`[MONITOR] [THREAD] [${process.pid}] Watchdog started`);
     }
     catch (e) {
-        lCore.debug('[MONITOR] Failed to start watchdog', e);
+        lCore.log({}, `WATCHDOG_START_FAILED ${lText.stringifyError(e)}`, '-monitor');
+        lCore.display('[MONITOR] Failed to start watchdog', e);
         scheduleWatchdogRestart();
     }
 }
 /**
  * --- 看门狗异常退出后延迟重启，避免失去阻塞检测能力 ---
+ * @returns 无返回值
  */
 function scheduleWatchdogRestart() {
     if (!timer || watchdog || watchdogRestartTimer) {
@@ -237,6 +307,7 @@ function scheduleWatchdogRestart() {
  * --- 注册一个活跃请求，返回追踪 ID ---
  * @param url 请求 URL
  * @param method 请求方法
+ * @returns 追踪 ID
  */
 export function track(url, method) {
     const now = Date.now();
@@ -250,28 +321,64 @@ export function track(url, method) {
         'startCpu': process.cpuUsage(),
         'startMem': process.memoryUsage.rss(),
     });
+    watchdog?.postMessage({ 'type': 'request', id, 'request': activeRequests.get(id) });
     return id;
 }
 /**
  * --- 移除已完成的请求追踪 ---
  * @param id 追踪 ID
+ * @returns 无返回值
  */
 export function untrack(id) {
+    const req = activeRequests.get(id);
+    if (req) {
+        const now = Date.now();
+        const completed = snapshotRequest(req, now, process.cpuUsage(), process.memoryUsage.rss());
+        completed.completedAt = now;
+        recentRequests.push(completed);
+        trimRecentRequests(now);
+        watchdog?.postMessage({ 'type': 'complete', id, 'request': completed });
+    }
     activeRequests.delete(id);
 }
+/**
+ * --- 获取请求存续期间的进程资源增量，不能据此认定请求独占资源 ---
+ * @param req 追踪中的请求
+ * @param now 当前时间戳 ms
+ * @param cpu 当前进程 CPU 累计用量
+ * @param rss 当前进程 RSS
+ * @returns 请求快照
+ */
+function snapshotRequest(req, now, cpu, rss) {
+    return {
+        'url': req.url, 'method': req.method, 'duration': now - req.start,
+        'cpuUser': cpu.user - req.startCpu.user, 'cpuSystem': cpu.system - req.startCpu.system,
+        'memDelta': rss - req.startMem,
+    };
+}
+/**
+ * --- 限制最近请求的保留时长及条数 ---
+ * @param now 当前时间戳 ms
+ * @returns 无返回值
+ */
+function trimRecentRequests(now) {
+    while (recentRequests.length > MAX_ACTIVE_REQUEST_DETAILS ||
+        recentRequests.length && now - (recentRequests[0].completedAt ?? 0) > 60_000) {
+        recentRequests.shift();
+    }
+}
 /**
  * --- 获取当前资源快照 ---
+ * @returns 资源与请求快照
  */
 export function getSnapshot() {
-    const mem = process.memoryUsage();
-    const heapStats = v8.getHeapStatistics();
     const cpuUsage = process.cpuUsage();
-    const now = Date.now();
+    const cpuTime = clockMs();
     // --- 计算进程 CPU 使用率（单核基准） ---
     let cpuProcess = 0;
     if (lastSnapshotCpuUsage && lastSnapshotCpuTime) {
         /** --- 经过的时间（微秒） --- */
-        const elapsed = (now - lastSnapshotCpuTime) * 1_000;
+        const elapsed = (cpuTime - lastSnapshotCpuTime) * 1_000;
         const userDiff = cpuUsage.user - lastSnapshotCpuUsage.user;
         const sysDiff = cpuUsage.system - lastSnapshotCpuUsage.system;
         if (elapsed > 0) {
@@ -279,15 +386,30 @@ export function getSnapshot() {
         }
     }
     lastSnapshotCpuUsage = cpuUsage;
-    lastSnapshotCpuTime = now;
+    lastSnapshotCpuTime = cpuTime;
     // --- 计算系统总 CPU 使用率 ---
     const osCpus = os.cpus();
     const cpuOs = getOsCpuPercent(lastSnapshotOsCpus, osCpus);
     lastSnapshotOsCpus = osCpus;
+    return createSnapshot(Math.round(cpuProcess * 100) / 100, cpuOs);
+}
+/**
+ * --- 组装资源与请求快照，不推进对外查询的 CPU 采样基准 ---
+ * @param cpuProcess 当前采样窗口进程 CPU 百分比
+ * @param cpuOs 当前采样窗口系统 CPU 百分比
+ * @returns 资源快照
+ */
+function createSnapshot(cpuProcess, cpuOs) {
+    const mem = process.memoryUsage();
+    const heapStats = v8.getHeapStatistics();
+    const cpuUsage = process.cpuUsage();
+    const now = Date.now();
     // --- 计算事件循环延迟（P99，纳秒转毫秒） ---
     let eloopLag = 0;
+    let eloopMax = 0;
     if (eloopHistogram) {
         eloopLag = Math.round(eloopHistogram.percentile(99) / 1_000_000);
+        eloopMax = Math.round(eloopHistogram.max / 1_000_000);
     }
     // --- 收集活跃请求 ---
     const requests = [];
@@ -295,16 +417,9 @@ export function getSnapshot() {
         if (requests.length >= MAX_ACTIVE_REQUEST_DETAILS) {
             break;
         }
-        const reqCpu = process.cpuUsage(req.startCpu);
-        requests.push({
-            'url': req.url,
-            'method': req.method,
-            'duration': now - req.start,
-            'cpuUser': reqCpu.user,
-            'cpuSystem': reqCpu.system,
-            'memDelta': mem.rss - req.startMem,
-        });
+        requests.push(snapshotRequest(req, now, cpuUsage, mem.rss));
     }
+    trimRecentRequests(now);
     return {
         'pid': process.pid,
         'time': now,
@@ -327,146 +442,239 @@ export function getSnapshot() {
             'free': os.freemem(),
         },
         'eloopLag': eloopLag,
+        'eloopMax': eloopMax,
         'activeRequests': requests,
         'activeCount': activeRequests.size,
+        'recentRequests': [...recentRequests],
     };
 }
 /**
- * --- 周期性检查，在超阈值时记录日志 ---
+ * --- 周期检查：独立确认各指标，按一次持续异常记录开始、现场及恢复 ---
+ * @returns 无返回值
  */
 function check() {
     const cpuUsage = process.cpuUsage();
     const now = Date.now();
-    // --- 更新心跳时间戳，让看门狗线程知道主线程还活着 ---
-    if (heartbeatView) {
-        Atomics.store(heartbeatView, 0, Math.floor(now / 1000));
-        // --- 更新调试模式标志，让看门狗线程知道当前是否在调试 ---
-        Atomics.store(heartbeatView, 1, isDebugMode() ? 1 : 0);
-    }
-    /** --- 计算本周期进程 CPU 使用率（单核基准） --- */
+    const cpuTime = clockMs();
+    const previousTime = lastCheckCpuTime;
+    const actualElapsed = cpuTime - previousTime;
     let cpuPercent = 0;
-    /** --- 实际经过的时间（ms），用于检测事件循环阻塞 --- */
-    let actualElapsed = INTERVAL;
-    if (lastCheckCpuUsage && lastCheckCpuTime) {
-        actualElapsed = now - lastCheckCpuTime;
-        const elapsedUs = actualElapsed * 1_000;
-        const userDiff = cpuUsage.user - lastCheckCpuUsage.user;
-        const sysDiff = cpuUsage.system - lastCheckCpuUsage.system;
-        if (elapsedUs > 0) {
-            cpuPercent = ((userDiff + sysDiff) / elapsedUs) * 100;
-        }
+    if (lastCheckCpuUsage && actualElapsed > 0) {
+        cpuPercent = (cpuUsage.user - lastCheckCpuUsage.user + cpuUsage.system - lastCheckCpuUsage.system)
+            / (actualElapsed * 1_000) * 100;
     }
     lastCheckCpuUsage = cpuUsage;
-    lastCheckCpuTime = now;
-    // --- 计算系统总 CPU 使用率 ---
+    lastCheckCpuTime = cpuTime;
     const osCpus = os.cpus();
     const cpuOs = getOsCpuPercent(lastCheckOsCpus, osCpus);
     lastCheckOsCpus = osCpus;
-    const memMB = process.memoryUsage.rss() / 1024 / 1024;
-    let eloopLag = 0;
-    if (eloopHistogram) {
-        eloopLag = Math.round(eloopHistogram.percentile(99) / 1_000_000);
-        // --- 重置直方图，避免历史数据影响 ---
-        eloopHistogram.reset();
-    }
-    // --- 检测事件循环阻塞：实际间隔远大于预期说明事件循环曾被阻塞 ---
-    if (actualElapsed > INTERVAL * 3) {
-        // --- 调试模式下断点暂停会导致计时差异，跳过阻塞检测 ---
-        if (isDebugMode()) {
-            spikeCounter = 0;
-            return;
+    if (heartbeatView) {
+        Atomics.store(heartbeatView, 0, BigInt(cpuTime));
+        Atomics.store(heartbeatView, 1, isDebugMode() ? 1n : 0n);
+    }
+    const snapshot = createSnapshot(Math.round(cpuPercent * 100) / 100, cpuOs);
+    // --- 采样时同步配置，框架 reload 后看门狗也使用当前日志格式 ---
+    watchdog?.postMessage({ 'type': 'snapshot', snapshot, 'logFormat': lCore.globalConfig.logFormat ?? 'jsonl' });
+    eloopHistogram?.reset();
+    // --- 调试暂停和同步诊断不能冒充业务阻塞，也不能连接暂停前后的异常计数 ---
+    if (isDebugMode()) {
+        spikeSince = { 'cpu': null, 'mem': null, 'eloop': null };
+        return;
+    }
+    if (diagnosticOverhead) {
+        diagnosticOverhead = false;
+        // --- 已确认事件继续等待正常采样恢复，未确认的连续性重新计时 ---
+        if (!incident?.reasons.includes('PROC_CPU')) {
+            spikeSince.cpu = null;
+        }
+        if (!incident?.reasons.includes('MEM')) {
+            spikeSince.mem = null;
+        }
+        if (!incident?.reasons.includes('ELOOP_LAG')) {
+            spikeSince.eloop = null;
         }
-        const blockMs = actualElapsed - INTERVAL;
-        const alerts = [
-            `ELOOP_BLOCKED ${Math.round(blockMs)}ms`
-        ];
-        // --- 事件循环阻塞是严重事件，跳过 spikeCounter 直接记录 ---
-        logSpike(alerts, cpuPercent, cpuOs, eloopLag, true);
-        spikeCounter = 0;
         return;
     }
-    // --- 判断是否超阈值 ---
+    const sample = {
+        'time': now, 'elapsed': actualElapsed, 'cpu': snapshot.cpuProcess, 'cpuOs': cpuOs,
+        'rss': snapshot.mem.rss, 'heapUsed': snapshot.mem.heapUsed,
+        'external': snapshot.mem.external, 'arrayBuffers': snapshot.mem.arrayBuffers,
+        'eloop': snapshot.eloopLag, 'eloopMax': snapshot.eloopMax,
+    };
+    sampleHistory.push(sample);
+    while (sampleHistory.length > 60) {
+        sampleHistory.shift();
+    }
     const alerts = [];
-    if (cpuPercent >= cpuThreshold) {
-        alerts.push(`PROC_CPU ${Math.round(cpuPercent * 100) / 100}% >= ${cpuThreshold}%`);
-    }
-    if (memMB >= memThreshold) {
-        alerts.push(`MEM ${Math.round(memMB)}MB >= ${memThreshold}MB`);
-    }
-    if (eloopLag >= eloopThreshold) {
-        alerts.push(`ELOOP_LAG ${eloopLag}ms >= ${eloopThreshold}ms`);
-    }
-    if (alerts.length) {
-        ++spikeCounter;
-        if (spikeCounter >= SPIKE_COUNT) {
-            // --- 超阈值持续达标，记录日志 ---
-            logSpike(alerts, cpuPercent, cpuOs, eloopLag);
-            // --- 重置计数，防止持续刷日志 ---
-            spikeCounter = 0;
-        }
+    const blocked = actualElapsed - INTERVAL >= watchdogThreshold;
+    if (blocked) {
+        alerts.push('ELOOP_BLOCKED');
+        // --- 延迟检查只能证明这一窗口发生过阻塞，不能证明资源持续超阈值 ---
+        spikeSince = { 'cpu': null, 'mem': null, 'eloop': null };
     }
     else {
-        spikeCounter = 0;
+        if (snapshot.cpuProcess >= cpuThreshold) {
+            spikeSince.cpu ??= previousTime;
+            if (cpuTime - spikeSince.cpu >= spikeDuration) {
+                alerts.push('PROC_CPU');
+            }
+        }
+        else {
+            spikeSince.cpu = null;
+        }
+        if (snapshot.mem.rss / 1024 / 1024 >= memThreshold) {
+            // --- 内存是时点值，首次超阈值之前的一个采样周期不能计入持续时间 ---
+            spikeSince.mem ??= cpuTime;
+            if (cpuTime - spikeSince.mem >= spikeDuration) {
+                alerts.push('MEM');
+            }
+        }
+        else {
+            spikeSince.mem = null;
+        }
+        if (snapshot.eloopLag >= eloopThreshold) {
+            spikeSince.eloop ??= previousTime;
+            if (cpuTime - spikeSince.eloop >= spikeDuration) {
+                alerts.push('ELOOP_LAG');
+            }
+        }
+        else {
+            spikeSince.eloop = null;
+        }
     }
+    if (!alerts.length) {
+        if (incident) {
+            incident.endedAt = now;
+            incident.endReason = 'recovered';
+            incident.activeReasons = [];
+            incident.recoveredSnapshot = snapshot;
+            saveIncident(incident);
+            lCore.log({}, `RECOVERED PID:${process.pid} EVENT:${incident.dir}`, '-monitor');
+            incident = null;
+        }
+        return;
+    }
+    recordIncident(alerts, snapshot, sample, blocked, previousTime);
 }
 /**
- * --- 将骤升信息写入日志，并采集诊断数据 ---
- * @param alerts 告警描述
- * @param cpuPercent 当前 CPU 百分比
- * @param cpuOs 系统 CPU 百分比
- * @param eloopLag 当前事件循环延迟
- * @param blocked 是否为事件循环阻塞后触发（watchdog 已在阻塞期间远程采集诊断）
+ * --- 串行保存事件摘要，重型诊断失败不影响基础现场保存 ---
+ * @param event 要保存的事件
+ * @returns 无返回值
  */
-function logSpike(alerts, cpuPercent, cpuOs, eloopLag, blocked = false) {
-    const mem = process.memoryUsage();
-    const heapStats = v8.getHeapStatistics();
-    // --- 收集活跃请求详情 ---
-    const now = Date.now();
-    const requestDetails = [];
-    for (const [, req] of activeRequests) {
-        if (requestDetails.length >= MAX_ACTIVE_REQUEST_DETAILS) {
-            break;
+function saveIncident(event) {
+    const content = lText.stringifyJson(event, 2);
+    eventWrites = eventWrites.then(async () => {
+        if (!await lFs.mkdir(event.dir, 0o700) || !await lFs.chmod(event.dir, 0o700) ||
+            !await lFs.putContent(`${event.dir}event.json`, content, { 'mode': 0o600 }) ||
+            !await lFs.chmod(`${event.dir}event.json`, 0o600)) {
+            lCore.display('[MONITOR] Failed to save event:', event.dir);
+        }
+    }).catch((error) => {
+        lCore.display('[MONITOR] Failed to save event:', error);
+    });
+}
+/**
+ * --- 记录一次已确认异常；同一事件定期更新，新事件不受上一事件冷却限制 ---
+ * @param alerts 已确认的指标
+ * @param snapshot 检测现场快照
+ * @param sample 当前采样
+ * @param blocked 是否刚从主线程阻塞恢复
+ * @param previousTime 本采样窗口起点的单调时钟毫秒数
+ * @returns 无返回值
+ */
+function recordIncident(alerts, snapshot, sample, blocked, previousTime) {
+    const now = snapshot.time;
+    const first = incident === null;
+    if (!incident) {
+        const date = new Date(now);
+        const dir = `${kebab.LOG_CWD}monitor/${lTime.format(null, 'Y/m/d/His', date)}-pid-${process.pid}-${now}/`;
+        incident = {
+            dir, 'startedAt': now - Math.max(0, clockMs() - Math.min(spikeSince.cpu ?? Infinity, spikeSince.mem ?? Infinity, spikeSince.eloop ?? Infinity, blocked ? previousTime : Infinity)),
+            'confirmedAt': now, 'endedAt': null, 'endReason': null, 'reasons': [], 'activeReasons': [], 'impact': 'resource-high',
+            'thresholds': {
+                'cpu': cpuThreshold, 'mem': memThreshold, 'eloop': eloopThreshold,
+                'duration': spikeDuration, 'blocked': watchdogThreshold,
+            },
+            'before': [...sampleHistory], 'samples': [],
+            'peaks': { 'cpu': 0, 'rss': 0, 'heapUsed': 0, 'eloop': 0 },
+            snapshot, 'diagnostics': [],
+        };
+        lastDiagnosticTime = 0;
+        lastEventSaveTime = 0;
+    }
+    const event = incident;
+    // --- 内存事件尚未恢复时，CPU 恢复后再次升高仍是新的采集时机 ---
+    const newReason = alerts.some(alert => !event.activeReasons.includes(alert));
+    event.activeReasons = [...alerts];
+    for (const alert of alerts) {
+        if (!event.reasons.includes(alert)) {
+            event.reasons.push(alert);
+        }
+    }
+    if (alerts.includes('ELOOP_BLOCKED') || alerts.includes('ELOOP_LAG')) {
+        event.impact = 'event-loop-delayed';
+    }
+    event.samples.push(sample);
+    if (event.samples.length > 300) {
+        event.samples.shift();
+    }
+    event.peaks.cpu = Math.max(event.peaks.cpu, sample.cpu);
+    event.peaks.rss = Math.max(event.peaks.rss, sample.rss);
+    event.peaks.heapUsed = Math.max(event.peaks.heapUsed, sample.heapUsed);
+    event.peaks.eloop = Math.max(event.peaks.eloop, sample.eloopMax, sample.elapsed - INTERVAL);
+    if (first || newReason) {
+        const msg = `RESOURCE_HIGH [${alerts.join(', ')}] PID:${process.pid} ` +
+            `PROC_CPU:${snapshot.cpuProcess}% RSS:${lText.sizeFormat(snapshot.mem.rss, '')} ` +
+            `ELOOP_LAG:${snapshot.eloopLag}ms ACTIVE_REQ:${snapshot.activeCount} EVENT:${event.dir}`;
+        lCore.log({}, msg, '-monitor');
+        lCore.display('[MONITOR]', msg);
+    }
+    if (first || newReason || clockMs() - lastDiagnosticTime >= DIAGNOSTIC_COOLDOWN) {
+        lastDiagnosticTime = clockMs();
+        const result = { 'time': now, 'status': 'pending', 'files': [], 'errors': [], snapshot };
+        event.diagnostics.push(result);
+        if (event.diagnostics.length > 60) {
+            event.diagnostics.shift();
+        }
+        const view = heartbeatView;
+        // --- 只有看门狗确实保存过这一窗口的现场时，才省略恢复后的重复 Profile ---
+        const watchdogCaptured = blocked && view !== null && Atomics.load(view, 4) >= BigInt(previousTime);
+        if (watchdogCaptured) {
+            const heartbeat = Number(Atomics.load(view, 4));
+            if (lastWatchdogDiagnostic?.heartbeat === heartbeat) {
+                result.status = `watchdog-${lastWatchdogDiagnostic.status}`;
+                result.files = [`${lastWatchdogDiagnostic.dir}/blocked-event.json`, ...lastWatchdogDiagnostic.files];
+                result.errors.push(...lastWatchdogDiagnostic.errors);
+            }
+            else {
+                result.status = 'watchdog-captured';
+                watchdogIncidents.set(heartbeat, { event, result });
+            }
+        }
+        else if (diagnosing || view && Atomics.load(view, 2) !== 0n) {
+            result.status = 'skipped-busy';
+            lastDiagnosticTime = clockMs() - DIAGNOSTIC_COOLDOWN + 5_000;
+        }
+        else {
+            diagnosing = true;
+            const ts = `${lTime.format(null, 'YmdHis', new Date(now))}-${now}`;
+            // --- 阻塞恢复后的 Profile 只能记录后续执行，摘要保留这个采集时机限制 ---
+            if (blocked) {
+                result.errors.push('The blocking stack was not captured; this profile starts after recovery.');
+            }
+            void collectDiagnostics(event.dir, ts, alerts.includes('PROC_CPU') || blocked, alerts.includes('MEM') && heapSnapshotEnabled, result, view).catch((error) => {
+                result.status = 'failed';
+                result.errors.push(lText.stringifyError(error));
+            }).finally(() => {
+                diagnosing = false;
+                saveIncident(event);
+            });
         }
-        const reqCpu = process.cpuUsage(req.startCpu);
-        const duration = now - req.start;
-        const cpuTotal = reqCpu.user + reqCpu.system;
-        const memDelta = mem.rss - req.startMem;
-        requestDetails.push(`[${req.method}] ${req.url} (${duration}ms, PROC_CPU_DELTA: ${cpuTotal}us, PROC_RSS_DELTA: ${memDelta >= 0 ? '+' : '-'}${lText.sizeFormat(Math.abs(memDelta), '')})`);
-    }
-    const omittedRequests = activeRequests.size - requestDetails.length;
-    if (omittedRequests > 0) {
-        requestDetails.push(`... ${omittedRequests} more`);
-    }
-    // --- 诊断采集策略 ---
-    // --- blocked=true：阻塞期间 watchdog 已通过 connectToMainThread() 远程采集了精确堆栈和 Profile ---
-    // --- blocked=false：持续性骤升，此处由主线程自行采集 Report/Profile/HeapSnapshot ---
-    if (!blocked && !diagnosing && now - lastDiagnosticTime >= DIAGNOSTIC_COOLDOWN) {
-        lastDiagnosticTime = now;
-        diagnosing = true;
-        const hasCpuSpike = cpuPercent >= cpuThreshold;
-        const hasMemSpike = (mem.rss / 1024 / 1024) >= memThreshold;
-        const diagnosticTime = new Date();
-        const ts = lTime.format(null, 'YmdHis', diagnosticTime);
-        const diagDir = `${kebab.LOG_CWD}monitor/${lTime.format(null, 'Y/m/d/His', diagnosticTime)}-pid-${process.pid}/`;
-        collectDiagnostics(diagDir, ts, hasCpuSpike, hasMemSpike && heapSnapshotEnabled).catch((e) => {
-            lCore.debug('[MONITOR] Diagnostic collection failed', e);
-        }).finally(() => {
-            diagnosing = false;
-        });
     }
-    const msg = `SPIKE [${alerts.join(', ')}] ` +
-        `PID:${process.pid} ` +
-        `PROC_CPU:${Math.round(cpuPercent * 100) / 100}% ` +
-        `OS_CPU:${cpuOs}% ` +
-        `RSS:${lText.sizeFormat(mem.rss, '')} ` +
-        `HEAP:${lText.sizeFormat(mem.heapUsed, '')}/${lText.sizeFormat(mem.heapTotal, '')} ` +
-        `HEAP_LIMIT:${lText.sizeFormat(heapStats.heap_size_limit, '')} ` +
-        `OS_MEM:${lText.sizeFormat(os.totalmem() - os.freemem(), '')}/${lText.sizeFormat(os.totalmem(), '')} ` +
-        `ELOOP_LAG:${eloopLag}ms ` +
-        `ACTIVE_REQ:${activeRequests.size} ` +
-        `REQUESTS:[${requestDetails.join(' | ')}]`;
-    lCore.log({}, msg, '-monitor');
-    lCore.display('[MONITOR]', msg);
+    if (first || newReason || now - lastEventSaveTime >= 5_000) {
+        lastEventSaveTime = now;
+        saveIncident(event);
+    }
 }
 /**
  * --- 通过 os.cpus() 两次采样的 Delta 计算系统总 CPU 使用率 ---
@@ -499,109 +707,178 @@ function getOsCpuPercent(previous, current) {
     return Math.round(Math.max(0, Math.min(100, percent)) * 100) / 100;
 }
 /**
- * --- 采集诊断报告及按需 Profile，避免并行采集互相污染数据 ---
- * @param dir 诊断文件输出目录
- * @param ts 时间戳字符串
- * @param hasCpuSpike 是否发生 CPU 骤升
+ * --- 独立保存 Report/Profile/HeapSnapshot，单项失败不阻止其他采集 ---
+ * @param dir 诊断目录
+ * @param ts 唯一采集时间标识
+ * @param hasCpuSpike 是否需要 CPU Profile
  * @param collectHeapSnapshot 是否采集堆快照
+ * @param result 写入事件摘要的采集结果
+ * @param view 本次监控实例的共享内存
+ * @returns 无返回值，失败原因保留在 result 中
  */
-async function collectDiagnostics(dir, ts, hasCpuSpike, collectHeapSnapshot) {
-    if (!await lFs.mkdir(dir, 0o700)) {
-        throw new Error(`Failed to create diagnostic directory: ${dir}`);
-    }
-    if (!await lFs.chmod(dir, 0o700)) {
-        throw new Error(`Failed to secure diagnostic directory: ${dir}`);
-    }
-    // --- Diagnostic Report 默认会包含命令行、环境变量和网络端点，落盘前显式脱敏 ---
-    const reportName = `report-${ts}.json`;
-    const report = process.report.getReport();
-    if (report.header) {
-        report.header.commandLine = [];
-        delete report.header.networkInterfaces;
-    }
-    delete report.environmentVariables;
-    for (const handle of report.libuv ?? []) {
-        delete handle.localEndpoint;
-        delete handle.remoteEndpoint;
-    }
-    const reportPath = `${dir}${reportName}`;
-    const written = await lFs.putContent(reportPath, lText.stringifyJson(report, 2), { 'encoding': 'utf8', 'mode': 0o600 });
-    if (!written) {
-        throw new Error(`Failed to write diagnostic report: ${reportPath}`);
-    }
-    if (!await lFs.chmod(reportPath, 0o600)) {
-        throw new Error(`Failed to secure diagnostic report: ${reportPath}`);
-    }
-    lCore.display(`[MONITOR] Diagnostic report: ${reportPath}`);
-    if (hasCpuSpike) {
-        await collectCpuProfile(dir, ts);
-    }
-    // --- 堆快照会同步阻塞并额外占用大量内存，仅在调用方明确开启时采集 ---
-    if (collectHeapSnapshot) {
-        const heapFile = v8.writeHeapSnapshot(`${dir}heap-${ts}.heapsnapshot`);
-        if (!await lFs.chmod(heapFile, 0o600)) {
-            throw new Error(`Failed to secure heap snapshot: ${heapFile}`);
+async function collectDiagnostics(dir, ts, hasCpuSpike, collectHeapSnapshot, result, view) {
+    try {
+        if (!await lFs.mkdir(dir, 0o700) || !await lFs.chmod(dir, 0o700)) {
+            result.status = 'failed';
+            result.errors.push('Failed to create or secure diagnostic directory.');
+            return;
+        }
+        if (view !== heartbeatView) {
+            result.status = 'cancelled';
+            return;
+        }
+        const reportPath = `${dir}report-${ts}.json`;
+        try {
+            // --- 同步诊断本身的阻塞不作为业务异常；Worker 在这段时间只保留采样 ---
+            diagnosticOverhead = true;
+            if (view) {
+                Atomics.store(view, 3, 1n);
+            }
+            const report = process.report.getReport();
+            if (report.header) {
+                report.header.commandLine = [];
+                delete report.header.networkInterfaces;
+            }
+            delete report.environmentVariables;
+            for (const handle of report.libuv ?? []) {
+                delete handle.localEndpoint;
+                delete handle.remoteEndpoint;
+            }
+            if (view) {
+                Atomics.store(view, 0, BigInt(clockMs()));
+                Atomics.store(view, 3, 0n);
+            }
+            if (await lFs.putContent(reportPath, lText.stringifyJson(report, 2), { 'mode': 0o600 }) &&
+                await lFs.chmod(reportPath, 0o600)) {
+                result.files.push(reportPath);
+            }
+            else {
+                result.errors.push('Failed to write or secure diagnostic report.');
+            }
         }
-        lCore.display(`[MONITOR] Heap snapshot: ${heapFile}`);
+        catch (error) {
+            result.errors.push(`Report: ${lText.stringifyError(error)}`);
+        }
+        finally {
+            if (view) {
+                Atomics.store(view, 0, BigInt(clockMs()));
+                Atomics.store(view, 3, 0n);
+            }
+        }
+        if (hasCpuSpike) {
+            if (await collectCpuProfile(dir, ts, result)) {
+                result.files.push(`${dir}cpu-${ts}.cpuprofile`);
+            }
+            else {
+                result.errors.push('CPU profile collection failed.');
+            }
+        }
+        if (view !== heartbeatView) {
+            result.status = 'cancelled';
+            return;
+        }
+        if (collectHeapSnapshot) {
+            // --- 堆快照会额外使用约两倍堆内存，不在剩余内存明显不足时尝试 ---
+            const constrained = process.constrainedMemory();
+            const available = constrained > 0
+                ? Math.min(os.freemem(), Math.max(0, constrained - process.memoryUsage.rss()))
+                : os.freemem();
+            if (available < v8.getHeapStatistics().total_heap_size * 2) {
+                result.errors.push('Heap snapshot skipped: insufficient available memory.');
+            }
+            else {
+                try {
+                    diagnosticOverhead = true;
+                    if (view) {
+                        Atomics.store(view, 3, 1n);
+                    }
+                    const heapFile = v8.writeHeapSnapshot(`${dir}heap-${ts}.heapsnapshot`);
+                    if (view) {
+                        Atomics.store(view, 0, BigInt(clockMs()));
+                        Atomics.store(view, 3, 0n);
+                    }
+                    if (await lFs.chmod(heapFile, 0o600)) {
+                        result.files.push(heapFile);
+                    }
+                    else {
+                        result.errors.push('Failed to secure heap snapshot.');
+                    }
+                }
+                catch (error) {
+                    result.errors.push(`Heap snapshot: ${lText.stringifyError(error)}`);
+                }
+                finally {
+                    if (view) {
+                        Atomics.store(view, 0, BigInt(clockMs()));
+                        Atomics.store(view, 3, 0n);
+                    }
+                }
+            }
+        }
+        result.status = result.errors.length ? 'partial' : 'complete';
+    }
+    catch (error) {
+        result.status = 'failed';
+        result.errors.push(lText.stringifyError(error));
     }
 }
 /**
- * --- 通过 Inspector 协议采集 CPU Profile ---
- * @param dir 诊断文件输出目录
- * @param ts 时间戳字符串
+ * --- 由 Worker 定时采集主线程 Profile，主线程再次阻塞也能按时停止并落盘 ---
+ * @param dir 诊断目录
+ * @param ts 唯一采集时间标识
+ * @param result 接收 Worker 的文件和失败原因
+ * @returns 是否保存成功；无可用 Worker、超时或监控停止时返回 false
  */
-async function collectCpuProfile(dir, ts) {
-    if (profiling) {
-        return;
+function collectCpuProfile(dir, ts, result) {
+    if (profiling || !watchdog) {
+        return Promise.resolve(false);
     }
+    const worker = watchdog;
     profiling = true;
-    const session = new inspector.Session();
-    try {
-        session.connect();
-        await new Promise((resolve, reject) => {
-            session.post('Profiler.enable', (err) => {
-                if (err) {
-                    reject(err);
-                    return;
-                }
-                resolve();
-            });
-        });
-        await new Promise((resolve, reject) => {
-            session.post('Profiler.start', (err) => {
-                if (err) {
-                    reject(err);
-                    return;
+    const file = `cpu-${ts}.cpuprofile`;
+    return new Promise((resolve) => {
+        let finished = false;
+        const onMessage = (message) => {
+            if (message.type === 'profile' && message.file === file) {
+                result.errors.push(...message.errors);
+                if (message.recorded) {
+                    result.files.push(`${dir}profile-${ts}.json`);
                 }
-                resolve();
-            });
-        });
-        // --- 采集指定时长 ---
-        await lCore.sleep(PROFILE_DURATION);
-        const profile = await new Promise((resolve, reject) => {
-            session.post('Profiler.stop', (err, result) => {
-                if (err) {
-                    reject(err);
-                    return;
-                }
-                resolve(result.profile);
-            });
-        });
-        const filePath = `${dir}cpu-${ts}.cpuprofile`;
-        const written = await lFs.putContent(filePath, lText.stringifyJson(profile), {
-            'encoding': 'utf8',
-            'mode': 0o600,
-        });
-        if (!written) {
-            throw new Error(`Failed to write CPU profile: ${filePath}`);
+                finish(message.saved);
+            }
+        };
+        const onExit = () => {
+            result.errors.push('Profile aborted because the watchdog exited.');
+            finish(false);
+        };
+        const timeout = setTimeout(() => {
+            result.errors.push('Profile Worker response timed out.');
+            finish(false);
+        }, PROFILE_DURATION + 15_000);
+        /**
+         * --- 正常响应、退出和超时共享清理，旧 Worker 不会继续持有监听器 ---
+         * @param saved 是否保存成功
+         * @returns 无返回值
+         */
+        function finish(saved) {
+            if (finished) {
+                return;
+            }
+            finished = true;
+            clearTimeout(timeout);
+            worker.off('message', onMessage);
+            worker.off('exit', onExit);
+            profiling = false;
+            resolve(saved);
         }
-        if (!await lFs.chmod(filePath, 0o600)) {
-            throw new Error(`Failed to secure CPU profile: ${filePath}`);
+        worker.on('message', onMessage);
+        worker.once('exit', onExit);
+        try {
+            worker.postMessage({ 'type': 'profile', dir, ts });
         }
-        lCore.display(`[MONITOR] CPU profile: ${filePath}`);
-    }
-    finally {
-        session.disconnect();
-        profiling = false;
-    }
+        catch (error) {
+            lCore.display('[MONITOR] Failed to request CPU profile:', error);
+            finish(false);
+        }
+    });
 }