@glassmkr/crucible 0.7.1 → 0.8.0

package/dist/alerts/__tests__/rules.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/alerts/__tests__/rules.test.js

@@ -0,0 +1,325 @@
+import { describe, it, expect } from "vitest";
+import { allRules } from "../rules.js";
+const baseThresholds = {
+    ram_percent: 90,
+    swap_alert: true,
+    disk_percent: 85,
+    iowait_percent: 20,
+    nvme_wear_percent: 85,
+    disk_latency_nvme_ms: 50,
+    disk_latency_hdd_ms: 200,
+    cpu_temp_warning_c: 80,
+    cpu_temp_critical_c: 90,
+    interface_utilization_percent: 90,
+};
+function emptySnap() {
+    return {
+        collector_version: "test",
+        timestamp: "2026-01-01T00:00:00Z",
+        system: { hostname: "h", ip: "1.2.3.4", os: "linux", kernel: "6.0", uptime_seconds: 1000 },
+        cpu: { user_percent: 0, system_percent: 0, iowait_percent: 0, idle_percent: 100, load_1m: 0, load_5m: 0, load_15m: 0 },
+        memory: { total_mb: 16384, used_mb: 1000, available_mb: 15000, swap_total_mb: 0, swap_used_mb: 0 },
+        disks: [],
+        smart: [],
+        network: [],
+        raid: [],
+        ipmi: { available: false, sensors: [], ecc_errors: { correctable: 0, uncorrectable: 0 }, sel_entries_count: 0, sel_events_recent: [], fans: [] },
+        os_alerts: { oom_kills_recent: 0, zombie_processes: 0, time_drift_ms: 0 },
+    };
+}
+const diskLatencyRule = allRules.find(r => r.type === "disk_latency_high");
+const swapHighRule = allRules.find(r => r.type === "swap_high");
+describe("swap_high (formerly swap_active)", () => {
+    it("does not fire when no swap is in use", () => {
+        const snap = emptySnap();
+        expect(swapHighRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("fires warning when swap is in use", () => {
+        const snap = emptySnap();
+        snap.memory.swap_used_mb = 128;
+        const out = swapHighRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].severity).toBe("warning");
+        expect(out[0].type).toBe("swap_high");
+        expect(out[0].evidence.swap_used_mb).toBe(128);
+    });
+    it("respects t.swap_alert=false", () => {
+        const snap = emptySnap();
+        snap.memory.swap_used_mb = 128;
+        expect(swapHighRule.evaluate(snap, { ...baseThresholds, swap_alert: false })).toEqual([]);
+    });
+});
+const cpuTempRule = allRules.find(r => r.type === "cpu_temperature_high");
+const eccRule = allRules.find(r => r.type === "ecc_errors");
+const psuRule = allRules.find(r => r.type === "psu_redundancy_loss");
+describe("ecc_errors (Dell-style SEL path)", () => {
+    it("fires when SEL ECC counts are higher than named-sensor counts", () => {
+        const snap = emptySnap();
+        snap.ipmi = {
+            available: true, sensors: [],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            ecc_errors_from_sel: { correctable: 3, uncorrectable: 0, newest_event_timestamp: "2026-04-05T14:31:00Z" },
+            sel_entries_count: 3, sel_events_recent: [], fans: [],
+        };
+        const out = eccRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].severity).toBe("warning");
+        expect(out[0].evidence.source).toBe("ipmi_sel");
+        expect(out[0].evidence.correctable).toBe(3);
+    });
+    it("uses named-sensor source when those counts exceed SEL", () => {
+        const snap = emptySnap();
+        snap.ipmi = {
+            available: true, sensors: [],
+            ecc_errors: { correctable: 5, uncorrectable: 0 },
+            ecc_errors_from_sel: { correctable: 0, uncorrectable: 0, newest_event_timestamp: null },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        const out = eccRule.evaluate(snap, baseThresholds);
+        expect(out[0].evidence.source).toBe("ipmi_sensors");
+        expect(out[0].evidence.correctable).toBe(5);
+    });
+    it("does not double-count when both sources are populated; uses max", () => {
+        const snap = emptySnap();
+        snap.ipmi = {
+            available: true, sensors: [],
+            ecc_errors: { correctable: 2, uncorrectable: 0 },
+            ecc_errors_from_sel: { correctable: 5, uncorrectable: 0, newest_event_timestamp: "2026-04-05T14:31:00Z" },
+            sel_entries_count: 5, sel_events_recent: [], fans: [],
+        };
+        const out = eccRule.evaluate(snap, baseThresholds);
+        expect(out[0].evidence.correctable).toBe(5); // max(2, 5)
+    });
+    it("escalates to critical on uncorrectable from SEL", () => {
+        const snap = emptySnap();
+        snap.ipmi = {
+            available: true, sensors: [],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            ecc_errors_from_sel: { correctable: 0, uncorrectable: 1, newest_event_timestamp: "2026-04-05T14:31:00Z" },
+            sel_entries_count: 1, sel_events_recent: [], fans: [],
+        };
+        const out = eccRule.evaluate(snap, baseThresholds);
+        expect(out[0].severity).toBe("critical");
+    });
+    it("does not fire when both sources are zero", () => {
+        const snap = emptySnap();
+        snap.ipmi.available = true;
+        expect(eccRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+});
+describe("psu_redundancy_loss (Dell + Supermicro)", () => {
+    it("fires from aggregate redundancy state on Dell even if individual PS sensors look OK", () => {
+        const snap = emptySnap();
+        snap.dmi = { available: true, vendor: "dell", raw_vendor: "Dell Inc.", product_name: "PowerEdge R740", bios_version: "2.21", bios_date: "2024-08-15", is_virtual: false };
+        snap.ipmi = {
+            available: true,
+            sensors: [
+                { name: "PS1 Status", value: "0x01", unit: "discrete", status: "ok" },
+                { name: "PS2 Status", value: "0x01", unit: "discrete", status: "ok" },
+            ],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            psu_redundancy_state: "redundancy_lost",
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        const out = psuRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.source).toBe("aggregate_sensor");
+        expect(out[0].evidence.redundancy_state).toBe("redundancy_lost");
+    });
+    it("fires from per-PSU status on Dell when redundancy state is not set", () => {
+        const snap = emptySnap();
+        snap.dmi = { available: true, vendor: "dell", raw_vendor: "Dell Inc.", product_name: "PowerEdge R740", bios_version: null, bios_date: null, is_virtual: false };
+        snap.ipmi = {
+            available: true,
+            sensors: [
+                { name: "PS1 Status", value: "0x01", unit: "discrete", status: "ok" },
+                { name: "PS2 Status", value: "Failure detected", unit: "discrete", status: "failure" },
+            ],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        const out = psuRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.source).toBe("per_psu_sensors");
+        expect(out[0].evidence.vendor).toBe("dell");
+    });
+    it("fires on Supermicro PSU1 Status critical (no regression from old behaviour)", () => {
+        const snap = emptySnap();
+        snap.dmi = { available: true, vendor: "supermicro", raw_vendor: "Supermicro", product_name: "X11", bios_version: null, bios_date: null, is_virtual: false };
+        snap.ipmi = {
+            available: true,
+            sensors: [
+                { name: "PSU1 Status", value: "absent", unit: "discrete", status: "absent" },
+                { name: "PSU2 Status", value: "OK", unit: "discrete", status: "ok" },
+            ],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        const out = psuRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.vendor).toBe("supermicro");
+    });
+    it("does not fire on a VM with no PSU sensors", () => {
+        const snap = emptySnap();
+        snap.dmi = { available: true, vendor: "virtual", raw_vendor: "QEMU", product_name: "Standard PC", bios_version: null, bios_date: null, is_virtual: true };
+        snap.ipmi = {
+            available: true, sensors: [], ecc_errors: { correctable: 0, uncorrectable: 0 },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        expect(psuRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("does not fire when fully redundant on Dell", () => {
+        const snap = emptySnap();
+        snap.dmi = { available: true, vendor: "dell", raw_vendor: "Dell Inc.", product_name: "PowerEdge R740", bios_version: null, bios_date: null, is_virtual: false };
+        snap.ipmi = {
+            available: true,
+            sensors: [
+                { name: "PS1 Status", value: "0x01", unit: "discrete", status: "ok" },
+                { name: "PS2 Status", value: "0x01", unit: "discrete", status: "ok" },
+            ],
+            ecc_errors: { correctable: 0, uncorrectable: 0 },
+            psu_redundancy_state: "fully_redundant",
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+        };
+        expect(psuRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+});
+describe("cpu_temperature_high (hwmon path)", () => {
+    it("fires from hwmon when no IPMI is available (Pi)", () => {
+        const snap = emptySnap();
+        snap.thermal = {
+            available: true, source: "hwmon",
+            cpu_readings: [{ label: "cpu_thermal temp1", value_celsius: 85, source_chip: "cpu_thermal", source: "hwmon" }],
+            other_readings: [], max_cpu_celsius: 85,
+        };
+        const out = cpuTempRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].severity).toBe("warning");
+        expect(out[0].evidence.source).toBe("hwmon");
+        expect(out[0].evidence.chip).toBe("cpu_thermal");
+    });
+    it("fires critical when value_celsius >= cpu_temp_critical_c", () => {
+        const snap = emptySnap();
+        snap.thermal = {
+            available: true, source: "hwmon",
+            cpu_readings: [{ label: "coretemp Package id 0", value_celsius: 95, source_chip: "coretemp", source: "hwmon" }],
+            other_readings: [], max_cpu_celsius: 95,
+        };
+        const out = cpuTempRule.evaluate(snap, baseThresholds);
+        expect(out[0].severity).toBe("critical");
+    });
+    it("does not fire when max_cpu_celsius is null (VM)", () => {
+        const snap = emptySnap();
+        snap.thermal = { available: true, source: "none", cpu_readings: [], other_readings: [], max_cpu_celsius: null };
+        expect(cpuTempRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("falls back to IPMI substring filter when hwmon is unavailable", () => {
+        const snap = emptySnap();
+        snap.ipmi = {
+            available: true, ecc_errors: { correctable: 0, uncorrectable: 0 },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+            sensors: [{ name: "CPU1 Temp", value: 85, unit: "degrees C", status: "ok" }],
+        };
+        const out = cpuTempRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.source).toBe("ipmi");
+    });
+    it("hwmon takes priority over IPMI when both are present", () => {
+        const snap = emptySnap();
+        snap.thermal = {
+            available: true, source: "hwmon",
+            cpu_readings: [{ label: "coretemp Package id 0", value_celsius: 88, source_chip: "coretemp", source: "hwmon" }],
+            other_readings: [], max_cpu_celsius: 88,
+        };
+        snap.ipmi = {
+            available: true, ecc_errors: { correctable: 0, uncorrectable: 0 },
+            sel_entries_count: 0, sel_events_recent: [], fans: [],
+            sensors: [{ name: "CPU1 Temp", value: 99, unit: "degrees C", status: "ok" }],
+        };
+        const out = cpuTempRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.source).toBe("hwmon");
+        expect(out[0].evidence.value).toBe(88);
+    });
+});
+describe("disk_latency_high", () => {
+    it("does not fire when io_latency is missing", () => {
+        const snap = emptySnap();
+        expect(diskLatencyRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("does not fire when io_latency is empty", () => {
+        const snap = emptySnap();
+        snap.io_latency = [];
+        expect(diskLatencyRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("does not fire on healthy NVMe (1ms)", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "nvme0n1", avg_read_latency_ms: 1, avg_write_latency_ms: 0.5, read_iops: 100, write_iops: 50 },
+        ];
+        expect(diskLatencyRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("fires on hot NVMe (60ms read on default 50ms threshold)", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "nvme0n1", avg_read_latency_ms: 60, avg_write_latency_ms: 1, read_iops: 100, write_iops: 50 },
+        ];
+        const out = diskLatencyRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].severity).toBe("warning");
+        expect(out[0].evidence.device).toBe("nvme0n1");
+        expect(out[0].evidence.threshold_ms).toBe(50);
+    });
+    it("fires on hot SATA HDD (250ms write on default 200ms threshold)", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "sda", avg_read_latency_ms: 10, avg_write_latency_ms: 250, read_iops: 5, write_iops: 20 },
+        ];
+        const out = diskLatencyRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(1);
+        expect(out[0].evidence.device).toBe("sda");
+        expect(out[0].evidence.threshold_ms).toBe(200);
+    });
+    it("does not fire on borderline-cold SATA (180ms vs 200ms threshold)", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "sda", avg_read_latency_ms: 180, avg_write_latency_ms: 50, read_iops: 5, write_iops: 5 },
+        ];
+        expect(diskLatencyRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+    it("fires once per hot device when multiple devices present", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "nvme0n1", avg_read_latency_ms: 1, avg_write_latency_ms: 1, read_iops: 100, write_iops: 50 }, // healthy
+            { device: "sda", avg_read_latency_ms: 300, avg_write_latency_ms: 10, read_iops: 5, write_iops: 5 }, // hot HDD
+            { device: "nvme1n1", avg_read_latency_ms: 80, avg_write_latency_ms: 0.5, read_iops: 100, write_iops: 50 }, // hot NVMe
+        ];
+        const out = diskLatencyRule.evaluate(snap, baseThresholds);
+        expect(out).toHaveLength(2);
+        expect(out.map(a => a.evidence.device).sort()).toEqual(["nvme1n1", "sda"]);
+    });
+    it("skips devices with zero IOPS over the interval (no samples)", () => {
+        const snap = emptySnap();
+        snap.io_latency = [
+            { device: "sda", avg_read_latency_ms: null, avg_write_latency_ms: null, read_iops: 0, write_iops: 0 },
+        ];
+        expect(diskLatencyRule.evaluate(snap, baseThresholds)).toEqual([]);
+    });
+});
+describe("ALL_RULE_IDS export sync", () => {
+    it("matches the actual rule definitions in allRules", async () => {
+        const { ALL_RULE_IDS } = await import("../rules.js");
+        const idsFromArray = allRules.map(r => r.type);
+        expect([...ALL_RULE_IDS]).toEqual(idsFromArray);
+    });
+    it("matches the static rule-ids.json file (npm-published metadata)", async () => {
+        const { ALL_RULE_IDS } = await import("../rules.js");
+        const fs = await import("node:fs/promises");
+        const url = await import("node:url");
+        const path = await import("node:path");
+        const here = path.dirname(url.fileURLToPath(import.meta.url));
+        const json = JSON.parse(await fs.readFile(path.resolve(here, "../../../rule-ids.json"), "utf-8"));
+        expect(json.rule_ids).toEqual([...ALL_RULE_IDS]);
+    });
+});
+//# sourceMappingURL=rules.test.js.map

package/dist/alerts/__tests__/rules.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rules.test.js","sourceRoot":"","sources":["../../../src/alerts/__tests__/rules.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,MAAM,cAAc,GAAG;IACrB,WAAW,EAAE,EAAE;IACf,UAAU,EAAE,IAAI;IAChB,YAAY,EAAE,EAAE;IAChB,cAAc,EAAE,EAAE;IAClB,iBAAiB,EAAE,EAAE;IACrB,oBAAoB,EAAE,EAAE;IACxB,mBAAmB,EAAE,GAAG;IACxB,kBAAkB,EAAE,EAAE;IACtB,mBAAmB,EAAE,EAAE;IACvB,6BAA6B,EAAE,EAAE;CAClC,CAAC;AAEF,SAAS,SAAS;IAChB,OAAO;QACL,iBAAiB,EAAE,MAAM;QACzB,SAAS,EAAE,sBAAsB;QACjC,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE;QAC1F,GAAG,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;QACtH,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;QAClG,KAAK,EAAE,EAAE;QACT,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QAChJ,SAAS,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;KAC1E,CAAC;AACJ,CAAC;AAED,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAE,CAAC;AAC5E,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAE,CAAC;AAEjE,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC;QAC/B,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC;QAC/B,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,cAAc,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,sBAAsB,CAAE,CAAC;AAC3E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAE,CAAC;AAC7D,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,qBAAqB,CAAE,CAAC;AAEtE,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAC5B,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,mBAAmB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,sBAAsB,EAAE,sBAAsB,EAAE;YACzG,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAC5B,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,mBAAmB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,EAAE;YACvF,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAC5B,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,mBAAmB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,sBAAsB,EAAE,sBAAsB,EAAE;YACzG,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAC5B,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,mBAAmB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,sBAAsB,EAAE,sBAAsB,EAAE;YACzG,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yCAAyC,EAAE,GAAG,EAAE;IACvD,EAAE,CAAC,qFAAqF,EAAE,GAAG,EAAE;QAC7F,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QAC1K,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI;YACf,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;aACtE;YACD,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,oBAAoB,EAAE,iBAAiB;YACvC,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QAChK,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI;YACf,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE;aACvF;YACD,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QAC5J,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI;YACf,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE;gBAC5E,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;aACrE;YACD,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC1J,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAC9E,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QAChK,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI;YACf,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;gBACrE,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;aACtE;YACD,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAChD,oBAAoB,EAAE,iBAAiB;YACvC,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;SACtD,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;IACjD,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;YAChC,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC9G,cAAc,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE;SACxC,CAAC;QACF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;YAChC,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC/G,cAAc,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE;SACxC,CAAC;QACF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;QAChH,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YACjE,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;YACrD,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SAC7E,CAAC;QACF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;YAChC,YAAY,EAAE,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC/G,cAAc,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE;SACxC,CAAC;QACF,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YACjE,iBAAiB,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE;YACrD,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SAC7E,CAAC;QACF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,EAAE,oBAAoB,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE;SACzG,CAAC;QACF,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,EAAE,oBAAoB,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE;SACxG,CAAC;QACF,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,EAAE,EAAE,oBAAoB,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE;SACpG,CAAC;QACF,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,oBAAoB,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;SACnG,CAAC;QACF,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,EAAE,oBAAoB,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,UAAU;YAClH,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,oBAAoB,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,UAAU;YAC9G,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,EAAE,EAAE,oBAAoB,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,WAAW;SACvH,CAAC;QACF,MAAM,GAAG,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG;YAChB,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;SACtG,CAAC;QACF,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACrD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAClG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/alerts/rules.d.ts

@@ -4,4 +4,12 @@ export interface AlertRule {
     type: string;
     evaluate(snap: Snapshot, thresholds: Config["thresholds"]): AlertResult[];
 }
+/**
+ * Stable, ordered list of every rule ID this collector ships. Exported as
+ * a public API so downstream tooling (Glassmkr's drift validator, Forge's
+ * UI registry) can verify both sides agree on what exists. When you add
+ * or remove a rule, this list updates automatically — but RULES.json in
+ * the Glassmkr monorepo is hand-maintained and must be updated separately.
+ */
+export declare const ALL_RULE_IDS: readonly string[];
 export declare const allRules: AlertRule[];

package/dist/alerts/rules.js

@@ -1,6 +1,29 @@
-// Alert rules for the collector are identical to the Forge evaluator.
-// Re-export from a shared definition to avoid duplication.
-// For the collector, we use the same 15 rules but with local thresholds from config.
+// Collector-side alert rules. Currently 23 rules covering RAM/swap/disk,
+// CPU, SMART, RAID, network, IPMI thermal/ECC/PSU/SEL/fan, and security.
+// Forge runs an additional set of server-side rules on top of these
+// (predictive, fleet-wide). See RULES_COUNT.md (TBD) for the canonical
+// customer-facing total.
+//
+// When you add or remove a rule, also update:
+//   - RULE_AUDIT.md (one section per rule)
+//   - the count in this header comment
+import { isPsuSensor } from "../lib/vendor-sensors.js";
+/**
+ * Stable, ordered list of every rule ID this collector ships. Exported as
+ * a public API so downstream tooling (Glassmkr's drift validator, Forge's
+ * UI registry) can verify both sides agree on what exists. When you add
+ * or remove a rule, this list updates automatically — but RULES.json in
+ * the Glassmkr monorepo is hand-maintained and must be updated separately.
+ */
+export const ALL_RULE_IDS = [
+    "ram_high", "swap_high", "disk_space_high", "cpu_iowait_high", "oom_kills",
+    "smart_failing", "nvme_wear_high", "raid_degraded", "disk_latency_high",
+    "interface_errors", "link_speed_mismatch", "interface_saturation",
+    "cpu_temperature_high", "ecc_errors", "psu_redundancy_loss",
+    "ipmi_sel_critical", "ipmi_fan_failure",
+    "ssh_root_password", "no_firewall", "pending_security_updates",
+    "kernel_vulnerabilities", "kernel_needs_reboot", "unattended_upgrades_disabled",
+];
 export const allRules = [
     // 1. RAM high
     { type: "ram_high", evaluate(snap, t) {
@@ -16,10 +39,10 @@ export const allRules = [
                     recommendation: "Check: ps aux --sort=-rss | head -20" }];
         } },
     // 2. Swap active
-    { type: "swap_active", evaluate(snap, t) {
+    { type: "swap_high", evaluate(snap, t) {
             if (t.swap_alert === false || !snap.memory || snap.memory.swap_used_mb <= 0)
                 return [];
-            return [{ type: "swap_active", severity: "warning", title: `Swap in use: ${snap.memory.swap_used_mb}MB`,
+            return [{ type: "swap_high", severity: "warning", title: `Swap in use: ${snap.memory.swap_used_mb}MB`,
                     message: "Server is using swap space, indicating memory pressure.",
                     evidence: { swap_used_mb: snap.memory.swap_used_mb },
                     recommendation: "Check: free -h && ps aux --sort=-rss | head -20" }];
@@ -87,19 +110,52 @@ export const allRules = [
                 recommendation: "Replace failed drive immediately." }));
         } },
     // 9. Disk latency
+    // Reads `snap.io_latency` (populated by `collectIoLatency` from /proc/diskstats
+    // deltas), not `snap.disks` (which never had a latency field populated). The
+    // pre-fix version of this rule referenced `d.latency_p99_ms` on `snap.disks`
+    // and never fired on any host, ever.
+    //
+    // io-latency reports avg_read_latency_ms / avg_write_latency_ms over the
+    // collection interval (not p99). We take max(read, write) per device and
+    // compare against the per-class threshold:
+    //   nvme*: t.disk_latency_nvme_ms (default 50ms)
+    //   sd*/vd*/xvd*/md*: t.disk_latency_hdd_ms (default 200ms)
+    // 50ms healthy NVMe is generous; SATA SSD and HDD use the 200ms bucket.
     { type: "disk_latency_high", evaluate(snap, t) {
-            if (!snap.disks)
+            if (!snap.io_latency || snap.io_latency.length === 0)
                 return [];
-            return snap.disks.filter(d => {
-                if (d.latency_p99_ms == null)
-                    return false;
-                const thresh = d.device.includes("nvme") ? (t.disk_latency_nvme_ms ?? 50) : (t.disk_latency_hdd_ms ?? 200);
-                return d.latency_p99_ms >= thresh;
-            }).map(d => ({ type: "disk_latency_high", severity: "warning",
-                title: `Disk ${d.device} latency ${d.latency_p99_ms.toFixed(1)}ms`,
-                message: `p99 I/O latency on ${d.device} is high.`,
-                evidence: { device: d.device, latency_p99_ms: d.latency_p99_ms },
-                recommendation: "Check: iotop -oP" }));
+            const findings = [];
+            for (const entry of snap.io_latency) {
+                const r = entry.avg_read_latency_ms;
+                const w = entry.avg_write_latency_ms;
+                if (r == null && w == null)
+                    continue;
+                // No samples this interval (read_iops + write_iops both 0): skip silently.
+                if ((entry.read_iops ?? 0) === 0 && (entry.write_iops ?? 0) === 0)
+                    continue;
+                const worst = Math.max(r ?? 0, w ?? 0);
+                if (worst <= 0)
+                    continue;
+                const isNvme = entry.device.startsWith("nvme");
+                const thresh = isNvme ? (t.disk_latency_nvme_ms ?? 50) : (t.disk_latency_hdd_ms ?? 200);
+                if (worst < thresh)
+                    continue;
+                findings.push({
+                    type: "disk_latency_high", severity: "warning",
+                    title: `Disk ${entry.device} latency ${worst.toFixed(1)}ms`,
+                    message: `Average I/O latency on ${entry.device} is high (read ${r ?? 0}ms, write ${w ?? 0}ms over interval).`,
+                    evidence: {
+                        device: entry.device,
+                        avg_read_latency_ms: r,
+                        avg_write_latency_ms: w,
+                        read_iops: entry.read_iops,
+                        write_iops: entry.write_iops,
+                        threshold_ms: thresh,
+                    },
+                    recommendation: "Check: iotop -oP",
+                });
+            }
+            return findings;
         } },
     // 10. Interface errors
     { type: "interface_errors", evaluate(snap) {
@@ -144,10 +200,28 @@ export const allRules = [
             });
         } },
     // 13. CPU temperature
+    // Primary: /sys/class/hwmon (vendor-agnostic, works on Pi + Dell + everything).
+    // Fallback: IPMI sensors with the historical "cpu" + "temp" substring filter,
+    // used only when hwmon produced no usable CPU reading.
     { type: "cpu_temperature_high", evaluate(snap, t) {
+            const warn = t.cpu_temp_warning_c ?? 80;
+            const crit = t.cpu_temp_critical_c ?? 90;
+            // Primary path: hwmon
+            if (snap.thermal?.available && snap.thermal.cpu_readings.length > 0 && snap.thermal.max_cpu_celsius != null) {
+                return snap.thermal.cpu_readings
+                    .filter(r => r.value_celsius >= warn)
+                    .map(r => ({
+                    type: "cpu_temperature_high",
+                    severity: r.value_celsius >= crit ? "critical" : "warning",
+                    title: `${r.label}: ${r.value_celsius}°C`,
+                    message: `CPU temperature above warning threshold (${r.source} ${r.source_chip}).`,
+                    evidence: { sensor: r.label, value: r.value_celsius, source: r.source, chip: r.source_chip },
+                    recommendation: "Check cooling, fans, airflow.",
+                }));
+            }
+            // Fallback path: IPMI substring filter (Supermicro/ASRock-style names).
             if (!snap.ipmi?.available || !snap.ipmi.sensors)
                 return [];
-            const warn = t.cpu_temp_warning_c ?? 80;
             return snap.ipmi.sensors.filter(s => {
                 const n = s.name.toLowerCase();
                 if (!n.includes("cpu") && !n.includes("temp"))
@@ -156,35 +230,67 @@ export const allRules = [
                 return !isNaN(v) && v >= warn;
             }).map(s => {
                 const v = typeof s.value === "number" ? s.value : parseFloat(String(s.value));
-                const crit = s.upper_critical ?? (t.cpu_temp_critical_c ?? 90);
-                return { type: "cpu_temperature_high", severity: v >= crit ? "critical" : "warning",
-                    title: `${s.name}: ${v}${s.unit}`, message: `Temperature above warning threshold.`,
-                    evidence: { sensor: s.name, value: v },
+                const sensorCrit = s.upper_critical ?? crit;
+                return { type: "cpu_temperature_high", severity: v >= sensorCrit ? "critical" : "warning",
+                    title: `${s.name}: ${v}${s.unit}`, message: `Temperature above warning threshold (IPMI sensor).`,
+                    evidence: { sensor: s.name, value: v, source: "ipmi" },
                     recommendation: "Check cooling, fans, airflow." };
             });
         } },
     // 14. ECC errors
+    // Reads max(named-sensor counts, SEL-derived counts). Dell iDRAC does
+    // not expose ECC as named numeric sensors; SEL is the only signal.
+    // Supermicro / HPE / ASRockRack expose them as named sensors.
+    // Caveat: SEL counts are cumulative since last SEL clear, not rate.
     { type: "ecc_errors", evaluate(snap) {
             if (!snap.ipmi?.ecc_errors)
                 return [];
-            const { correctable, uncorrectable } = snap.ipmi.ecc_errors;
+            const named = snap.ipmi.ecc_errors;
+            const sel = snap.ipmi.ecc_errors_from_sel ?? { correctable: 0, uncorrectable: 0, newest_event_timestamp: null };
+            const correctable = Math.max(named.correctable, sel.correctable);
+            const uncorrectable = Math.max(named.uncorrectable, sel.uncorrectable);
             if (correctable <= 0 && uncorrectable <= 0)
                 return [];
+            const sourceUsed = (sel.correctable > named.correctable || sel.uncorrectable > named.uncorrectable) ? "ipmi_sel" : "ipmi_sensors";
+            const sourceLabel = sourceUsed === "ipmi_sel" ? "IPMI SEL events" : "IPMI named sensors";
             if (uncorrectable > 0)
                 return [{ type: "ecc_errors", severity: "critical",
-                        title: `${uncorrectable} uncorrectable ECC error(s)`, message: "Data corruption possible. DIMM failing.",
-                        evidence: { correctable, uncorrectable },
-                        recommendation: "Replace DIMM immediately. Run: ipmitool sdr type Memory" }];
+                        title: `${uncorrectable} uncorrectable ECC error(s)`,
+                        message: `${uncorrectable} uncorrectable ECC error(s) from ${sourceLabel}. Data corruption possible. DIMM failing.`,
+                        evidence: { correctable, uncorrectable, source: sourceUsed, named, sel },
+                        recommendation: "Replace DIMM immediately. Run: ipmitool sel elist | grep -i memory" }];
             return [{ type: "ecc_errors", severity: "warning",
-                    title: `${correctable} correctable ECC error(s)`, message: "Early warning of DIMM failure.",
-                    evidence: { correctable, uncorrectable },
-                    recommendation: "Schedule DIMM replacement. Run: ipmitool sdr type Memory" }];
+                    title: `${correctable} correctable ECC error(s)`,
+                    message: `${correctable} correctable ECC error(s) from ${sourceLabel}. Early warning of DIMM failure.`,
+                    evidence: { correctable, uncorrectable, source: sourceUsed, named, sel },
+                    recommendation: "Schedule DIMM replacement. Run: ipmitool sel elist | grep -i memory" }];
         } },
     // 15. PSU redundancy
+    // Two paths:
+    //   A. Per-PSU status: filter individual PSU sensors via vendor-aware
+    //      classifier (covers Supermicro `PSU1 Status`, HPE `Power Supply 1`,
+    //      Dell `PS1 Status`). If 2+ PSUs and any has failed/absent, fire.
+    //   B. Aggregate redundancy state (Dell `PS Redundancy` only today): if
+    //      anything other than fully_redundant or unknown, fire — even when
+    //      individual PS sensors look OK. This catches "redundancy degraded"
+    //      cases the per-PSU path would miss.
     { type: "psu_redundancy_loss", evaluate(snap) {
-            if (!snap.ipmi?.available || !snap.ipmi.sensors)
+            if (!snap.ipmi?.available)
+                return [];
+            const vendor = snap.dmi?.vendor ?? "generic";
+            // Path B: aggregate redundancy state
+            const redundancyState = snap.ipmi.psu_redundancy_state;
+            if (redundancyState && redundancyState !== "fully_redundant" && redundancyState !== "unknown") {
+                return [{ type: "psu_redundancy_loss", severity: "critical",
+                        title: "PSU redundancy lost",
+                        message: `BMC reports redundancy state: ${redundancyState.replace(/_/g, " ")}.`,
+                        evidence: { redundancy_state: redundancyState, source: "aggregate_sensor", vendor },
+                        recommendation: "Replace failed PSU. Check power connections and BMC `ipmitool chassis status`." }];
+            }
+            // Path A: per-PSU sensor status
+            if (!snap.ipmi.sensors)
                 return [];
-            const psus = snap.ipmi.sensors.filter(s => { const n = s.name.toLowerCase(); return n.includes("psu") || n.includes("power supply"); });
+            const psus = snap.ipmi.sensors.filter(s => isPsuSensor(s.name, vendor));
             if (psus.length < 2)
                 return [];
             const failed = psus.filter(s => {
@@ -195,8 +301,9 @@ export const allRules = [
             if (failed.length === 0)
                 return [];
             return [{ type: "psu_redundancy_loss", severity: "critical",
-                    title: "PSU redundancy lost", message: `${failed.length} PSU(s) failed/absent: ${failed.map(p => p.name).join(", ")}.`,
-                    evidence: { failed: failed.map(p => ({ name: p.name, status: p.status })) },
+                    title: "PSU redundancy lost",
+                    message: `${failed.length} PSU(s) failed/absent: ${failed.map(p => p.name).join(", ")}.`,
+                    evidence: { failed: failed.map(p => ({ name: p.name, status: p.status })), source: "per_psu_sensors", vendor },
                     recommendation: "Replace failed PSU. Check power connections." }];
         } },
     // 19. IPMI SEL critical events