@glassmkr/crucible 0.6.1 → 0.6.3

package/src/collect/__tests__/ipmi.test.ts

@@ -0,0 +1,96 @@
+import { describe, it, expect } from "vitest";
+import { classifySensor, deriveSelSeverity, parseSelTimestamp, parseFanStatus } from "../ipmi.js";
+
+describe("classifySensor", () => {
+  it("recognizes memory sensors", () => {
+    expect(classifySensor("DIMM_A1")).toBe("memory");
+    expect(classifySensor("Memory ECC")).toBe("memory");
+  });
+  it("recognizes power supplies", () => {
+    expect(classifySensor("PSU1 Status")).toBe("power");
+    expect(classifySensor("Power Supply 1")).toBe("power");
+  });
+  it("recognizes fans, watchdog, processors, temps, voltage, storage, chassis", () => {
+    expect(classifySensor("Fan1")).toBe("fan");
+    expect(classifySensor("Watchdog")).toBe("watchdog");
+    expect(classifySensor("Processor 0")).toBe("processor");
+    // CPU-named temperature sensors classify as processor (cpu check wins over temp).
+    expect(classifySensor("CPU1 Temp")).toBe("processor");
+    expect(classifySensor("Inlet Temp")).toBe("temperature");
+    expect(classifySensor("VCore Voltage")).toBe("voltage");
+    expect(classifySensor("Drive Slot 1")).toBe("storage");
+    expect(classifySensor("Chassis Intrusion")).toBe("chassis");
+  });
+  it("falls back to 'other'", () => {
+    expect(classifySensor("Weird Sensor")).toBe("other");
+  });
+});
+
+describe("deriveSelSeverity", () => {
+  it("treats uncorrectable, thermal trip, AC lost as critical", () => {
+    expect(deriveSelSeverity("Uncorrectable ECC", "memory")).toBe("critical");
+    expect(deriveSelSeverity("Thermal trip", "processor")).toBe("critical");
+    expect(deriveSelSeverity("AC lost", "power")).toBe("critical");
+    expect(deriveSelSeverity("Machine check", "processor")).toBe("critical");
+  });
+  it("treats correctable ECC and redundancy lost as warning", () => {
+    expect(deriveSelSeverity("Correctable ECC", "memory")).toBe("warning");
+    expect(deriveSelSeverity("Redundancy lost", "power")).toBe("warning");
+  });
+  it("treats presence detected as info", () => {
+    expect(deriveSelSeverity("Presence detected", "memory")).toBe("info");
+  });
+  it("defaults to warning for memory/power/fan/processor sensor types", () => {
+    expect(deriveSelSeverity("Some odd event", "memory")).toBe("warning");
+    expect(deriveSelSeverity("Some odd event", "fan")).toBe("warning");
+  });
+  it("defaults to info for other sensor types", () => {
+    expect(deriveSelSeverity("Some odd event", "other")).toBe("info");
+  });
+});
+
+describe("parseSelTimestamp", () => {
+  it("formats a known date/time", () => {
+    expect(parseSelTimestamp("04/05/2026", "14:23:05")).toBe("2026-04-05T14:23:05Z");
+  });
+  it("pads single digit month/day", () => {
+    expect(parseSelTimestamp("4/5/2026", "09:00:00")).toBe("2026-04-05T09:00:00Z");
+  });
+  it("returns an ISO string for bad input (does not crash)", () => {
+    const out = parseSelTimestamp("", "");
+    expect(typeof out).toBe("string");
+    expect(out.length).toBeGreaterThan(10);
+  });
+});
+
+describe("parseFanStatus", () => {
+  it("parses healthy fan output", () => {
+    const raw = [
+      "FAN1       | 30h | ok  |  7.1 | 5000 RPM",
+      "FAN2       | 31h | ok  |  7.2 | 5100 RPM",
+    ].join("\n");
+    const fans = parseFanStatus(raw);
+    expect(fans).toHaveLength(2);
+    expect(fans[0]).toMatchObject({ name: "FAN1", rpm: 5000, status: "ok" });
+    expect(fans[1].rpm).toBe(5100);
+  });
+
+  it("marks critical fans (cr/nr) as critical", () => {
+    const raw = "FAN1 | 30h | cr  | 7.1 | 0 RPM";
+    const fans = parseFanStatus(raw);
+    expect(fans[0].status).toBe("critical");
+  });
+
+  it("marks absent/no-reading fans as absent", () => {
+    const raw = "FAN3 | 30h | ns  | 7.1 | no reading";
+    const fans = parseFanStatus(raw);
+    expect(fans[0].status).toBe("absent");
+    expect(fans[0].rpm).toBe(0);
+  });
+
+  it("treats 0 RPM with no explicit status as critical", () => {
+    const raw = "FAN1 | 30h | 7.1 | 0 RPM";
+    const fans = parseFanStatus(raw);
+    expect(fans[0].status).toBe("critical");
+  });
+});

package/src/collect/__tests__/smart.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect } from "vitest";
+import { parseSmartctlJson } from "../smart.js";
+
+describe("parseSmartctlJson", () => {
+  it("parses a healthy SATA SSD", () => {
+    const data = {
+      model_name: "Samsung SSD 970 EVO 1TB",
+      smart_status: { passed: true },
+      temperature: { current: 38 },
+      power_on_time: { hours: 9000 },
+      ata_smart_attributes: {
+        table: [
+          { id: 5, name: "Reallocated_Sector_Ct", raw: { value: 0 } },
+          { id: 197, name: "Current_Pending_Sector", raw: { value: 0 } },
+        ],
+      },
+    };
+    const info = parseSmartctlJson(data, "/dev/sda");
+    expect(info).toMatchObject({
+      device: "/dev/sda",
+      model: "Samsung SSD 970 EVO 1TB",
+      health: "PASSED",
+      temperature_c: 38,
+      power_on_hours: 9000,
+      reallocated_sectors: 0,
+      pending_sectors: 0,
+    });
+  });
+
+  it("parses a failing SATA drive with reallocated sectors", () => {
+    const data = {
+      model_name: "WD Red 4TB",
+      smart_status: { passed: false },
+      ata_smart_attributes: {
+        table: [
+          { id: 5, raw: { value: 12 } },
+          { id: 197, raw: { value: 3 } },
+        ],
+      },
+    };
+    const info = parseSmartctlJson(data, "/dev/sdb");
+    expect(info.health).toBe("FAILED");
+    expect(info.reallocated_sectors).toBe(12);
+    expect(info.pending_sectors).toBe(3);
+  });
+
+  it("parses an NVMe drive with percentage_used", () => {
+    const data = {
+      model_name: "Samsung 980 PRO",
+      smart_status: { passed: true },
+      nvme_smart_health_information_log: { percentage_used: 22, temperature: 41 },
+    };
+    const info = parseSmartctlJson(data, "/dev/nvme0n1");
+    expect(info.percentage_used).toBe(22);
+    expect(info.temperature_c).toBe(41);
+    expect(info.health).toBe("PASSED");
+  });
+
+  it("falls back to 'unknown' model when absent", () => {
+    const info = parseSmartctlJson({ smart_status: { passed: true } }, "/dev/sdc");
+    expect(info.model).toBe("unknown");
+  });
+
+  it("treats missing smart_status as FAILED (safer default)", () => {
+    const info = parseSmartctlJson({}, "/dev/sdd");
+    expect(info.health).toBe("FAILED");
+  });
+});

package/src/collect/__tests__/zfs.test.ts

@@ -0,0 +1,72 @@
+import { describe, it, expect } from "vitest";
+import { parseZpoolStatus } from "../zfs.js";
+
+describe("parseZpoolStatus", () => {
+  it("parses a healthy pool", () => {
+    const raw = `  pool: tank
+ state: ONLINE
+  scan: scrub repaired 0B in 01:23:45 with 0 errors on Sun Apr  5 12:34:56 2026
+config:
+
+        NAME        STATE     READ WRITE CKSUM
+        tank        ONLINE       0     0     0
+          mirror-0  ONLINE       0     0     0
+
+errors: No known data errors
+`;
+    const pools = parseZpoolStatus(raw);
+    expect(pools).toHaveLength(1);
+    expect(pools[0]).toMatchObject({
+      name: "tank",
+      state: "ONLINE",
+      errors_text: "No known data errors",
+      scrub_errors: 0,
+      scrub_repaired: "0B",
+    });
+    expect(pools[0].last_scrub_date).toContain("2026");
+  });
+
+  it("parses a DEGRADED pool", () => {
+    const raw = `  pool: tank
+ state: DEGRADED
+  scan: scrub repaired 16K in 02:00:00 with 3 errors on Sun Apr  5 12:34:56 2026
+
+errors: 3 data errors, use '-v' for a list
+`;
+    const [p] = parseZpoolStatus(raw);
+    expect(p.state).toBe("DEGRADED");
+    expect(p.scrub_errors).toBe(3);
+    expect(p.scrub_repaired).toBe("16K");
+  });
+
+  it("flags never-scrubbed pools", () => {
+    const raw = `  pool: tank
+ state: ONLINE
+  scan: none requested
+
+errors: No known data errors
+`;
+    const [p] = parseZpoolStatus(raw);
+    expect(p.scrub_never_run).toBe(true);
+    expect(p.scrub_errors).toBeUndefined();
+  });
+
+  it("returns empty for no pools", () => {
+    expect(parseZpoolStatus("no pools available")).toEqual([]);
+  });
+
+  it("parses multiple pools", () => {
+    const raw = `  pool: tank
+ state: ONLINE
+  scan: none requested
+errors: No known data errors
+  pool: data
+ state: FAULTED
+  scan: none requested
+errors: 2 data errors
+`;
+    const pools = parseZpoolStatus(raw);
+    expect(pools.map((p) => p.name)).toEqual(["tank", "data"]);
+    expect(pools[1].state).toBe("FAULTED");
+  });
+});

package/src/collect/ipmi.ts

@@ -104,7 +104,7 @@ async function collectSelEvents(): Promise<SelEvent[]> {
   return events.slice(-20).reverse();
 }
 
-function parseSelTimestamp(date: string, time: string): string {
+export function parseSelTimestamp(date: string, time: string): string {
   if (!date || !time) return new Date().toISOString();
   // Format: "04/05/2026" and "14:23:05"
   const parts = date.split("/");
@@ -113,7 +113,7 @@ function parseSelTimestamp(date: string, time: string): string {
   return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}T${time}Z`;
 }
 
-function classifySensor(sensor: string): string {
+export function classifySensor(sensor: string): string {
   const lower = sensor.toLowerCase();
   if (lower.includes("memory") || lower.includes("dimm")) return "memory";
   if (lower.includes("power supply") || lower.includes("psu")) return "power";
@@ -127,7 +127,7 @@ function classifySensor(sensor: string): string {
   return "other";
 }
 
-function deriveSelSeverity(event: string, sensorType: string): string {
+export function deriveSelSeverity(event: string, sensorType: string): string {
   const lower = event.toLowerCase();
 
   // Critical events
@@ -163,7 +163,10 @@ function deriveSelSeverity(event: string, sensorType: string): string {
 async function collectFanStatus(): Promise<FanStatus[]> {
   const output = await run("ipmitool", ["sdr", "type", "Fan"]);
   if (!output) return [];
+  return parseFanStatus(output);
+}
 
+export function parseFanStatus(output: string): FanStatus[] {
   const fans: FanStatus[] = [];
   const lines = output.trim().split("\n");
 

package/src/collect/ntp.ts

@@ -5,16 +5,37 @@ export interface NtpData {
   offset_seconds: number;
   source: string;
   daemon_running: boolean;
+  daemon_name: string;
+}
+
+// Check whether a systemd unit is active. Returns false for missing units, not-found, etc.
+async function isUnitActive(unit: string): Promise<boolean> {
+  const out = await run("systemctl", ["is-active", unit], 3000);
+  return out?.trim() === "active";
+}
+
+// Detect which time-sync daemon unit is currently active on the host, if any.
+// Returns "" when none are. We check the common names in order of preference.
+async function detectActiveDaemon(): Promise<string> {
+  const candidates = ["chrony", "chronyd", "systemd-timesyncd", "ntp", "ntpsec", "ntpd"];
+  for (const unit of candidates) {
+    if (await isUnitActive(unit)) return unit;
+  }
+  return "";
 }
 
 export async function collectNtp(): Promise<NtpData> {
-  // Try timedatectl first (systemd-timesyncd)
+  // The authoritative "is the daemon running" check is systemctl is-active,
+  // not any derived flag from timedatectl. This catches daemon crashes and
+  // manual stops where the kernel clock is still synced.
+  const daemonName = await detectActiveDaemon();
+  const daemonRunning = daemonName !== "";
+
+  // Try timedatectl first (works for systemd-timesyncd and records the kernel
+  // NTPSynchronized flag regardless of which daemon set it).
   const tdctl = await run("timedatectl", ["show", "--property=NTPSynchronized", "--value"]);
-  // Only trust timedatectl if it returns a clear "yes" or "no"
   if (tdctl !== null && (tdctl.trim() === "yes" || tdctl.trim() === "no")) {
     const synced = tdctl.trim() === "yes";
-    const statusOut = await run("timedatectl", ["show", "--property=NTP", "--value"]);
-    const ntpEnabled = statusOut?.trim() === "yes";
 
     let offset = 0;
     try {
@@ -31,20 +52,17 @@ export async function collectNtp(): Promise<NtpData> {
       }
     } catch { /* offset stays 0 */ }
 
-    return {
-      synced,
-      offset_seconds: offset,
-      source: "systemd-timesyncd",
-      daemon_running: ntpEnabled || synced,
-    };
+    // Prefer an explicitly detected daemon name; fall back to systemd-timesyncd
+    // since timedatectl is most commonly the timesyncd frontend.
+    const source = daemonName || "systemd-timesyncd";
+    return { synced, offset_seconds: offset, source, daemon_running: daemonRunning, daemon_name: daemonName };
   }
 
-  // Try chrony - validate output contains expected "Leap status" field
+  // Chrony tracking. Validate Leap status so we do not misread error text.
   const chronyOut = await run("chronyc", ["tracking"]);
   if (chronyOut) {
     const leapMatch = chronyOut.match(/Leap status\s*:\s*(.+)/);
     if (leapMatch) {
-      // Output looks like real chrony tracking data
       const synced = leapMatch[1].trim() === "Normal";
       let offset = 0;
       const offsetMatch = chronyOut.match(/Last offset\s*:\s*([+-]?\d+\.?\d*)\s*seconds/);
@@ -52,20 +70,18 @@ export async function collectNtp(): Promise<NtpData> {
       return {
         synced,
         offset_seconds: Math.abs(offset),
-        source: "chrony",
-        daemon_running: true,
+        source: daemonName || "chrony",
+        daemon_running: daemonRunning,
+        daemon_name: daemonName,
       };
     }
-    // chronyc returned output but not tracking data (error message, daemon not running)
-    // Fall through to ntpq
   }
 
-  // Try ntpq - validate output contains the header line with "remote"
+  // ntpq peer table. Header check avoids false positives on error messages.
   const ntpqOut = await run("ntpq", ["-pn"]);
   if (ntpqOut) {
     const hasHeader = ntpqOut.split("\n").some((line) => line.includes("remote"));
     if (hasHeader) {
-      // Output looks like real ntpq peer table
       const synced = ntpqOut.split("\n").some((line) => line.startsWith("*"));
       let offset = 0;
       const selectedLine = ntpqOut.split("\n").find((line) => line.startsWith("*"));
@@ -79,19 +95,20 @@ export async function collectNtp(): Promise<NtpData> {
       return {
         synced,
         offset_seconds: offset,
-        source: "ntpd",
-        daemon_running: true,
+        source: daemonName || "ntpd",
+        daemon_running: daemonRunning,
+        daemon_name: daemonName,
       };
     }
-    // ntpq returned output but not peer table (error message, daemon not running)
-    // Fall through to "none"
   }
 
-  // No time sync daemon found (or all probes returned invalid output)
+  // No usable probe output. If systemd still reports a daemon as active, trust that;
+  // otherwise report fully down.
   return {
     synced: false,
     offset_seconds: 0,
-    source: "none",
-    daemon_running: false,
+    source: daemonName || "none",
+    daemon_running: daemonRunning,
+    daemon_name: daemonName,
   };
 }

package/src/collect/smart.ts

@@ -22,34 +22,7 @@ export async function collectSmart(): Promise<SmartInfo[]> {
     if (!output) continue;
 
     try {
-      const data = JSON.parse(output);
-      const info: SmartInfo = {
-        device,
-        model: data.model_name || data.model_family || "unknown",
-        health: data.smart_status?.passed ? "PASSED" : "FAILED",
-        temperature_c: data.temperature?.current,
-        power_on_hours: data.power_on_time?.hours,
-      };
-
-      // NVMe specific
-      if (data.nvme_smart_health_information_log) {
-        const nvme = data.nvme_smart_health_information_log;
-        info.percentage_used = nvme.percentage_used;
-        info.temperature_c = nvme.temperature;
-      }
-
-      // SATA specific
-      if (data.ata_smart_attributes?.table) {
-        for (const attr of data.ata_smart_attributes.table) {
-          if (attr.id === 5 || attr.name === "Reallocated_Sector_Ct") {
-            info.reallocated_sectors = attr.raw?.value || 0;
-          }
-          if (attr.id === 197 || attr.name === "Current_Pending_Sector") {
-            info.pending_sectors = attr.raw?.value || 0;
-          }
-        }
-      }
-
+      const info = parseSmartctlJson(JSON.parse(output), device);
       results.push(info);
     } catch {
       // Failed to parse, skip this device
@@ -58,3 +31,42 @@ export async function collectSmart(): Promise<SmartInfo[]> {
 
   return results;
 }
+
+export function parseSmartctlJson(data: Record<string, unknown> & {
+  model_name?: string;
+  model_family?: string;
+  smart_status?: { passed?: boolean };
+  temperature?: { current?: number };
+  power_on_time?: { hours?: number };
+  nvme_smart_health_information_log?: { percentage_used?: number; temperature?: number };
+  ata_smart_attributes?: { table?: Array<{ id?: number; name?: string; raw?: { value?: number } }> };
+}, device: string): SmartInfo {
+  const info: SmartInfo = {
+    device,
+    model: data.model_name || data.model_family || "unknown",
+    health: data.smart_status?.passed ? "PASSED" : "FAILED",
+    temperature_c: data.temperature?.current,
+    power_on_hours: data.power_on_time?.hours,
+  };
+
+  // NVMe specific
+  if (data.nvme_smart_health_information_log) {
+    const nvme = data.nvme_smart_health_information_log;
+    info.percentage_used = nvme.percentage_used;
+    info.temperature_c = nvme.temperature;
+  }
+
+  // SATA specific
+  if (data.ata_smart_attributes?.table) {
+    for (const attr of data.ata_smart_attributes.table) {
+      if (attr.id === 5 || attr.name === "Reallocated_Sector_Ct") {
+        info.reallocated_sectors = attr.raw?.value || 0;
+      }
+      if (attr.id === 197 || attr.name === "Current_Pending_Sector") {
+        info.pending_sectors = attr.raw?.value || 0;
+      }
+    }
+  }
+
+  return info;
+}

package/src/collect/zfs.ts

@@ -9,6 +9,12 @@ export async function collectZfs(): Promise<ZfsData | null> {
   const zpoolStatus = await run("zpool", ["status"], 10000);
   if (!zpoolStatus || !zpoolStatus.trim()) return null;
 
+  const pools = parseZpoolStatus(zpoolStatus);
+  if (pools.length === 0) return null;
+  return { pools };
+}
+
+export function parseZpoolStatus(zpoolStatus: string): ZfsPool[] {
   const pools: ZfsPool[] = [];
   let current: ZfsPool | null = null;
 
@@ -56,6 +62,5 @@ export async function collectZfs(): Promise<ZfsData | null> {
     }
   }
 
-  if (pools.length === 0) return null;
-  return { pools };
+  return pools;
 }

package/src/index.ts

@@ -3,7 +3,7 @@
 import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { dirname, join } from "node:path";
-import { loadConfig } from "./config.js";
+import { parseCliArgs } from "./cli.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PKG_VERSION = (() => {
@@ -14,6 +14,17 @@ const PKG_VERSION = (() => {
     return "0.0.0";
   }
 })();
+
+// Handle --version and --help before importing collectors, loading config, or
+// starting the Prometheus server. This keeps the CLI responsive even on hosts
+// missing the config file or external tools.
+const { result: cliArgs, output: cliOutput } = parseCliArgs(process.argv.slice(2), PKG_VERSION);
+if (cliArgs.mode !== "run") {
+  console.log(cliOutput);
+  process.exit(0);
+}
+
+import { loadConfig } from "./config.js";
 import { checkForUpdates } from "./lib/version-check.js";
 import { startMetricsServer, updateMetrics } from "./metrics-server.js";
 import { collectSystem } from "./collect/system.js";
@@ -41,8 +52,7 @@ import { collectNtp } from "./collect/ntp.js";
 import { collectFileDescriptors } from "./collect/fd.js";
 import type { Snapshot, IpmiInfo } from "./lib/types.js";
 
-const configPath = process.argv[2] || "/etc/glassmkr/collector.yaml";
-const config = loadConfig(configPath);
+const config = loadConfig(cliArgs.configPath);
 
 console.log(`[collector] Starting. Server: ${config.server_name}. Interval: ${config.collection.interval_seconds}s`);
 console.log(`[collector] IPMI: ${config.collection.ipmi ? "enabled" : "disabled"}, SMART: ${config.collection.smart ? "enabled" : "disabled"}`);

package/src/lib/__tests__/parse.test.ts

@@ -0,0 +1,28 @@
+import { describe, it, expect } from "vitest";
+import { parseKeyValue, parseKb } from "../parse.js";
+
+describe("parseKeyValue", () => {
+  it("parses colon-delimited key/value lines", () => {
+    const out = parseKeyValue("Name: foo\nVersion: 1.2.3\n");
+    expect(out).toEqual({ Name: "foo", Version: "1.2.3" });
+  });
+  it("ignores lines with no colon", () => {
+    expect(parseKeyValue("no colon here\nA: 1\n")).toEqual({ A: "1" });
+  });
+  it("trims whitespace around keys and values", () => {
+    expect(parseKeyValue("   A   :    1   \n")).toEqual({ A: "1" });
+  });
+});
+
+describe("parseKb", () => {
+  it("parses a numeric kB value", () => {
+    expect(parseKb("16384 kB")).toBe(16384);
+  });
+  it("parses without unit", () => {
+    expect(parseKb("4096")).toBe(4096);
+  });
+  it("returns 0 for undefined/bad input", () => {
+    expect(parseKb(undefined)).toBe(0);
+    expect(parseKb("not a number")).toBe(0);
+  });
+});

package/vitest.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["src/**/*.test.ts"],
+    exclude: ["node_modules", "dist"],
+  },
+  resolve: {
+    // .js specifier inside TS (NodeNext) needs to resolve to .ts in tests
+    extensions: [".ts", ".js"],
+  },
+});