@glassmkr/crucible 0.6.5 → 0.7.0

package/src/collect/network.ts

@@ -13,10 +13,26 @@ interface PreviousCounters {
   tx_errors: number;
   rx_drops: number;
   tx_drops: number;
+  rx_packets: number;
+  tx_packets: number;
+  rx_crc_errors?: number;
+  rx_frame_errors?: number;
+  rx_length_errors?: number;
+  tx_carrier_errors?: number;
 }
 
 const previousCounters = new Map<string, PreviousCounters>();
 
+function readStatCounter(iface: string, name: string): number | undefined {
+  try {
+    const raw = readFileSync(`/sys/class/net/${iface}/statistics/${name}`, "utf-8").trim();
+    const val = parseInt(raw, 10);
+    return Number.isFinite(val) ? val : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 function parseNetDev(): Record<string, IfaceStats> {
   const raw = readProcFile("/proc/net/dev") || "";
   const result: Record<string, IfaceStats> = {};
@@ -66,6 +82,14 @@ function getBondMaster(iface: string): string | undefined {
   return undefined;
 }
 
+function isBondMaster(iface: string): boolean {
+  try {
+    return readdirSync("/proc/net/bonding/").includes(iface);
+  } catch {
+    return false;
+  }
+}
+
 // Compute delta, handling counter wraps (current < previous means reset, use current as delta)
 function delta(current: number, previous: number): number {
   if (current >= previous) return current - previous;
@@ -87,17 +111,37 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
 
     const prev = previousCounters.get(name);
 
+    // /sys/class/net/*/statistics/ exposes finer-grained RX/TX subtype
+    // counters than /proc/net/dev. Read cumulative values here; delta is
+    // derived below against the previous cycle's snapshot.
+    const rxCrcCum = readStatCounter(name, "rx_crc_errors");
+    const rxFrameCum = readStatCounter(name, "rx_frame_errors");
+    const rxLenCum = readStatCounter(name, "rx_length_errors");
+    const txCarrierCum = readStatCounter(name, "tx_carrier_errors");
+
     // Compute error/drop deltas (0 on first cycle after start or new interface)
     let rxErrorsDelta = 0;
     let txErrorsDelta = 0;
     let rxDropsDelta = 0;
     let txDropsDelta = 0;
+    let rxPacketsDelta = 0;
+    let txPacketsDelta = 0;
+    let rxCrcDelta: number | undefined;
+    let rxFrameDelta: number | undefined;
+    let rxLenDelta: number | undefined;
+    let txCarrierDelta: number | undefined;
 
     if (prev) {
       rxErrorsDelta = delta(s2.rx_errors, prev.rx_errors);
       txErrorsDelta = delta(s2.tx_errors, prev.tx_errors);
       rxDropsDelta = delta(s2.rx_drops, prev.rx_drops);
       txDropsDelta = delta(s2.tx_drops, prev.tx_drops);
+      rxPacketsDelta = delta(s2.rx_packets, prev.rx_packets);
+      txPacketsDelta = delta(s2.tx_packets, prev.tx_packets);
+      if (rxCrcCum != null && prev.rx_crc_errors != null) rxCrcDelta = delta(rxCrcCum, prev.rx_crc_errors);
+      if (rxFrameCum != null && prev.rx_frame_errors != null) rxFrameDelta = delta(rxFrameCum, prev.rx_frame_errors);
+      if (rxLenCum != null && prev.rx_length_errors != null) rxLenDelta = delta(rxLenCum, prev.rx_length_errors);
+      if (txCarrierCum != null && prev.tx_carrier_errors != null) txCarrierDelta = delta(txCarrierCum, prev.tx_carrier_errors);
     }
 
     // Store current cumulative values for next cycle
@@ -106,6 +150,12 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
       tx_errors: s2.tx_errors,
       rx_drops: s2.rx_drops,
       tx_drops: s2.tx_drops,
+      rx_packets: s2.rx_packets,
+      tx_packets: s2.tx_packets,
+      rx_crc_errors: rxCrcCum,
+      rx_frame_errors: rxFrameCum,
+      rx_length_errors: rxLenCum,
+      tx_carrier_errors: txCarrierCum,
     });
 
     const entry: NetworkInfo = {
@@ -117,10 +167,18 @@ export async function collectNetwork(): Promise<NetworkInfo[]> {
       tx_errors: txErrorsDelta,
       rx_drops: rxDropsDelta,
       tx_drops: txDropsDelta,
+      rx_packets: rxPacketsDelta,
+      tx_packets: txPacketsDelta,
       operstate: getOperstate(name),
     };
+    if (rxCrcDelta !== undefined) entry.rx_crc_errors = rxCrcDelta;
+    if (rxFrameDelta !== undefined) entry.rx_frame_errors = rxFrameDelta;
+    if (rxLenDelta !== undefined) entry.rx_length_errors = rxLenDelta;
+    if (txCarrierDelta !== undefined) entry.tx_carrier_errors = txCarrierDelta;
     const master = getBondMaster(name);
     if (master) entry.bond_master = master;
+    // Identify bond masters (have at least one slave pointing at them).
+    if (isBondMaster(name)) entry.is_bond_master = true;
     results.push(entry);
   }
 

package/src/collect/system.ts

@@ -3,9 +3,18 @@ import { readProcFile } from "../lib/parse.js";
 import { run } from "../lib/exec.js";
 import type { SystemInfo } from "../lib/types.js";
 
+// Matches KEY=value with optional surrounding double quotes. Handles both
+// `ID=ubuntu` and `ID="rocky"` styles found in the wild.
+export function readOsReleaseField(osRelease: string, key: string): string | undefined {
+  const m = osRelease.match(new RegExp(`^${key}=("?)(.+?)\\1$`, "m"));
+  return m ? m[2].toLowerCase() : undefined;
+}
+
 export async function collectSystem(): Promise<SystemInfo> {
   const osRelease = readProcFile("/etc/os-release") || "";
   const osName = osRelease.match(/PRETTY_NAME="(.+?)"/)?.[1] || "Unknown";
+  const os_id = readOsReleaseField(osRelease, "ID");
+  const os_id_like = readOsReleaseField(osRelease, "ID_LIKE");
   const kernel = (await run("uname", ["-r"]))?.trim() || "unknown";
   const uptimeRaw = readProcFile("/proc/uptime") || "0";
   const uptimeSeconds = Math.floor(parseFloat(uptimeRaw.split(" ")[0]));
@@ -15,6 +24,8 @@ export async function collectSystem(): Promise<SystemInfo> {
     hostname: hostname(),
     ip,
     os: osName,
+    ...(os_id ? { os_id } : {}),
+    ...(os_id_like ? { os_id_like } : {}),
     kernel,
     uptime_seconds: uptimeSeconds,
   };

package/src/index.ts

@@ -15,14 +15,44 @@ const PKG_VERSION = (() => {
   }
 })();
 
-// Handle --version and --help before importing collectors, loading config, or
-// starting the Prometheus server. This keeps the CLI responsive even on hosts
-// missing the config file or external tools.
+// Handle --version, --help, and planned-reboot subcommands before
+// importing collectors, loading config, or starting the Prometheus
+// server. Keeps the CLI responsive even on hosts missing the config
+// file or external tools.
 const { result: cliArgs, output: cliOutput } = parseCliArgs(process.argv.slice(2), PKG_VERSION);
-if (cliArgs.mode !== "run") {
+if (cliArgs.mode === "version" || cliArgs.mode === "help") {
   console.log(cliOutput);
   process.exit(0);
 }
+if (cliArgs.mode === "mark-reboot" || cliArgs.mode === "reboot") {
+  const { writeRebootMarker, parseDuration, DEFAULT_TTL_MS } = await import("./lib/reboot-marker.js");
+  const ttlMs = cliArgs.ttl ? parseDuration(cliArgs.ttl) : DEFAULT_TTL_MS;
+  if (ttlMs === null) {
+    console.error(`[mark-reboot] invalid --ttl value: ${cliArgs.ttl}. Use e.g. 10m, 2h, 600s.`);
+    process.exit(2);
+  }
+  try {
+    const { path, expires_at } = writeRebootMarker({
+      reason: cliArgs.reason, ttlMs,
+    });
+    console.log(`[${cliArgs.mode}] marker written: ${path} (expires ${expires_at}${cliArgs.reason ? `, reason: ${cliArgs.reason}` : ""})`);
+  } catch (err: any) {
+    console.error(`[${cliArgs.mode}] failed to write marker: ${err?.message || err}`);
+    console.error(`  Most likely cause: need root privileges to write under /var/lib/crucible/.`);
+    process.exit(1);
+  }
+  if (cliArgs.mode === "reboot") {
+    const { execFileSync } = await import("node:child_process");
+    console.log("[reboot] invoking systemctl reboot");
+    try {
+      execFileSync("systemctl", ["reboot"], { stdio: "inherit" });
+    } catch (err: any) {
+      console.error(`[reboot] systemctl reboot failed: ${err?.message || err}`);
+      process.exit(1);
+    }
+  }
+  process.exit(0);
+}
 
 import { loadConfig } from "./config.js";
 import { checkForUpdates } from "./lib/version-check.js";
@@ -51,6 +81,17 @@ import { collectSystemd } from "./collect/systemd.js";
 import { collectNtp } from "./collect/ntp.js";
 import { collectFileDescriptors } from "./collect/fd.js";
 import type { Snapshot, IpmiInfo } from "./lib/types.js";
+import { consumeRebootMarker, type PlannedReboot } from "./lib/reboot-marker.js";
+
+// Consume the planned-reboot marker once at startup. If the operator ran
+// `crucible-agent mark-reboot` / `reboot` before this boot, the marker
+// exists, we flag it on the first snapshot, and we delete the file (so
+// subsequent snapshots don't keep claiming the reboot was planned).
+const plannedRebootFlag: PlannedReboot | null = consumeRebootMarker();
+if (plannedRebootFlag) {
+  console.log(`[collector] Planned reboot acknowledged${plannedRebootFlag.reason ? `: ${plannedRebootFlag.reason}` : ""}`);
+}
+let plannedRebootConsumed = false;
 
 const config = loadConfig(cliArgs.configPath);
 
@@ -106,6 +147,14 @@ async function collect() {
     security: cachedSecurity,
   };
 
+  // Single-shot: the very first snapshot after a marked reboot carries
+  // the flag, subsequent snapshots do not.
+  if (plannedRebootFlag && !plannedRebootConsumed) {
+    (snapshot as any).expected_reboot = true;
+    if (plannedRebootFlag.reason) (snapshot as any).expected_reboot_reason = plannedRebootFlag.reason;
+    plannedRebootConsumed = true;
+  }
+
   // ZFS and I/O errors: collect every cycle (lightweight checks)
   try { snapshot.zfs = await collectZfs() ?? undefined; } catch { /* skip if ZFS not available */ }
   try { snapshot.io_errors = await collectIoErrors() ?? undefined; } catch { /* skip on error */ }

package/src/lib/reboot-marker.ts

@@ -0,0 +1,88 @@
+// Planned-reboot marker handling.
+//
+// An operator signals "the next reboot is expected, don't page me"
+// by writing a short-lived JSON file to disk BEFORE rebooting. The
+// collector reads and deletes it on agent startup; the first
+// post-boot snapshot then carries `expected_reboot: true` so Forge's
+// unexpected_reboot rule stays quiet.
+//
+// Single-use (deleted on read regardless of validity) and TTL-guarded
+// (default 10 min) so a forgotten marker cannot silence a genuine
+// crash reboot weeks later.
+
+import { existsSync, readFileSync, unlinkSync, writeFileSync, mkdirSync, chmodSync } from "node:fs";
+import { dirname } from "node:path";
+
+export const DEFAULT_MARKER_PATH = "/var/lib/crucible/reboot-expected";
+export const DEFAULT_TTL_MS = 10 * 60 * 1000;
+
+export interface PlannedReboot {
+  expected: true;
+  reason?: string;
+}
+
+export interface RebootMarker {
+  expires_at: string; // ISO timestamp
+  reason?: string;
+}
+
+/**
+ * Read and delete the marker at `path`. Returns the resolved reboot flag
+ * if the file existed, was parseable JSON, and hasn't expired; otherwise
+ * returns null. The file is unlinked in every branch where it existed,
+ * so a malformed or stale marker is one-shot (can't linger).
+ */
+export function consumeRebootMarker(
+  path: string = DEFAULT_MARKER_PATH,
+  now: Date = new Date(),
+): PlannedReboot | null {
+  if (!existsSync(path)) return null;
+  let raw: string;
+  try { raw = readFileSync(path, "utf-8"); } catch { try { unlinkSync(path); } catch {} return null; }
+  // Always delete after read, regardless of validity.
+  try { unlinkSync(path); } catch {}
+
+  let parsed: RebootMarker;
+  try { parsed = JSON.parse(raw); } catch { return null; }
+  if (!parsed || typeof parsed !== "object" || typeof parsed.expires_at !== "string") return null;
+  const expiresAt = new Date(parsed.expires_at);
+  if (isNaN(expiresAt.getTime())) return null;
+  if (expiresAt.getTime() <= now.getTime()) return null; // stale
+  return { expected: true, reason: parsed.reason };
+}
+
+/**
+ * Write a planned-reboot marker. Used by the `mark-reboot` and `reboot`
+ * CLI subcommands. `ttlMs` defaults to 10 minutes. Creates the parent
+ * directory if needed. Chmod 600 so other users on the host can't read
+ * or modify it.
+ */
+export function writeRebootMarker(opts: {
+  reason?: string;
+  ttlMs?: number;
+  path?: string;
+  now?: Date;
+}): { path: string; expires_at: string } {
+  const path = opts.path ?? DEFAULT_MARKER_PATH;
+  const now = opts.now ?? new Date();
+  const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;
+  const expiresAt = new Date(now.getTime() + ttlMs);
+  const body: RebootMarker = { expires_at: expiresAt.toISOString() };
+  if (opts.reason) body.reason = opts.reason;
+  try { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); } catch {}
+  writeFileSync(path, JSON.stringify(body), { mode: 0o600 });
+  try { chmodSync(path, 0o600); } catch {}
+  return { path, expires_at: body.expires_at };
+}
+
+/** Parse a duration like "10m", "2h", "600s" into milliseconds. Used by
+ *  the CLI for the `--ttl` flag. */
+export function parseDuration(s: string): number | null {
+  const m = /^(\d+)\s*(ms|s|m|h)?$/.exec(s.trim());
+  if (!m) return null;
+  const n = parseInt(m[1], 10);
+  if (!Number.isFinite(n) || n < 0) return null;
+  const unit = m[2] ?? "s";
+  const mult = unit === "ms" ? 1 : unit === "s" ? 1000 : unit === "m" ? 60_000 : 3_600_000;
+  return n * mult;
+}

package/src/lib/types.ts

@@ -18,6 +18,12 @@ export interface Snapshot {
   systemd?: SystemdData;
   ntp?: NtpData;
   file_descriptors?: FileDescriptorData;
+  // Planned-reboot flag: set only on the first snapshot after a reboot
+  // that was marked with `crucible-agent mark-reboot` / `reboot`. Forge
+  // reads this to suppress the `unexpected_reboot` rule. Single-use:
+  // subsequent snapshots don't carry it.
+  expected_reboot?: boolean;
+  expected_reboot_reason?: string;
 }
 
 export interface ConntrackData {
@@ -73,6 +79,12 @@ export interface SystemInfo {
   hostname: string;
   ip: string;
   os: string;
+  /** `ID=` from /etc/os-release, lowercased. e.g. "ubuntu", "debian", "rocky", "arch", "alpine". */
+  os_id?: string;
+  /** `ID_LIKE=` from /etc/os-release, lowercased, space-separated. Used by Forge
+   *  to pick distro-family-specific fix command variants. e.g. on Rocky this
+   *  is "rhel centos fedora"; on Ubuntu it is "debian". */
+  os_id_like?: string;
   kernel: string;
   uptime_seconds: number;
 }
@@ -139,12 +151,23 @@ export interface NetworkInfo {
   speed_mbps: number;
   rx_bytes_sec: number;
   tx_bytes_sec: number;
+  /** Delta over the collection interval (rx_errors + any subtype counter). */
   rx_errors: number;
   tx_errors: number;
   rx_drops: number;
   tx_drops: number;
+  /** Delta over the collection interval. Null if counter not available on this NIC. */
+  rx_packets?: number;
+  tx_packets?: number;
+  /** Fine-grained RX hardware-error subtypes (deltas). Null if unavailable. */
+  rx_crc_errors?: number;
+  rx_frame_errors?: number;
+  rx_length_errors?: number;
+  /** TX physical-layer fault counter (delta). Null if unavailable. */
+  tx_carrier_errors?: number;
   operstate?: string; // "up", "down", "unknown", etc. from /sys/class/net/{iface}/operstate
   bond_master?: string; // if this interface is a bond slave, the bond name
+  is_bond_master?: boolean; // true when this entry represents the bond aggregate
 }
 
 export interface RaidInfo {