@glassmkr/crucible 0.7.1 → 0.8.0

package/src/lib/types.ts

@@ -1,226 +0,0 @@
-export interface Snapshot {
-  collector_version: string;
-  timestamp: string;
-  system: SystemInfo;
-  cpu: CpuInfo;
-  memory: MemoryInfo;
-  disks: DiskInfo[];
-  smart: SmartInfo[];
-  network: NetworkInfo[];
-  raid: RaidInfo[];
-  ipmi: IpmiInfo;
-  os_alerts: OsAlerts;
-  security?: SecurityData;
-  zfs?: ZfsData;
-  io_errors?: { count: number; devices: string[] };
-  io_latency?: Array<{ device: string; avg_read_latency_ms: number | null; avg_write_latency_ms: number | null; read_iops: number; write_iops: number }>;
-  conntrack?: ConntrackData;
-  systemd?: SystemdData;
-  ntp?: NtpData;
-  file_descriptors?: FileDescriptorData;
-  // Planned-reboot flag: set only on the first snapshot after a reboot
-  // that was marked with `crucible-agent mark-reboot` / `reboot`. Forge
-  // reads this to suppress the `unexpected_reboot` rule. Single-use:
-  // subsequent snapshots don't carry it.
-  expected_reboot?: boolean;
-  expected_reboot_reason?: string;
-}
-
-export interface ConntrackData {
-  available: boolean;
-  count: number;
-  max: number;
-  percent: number;
-}
-
-export interface SystemdData {
-  failed_units: string[];
-  failed_count: number;
-}
-
-export interface NtpData {
-  synced: boolean;
-  offset_seconds: number;
-  source: string;
-  daemon_running: boolean;
-}
-
-export interface FileDescriptorData {
-  allocated: number;
-  free: number;
-  max: number;
-  percent: number;
-}
-
-export interface ZfsPool {
-  name: string;
-  state: string;
-  errors_text: string;
-  scrub_errors?: number;
-  scrub_repaired?: string;
-  last_scrub_date?: string;
-  scrub_never_run?: boolean;
-}
-
-export interface ZfsData {
-  pools: ZfsPool[];
-}
-
-export interface SecurityData {
-  ssh: { permitRootLogin: string; passwordAuthentication: string; rootPasswordExposed: boolean } | null;
-  firewall: { active: boolean; source: string; details: string };
-  pending_updates: { distro: string; pendingCount: number; available: boolean } | null;
-  kernel_vulns: Array<{ name: string; status: string; mitigated: boolean }>;
-  kernel_reboot: { running: string; installed: string; needsReboot: boolean } | null;
-  auto_updates: { configured: boolean; mechanism: string; details: string };
-}
-
-export interface SystemInfo {
-  hostname: string;
-  ip: string;
-  os: string;
-  /** `ID=` from /etc/os-release, lowercased. e.g. "ubuntu", "debian", "rocky", "arch", "alpine". */
-  os_id?: string;
-  /** `ID_LIKE=` from /etc/os-release, lowercased, space-separated. Used by Forge
-   *  to pick distro-family-specific fix command variants. e.g. on Rocky this
-   *  is "rhel centos fedora"; on Ubuntu it is "debian". */
-  os_id_like?: string;
-  kernel: string;
-  uptime_seconds: number;
-}
-
-export interface CpuCoreInfo {
-  core: number;
-  user_percent: number;
-  system_percent: number;
-  iowait_percent: number;
-  idle_percent: number;
-  irq_percent: number;
-  softirq_percent: number;
-}
-
-export interface CpuInfo {
-  user_percent: number;
-  system_percent: number;
-  iowait_percent: number;
-  idle_percent: number;
-  load_1m: number;
-  load_5m: number;
-  load_15m: number;
-  cores?: CpuCoreInfo[];
-}
-
-export interface MemoryInfo {
-  total_mb: number;
-  used_mb: number;
-  available_mb: number;
-  swap_total_mb: number;
-  swap_used_mb: number;
-}
-
-export interface DiskInfo {
-  device: string;
-  mount: string;
-  total_gb: number;
-  used_gb: number;
-  available_gb: number;
-  percent_used: number;
-  fstype?: string;
-  options?: string;
-  inodes_total?: number;
-  inodes_used?: number;
-  inodes_free?: number;
-  io_read_mb_s?: number;
-  io_write_mb_s?: number;
-  latency_p99_ms?: number;
-}
-
-export interface SmartInfo {
-  device: string;
-  model: string;
-  health: string;
-  temperature_c?: number;
-  percentage_used?: number;
-  reallocated_sectors?: number;
-  pending_sectors?: number;
-  power_on_hours?: number;
-}
-
-export interface NetworkInfo {
-  interface: string;
-  speed_mbps: number;
-  rx_bytes_sec: number;
-  tx_bytes_sec: number;
-  /** Delta over the collection interval (rx_errors + any subtype counter). */
-  rx_errors: number;
-  tx_errors: number;
-  rx_drops: number;
-  tx_drops: number;
-  /** Delta over the collection interval. Null if counter not available on this NIC. */
-  rx_packets?: number;
-  tx_packets?: number;
-  /** Fine-grained RX hardware-error subtypes (deltas). Null if unavailable. */
-  rx_crc_errors?: number;
-  rx_frame_errors?: number;
-  rx_length_errors?: number;
-  /** TX physical-layer fault counter (delta). Null if unavailable. */
-  tx_carrier_errors?: number;
-  operstate?: string; // "up", "down", "unknown", etc. from /sys/class/net/{iface}/operstate
-  bond_master?: string; // if this interface is a bond slave, the bond name
-  is_bond_master?: boolean; // true when this entry represents the bond aggregate
-}
-
-export interface RaidInfo {
-  device: string;
-  level: string;
-  status: string;
-  degraded: boolean;
-  disks: string[];
-  failed_disks: string[];
-}
-
-export interface SelEvent {
-  id: number;
-  timestamp: string;
-  sensor: string;
-  sensor_type: string;
-  event: string;
-  direction: string;
-  severity: string;
-}
-
-export interface FanStatus {
-  name: string;
-  rpm: number;
-  status: string;
-}
-
-export interface IpmiInfo {
-  available: boolean;
-  sensors: Array<{
-    name: string;
-    value: number | string;
-    unit: string;
-    status: string;
-    upper_critical?: number;
-  }>;
-  ecc_errors: { correctable: number; uncorrectable: number };
-  sel_entries_count: number;
-  sel_events_recent: SelEvent[];
-  fans: FanStatus[];
-}
-
-export interface OsAlerts {
-  oom_kills_recent: number;
-  zombie_processes: number;
-  time_drift_ms: number;
-}
-
-export interface AlertResult {
-  type: string;
-  severity: "critical" | "warning";
-  title: string;
-  message: string;
-  evidence: Record<string, unknown>;
-  recommendation: string;
-}

package/src/lib/version-check.ts

@@ -1,39 +0,0 @@
-import { CRUCIBLE_VERSION as CURRENT_VERSION } from "./version.js";
-
-let lastCheckTime = 0;
-let lastResult: { updateAvailable: boolean; latest: string; changelog: string } | null = null;
-const CHECK_INTERVAL = 6 * 60 * 60 * 1000; // check every 6 hours
-
-export function getCurrentVersion(): string {
-  return CURRENT_VERSION;
-}
-
-export async function checkForUpdates(forgeUrl?: string): Promise<void> {
-  const now = Date.now();
-  if (now - lastCheckTime < CHECK_INTERVAL) return;
-  lastCheckTime = now;
-
-  const url = forgeUrl || "https://forge.glassmkr.com";
-  try {
-    const res = await fetch(`${url}/api/v1/version`, { signal: AbortSignal.timeout(5000) });
-    if (!res.ok) return;
-    const data = await res.json() as { crucible?: { latest?: string; min_supported?: string; changelog_url?: string } };
-    const latest = data.crucible?.latest;
-    if (!latest) return;
-
-    if (latest !== CURRENT_VERSION) {
-      console.log(`[update] New Crucible version available: ${latest} (current: ${CURRENT_VERSION})`);
-      console.log(`[update] Changelog: ${data.crucible?.changelog_url || "https://github.com/glassmkr/crucible/releases"}`);
-      console.log(`[update] Run: npm update -g @glassmkr/crucible && sudo systemctl restart glassmkr-crucible`);
-      lastResult = { updateAvailable: true, latest, changelog: data.crucible?.changelog_url || "" };
-    } else {
-      lastResult = { updateAvailable: false, latest, changelog: "" };
-    }
-  } catch {
-    // Version check is non-critical, fail silently
-  }
-}
-
-export function getUpdateStatus() {
-  return lastResult;
-}

package/src/lib/version.ts

@@ -1,33 +0,0 @@
-// Single source of truth for the Crucible version string at runtime.
-// Read from package.json so a release bump propagates everywhere
-// (notification footers, version-check log lines, --version flag, the
-// `collector_version` field on every snapshot) without anyone having to
-// remember to update a hardcoded constant.
-//
-// Returns "0.0.0" on read failure rather than throwing; the agent has to
-// keep running even if its package.json is somehow missing.
-
-import { readFileSync } from "node:fs";
-import { fileURLToPath } from "node:url";
-import { dirname, join } from "node:path";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-function readPkgVersion(): string {
-  // src/lib/version.ts -> ../../package.json under src layout, but at
-  // runtime (compiled to dist/lib/version.js) it's still ../../package.json.
-  // Both paths resolve correctly because package.json sits one level above
-  // dist/ AND one level above src/.
-  for (const candidate of [
-    join(__dirname, "..", "..", "package.json"),
-    join(__dirname, "..", "package.json"),
-  ]) {
-    try {
-      const pkg = JSON.parse(readFileSync(candidate, "utf8"));
-      if (pkg && typeof pkg.version === "string") return pkg.version;
-    } catch { /* try next */ }
-  }
-  return "0.0.0";
-}
-
-export const CRUCIBLE_VERSION = readPkgVersion();

package/src/metrics-server.ts

@@ -1,123 +0,0 @@
-import { createServer } from "http";
-import type { Snapshot } from "./lib/types.js";
-
-let latestSnapshot: Snapshot | null = null;
-
-export function updateMetrics(snapshot: Snapshot) {
-  latestSnapshot = snapshot;
-}
-
-export function startMetricsServer(port: number) {
-  const server = createServer((req, res) => {
-    if (req.url === "/metrics" && req.method === "GET") {
-      if (!latestSnapshot) {
-        res.writeHead(503);
-        res.end("# No data collected yet\n");
-        return;
-      }
-      res.writeHead(200, { "Content-Type": "text/plain; version=0.0.4" });
-      res.end(formatPrometheus(latestSnapshot));
-    } else if (req.url === "/health") {
-      res.writeHead(200);
-      res.end("ok\n");
-    } else {
-      res.writeHead(404);
-      res.end("Not found\n");
-    }
-  });
-
-  server.listen(port, "0.0.0.0", () => {
-    console.log(`[metrics] Prometheus endpoint listening on :${port}/metrics`);
-  });
-}
-
-function formatPrometheus(snap: Snapshot): string {
-  const lines: string[] = [];
-
-  // CPU
-  lines.push("# HELP glassmkr_cpu_user_percent CPU user utilization");
-  lines.push("# TYPE glassmkr_cpu_user_percent gauge");
-  lines.push(`glassmkr_cpu_user_percent ${snap.cpu.user_percent}`);
-  lines.push(`glassmkr_cpu_system_percent ${snap.cpu.system_percent}`);
-  lines.push(`glassmkr_cpu_iowait_percent ${snap.cpu.iowait_percent}`);
-  lines.push(`glassmkr_cpu_idle_percent ${snap.cpu.idle_percent}`);
-  lines.push(`glassmkr_load_1m ${snap.cpu.load_1m}`);
-  lines.push(`glassmkr_load_5m ${snap.cpu.load_5m}`);
-  lines.push(`glassmkr_load_15m ${snap.cpu.load_15m}`);
-
-  // Memory
-  lines.push("# HELP glassmkr_memory_used_mb Memory used in MB");
-  lines.push("# TYPE glassmkr_memory_used_mb gauge");
-  lines.push(`glassmkr_memory_used_mb ${snap.memory.used_mb}`);
-  lines.push(`glassmkr_memory_total_mb ${snap.memory.total_mb}`);
-  lines.push(`glassmkr_memory_available_mb ${snap.memory.available_mb}`);
-  lines.push(`glassmkr_swap_used_mb ${snap.memory.swap_used_mb}`);
-
-  // Disks
-  lines.push("# HELP glassmkr_disk_used_percent Disk usage percentage");
-  lines.push("# TYPE glassmkr_disk_used_percent gauge");
-  for (const disk of snap.disks) {
-    const labels = `mount="${disk.mount}",device="${disk.device}"`;
-    lines.push(`glassmkr_disk_used_percent{${labels}} ${disk.percent_used}`);
-    lines.push(`glassmkr_disk_total_gb{${labels}} ${disk.total_gb}`);
-    lines.push(`glassmkr_disk_used_gb{${labels}} ${disk.used_gb}`);
-  }
-
-  // Network
-  lines.push("# HELP glassmkr_net_rx_bytes_sec Network receive bytes per second");
-  lines.push("# TYPE glassmkr_net_rx_bytes_sec gauge");
-  for (const iface of snap.network) {
-    const labels = `interface="${iface.interface}"`;
-    lines.push(`glassmkr_net_rx_bytes_sec{${labels}} ${iface.rx_bytes_sec}`);
-    lines.push(`glassmkr_net_tx_bytes_sec{${labels}} ${iface.tx_bytes_sec}`);
-    lines.push(`glassmkr_net_rx_errors{${labels}} ${iface.rx_errors}`);
-    lines.push(`glassmkr_net_tx_errors{${labels}} ${iface.tx_errors}`);
-    lines.push(`glassmkr_net_speed_mbps{${labels}} ${iface.speed_mbps}`);
-  }
-
-  // SMART
-  for (const drive of snap.smart) {
-    const labels = `device="${drive.device}",model="${drive.model}"`;
-    if (drive.temperature_c != null) lines.push(`glassmkr_smart_temperature_c{${labels}} ${drive.temperature_c}`);
-    if (drive.percentage_used != null) lines.push(`glassmkr_smart_percentage_used{${labels}} ${drive.percentage_used}`);
-    if (drive.reallocated_sectors != null) lines.push(`glassmkr_smart_reallocated_sectors{${labels}} ${drive.reallocated_sectors}`);
-  }
-
-  // IPMI
-  if (snap.ipmi?.available) {
-    for (const sensor of snap.ipmi.sensors) {
-      if (typeof sensor.value === "number") {
-        const sensorName = sensor.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
-        lines.push(`glassmkr_ipmi_sensor{sensor="${sensor.name}",unit="${sensor.unit}"} ${sensor.value}`);
-      }
-    }
-    lines.push(`glassmkr_ipmi_ecc_correctable ${snap.ipmi.ecc_errors.correctable}`);
-    lines.push(`glassmkr_ipmi_ecc_uncorrectable ${snap.ipmi.ecc_errors.uncorrectable}`);
-
-    // Fans
-    if (snap.ipmi.fans) {
-      for (const fan of snap.ipmi.fans) {
-        lines.push(`glassmkr_ipmi_fan_rpm{fan="${fan.name}",status="${fan.status}"} ${fan.rpm}`);
-      }
-    }
-  }
-
-  // OS alerts
-  lines.push(`glassmkr_oom_kills_recent ${snap.os_alerts.oom_kills_recent}`);
-  lines.push(`glassmkr_zombie_processes ${snap.os_alerts.zombie_processes}`);
-
-  // Security
-  if (snap.security) {
-    lines.push(`glassmkr_ssh_root_password_exposed ${snap.security.ssh?.rootPasswordExposed ? 1 : 0}`);
-    lines.push(`glassmkr_firewall_active ${snap.security.firewall.active ? 1 : 0}`);
-    if (snap.security.pending_updates?.available) {
-      lines.push(`glassmkr_pending_security_updates ${snap.security.pending_updates.pendingCount}`);
-    }
-    const unmitigated = snap.security.kernel_vulns.filter(v => !v.mitigated).length;
-    lines.push(`glassmkr_kernel_vulns_unmitigated ${unmitigated}`);
-    lines.push(`glassmkr_kernel_needs_reboot ${snap.security.kernel_reboot?.needsReboot ? 1 : 0}`);
-    lines.push(`glassmkr_auto_updates_configured ${snap.security.auto_updates.configured ? 1 : 0}`);
-  }
-
-  return lines.join("\n") + "\n";
-}

package/src/notify/email.ts

@@ -1,69 +0,0 @@
-import { execFile } from "child_process";
-import { promisify } from "util";
-import type { AlertResult } from "../lib/types.js";
-import { CRUCIBLE_VERSION } from "../lib/version.js";
-
-const execFileAsync = promisify(execFile);
-
-export async function sendEmail(
-  config: { to: string },
-  newAlerts: AlertResult[],
-  resolvedAlerts: AlertResult[],
-  serverName: string
-): Promise<boolean> {
-  if (!config.to) return false;
-
-  const subject = buildSubject(newAlerts, resolvedAlerts, serverName);
-  const body = buildBody(newAlerts, resolvedAlerts, serverName);
-
-  const email = [
-    `To: ${config.to}`,
-    `From: glassmkr-crucible@${serverName}`,
-    `Subject: ${subject}`,
-    `Content-Type: text/plain; charset=utf-8`,
-    "",
-    body,
-  ].join("\n");
-
-  try {
-    const child = execFileAsync("/usr/sbin/sendmail", ["-t"], { timeout: 10000 });
-    child.child.stdin?.write(email);
-    child.child.stdin?.end();
-    await child;
-    return true;
-  } catch {
-    console.error("[email] Failed to send. Is sendmail/postfix/msmtp installed?");
-    return false;
-  }
-}
-
-function buildSubject(newAlerts: AlertResult[], resolvedAlerts: AlertResult[], serverName: string): string {
-  if (newAlerts.length > 0) {
-    const worst = newAlerts.find((a) => a.severity === "critical") ? "CRITICAL" : "WARNING";
-    return `[${worst}] ${serverName}: ${newAlerts.length} alert(s)`;
-  }
-  return `[RESOLVED] ${serverName}: ${resolvedAlerts.length} alert(s) cleared`;
-}
-
-function buildBody(newAlerts: AlertResult[], resolvedAlerts: AlertResult[], serverName: string): string {
-  const lines: string[] = [];
-  lines.push(`Server: ${serverName}`);
-  lines.push(`Time: ${new Date().toISOString()}`);
-  lines.push("");
-
-  for (const a of newAlerts) {
-    lines.push(`[${a.severity.toUpperCase()}] ${a.title}`);
-    lines.push(a.message);
-    lines.push(`Action: ${a.recommendation}`);
-    lines.push("");
-  }
-
-  for (const a of resolvedAlerts) {
-    lines.push(`[RESOLVED] ${a.title}`);
-    lines.push("");
-  }
-
-  lines.push("---");
-  lines.push(`Glassmkr Crucible v${CRUCIBLE_VERSION}`);
-  return lines.join("\n");
-}

package/src/notify/slack.ts

@@ -1,47 +0,0 @@
-import type { AlertResult } from "../lib/types.js";
-import { CRUCIBLE_VERSION } from "../lib/version.js";
-
-export async function sendSlack(
-  webhookUrl: string,
-  newAlerts: AlertResult[],
-  resolvedAlerts: AlertResult[],
-  serverName: string
-): Promise<boolean> {
-  const blocks: any[] = [];
-
-  if (newAlerts.length > 0) {
-    const criticals = newAlerts.filter((a) => a.severity === "critical");
-    const warnings = newAlerts.filter((a) => a.severity === "warning");
-
-    if (criticals.length > 0) {
-      blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u{1F534} *${criticals.length} CRITICAL* on *${serverName}*` } });
-      for (const a of criticals) blocks.push({ type: "section", text: { type: "mrkdwn", text: `*${a.title}*\n${a.recommendation}` } });
-    }
-    if (warnings.length > 0) {
-      blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u{1F7E1} *${warnings.length} WARNING* on *${serverName}*` } });
-      for (const a of warnings) blocks.push({ type: "section", text: { type: "mrkdwn", text: `*${a.title}*\n${a.recommendation}` } });
-    }
-  }
-
-  if (resolvedAlerts.length > 0) {
-    blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u2705 *${resolvedAlerts.length} resolved* on *${serverName}*` } });
-  }
-
-  if (blocks.length === 0) return true;
-
-  blocks.push({ type: "divider" });
-  blocks.push({ type: "context", elements: [{ type: "mrkdwn", text: `Glassmkr Crucible v${CRUCIBLE_VERSION}` }] });
-
-  try {
-    const res = await fetch(webhookUrl, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ blocks }),
-      signal: AbortSignal.timeout(10000),
-    });
-    return res.ok;
-  } catch {
-    console.error("[slack] Failed to send notification");
-    return false;
-  }
-}

package/src/notify/telegram.ts

@@ -1,65 +0,0 @@
-import type { AlertResult } from "../lib/types.js";
-
-const PRIORITY_MAP: Record<string, string> = {
-  raid_degraded: "P1", smart_failing: "P1", ecc_errors: "P1", psu_redundancy_loss: "P1", ipmi_fan_failure: "P1",
-  oom_kills: "P2", ram_high: "P2", disk_space_high: "P2", ipmi_sel_critical: "P2", disk_io_errors: "P2", zfs_pool_unhealthy: "P2",
-  cpu_iowait_high: "P3", nvme_wear_high: "P3", disk_latency_high: "P3", cpu_temperature_high: "P3",
-  ssh_root_password: "P3", pending_security_updates: "P3", kernel_vulnerabilities: "P3", zfs_scrub_errors: "P3",
-  swap_active: "P4", no_firewall: "P4", kernel_needs_reboot: "P4", unattended_upgrades_disabled: "P4",
-  interface_errors: "P4", link_speed_mismatch: "P4", interface_saturation: "P4",
-};
-
-const PRIORITY_LABELS: Record<string, string> = {
-  P1: "\u{1F534} P1 Urgent", P2: "\u{1F7E0} P2 High", P3: "\u{1F7E1} P3 Medium", P4: "\u{1F535} P4 Low",
-};
-
-function getPriority(alertType: string): string {
-  return PRIORITY_MAP[alertType] || "P3";
-}
-
-export async function sendTelegram(
-  botToken: string,
-  chatId: string,
-  newAlerts: AlertResult[],
-  resolvedAlerts: AlertResult[],
-  serverName: string
-): Promise<boolean> {
-  const parts: string[] = [];
-
-  if (newAlerts.length > 0) {
-    // Group by priority
-    const byPriority: Record<string, AlertResult[]> = {};
-    for (const a of newAlerts) {
-      const p = getPriority(a.type);
-      if (!byPriority[p]) byPriority[p] = [];
-      byPriority[p].push(a);
-    }
-
-    for (const p of ["P1", "P2", "P3", "P4"]) {
-      const alerts = byPriority[p];
-      if (!alerts?.length) continue;
-      parts.push(`${PRIORITY_LABELS[p]} on <b>${serverName}</b>:\n`);
-      for (const a of alerts) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
-    }
-  }
-
-  if (resolvedAlerts.length > 0) {
-    parts.push(`\u2705 <b>${resolvedAlerts.length} resolved</b> on <b>${serverName}</b>:\n`);
-    for (const a of resolvedAlerts) parts.push(`  \u2022 ${a.title}\n`);
-  }
-
-  if (parts.length === 0) return true;
-
-  try {
-    const res = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ chat_id: chatId, text: parts.join("\n"), parse_mode: "HTML", disable_web_page_preview: true }),
-      signal: AbortSignal.timeout(10000),
-    });
-    return res.ok;
-  } catch {
-    console.error("[telegram] Failed to send notification");
-    return false;
-  }
-}