@glassmkr/crucible 0.1.0 → 0.3.1

package/src/index.ts

@@ -1,6 +1,21 @@
 #!/usr/bin/env node
 
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
 import { loadConfig } from "./config.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PKG_VERSION = (() => {
+  try {
+    const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf8"));
+    return pkg.version || "0.0.0";
+  } catch {
+    return "0.0.0";
+  }
+})();
+import { checkForUpdates } from "./lib/version-check.js";
+import { startMetricsServer, updateMetrics } from "./metrics-server.js";
 import { collectSystem } from "./collect/system.js";
 import { collectCpu } from "./collect/cpu.js";
 import { collectMemory } from "./collect/memory.js";
@@ -15,7 +30,8 @@ import { updateAlertState } from "./alerts/state.js";
 import { sendTelegram } from "./notify/telegram.js";
 import { sendSlack } from "./notify/slack.js";
 import { sendEmail } from "./notify/email.js";
-import { pushToForge } from "./push/forge.js";
+import { pushToForge, initForgeAgent } from "./push/forge.js";
+import { collectSecurity, type SecurityData } from "./collect/security.js";
 import type { Snapshot, IpmiInfo } from "./lib/types.js";
 
 const configPath = process.argv[2] || "/etc/glassmkr/collector.yaml";
@@ -24,8 +40,24 @@ const config = loadConfig(configPath);
 console.log(`[collector] Starting. Server: ${config.server_name}. Interval: ${config.collection.interval_seconds}s`);
 console.log(`[collector] IPMI: ${config.collection.ipmi ? "enabled" : "disabled"}, SMART: ${config.collection.smart ? "enabled" : "disabled"}`);
 console.log(`[collector] Forge: ${config.forge.enabled ? config.forge.url : "disabled"}`);
+console.log(`[collector] Prometheus: ${config.prometheus.enabled ? `:${config.prometheus.port}/metrics` : "disabled"}`);
+
+// Start Prometheus metrics server if enabled
+if (config.prometheus.enabled) {
+  startMetricsServer(config.prometheus.port);
+}
+
+// Initialize TLS pinning for Forge if configured
+if (config.forge.tls_pin) {
+  initForgeAgent(config.forge.tls_pin);
+  console.log("[collector] TLS pinning enabled for Forge");
+}
 
-const emptyIpmi: IpmiInfo = { available: false, sensors: [], ecc_errors: { correctable: 0, uncorrectable: 0 }, sel_entries_count: 0 };
+const emptyIpmi: IpmiInfo = { available: false, sensors: [], ecc_errors: { correctable: 0, uncorrectable: 0 }, sel_entries_count: 0, sel_events_recent: [], fans: [] };
+
+// Security checks run once per hour (every 12th cycle at 5-min intervals)
+let securityCycleCount = 0;
+let cachedSecurity: SecurityData | undefined;
 
 async function collect() {
   const startTime = Date.now();
@@ -43,12 +75,23 @@ async function collect() {
     collectOsAlerts(),
   ]);
 
+  // Security checks: run once per hour, reuse cached data between runs
+  securityCycleCount++;
+  if (securityCycleCount >= 12 || !cachedSecurity) {
+    securityCycleCount = 0;
+    try { cachedSecurity = await collectSecurity(); } catch (err) { console.error("[security] Collection error:", err); }
+  }
+
   const snapshot: Snapshot = {
-    collector_version: "0.1.0",
+    collector_version: PKG_VERSION,
     timestamp: new Date().toISOString(),
     system, cpu, memory, disks, smart, network, raid, ipmi, os_alerts: osAlerts,
+    security: cachedSecurity,
   };
 
+  // Update Prometheus metrics
+  updateMetrics(snapshot);
+
   // Evaluate alerts
   const alertResults = evaluateAlerts(snapshot, config.thresholds);
   const { newAlerts, resolvedAlerts } = updateAlertState(alertResults);
@@ -74,6 +117,9 @@ async function collect() {
     pushToForge(config.forge.url, config.forge.api_key, snapshot);
   }
 
+  // Check for updates (every 6 hours, non-blocking)
+  checkForUpdates(config.forge.enabled ? config.forge.url : undefined);
+
   // Print summary on first run
   if (firstRun) {
     firstRun = false;

package/src/lib/types.ts

@@ -10,6 +10,16 @@ export interface Snapshot {
   raid: RaidInfo[];
   ipmi: IpmiInfo;
   os_alerts: OsAlerts;
+  security?: SecurityData;
+}
+
+export interface SecurityData {
+  ssh: { permitRootLogin: string; passwordAuthentication: string; rootPasswordExposed: boolean } | null;
+  firewall: { active: boolean; source: string; details: string };
+  pending_updates: { distro: string; pendingCount: number; available: boolean } | null;
+  kernel_vulns: Array<{ name: string; status: string; mitigated: boolean }>;
+  kernel_reboot: { running: string; installed: string; needsReboot: boolean } | null;
+  auto_updates: { configured: boolean; mechanism: string; details: string };
 }
 
 export interface SystemInfo {
@@ -20,6 +30,16 @@ export interface SystemInfo {
   uptime_seconds: number;
 }
 
+export interface CpuCoreInfo {
+  core: number;
+  user_percent: number;
+  system_percent: number;
+  iowait_percent: number;
+  idle_percent: number;
+  irq_percent: number;
+  softirq_percent: number;
+}
+
 export interface CpuInfo {
   user_percent: number;
   system_percent: number;
@@ -28,6 +48,7 @@ export interface CpuInfo {
   load_1m: number;
   load_5m: number;
   load_15m: number;
+  cores?: CpuCoreInfo[];
 }
 
 export interface MemoryInfo {
@@ -45,6 +66,11 @@ export interface DiskInfo {
   used_gb: number;
   available_gb: number;
   percent_used: number;
+  fstype?: string;
+  options?: string;
+  inodes_total?: number;
+  inodes_used?: number;
+  inodes_free?: number;
   io_read_mb_s?: number;
   io_write_mb_s?: number;
   latency_p99_ms?: number;
@@ -81,6 +107,22 @@ export interface RaidInfo {
   failed_disks: string[];
 }
 
+export interface SelEvent {
+  id: number;
+  timestamp: string;
+  sensor: string;
+  sensor_type: string;
+  event: string;
+  direction: string;
+  severity: string;
+}
+
+export interface FanStatus {
+  name: string;
+  rpm: number;
+  status: string;
+}
+
 export interface IpmiInfo {
   available: boolean;
   sensors: Array<{
@@ -92,6 +134,8 @@ export interface IpmiInfo {
   }>;
   ecc_errors: { correctable: number; uncorrectable: number };
   sel_entries_count: number;
+  sel_events_recent: SelEvent[];
+  fans: FanStatus[];
 }
 
 export interface OsAlerts {

package/src/lib/version-check.ts

@@ -0,0 +1,38 @@
+const CURRENT_VERSION = "0.1.0";
+let lastCheckTime = 0;
+let lastResult: { updateAvailable: boolean; latest: string; changelog: string } | null = null;
+const CHECK_INTERVAL = 6 * 60 * 60 * 1000; // check every 6 hours
+
+export function getCurrentVersion(): string {
+  return CURRENT_VERSION;
+}
+
+export async function checkForUpdates(forgeUrl?: string): Promise<void> {
+  const now = Date.now();
+  if (now - lastCheckTime < CHECK_INTERVAL) return;
+  lastCheckTime = now;
+
+  const url = forgeUrl || "https://forge.glassmkr.com";
+  try {
+    const res = await fetch(`${url}/api/v1/version`, { signal: AbortSignal.timeout(5000) });
+    if (!res.ok) return;
+    const data = await res.json() as { crucible?: { latest?: string; min_supported?: string; changelog_url?: string } };
+    const latest = data.crucible?.latest;
+    if (!latest) return;
+
+    if (latest !== CURRENT_VERSION) {
+      console.log(`[update] New Crucible version available: ${latest} (current: ${CURRENT_VERSION})`);
+      console.log(`[update] Changelog: ${data.crucible?.changelog_url || "https://github.com/glassmkr/crucible/releases"}`);
+      console.log(`[update] Run: npm update -g @glassmkr/crucible && sudo systemctl restart glassmkr-collector`);
+      lastResult = { updateAvailable: true, latest, changelog: data.crucible?.changelog_url || "" };
+    } else {
+      lastResult = { updateAvailable: false, latest, changelog: "" };
+    }
+  } catch {
+    // Version check is non-critical, fail silently
+  }
+}
+
+export function getUpdateStatus() {
+  return lastResult;
+}

package/src/metrics-server.ts

@@ -0,0 +1,123 @@
+import { createServer } from "http";
+import type { Snapshot } from "./lib/types.js";
+
+let latestSnapshot: Snapshot | null = null;
+
+export function updateMetrics(snapshot: Snapshot) {
+  latestSnapshot = snapshot;
+}
+
+export function startMetricsServer(port: number) {
+  const server = createServer((req, res) => {
+    if (req.url === "/metrics" && req.method === "GET") {
+      if (!latestSnapshot) {
+        res.writeHead(503);
+        res.end("# No data collected yet\n");
+        return;
+      }
+      res.writeHead(200, { "Content-Type": "text/plain; version=0.0.4" });
+      res.end(formatPrometheus(latestSnapshot));
+    } else if (req.url === "/health") {
+      res.writeHead(200);
+      res.end("ok\n");
+    } else {
+      res.writeHead(404);
+      res.end("Not found\n");
+    }
+  });
+
+  server.listen(port, "0.0.0.0", () => {
+    console.log(`[metrics] Prometheus endpoint listening on :${port}/metrics`);
+  });
+}
+
+function formatPrometheus(snap: Snapshot): string {
+  const lines: string[] = [];
+
+  // CPU
+  lines.push("# HELP glassmkr_cpu_user_percent CPU user utilization");
+  lines.push("# TYPE glassmkr_cpu_user_percent gauge");
+  lines.push(`glassmkr_cpu_user_percent ${snap.cpu.user_percent}`);
+  lines.push(`glassmkr_cpu_system_percent ${snap.cpu.system_percent}`);
+  lines.push(`glassmkr_cpu_iowait_percent ${snap.cpu.iowait_percent}`);
+  lines.push(`glassmkr_cpu_idle_percent ${snap.cpu.idle_percent}`);
+  lines.push(`glassmkr_load_1m ${snap.cpu.load_1m}`);
+  lines.push(`glassmkr_load_5m ${snap.cpu.load_5m}`);
+  lines.push(`glassmkr_load_15m ${snap.cpu.load_15m}`);
+
+  // Memory
+  lines.push("# HELP glassmkr_memory_used_mb Memory used in MB");
+  lines.push("# TYPE glassmkr_memory_used_mb gauge");
+  lines.push(`glassmkr_memory_used_mb ${snap.memory.used_mb}`);
+  lines.push(`glassmkr_memory_total_mb ${snap.memory.total_mb}`);
+  lines.push(`glassmkr_memory_available_mb ${snap.memory.available_mb}`);
+  lines.push(`glassmkr_swap_used_mb ${snap.memory.swap_used_mb}`);
+
+  // Disks
+  lines.push("# HELP glassmkr_disk_used_percent Disk usage percentage");
+  lines.push("# TYPE glassmkr_disk_used_percent gauge");
+  for (const disk of snap.disks) {
+    const labels = `mount="${disk.mount}",device="${disk.device}"`;
+    lines.push(`glassmkr_disk_used_percent{${labels}} ${disk.percent_used}`);
+    lines.push(`glassmkr_disk_total_gb{${labels}} ${disk.total_gb}`);
+    lines.push(`glassmkr_disk_used_gb{${labels}} ${disk.used_gb}`);
+  }
+
+  // Network
+  lines.push("# HELP glassmkr_net_rx_bytes_sec Network receive bytes per second");
+  lines.push("# TYPE glassmkr_net_rx_bytes_sec gauge");
+  for (const iface of snap.network) {
+    const labels = `interface="${iface.interface}"`;
+    lines.push(`glassmkr_net_rx_bytes_sec{${labels}} ${iface.rx_bytes_sec}`);
+    lines.push(`glassmkr_net_tx_bytes_sec{${labels}} ${iface.tx_bytes_sec}`);
+    lines.push(`glassmkr_net_rx_errors{${labels}} ${iface.rx_errors}`);
+    lines.push(`glassmkr_net_tx_errors{${labels}} ${iface.tx_errors}`);
+    lines.push(`glassmkr_net_speed_mbps{${labels}} ${iface.speed_mbps}`);
+  }
+
+  // SMART
+  for (const drive of snap.smart) {
+    const labels = `device="${drive.device}",model="${drive.model}"`;
+    if (drive.temperature_c != null) lines.push(`glassmkr_smart_temperature_c{${labels}} ${drive.temperature_c}`);
+    if (drive.percentage_used != null) lines.push(`glassmkr_smart_percentage_used{${labels}} ${drive.percentage_used}`);
+    if (drive.reallocated_sectors != null) lines.push(`glassmkr_smart_reallocated_sectors{${labels}} ${drive.reallocated_sectors}`);
+  }
+
+  // IPMI
+  if (snap.ipmi?.available) {
+    for (const sensor of snap.ipmi.sensors) {
+      if (typeof sensor.value === "number") {
+        const sensorName = sensor.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
+        lines.push(`glassmkr_ipmi_sensor{sensor="${sensor.name}",unit="${sensor.unit}"} ${sensor.value}`);
+      }
+    }
+    lines.push(`glassmkr_ipmi_ecc_correctable ${snap.ipmi.ecc_errors.correctable}`);
+    lines.push(`glassmkr_ipmi_ecc_uncorrectable ${snap.ipmi.ecc_errors.uncorrectable}`);
+
+    // Fans
+    if (snap.ipmi.fans) {
+      for (const fan of snap.ipmi.fans) {
+        lines.push(`glassmkr_ipmi_fan_rpm{fan="${fan.name}",status="${fan.status}"} ${fan.rpm}`);
+      }
+    }
+  }
+
+  // OS alerts
+  lines.push(`glassmkr_oom_kills_recent ${snap.os_alerts.oom_kills_recent}`);
+  lines.push(`glassmkr_zombie_processes ${snap.os_alerts.zombie_processes}`);
+
+  // Security
+  if (snap.security) {
+    lines.push(`glassmkr_ssh_root_password_exposed ${snap.security.ssh?.rootPasswordExposed ? 1 : 0}`);
+    lines.push(`glassmkr_firewall_active ${snap.security.firewall.active ? 1 : 0}`);
+    if (snap.security.pending_updates?.available) {
+      lines.push(`glassmkr_pending_security_updates ${snap.security.pending_updates.pendingCount}`);
+    }
+    const unmitigated = snap.security.kernel_vulns.filter(v => !v.mitigated).length;
+    lines.push(`glassmkr_kernel_vulns_unmitigated ${unmitigated}`);
+    lines.push(`glassmkr_kernel_needs_reboot ${snap.security.kernel_reboot?.needsReboot ? 1 : 0}`);
+    lines.push(`glassmkr_auto_updates_configured ${snap.security.auto_updates.configured ? 1 : 0}`);
+  }
+
+  return lines.join("\n") + "\n";
+}

package/src/notify/telegram.ts

@@ -1,5 +1,22 @@
 import type { AlertResult } from "../lib/types.js";
 
+const PRIORITY_MAP: Record<string, string> = {
+  raid_degraded: "P1", smart_failing: "P1", ecc_errors: "P1", psu_redundancy_loss: "P1", ipmi_fan_failure: "P1",
+  oom_kills: "P2", ram_high: "P2", disk_space_high: "P2", ipmi_sel_critical: "P2", disk_io_errors: "P2", zfs_pool_unhealthy: "P2",
+  cpu_iowait_high: "P3", nvme_wear_high: "P3", disk_latency_high: "P3", cpu_temperature_high: "P3",
+  ssh_root_password: "P3", pending_security_updates: "P3", kernel_vulnerabilities: "P3", zfs_scrub_errors: "P3",
+  swap_active: "P4", no_firewall: "P4", kernel_needs_reboot: "P4", unattended_upgrades_disabled: "P4",
+  interface_errors: "P4", link_speed_mismatch: "P4", interface_saturation: "P4",
+};
+
+const PRIORITY_LABELS: Record<string, string> = {
+  P1: "\u{1F534} P1 Urgent", P2: "\u{1F7E0} P2 High", P3: "\u{1F7E1} P3 Medium", P4: "\u{1F535} P4 Low",
+};
+
+function getPriority(alertType: string): string {
+  return PRIORITY_MAP[alertType] || "P3";
+}
+
 export async function sendTelegram(
   botToken: string,
   chatId: string,
@@ -10,16 +27,19 @@ export async function sendTelegram(
   const parts: string[] = [];
 
   if (newAlerts.length > 0) {
-    const criticals = newAlerts.filter((a) => a.severity === "critical");
-    const warnings = newAlerts.filter((a) => a.severity === "warning");
-
-    if (criticals.length > 0) {
-      parts.push(`\u{1F534} <b>${criticals.length} CRITICAL</b> on <b>${serverName}</b>:\n`);
-      for (const a of criticals) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
+    // Group by priority
+    const byPriority: Record<string, AlertResult[]> = {};
+    for (const a of newAlerts) {
+      const p = getPriority(a.type);
+      if (!byPriority[p]) byPriority[p] = [];
+      byPriority[p].push(a);
     }
-    if (warnings.length > 0) {
-      parts.push(`\u{1F7E1} <b>${warnings.length} WARNING</b> on <b>${serverName}</b>:\n`);
-      for (const a of warnings) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
+
+    for (const p of ["P1", "P2", "P3", "P4"]) {
+      const alerts = byPriority[p];
+      if (!alerts?.length) continue;
+      parts.push(`${PRIORITY_LABELS[p]} on <b>${serverName}</b>:\n`);
+      for (const a of alerts) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
     }
   }
 

package/src/push/forge.ts

@@ -1,18 +1,55 @@
+import https from "https";
+import tls from "tls";
+import crypto from "crypto";
 import type { Snapshot } from "../lib/types.js";
 
+let agent: https.Agent | undefined;
+
+export function initForgeAgent(tlsPin?: string): void {
+  if (!tlsPin) {
+    agent = undefined; // Use default (Node built-in fetch)
+    return;
+  }
+
+  agent = new https.Agent({
+    rejectUnauthorized: true,
+    checkServerIdentity: (hostname: string, cert: any) => {
+      const err = tls.checkServerIdentity(hostname, cert);
+      if (err) return err;
+
+      const pubkey = cert.pubkey;
+      if (!pubkey) return new Error("Certificate has no public key");
+
+      const hash = crypto.createHash("sha256").update(pubkey).digest("base64");
+      if (hash !== tlsPin) {
+        return new Error(
+          `TLS pin mismatch for ${hostname}. ` +
+          `Expected: ${tlsPin}, Got: ${hash}. ` +
+          `If the server certificate was rotated with a new key, update tls_pin in collector.yaml.`
+        );
+      }
+
+      return undefined;
+    },
+  });
+}
+
 export async function pushToForge(url: string, apiKey: string, snapshot: Snapshot): Promise<boolean> {
+  // If TLS pinning is enabled, use https.request (fetch doesn't support custom agents)
+  if (agent) {
+    return pushWithAgent(url, apiKey, snapshot);
+  }
+
+  // Default: use fetch (no pinning)
   try {
     const response = await fetch(`${url}/api/v1/ingest`, {
       method: "POST",
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-        "Content-Type": "application/json",
-      },
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
       body: JSON.stringify(snapshot),
       signal: AbortSignal.timeout(10000),
     });
     if (response.ok) {
-      const data = await response.json() as { new_alerts?: number; active_alerts?: number };
+      const data = await response.json() as { active_alerts?: number };
       console.log(`[forge] Push successful. Active alerts: ${data.active_alerts ?? 0}`);
     } else {
       console.error(`[forge] Push failed: ${response.status} ${response.statusText}`);
@@ -23,3 +60,50 @@ export async function pushToForge(url: string, apiKey: string, snapshot: Snapsho
     return false;
   }
 }
+
+function pushWithAgent(url: string, apiKey: string, snapshot: Snapshot): Promise<boolean> {
+  return new Promise((resolve) => {
+    const parsed = new URL(`${url}/api/v1/ingest`);
+    const body = JSON.stringify(snapshot);
+
+    const req = https.request({
+      hostname: parsed.hostname,
+      port: parsed.port ? parseInt(parsed.port) : 443,
+      path: parsed.pathname,
+      method: "POST",
+      agent,
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(body),
+      },
+      timeout: 10000,
+    }, (res) => {
+      let data = "";
+      res.on("data", (chunk) => data += chunk);
+      res.on("end", () => {
+        if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
+          try {
+            const parsed = JSON.parse(data);
+            console.log(`[forge] Push successful (pinned). Active alerts: ${parsed.active_alerts ?? 0}`);
+          } catch { /* ignore parse errors */ }
+          resolve(true);
+        } else {
+          console.error(`[forge] Push failed (pinned): ${res.statusCode}`);
+          resolve(false);
+        }
+      });
+    });
+
+    req.on("error", (err) => {
+      console.error(`[forge] Push failed (pinned): ${err.message}`);
+      resolve(false);
+    });
+    req.on("timeout", () => {
+      req.destroy(new Error("Request timed out"));
+      resolve(false);
+    });
+    req.write(body);
+    req.end();
+  });
+}