@glassmkr/crucible 0.1.0

package/src/config.ts

@@ -0,0 +1,60 @@
+import { readFileSync } from "fs";
+import { parse } from "yaml";
+import { z } from "zod";
+
+const ConfigSchema = z.object({
+  server_name: z.string().default("unnamed-server"),
+  collection: z.object({
+    interval_seconds: z.number().min(60).max(3600).default(300),
+    ipmi: z.boolean().default(true),
+    smart: z.boolean().default(true),
+  }).default({}),
+  forge: z.object({
+    enabled: z.boolean().default(false),
+    url: z.string().default("https://forge.glassmkr.com"),
+    api_key: z.string().default(""),
+  }).default({}),
+  thresholds: z.object({
+    ram_percent: z.number().default(90),
+    swap_alert: z.boolean().default(true),
+    disk_percent: z.number().default(85),
+    iowait_percent: z.number().default(20),
+    nvme_wear_percent: z.number().default(85),
+    disk_latency_nvme_ms: z.number().default(50),
+    disk_latency_hdd_ms: z.number().default(200),
+    cpu_temp_warning_c: z.number().default(80),
+    cpu_temp_critical_c: z.number().default(90),
+    interface_utilization_percent: z.number().default(90),
+  }).default({}),
+  channels: z.object({
+    telegram: z.object({
+      enabled: z.boolean().default(false),
+      bot_token: z.string().default(""),
+      chat_id: z.string().default(""),
+    }).default({}),
+    email: z.object({
+      enabled: z.boolean().default(false),
+      to: z.string().default(""),
+    }).default({}),
+    slack: z.object({
+      enabled: z.boolean().default(false),
+      webhook_url: z.string().default(""),
+    }).default({}),
+  }).default({}),
+});
+
+export type Config = z.infer<typeof ConfigSchema>;
+
+export function loadConfig(path: string): Config {
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const parsed = parse(raw);
+    return ConfigSchema.parse(parsed);
+  } catch (err: any) {
+    if (err.code === "ENOENT") {
+      console.log(`[config] No config file at ${path}, using defaults`);
+      return ConfigSchema.parse({});
+    }
+    throw err;
+  }
+}

package/src/index.ts

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+
+import { loadConfig } from "./config.js";
+import { collectSystem } from "./collect/system.js";
+import { collectCpu } from "./collect/cpu.js";
+import { collectMemory } from "./collect/memory.js";
+import { collectDisks } from "./collect/disks.js";
+import { collectSmart } from "./collect/smart.js";
+import { collectNetwork } from "./collect/network.js";
+import { collectRaid } from "./collect/raid.js";
+import { collectIpmi } from "./collect/ipmi.js";
+import { collectOsAlerts } from "./collect/os-alerts.js";
+import { evaluateAlerts } from "./alerts/evaluator.js";
+import { updateAlertState } from "./alerts/state.js";
+import { sendTelegram } from "./notify/telegram.js";
+import { sendSlack } from "./notify/slack.js";
+import { sendEmail } from "./notify/email.js";
+import { pushToForge } from "./push/forge.js";
+import type { Snapshot, IpmiInfo } from "./lib/types.js";
+
+const configPath = process.argv[2] || "/etc/glassmkr/collector.yaml";
+const config = loadConfig(configPath);
+
+console.log(`[collector] Starting. Server: ${config.server_name}. Interval: ${config.collection.interval_seconds}s`);
+console.log(`[collector] IPMI: ${config.collection.ipmi ? "enabled" : "disabled"}, SMART: ${config.collection.smart ? "enabled" : "disabled"}`);
+console.log(`[collector] Forge: ${config.forge.enabled ? config.forge.url : "disabled"}`);
+
+const emptyIpmi: IpmiInfo = { available: false, sensors: [], ecc_errors: { correctable: 0, uncorrectable: 0 }, sel_entries_count: 0 };
+
+async function collect() {
+  const startTime = Date.now();
+  console.log(`[collector] Collecting...`);
+
+  const [system, cpu, memory, disks, smart, network, raid, ipmi, osAlerts] = await Promise.all([
+    collectSystem(),
+    collectCpu(),
+    collectMemory(),
+    collectDisks(),
+    config.collection.smart ? collectSmart() : Promise.resolve([]),
+    collectNetwork(),
+    collectRaid(),
+    config.collection.ipmi ? collectIpmi() : Promise.resolve(emptyIpmi),
+    collectOsAlerts(),
+  ]);
+
+  const snapshot: Snapshot = {
+    collector_version: "0.1.0",
+    timestamp: new Date().toISOString(),
+    system, cpu, memory, disks, smart, network, raid, ipmi, os_alerts: osAlerts,
+  };
+
+  // Evaluate alerts
+  const alertResults = evaluateAlerts(snapshot, config.thresholds);
+  const { newAlerts, resolvedAlerts } = updateAlertState(alertResults);
+
+  const elapsed = Date.now() - startTime;
+  console.log(`[collector] Collected in ${elapsed}ms. Alerts: ${alertResults.length} active, ${newAlerts.length} new, ${resolvedAlerts.length} resolved`);
+
+  // Send notifications for new/resolved alerts
+  if (newAlerts.length > 0 || resolvedAlerts.length > 0) {
+    if (config.channels.telegram.enabled && config.channels.telegram.bot_token && config.channels.telegram.chat_id) {
+      await sendTelegram(config.channels.telegram.bot_token, config.channels.telegram.chat_id, newAlerts, resolvedAlerts, config.server_name);
+    }
+    if (config.channels.slack.enabled && config.channels.slack.webhook_url) {
+      await sendSlack(config.channels.slack.webhook_url, newAlerts, resolvedAlerts, config.server_name);
+    }
+    if (config.channels.email.enabled && config.channels.email.to) {
+      await sendEmail(config.channels.email, newAlerts, resolvedAlerts, config.server_name);
+    }
+  }
+
+  // Push to Forge (non-blocking)
+  if (config.forge.enabled && config.forge.api_key) {
+    pushToForge(config.forge.url, config.forge.api_key, snapshot);
+  }
+
+  // Print summary on first run
+  if (firstRun) {
+    firstRun = false;
+    console.log("");
+    console.log("=== First collection complete ===");
+    console.log(`Server: ${system.hostname} (${system.os})`);
+    console.log(`CPU:    ${cpu.user_percent.toFixed(1)}% (load: ${cpu.load_1m})`);
+    const ramPct = memory.total_mb > 0 ? ((memory.used_mb / memory.total_mb) * 100).toFixed(1) : "0";
+    console.log(`RAM:    ${ramPct}% (${memory.used_mb} / ${memory.total_mb} MB)`);
+    if (disks.length > 0) console.log(`Disk:   ${disks[0].percent_used}% (${disks[0].mount})`);
+    console.log(`SMART:  ${smart.length > 0 ? `${smart.length} drive(s) checked` : "not available"}`);
+    console.log(`Network: ${network.map((n) => n.interface).join(", ") || "none detected"}`);
+    console.log(`IPMI:   ${ipmi.available ? "available" : "not available"}`);
+    console.log(`Active alerts: ${alertResults.length}`);
+    console.log(`Forge: ${config.forge.enabled ? "enabled" : "disabled"}`);
+    console.log("");
+  }
+}
+
+let firstRun = true;
+
+// Run immediately
+collect();
+
+// Then on interval
+setInterval(collect, config.collection.interval_seconds * 1000);
+
+process.on("SIGTERM", () => {
+  console.log("[collector] Received SIGTERM, shutting down");
+  process.exit(0);
+});
+
+process.on("SIGINT", () => {
+  console.log("[collector] Received SIGINT, shutting down");
+  process.exit(0);
+});

package/src/lib/exec.ts

@@ -0,0 +1,16 @@
+import { execFile } from "child_process";
+import { promisify } from "util";
+
+const execFileAsync = promisify(execFile);
+
+export async function run(cmd: string, args: string[], timeoutMs = 10000): Promise<string | null> {
+  try {
+    const { stdout } = await execFileAsync(cmd, args, { timeout: timeoutMs });
+    return stdout;
+  } catch (err: any) {
+    if (err.code === "ENOENT") return null; // command not installed
+    if (err.killed) return null; // timeout
+    if (err.stdout) return err.stdout; // non-zero exit but has output
+    return null;
+  }
+}

package/src/lib/parse.ts

@@ -0,0 +1,29 @@
+import { readFileSync } from "fs";
+
+export function readProcFile(path: string): string | null {
+  try {
+    return readFileSync(path, "utf-8");
+  } catch {
+    return null;
+  }
+}
+
+export function parseKeyValue(raw: string): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const line of raw.split("\n")) {
+    const idx = line.indexOf(":");
+    if (idx === -1) continue;
+    result[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+  }
+  return result;
+}
+
+export function parseKb(val: string | undefined): number {
+  if (!val) return 0;
+  const num = parseInt(val.replace(/\s*kB$/i, ""), 10);
+  return isNaN(num) ? 0 : num;
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}

package/src/lib/types.ts

@@ -0,0 +1,110 @@
+export interface Snapshot {
+  collector_version: string;
+  timestamp: string;
+  system: SystemInfo;
+  cpu: CpuInfo;
+  memory: MemoryInfo;
+  disks: DiskInfo[];
+  smart: SmartInfo[];
+  network: NetworkInfo[];
+  raid: RaidInfo[];
+  ipmi: IpmiInfo;
+  os_alerts: OsAlerts;
+}
+
+export interface SystemInfo {
+  hostname: string;
+  ip: string;
+  os: string;
+  kernel: string;
+  uptime_seconds: number;
+}
+
+export interface CpuInfo {
+  user_percent: number;
+  system_percent: number;
+  iowait_percent: number;
+  idle_percent: number;
+  load_1m: number;
+  load_5m: number;
+  load_15m: number;
+}
+
+export interface MemoryInfo {
+  total_mb: number;
+  used_mb: number;
+  available_mb: number;
+  swap_total_mb: number;
+  swap_used_mb: number;
+}
+
+export interface DiskInfo {
+  device: string;
+  mount: string;
+  total_gb: number;
+  used_gb: number;
+  available_gb: number;
+  percent_used: number;
+  io_read_mb_s?: number;
+  io_write_mb_s?: number;
+  latency_p99_ms?: number;
+}
+
+export interface SmartInfo {
+  device: string;
+  model: string;
+  health: string;
+  temperature_c?: number;
+  percentage_used?: number;
+  reallocated_sectors?: number;
+  pending_sectors?: number;
+  power_on_hours?: number;
+}
+
+export interface NetworkInfo {
+  interface: string;
+  speed_mbps: number;
+  rx_bytes_sec: number;
+  tx_bytes_sec: number;
+  rx_errors: number;
+  tx_errors: number;
+  rx_drops: number;
+  tx_drops: number;
+}
+
+export interface RaidInfo {
+  device: string;
+  level: string;
+  status: string;
+  degraded: boolean;
+  disks: string[];
+  failed_disks: string[];
+}
+
+export interface IpmiInfo {
+  available: boolean;
+  sensors: Array<{
+    name: string;
+    value: number | string;
+    unit: string;
+    status: string;
+    upper_critical?: number;
+  }>;
+  ecc_errors: { correctable: number; uncorrectable: number };
+  sel_entries_count: number;
+}
+
+export interface OsAlerts {
+  oom_kills_recent: number;
+  zombie_processes: number;
+  time_drift_ms: number;
+}
+
+export interface AlertResult {
+  type: string;
+  severity: "critical" | "warning";
+  title: string;
+  message: string;
+  evidence: Record<string, unknown>;
+  recommendation: string;
+}

package/src/notify/email.ts

@@ -0,0 +1,68 @@
+import { execFile } from "child_process";
+import { promisify } from "util";
+import type { AlertResult } from "../lib/types.js";
+
+const execFileAsync = promisify(execFile);
+
+export async function sendEmail(
+  config: { to: string },
+  newAlerts: AlertResult[],
+  resolvedAlerts: AlertResult[],
+  serverName: string
+): Promise<boolean> {
+  if (!config.to) return false;
+
+  const subject = buildSubject(newAlerts, resolvedAlerts, serverName);
+  const body = buildBody(newAlerts, resolvedAlerts, serverName);
+
+  const email = [
+    `To: ${config.to}`,
+    `From: glassmkr-collector@${serverName}`,
+    `Subject: ${subject}`,
+    `Content-Type: text/plain; charset=utf-8`,
+    "",
+    body,
+  ].join("\n");
+
+  try {
+    const child = execFileAsync("/usr/sbin/sendmail", ["-t"], { timeout: 10000 });
+    child.child.stdin?.write(email);
+    child.child.stdin?.end();
+    await child;
+    return true;
+  } catch {
+    console.error("[email] Failed to send. Is sendmail/postfix/msmtp installed?");
+    return false;
+  }
+}
+
+function buildSubject(newAlerts: AlertResult[], resolvedAlerts: AlertResult[], serverName: string): string {
+  if (newAlerts.length > 0) {
+    const worst = newAlerts.find((a) => a.severity === "critical") ? "CRITICAL" : "WARNING";
+    return `[${worst}] ${serverName}: ${newAlerts.length} alert(s)`;
+  }
+  return `[RESOLVED] ${serverName}: ${resolvedAlerts.length} alert(s) cleared`;
+}
+
+function buildBody(newAlerts: AlertResult[], resolvedAlerts: AlertResult[], serverName: string): string {
+  const lines: string[] = [];
+  lines.push(`Server: ${serverName}`);
+  lines.push(`Time: ${new Date().toISOString()}`);
+  lines.push("");
+
+  for (const a of newAlerts) {
+    lines.push(`[${a.severity.toUpperCase()}] ${a.title}`);
+    lines.push(a.message);
+    lines.push(`Action: ${a.recommendation}`);
+    lines.push("");
+  }
+
+  for (const a of resolvedAlerts) {
+    lines.push(`[RESOLVED] ${a.title}`);
+    lines.push("");
+  }
+
+  lines.push("---");
+  lines.push("Glassmkr Collector v0.1.0");
+  return lines.join("\n");
+}

package/src/notify/slack.ts

@@ -0,0 +1,46 @@
+import type { AlertResult } from "../lib/types.js";
+
+export async function sendSlack(
+  webhookUrl: string,
+  newAlerts: AlertResult[],
+  resolvedAlerts: AlertResult[],
+  serverName: string
+): Promise<boolean> {
+  const blocks: any[] = [];
+
+  if (newAlerts.length > 0) {
+    const criticals = newAlerts.filter((a) => a.severity === "critical");
+    const warnings = newAlerts.filter((a) => a.severity === "warning");
+
+    if (criticals.length > 0) {
+      blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u{1F534} *${criticals.length} CRITICAL* on *${serverName}*` } });
+      for (const a of criticals) blocks.push({ type: "section", text: { type: "mrkdwn", text: `*${a.title}*\n${a.recommendation}` } });
+    }
+    if (warnings.length > 0) {
+      blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u{1F7E1} *${warnings.length} WARNING* on *${serverName}*` } });
+      for (const a of warnings) blocks.push({ type: "section", text: { type: "mrkdwn", text: `*${a.title}*\n${a.recommendation}` } });
+    }
+  }
+
+  if (resolvedAlerts.length > 0) {
+    blocks.push({ type: "section", text: { type: "mrkdwn", text: `\u2705 *${resolvedAlerts.length} resolved* on *${serverName}*` } });
+  }
+
+  if (blocks.length === 0) return true;
+
+  blocks.push({ type: "divider" });
+  blocks.push({ type: "context", elements: [{ type: "mrkdwn", text: "Glassmkr Collector v0.1.0" }] });
+
+  try {
+    const res = await fetch(webhookUrl, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ blocks }),
+      signal: AbortSignal.timeout(10000),
+    });
+    return res.ok;
+  } catch {
+    console.error("[slack] Failed to send notification");
+    return false;
+  }
+}

package/src/notify/telegram.ts

@@ -0,0 +1,45 @@
+import type { AlertResult } from "../lib/types.js";
+
+export async function sendTelegram(
+  botToken: string,
+  chatId: string,
+  newAlerts: AlertResult[],
+  resolvedAlerts: AlertResult[],
+  serverName: string
+): Promise<boolean> {
+  const parts: string[] = [];
+
+  if (newAlerts.length > 0) {
+    const criticals = newAlerts.filter((a) => a.severity === "critical");
+    const warnings = newAlerts.filter((a) => a.severity === "warning");
+
+    if (criticals.length > 0) {
+      parts.push(`\u{1F534} <b>${criticals.length} CRITICAL</b> on <b>${serverName}</b>:\n`);
+      for (const a of criticals) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
+    }
+    if (warnings.length > 0) {
+      parts.push(`\u{1F7E1} <b>${warnings.length} WARNING</b> on <b>${serverName}</b>:\n`);
+      for (const a of warnings) parts.push(`  \u2022 <b>${a.title}</b>\n  ${a.recommendation}\n`);
+    }
+  }
+
+  if (resolvedAlerts.length > 0) {
+    parts.push(`\u2705 <b>${resolvedAlerts.length} resolved</b> on <b>${serverName}</b>:\n`);
+    for (const a of resolvedAlerts) parts.push(`  \u2022 ${a.title}\n`);
+  }
+
+  if (parts.length === 0) return true;
+
+  try {
+    const res = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ chat_id: chatId, text: parts.join("\n"), parse_mode: "HTML", disable_web_page_preview: true }),
+      signal: AbortSignal.timeout(10000),
+    });
+    return res.ok;
+  } catch {
+    console.error("[telegram] Failed to send notification");
+    return false;
+  }
+}

package/src/push/forge.ts

@@ -0,0 +1,25 @@
+import type { Snapshot } from "../lib/types.js";
+
+export async function pushToForge(url: string, apiKey: string, snapshot: Snapshot): Promise<boolean> {
+  try {
+    const response = await fetch(`${url}/api/v1/ingest`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(snapshot),
+      signal: AbortSignal.timeout(10000),
+    });
+    if (response.ok) {
+      const data = await response.json() as { new_alerts?: number; active_alerts?: number };
+      console.log(`[forge] Push successful. Active alerts: ${data.active_alerts ?? 0}`);
+    } else {
+      console.error(`[forge] Push failed: ${response.status} ${response.statusText}`);
+    }
+    return response.ok;
+  } catch (err) {
+    console.error("[forge] Push failed, will retry next cycle");
+    return false;
+  }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "declaration": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"]
+}