mergen-server 1.4.0

package/dist/sensor/heartbeat-monitor.js

@@ -0,0 +1,133 @@
+import fs from "fs";
+import { DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const HEARTBEAT_FILE = `${DATA_DIR}/heartbeats.json`;
+const CHECK_INTERVAL_MS = 6e4;
+let _state = {};
+let _loaded = false;
+function load() {
+  if (_loaded) return;
+  _loaded = true;
+  if (!fs.existsSync(HEARTBEAT_FILE)) return;
+  try {
+    const raw = JSON.parse(fs.readFileSync(HEARTBEAT_FILE, "utf8"));
+    if (raw?.version === 1 && raw.heartbeats) _state = raw.heartbeats;
+  } catch (err) {
+    logger.warn({ err }, "heartbeat-monitor: failed to load state \u2014 starting fresh");
+  }
+}
+let _tmpCounter = 0;
+function persist() {
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    const payload = { version: 1, heartbeats: _state };
+    _tmpCounter = _tmpCounter + 1 >>> 0;
+    const tmp = `${HEARTBEAT_FILE}.tmp.${process.pid}.${Date.now().toString(36)}.${_tmpCounter}`;
+    fs.writeFileSync(tmp, JSON.stringify(payload), "utf8");
+    fs.renameSync(tmp, HEARTBEAT_FILE);
+  } catch (err) {
+    logger.warn({ err }, "heartbeat-monitor: persist failed");
+  }
+}
+function computeStatus(hb) {
+  const now = Date.now();
+  const deadline = (hb.intervalSeconds + hb.graceSeconds) * 1e3;
+  if (!hb.lastPingAt) {
+    const age = now - hb.createdAt;
+    return {
+      ...hb,
+      status: age > deadline ? "never-pinged" : "ok",
+      secondsSinceLastPing: null,
+      nextExpectedAt: hb.createdAt + deadline,
+      overdueSeconds: age > deadline ? Math.round((age - deadline) / 1e3) : null
+    };
+  }
+  const elapsed = now - hb.lastPingAt;
+  const isLate = elapsed > hb.intervalSeconds * 1e3;
+  const isMissing = elapsed > deadline;
+  return {
+    ...hb,
+    status: isMissing ? "missing" : isLate ? "late" : "ok",
+    secondsSinceLastPing: Math.round(elapsed / 1e3),
+    nextExpectedAt: hb.lastPingAt + hb.intervalSeconds * 1e3,
+    overdueSeconds: isMissing ? Math.round((elapsed - deadline) / 1e3) : null
+  };
+}
+let _alertFn = null;
+function setHeartbeatAlertFn(fn) {
+  _alertFn = fn;
+}
+function checkHeartbeats() {
+  load();
+  const now = Date.now();
+  for (const hb of Object.values(_state)) {
+    const report = computeStatus(hb);
+    if (report.status !== "missing" && report.status !== "never-pinged") continue;
+    const alertCooldown = hb.intervalSeconds * 1e3;
+    if (hb.lastAlertedAt && now - hb.lastAlertedAt < alertCooldown) continue;
+    const msg = report.status === "never-pinged" ? `Heartbeat \`${hb.name}\` has never received a ping \u2014 expected within ${hb.intervalSeconds}s of registration.` : `Heartbeat \`${hb.name}\` missed \u2014 ${report.overdueSeconds}s overdue (interval: ${hb.intervalSeconds}s, grace: ${hb.graceSeconds}s).`;
+    logger.warn({ name: hb.name, status: report.status, overdueSeconds: report.overdueSeconds }, "heartbeat-monitor: missed heartbeat");
+    if (_alertFn) _alertFn(hb.name, hb.description ? `${msg} \u2014 ${hb.description}` : msg);
+    _state[hb.name] = { ...hb, lastAlertedAt: now };
+    persist();
+  }
+}
+let _checkHandle = null;
+function startHeartbeatMonitor() {
+  _checkHandle = setInterval(checkHeartbeats, CHECK_INTERVAL_MS);
+  _checkHandle.unref();
+  logger.info("heartbeat-monitor: started");
+  return () => {
+    if (_checkHandle) {
+      clearInterval(_checkHandle);
+      _checkHandle = null;
+    }
+  };
+}
+function ping(name, intervalSeconds = 86400, graceSeconds, description) {
+  load();
+  const grace = graceSeconds ?? Math.max(60, Math.round(intervalSeconds * 0.1));
+  const existing = _state[name];
+  const updated = {
+    name,
+    intervalSeconds,
+    graceSeconds: grace,
+    createdAt: existing?.createdAt ?? Date.now(),
+    lastPingAt: Date.now(),
+    pingCount: (existing?.pingCount ?? 0) + 1,
+    lastAlertedAt: existing?.lastAlertedAt ?? null,
+    description: description ?? existing?.description
+  };
+  _state[name] = updated;
+  persist();
+  logger.debug({ name, pingCount: updated.pingCount }, "heartbeat-monitor: ping received");
+  return updated;
+}
+function getReport(name) {
+  load();
+  const hb = _state[name];
+  if (!hb) return null;
+  return computeStatus(hb);
+}
+function getAllReports() {
+  load();
+  return Object.values(_state).map(computeStatus).sort((a, b) => {
+    const order = { missing: 0, "never-pinged": 1, late: 2, ok: 3 };
+    return order[a.status] - order[b.status] || a.name.localeCompare(b.name);
+  });
+}
+function removeHeartbeat(name) {
+  load();
+  if (!_state[name]) return false;
+  delete _state[name];
+  persist();
+  return true;
+}
+export {
+  getAllReports,
+  getReport,
+  ping,
+  removeHeartbeat,
+  setHeartbeatAlertFn,
+  startHeartbeatMonitor
+};

package/dist/sensor/incident-store.js

@@ -0,0 +1,340 @@
+import initSqlJs from "sql.js";
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+import { DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const INCIDENT_DB = path.join(DATA_DIR, "incidents.db");
+class IncidentStore {
+  db = null;
+  resolveWasmPath() {
+    if (process.env.MERGEN_WASM_PATH) return process.env.MERGEN_WASM_PATH;
+    const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+    const fromModule = path.resolve(moduleDir, "../../node_modules/sql.js/dist/sql-wasm.wasm");
+    if (fs.existsSync(fromModule)) return fromModule;
+    try {
+      const req = createRequire(import.meta.url);
+      const resolved = path.join(path.dirname(req.resolve("sql.js")), "sql-wasm.wasm");
+      if (fs.existsSync(resolved)) return resolved;
+    } catch {
+    }
+    return fromModule;
+  }
+  async init() {
+    try {
+      const wasmBinary = fs.readFileSync(this.resolveWasmPath());
+      const SQL = await initSqlJs({ wasmBinary });
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      let fileBuffer;
+      if (fs.existsSync(INCIDENT_DB)) {
+        try {
+          fileBuffer = fs.readFileSync(INCIDENT_DB);
+        } catch {
+        }
+      }
+      this.db = fileBuffer ? new SQL.Database(fileBuffer) : new SQL.Database();
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS incidents (
+          pid                    TEXT PRIMARY KEY,
+          hypothesis             TEXT NOT NULL,
+          tag                    TEXT NOT NULL DEFAULT '',
+          status                 TEXT NOT NULL DEFAULT 'open',
+          assignee               TEXT,
+          notes                  TEXT NOT NULL DEFAULT '[]',
+          sha                    TEXT,
+          environment            TEXT,
+          service                TEXT,
+          cluster                TEXT,
+          confidence             REAL NOT NULL DEFAULT 0,
+          created_at             INTEGER NOT NULL,
+          updated_at             INTEGER NOT NULL,
+          acknowledged_by        TEXT,
+          resolved_at            INTEGER,
+          resolved_autonomously  INTEGER NOT NULL DEFAULT 0,
+          causally_correct       INTEGER NOT NULL DEFAULT 0
+        );
+      `);
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS service_edges (
+          source          TEXT NOT NULL,
+          target          TEXT NOT NULL,
+          weight          INTEGER NOT NULL DEFAULT 1,
+          last_incident_at INTEGER NOT NULL,
+          PRIMARY KEY (source, target)
+        );
+      `);
+      try {
+        this.db.run(`ALTER TABLE incidents ADD COLUMN service TEXT`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE incidents ADD COLUMN cluster TEXT`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE incidents ADD COLUMN causally_correct INTEGER NOT NULL DEFAULT 0`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE incidents ADD COLUMN context_brief_viewed_at INTEGER`);
+      } catch {
+      }
+      this._flush();
+      logger.info({ path: INCIDENT_DB }, "incident store initialised");
+    } catch (err) {
+      logger.warn({ err }, "incident store failed to init \u2014 running without persistence");
+    }
+  }
+  _flush() {
+    if (!this.db) return;
+    try {
+      const data = this.db.export();
+      fs.writeFileSync(INCIDENT_DB, Buffer.from(data));
+    } catch (err) {
+      logger.warn({ err }, "incident store flush failed");
+    }
+  }
+  _row(row) {
+    return {
+      pid: String(row.pid ?? ""),
+      hypothesis: String(row.hypothesis ?? ""),
+      tag: String(row.tag ?? ""),
+      status: row.status ?? "open",
+      assignee: row.assignee ? String(row.assignee) : null,
+      notes: (() => {
+        try {
+          return JSON.parse(String(row.notes ?? "[]"));
+        } catch {
+          return [];
+        }
+      })(),
+      sha: row.sha ? String(row.sha) : null,
+      environment: row.environment ? String(row.environment) : null,
+      service: row.service ? String(row.service) : null,
+      cluster: row.cluster ? String(row.cluster) : null,
+      confidence: Number(row.confidence ?? 0),
+      createdAt: Number(row.created_at ?? 0),
+      updatedAt: Number(row.updated_at ?? 0),
+      acknowledgedBy: row.acknowledged_by ? String(row.acknowledged_by) : null,
+      resolvedAt: row.resolved_at ? Number(row.resolved_at) : null,
+      resolvedAutonomously: Boolean(row.resolved_autonomously),
+      causallyCorrect: Boolean(row.causally_correct),
+      contextBriefViewedAt: row.context_brief_viewed_at ? Number(row.context_brief_viewed_at) : null
+    };
+  }
+  upsert(pid, fields) {
+    if (!this.db) {
+      const now2 = Date.now();
+      return {
+        pid,
+        status: "open",
+        hypothesis: "",
+        tag: "",
+        assignee: null,
+        notes: [],
+        sha: null,
+        environment: null,
+        service: null,
+        cluster: null,
+        confidence: 0,
+        createdAt: now2,
+        updatedAt: now2,
+        acknowledgedBy: null,
+        resolvedAt: null,
+        resolvedAutonomously: false,
+        causallyCorrect: false,
+        contextBriefViewedAt: null,
+        ...fields
+      };
+    }
+    const existing = this.get(pid);
+    const now = Date.now();
+    if (!existing) {
+      this.db.run(
+        `INSERT INTO incidents (pid, hypothesis, tag, status, assignee, notes, sha, environment, service, cluster, confidence, created_at, updated_at, acknowledged_by, resolved_at, resolved_autonomously, causally_correct)
+         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+        [
+          pid,
+          fields.hypothesis ?? "",
+          fields.tag ?? "",
+          fields.status ?? "open",
+          fields.assignee ?? null,
+          JSON.stringify(fields.notes ?? []),
+          fields.sha ?? null,
+          fields.environment ?? null,
+          fields.service ?? null,
+          fields.cluster ?? null,
+          fields.confidence ?? 0,
+          now,
+          now,
+          fields.acknowledgedBy ?? null,
+          fields.resolvedAt ?? null,
+          fields.resolvedAutonomously ? 1 : 0,
+          fields.causallyCorrect ? 1 : 0
+        ]
+      );
+    } else {
+      const merged = { ...existing, ...fields };
+      this.db.run(
+        `UPDATE incidents SET hypothesis=?,tag=?,status=?,assignee=?,notes=?,sha=?,environment=?,service=?,cluster=?,confidence=?,updated_at=?,acknowledged_by=?,resolved_at=?,resolved_autonomously=?,causally_correct=? WHERE pid=?`,
+        [
+          merged.hypothesis,
+          merged.tag,
+          merged.status,
+          merged.assignee,
+          JSON.stringify(merged.notes),
+          merged.sha,
+          merged.environment,
+          merged.service,
+          merged.cluster,
+          merged.confidence,
+          now,
+          merged.acknowledgedBy,
+          merged.resolvedAt,
+          merged.resolvedAutonomously ? 1 : 0,
+          merged.causallyCorrect ? 1 : 0,
+          pid
+        ]
+      );
+    }
+    this._flush();
+    const result = this.get(pid);
+    if (result.service) {
+      this.updateServiceEdges(result.service, result.updatedAt);
+    }
+    return result;
+  }
+  get(pid) {
+    if (!this.db) return null;
+    try {
+      const res = this.db.exec("SELECT * FROM incidents WHERE pid=?", [pid]);
+      if (!res[0]?.values?.length) return null;
+      const cols = res[0].columns;
+      const vals = res[0].values[0];
+      const row = {};
+      cols.forEach((c, i) => {
+        row[c] = vals[i];
+      });
+      return this._row(row);
+    } catch {
+      return null;
+    }
+  }
+  list(status, limit = 50) {
+    if (!this.db) return [];
+    try {
+      const sql = status ? `SELECT * FROM incidents WHERE status=? ORDER BY updated_at DESC LIMIT ?` : `SELECT * FROM incidents ORDER BY updated_at DESC LIMIT ?`;
+      const params = status ? [status, limit] : [limit];
+      const res = this.db.exec(sql, params);
+      if (!res[0]?.values) return [];
+      return res[0].values.map((vals) => {
+        const row = {};
+        res[0].columns.forEach((c, i) => {
+          row[c] = vals[i];
+        });
+        return this._row(row);
+      });
+    } catch {
+      return [];
+    }
+  }
+  markContextViewed(pid) {
+    if (!this.db) return;
+    const existing = this.get(pid);
+    if (!existing || existing.contextBriefViewedAt != null) return;
+    try {
+      this.db.run(
+        `UPDATE incidents SET context_brief_viewed_at=? WHERE pid=?`,
+        [Date.now(), pid]
+      );
+      this._flush();
+    } catch (err) {
+      logger.warn({ err, pid }, "incident store: markContextViewed failed");
+    }
+  }
+  /**
+   * Co-occurring services: other services that had incidents within windowMs of
+   * any incident for the given service. Pure SQL — no JS O(n²) scan.
+   * Returns up to `limit` services sorted by co-occurrence count descending.
+   */
+  coOccurringServices(service, windowMs = 10 * 60 * 1e3, limit = 4) {
+    if (!this.db) return [];
+    try {
+      const res = this.db.exec(
+        `SELECT other.service, COUNT(*) AS cnt
+         FROM incidents AS base
+         JOIN incidents AS other
+           ON other.service != base.service
+          AND other.service IS NOT NULL
+          AND other.created_at BETWEEN base.created_at - ? AND base.created_at + ?
+         WHERE base.service = ?
+         GROUP BY other.service
+         ORDER BY cnt DESC
+         LIMIT ?`,
+        [windowMs, windowMs, service, limit]
+      );
+      if (!res[0]?.values) return [];
+      return res[0].values.map((v) => ({
+        service: String(v[0] ?? ""),
+        count: Number(v[1] ?? 0)
+      }));
+    } catch {
+      return [];
+    }
+  }
+  addNote(pid, note, author) {
+    const inc = this.get(pid);
+    if (!inc) return null;
+    const entry = author ? `[${author}] ${note}` : note;
+    return this.upsert(pid, { notes: [...inc.notes, entry] });
+  }
+  /**
+   * Persist a co-occurrence edge between `service` and every other service
+   * that had an incident within windowMs. Called on every upsert so the
+   * graph accumulates weight over time without a separate indexing job.
+   */
+  updateServiceEdges(service, at, windowMs = 10 * 60 * 1e3) {
+    if (!this.db) return;
+    try {
+      const coServices = this.coOccurringServices(service, windowMs, 20);
+      for (const { service: target } of coServices) {
+        for (const [src, tgt] of [[service, target], [target, service]]) {
+          this.db.run(
+            `INSERT INTO service_edges (source, target, weight, last_incident_at) VALUES (?,?,1,?)
+             ON CONFLICT(source, target) DO UPDATE SET weight = weight + 1, last_incident_at = excluded.last_incident_at`,
+            [src, tgt, at]
+          );
+        }
+      }
+      this._flush();
+    } catch (err) {
+      logger.warn({ err, service }, "incident store: updateServiceEdges failed");
+    }
+  }
+  /**
+   * Return the full interaction graph, optionally filtered to edges touching `service`.
+   * Sorted by weight descending — strongest co-occurrence relationships first.
+   */
+  getInteractionGraph(service) {
+    if (!this.db) return [];
+    try {
+      const sql = service ? `SELECT source, target, weight, last_incident_at FROM service_edges WHERE source=? OR target=? ORDER BY weight DESC LIMIT 100` : `SELECT source, target, weight, last_incident_at FROM service_edges ORDER BY weight DESC LIMIT 200`;
+      const params = service ? [service, service] : [];
+      const res = this.db.exec(sql, params);
+      if (!res[0]?.values) return [];
+      return res[0].values.map((v) => ({
+        source: String(v[0] ?? ""),
+        target: String(v[1] ?? ""),
+        weight: Number(v[2] ?? 0),
+        lastIncidentAt: Number(v[3] ?? 0)
+      }));
+    } catch {
+      return [];
+    }
+  }
+}
+const incidentStore = new IncidentStore();
+export {
+  incidentStore
+};