@cohortapp/agent-sdk 2.3.0

package/lib/diagnostics/alerts.mjs

@@ -0,0 +1,424 @@
+/**
+ * lib/diagnostics/alerts.mjs — behavioral alert rules over the diagnostics spine.
+ *
+ * The alerting half of gaps-enterprise-ops G14. The audit's load-bearing
+ * insight: "Alert on behavioral PATTERNS, not single failures: agents fail
+ * silently by completing with wrong output." A single guard fail-open, one
+ * refusal, one expensive session — none of those is an incident. A *spike* of
+ * refusals, a guard that has fallen open dozens of times today, a cost-per-session
+ * p99 blowing past its ceiling, an agent that stopped heartbeating, a daemon
+ * whose health file went cold — those are. This module evaluates that taxonomy
+ * over the already-collected signals and hands the firing alerts to an INJECTED
+ * sink (a webhook / Alertmanager poster) — it never makes a live call itself, so
+ * it stays pure, testable, and impossible to wedge on a network hang.
+ *
+ * Inputs (all injected; nothing is read from the network here):
+ *   - snapshot  the diagnostics counters snapshot (lib/diagnostics/counters.mjs
+ *               `snapshot()`): `{ "<counter>": <total-today>, ... }`. Refusal +
+ *               guard fail-open spikes are read from here.
+ *   - ledger    today's cost rollup (lib/model-router/ledger.mjs `sumLedgerToday`
+ *               shape `{ spentUSD, sessions, byModel }`) PLUS an optional
+ *               `perSessionUSD: number[]` for the p99 rule. When only the rollup
+ *               is present we fall back to mean-per-session.
+ *   - presence  optional liveness facts: `{ heartbeatAgeSec, daemonHealthAgeSec }`
+ *               (or `daemonDown:true`). Absent-agent staleness + daemon-down read
+ *               from here. Caller derives the ages from presence.mjs / readHealth.
+ *   - thresholds  per-rule knobs (see DEFAULT_THRESHOLDS); operator overrides
+ *               from config/alerts.yaml via `loadThresholds`.
+ *   - now       clock in epoch ms (default Date.now()).
+ *
+ * Output: a stable, deterministic array `[{ id, severity, detail }, ...]` sorted
+ * by severity then id, so the same inputs always produce the same alert list (a
+ * de-dupe upstream can key on `id`). Severity is one of "warning" | "critical".
+ *
+ * Invariant: NEITHER `evaluateAlerts` NOR `emitAlerts` EVER throws. A bad
+ * snapshot, a missing ledger, a sink that rejects/throws — all degrade to a
+ * return value, never an exception. An alerting path that can crash the daemon it
+ * watches is worse than no alerting at all (the same fail-open contract as
+ * counters.mjs / events.mjs). emitAlerts is best-effort delivery: it reports
+ * per-alert success and swallows sink failures.
+ *
+ * Constraints (CLAUDE.md): ESM, Node built-ins + js-yaml (lazy, optional) only,
+ * everything injectable so tests are hermetic and make no live calls.
+ *
+ * @module lib/diagnostics/alerts
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+/**
+ * Built-in alert thresholds. Each rule is documented inline. These mirror
+ * config/alerts.yaml; an operator override merges over these (loadThresholds).
+ * Tuned to be quiet by default — alert on a sustained pattern, not noise.
+ * @type {Readonly<object>}
+ */
+export const DEFAULT_THRESHOLDS = Object.freeze({
+  refusalSpike: Object.freeze({
+    // Counter name(s) that count refusals (a guard or model declined to act).
+    // Summed across the listed counters for the day.
+    counters: Object.freeze(["refusal", "policy.refusal", "classifier.refusal"]),
+    warnCount: 10, // warn at this many refusals today
+    critCount: 25, // critical at this many
+  }),
+  guardFailSpike: Object.freeze({
+    // Any counter ending ".fail_open" / ".persist_failed" / ".degraded" is a
+    // guard that fell open silently. Named counters are summed; the suffix match
+    // catches new guards without a config change. "guard.fail" is the name the
+    // live daemon onGuardFail hook-bus subscriber bumps (diag-pipeline wiring),
+    // so it is listed explicitly to keep the wired signal and the rule aligned.
+    counters: Object.freeze(["guard.fail"]),
+    suffixes: Object.freeze([".fail_open", ".persist_failed", ".degraded"]),
+    warnCount: 5,
+    critCount: 20,
+  }),
+  costPerSessionP99: Object.freeze({
+    // Cost-per-session p99 (USD). Falls back to mean-per-session when no
+    // per-session distribution is supplied.
+    warnUSD: 1.5,
+    critUSD: 3.0,
+    minSessions: 5, // don't fire on a tiny sample
+  }),
+  absentAgent: Object.freeze({
+    // No session/agent heartbeat for this long → the agent is silently absent.
+    warnSec: 600, // 10 min
+    critSec: 1800, // 30 min
+  }),
+  daemonDown: Object.freeze({
+    // The consumer's health.json heartbeat (state/cadence-bus/health.json) went
+    // cold. The daemon is the sole cadence orchestrator — a cold heartbeat is a
+    // dark agent.
+    warnSec: 120, // 2 min (heartbeat refreshes ~15s)
+    critSec: 300, // 5 min
+  }),
+});
+
+const SEVERITY_RANK = { critical: 0, warning: 1 };
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve config/alerts.yaml for the agent (explicit path wins for tests).
+ * @param {object} cfg
+ * @returns {string}
+ */
+function alertsConfigPath(cfg = {}) {
+  if (cfg.configPath) return resolve(cfg.configPath);
+  const root = resolve(
+    cfg.agentRoot ||
+      process.env.AGENT_ROOT ||
+      process.env.AGENT_DIR ||
+      process.cwd()
+  );
+  return join(root, "config", "alerts.yaml");
+}
+
+/** Deep-clone a JSON-safe value (the frozen defaults must not leak by reference). */
+function clone(v) {
+  return JSON.parse(JSON.stringify(v));
+}
+
+/**
+ * Recursively merge `override` onto `base` (plain objects deep-merge; arrays and
+ * scalars are last-wins). Returns a new object; inputs are not mutated. Mirrors
+ * lib/collective/config.mjs `mergeConfig`.
+ */
+export function mergeThresholds(base, override) {
+  if (override == null) return clone(base);
+  if (Array.isArray(base) || Array.isArray(override)) return clone(override);
+  if (typeof base !== "object" || typeof override !== "object") return clone(override);
+  const out = clone(base);
+  for (const k of Object.keys(override)) {
+    const b = out[k];
+    const o = override[k];
+    if (b && o && typeof b === "object" && typeof o === "object" && !Array.isArray(b) && !Array.isArray(o)) {
+      out[k] = mergeThresholds(b, o);
+    } else if (o !== undefined) {
+      out[k] = clone(o);
+    }
+  }
+  return out;
+}
+
+/**
+ * Load alert thresholds: DEFAULT_THRESHOLDS merged with config/alerts.yaml when
+ * present. NEVER throws — a missing/malformed file yields the defaults. js-yaml
+ * is loaded lazily (already a maestro dep) so the module imports cleanly even if
+ * yaml is unavailable.
+ *
+ * @param {object} [cfg] { agentRoot?, configPath? }
+ * @returns {Promise<object>} merged thresholds
+ */
+export async function loadThresholds(cfg = {}) {
+  const defaults = clone(DEFAULT_THRESHOLDS);
+  try {
+    const p = alertsConfigPath(cfg);
+    if (!existsSync(p)) return defaults;
+    const body = readFileSync(p, "utf-8");
+    const mod = await import("js-yaml");
+    const yaml = mod.default || mod;
+    const raw = yaml.load(body);
+    if (!raw || typeof raw !== "object") return defaults;
+    // Operators write knobs under an `alerts:` key (matching the file name) or at
+    // top level; accept either so the file is forgiving.
+    const override = raw.alerts && typeof raw.alerts === "object" ? raw.alerts : raw;
+    return mergeThresholds(defaults, override);
+  } catch {
+    return defaults;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Rule helpers
+// ---------------------------------------------------------------------------
+
+function num(v, fallback = 0) {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : fallback;
+}
+
+/** Sum the listed counters from a snapshot (missing counters contribute 0). */
+function sumCounters(snapshot, names) {
+  if (!snapshot || typeof snapshot !== "object") return 0;
+  if (!Array.isArray(names)) return 0;
+  let total = 0;
+  for (const name of names) {
+    if (typeof name !== "string") continue;
+    total += num(snapshot[name]);
+  }
+  return total;
+}
+
+/** Sum every snapshot counter whose name ends with one of `suffixes`. */
+function sumBySuffix(snapshot, suffixes) {
+  if (!snapshot || typeof snapshot !== "object") return 0;
+  if (!Array.isArray(suffixes) || suffixes.length === 0) return 0;
+  let total = 0;
+  for (const [name, val] of Object.entries(snapshot)) {
+    if (typeof name !== "string") continue;
+    if (suffixes.some((s) => typeof s === "string" && s && name.endsWith(s))) {
+      total += num(val);
+    }
+  }
+  return total;
+}
+
+/**
+ * Nearest-rank p99 of a numeric sample. Returns null on an empty/invalid sample.
+ * Pure; tolerates non-numeric entries (they're dropped).
+ */
+function p99(values) {
+  if (!Array.isArray(values)) return null;
+  const nums = values.map((v) => Number(v)).filter((n) => Number.isFinite(n));
+  if (nums.length === 0) return null;
+  nums.sort((a, b) => a - b);
+  // Nearest-rank: ceil(p/100 * N), 1-indexed.
+  const rank = Math.max(1, Math.ceil(0.99 * nums.length));
+  return nums[rank - 1];
+}
+
+/** Map a value to a severity given warn/crit thresholds (higher = worse). */
+function severityFor(value, warn, crit) {
+  if (Number.isFinite(crit) && value >= crit) return "critical";
+  if (Number.isFinite(warn) && value >= warn) return "warning";
+  return null;
+}
+
+function round2(n) {
+  return Math.round(n * 100) / 100;
+}
+
+// ---------------------------------------------------------------------------
+// evaluateAlerts
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluate the behavioral alert rules and return the firing alerts.
+ *
+ * @param {object} [args]
+ * @param {object} [args.snapshot]   diagnostics counters snapshot (today's totals)
+ * @param {object} [args.ledger]     cost rollup `{ spentUSD, sessions, byModel }`
+ *                                   optionally with `perSessionUSD: number[]`
+ * @param {object} [args.presence]   `{ heartbeatAgeSec?, daemonHealthAgeSec?,
+ *                                       daemonDown? }`
+ * @param {object} [args.thresholds] merged thresholds (default DEFAULT_THRESHOLDS)
+ * @param {number} [args.now]        clock in epoch ms (reserved; rules are
+ *                                   age-based via injected `*AgeSec`)
+ * @returns {Array<{id:string, severity:"warning"|"critical", detail:string}>}
+ */
+export function evaluateAlerts(args = {}) {
+  const alerts = [];
+  try {
+    const snapshot = args.snapshot && typeof args.snapshot === "object" ? args.snapshot : {};
+    const ledger = args.ledger && typeof args.ledger === "object" ? args.ledger : {};
+    const presence = args.presence && typeof args.presence === "object" ? args.presence : {};
+    const t = mergeThresholds(DEFAULT_THRESHOLDS, args.thresholds || {});
+
+    push(alerts, refusalSpikeRule(snapshot, t.refusalSpike));
+    push(alerts, guardFailSpikeRule(snapshot, t.guardFailSpike));
+    push(alerts, costPerSessionRule(ledger, t.costPerSessionP99));
+    push(alerts, absentAgentRule(presence, t.absentAgent));
+    push(alerts, daemonDownRule(presence, t.daemonDown));
+
+    // Deterministic order: critical first, then by id, so identical inputs always
+    // produce identical output (a de-dupe upstream can key on id).
+    alerts.sort((a, b) => {
+      const s = (SEVERITY_RANK[a.severity] ?? 9) - (SEVERITY_RANK[b.severity] ?? 9);
+      return s !== 0 ? s : a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+    });
+    return alerts;
+  } catch {
+    // Never throw: whatever we accumulated before the failure is still valid.
+    return alerts;
+  }
+}
+
+function push(arr, alert) {
+  if (alert) arr.push(alert);
+}
+
+function refusalSpikeRule(snapshot, cfg = {}) {
+  const count = sumCounters(snapshot, cfg.counters);
+  const sev = severityFor(count, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
+  if (!sev) return null;
+  return {
+    id: "refusal_spike",
+    severity: sev,
+    detail: `${count} refusals today (warn ${num(cfg.warnCount)}, crit ${num(cfg.critCount)}) — behavioral spike, not a single decline.`,
+  };
+}
+
+function guardFailSpikeRule(snapshot, cfg = {}) {
+  const named = sumCounters(snapshot, cfg.counters);
+  const bySuffix = sumBySuffix(snapshot, cfg.suffixes);
+  const count = named + bySuffix;
+  const sev = severityFor(count, num(cfg.warnCount, Infinity), num(cfg.critCount, Infinity));
+  if (!sev) return null;
+  return {
+    id: "guard_fail_spike",
+    severity: sev,
+    detail: `${count} guard fail-opens today (warn ${num(cfg.warnCount)}, crit ${num(cfg.critCount)}) — a guard is silently degrading, not erroring.`,
+  };
+}
+
+function costPerSessionRule(ledger, cfg = {}) {
+  const sessions = num(ledger.sessions);
+  const minSessions = num(cfg.minSessions, 0);
+  if (sessions < Math.max(1, minSessions)) return null; // sample too small / no work
+
+  // Prefer a real per-session distribution for the p99; fall back to the mean
+  // when only the rollup is available (still a useful "spend per session" signal).
+  let value;
+  let basis;
+  const dist = Array.isArray(ledger.perSessionUSD) ? ledger.perSessionUSD : null;
+  const p = dist ? p99(dist) : null;
+  if (p != null) {
+    value = p;
+    basis = "p99";
+  } else {
+    value = num(ledger.spentUSD) / sessions;
+    basis = "mean";
+  }
+
+  const sev = severityFor(value, num(cfg.warnUSD, Infinity), num(cfg.critUSD, Infinity));
+  if (!sev) return null;
+  return {
+    id: "cost_per_session_p99",
+    severity: sev,
+    detail: `cost-per-session ${basis} $${round2(value)} over ${sessions} sessions (warn $${num(cfg.warnUSD)}, crit $${num(cfg.critUSD)}).`,
+  };
+}
+
+function absentAgentRule(presence, cfg = {}) {
+  // Only fire when we actually have a heartbeat age to judge — a missing signal
+  // is not "absent" here (daemonDownRule covers the cold-daemon case).
+  if (!Number.isFinite(presence.heartbeatAgeSec)) return null;
+  const age = presence.heartbeatAgeSec;
+  const sev = severityFor(age, num(cfg.warnSec, Infinity), num(cfg.critSec, Infinity));
+  if (!sev) return null;
+  return {
+    id: "absent_agent",
+    severity: sev,
+    detail: `no agent heartbeat for ${Math.round(age)}s (warn ${num(cfg.warnSec)}s, crit ${num(cfg.critSec)}s) — agent silently absent.`,
+  };
+}
+
+function daemonDownRule(presence, cfg = {}) {
+  // Explicit daemonDown flag is the unambiguous signal (no health file at all).
+  if (presence.daemonDown === true) {
+    return {
+      id: "daemon_down",
+      severity: "critical",
+      detail: "daemon health heartbeat absent — the cadence orchestrator is down (no health.json).",
+    };
+  }
+  if (!Number.isFinite(presence.daemonHealthAgeSec)) return null;
+  const age = presence.daemonHealthAgeSec;
+  const sev = severityFor(age, num(cfg.warnSec, Infinity), num(cfg.critSec, Infinity));
+  if (!sev) return null;
+  return {
+    id: "daemon_down",
+    severity: sev,
+    detail: `daemon health heartbeat ${Math.round(age)}s stale (warn ${num(cfg.warnSec)}s, crit ${num(cfg.critSec)}s) — cadence orchestrator may be wedged.`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// emitAlerts
+// ---------------------------------------------------------------------------
+
+/**
+ * Deliver alerts to an INJECTED sink. The sink is a function
+ * `(alert) => boolean | Promise<boolean>` — typically a webhook / Alertmanager
+ * poster supplied by the caller; this module never makes a live network call, so
+ * tests pass a spy and CI never reaches the wire.
+ *
+ * Best-effort and never-throwing: each alert is delivered independently; a sink
+ * that throws or rejects for one alert counts as a failure for that alert and
+ * does not abort the rest. Returns a summary `{ attempted, delivered, failed,
+ * results }` so the caller (doctor, a cadence handler) can log delivery health.
+ *
+ * @param {Array<object>} alerts  the alerts from evaluateAlerts
+ * @param {object} [opts]
+ * @param {(alert:object) => boolean|Promise<boolean>} [opts.sink]  delivery fn
+ * @returns {Promise<{attempted:number, delivered:number, failed:number,
+ *           results: Array<{id:string, ok:boolean}>}>}
+ */
+export async function emitAlerts(alerts, opts = {}) {
+  const summary = { attempted: 0, delivered: 0, failed: 0, results: [] };
+  try {
+    const list = Array.isArray(alerts) ? alerts : [];
+    const sink = typeof opts.sink === "function" ? opts.sink : null;
+    for (const alert of list) {
+      if (!alert || typeof alert !== "object") continue;
+      summary.attempted += 1;
+      const id = typeof alert.id === "string" ? alert.id : "unknown";
+      if (!sink) {
+        // No sink wired: count as a failed delivery so the caller can tell the
+        // alert was raised but went nowhere (a misconfiguration, not silence).
+        summary.failed += 1;
+        summary.results.push({ id, ok: false });
+        continue;
+      }
+      let ok = false;
+      try {
+        const r = await sink(alert);
+        ok = r !== false; // a sink that returns undefined/true is a success
+      } catch {
+        ok = false; // a throwing/rejecting sink is a per-alert failure, not fatal
+      }
+      if (ok) summary.delivered += 1;
+      else summary.failed += 1;
+      summary.results.push({ id, ok });
+    }
+    return summary;
+  } catch {
+    return summary;
+  }
+}
+
+export const _internals = { p99, sumCounters, sumBySuffix, severityFor };
+
+export default { evaluateAlerts, emitAlerts, loadThresholds, mergeThresholds, DEFAULT_THRESHOLDS };