@cohortapp/agent-sdk 2.3.0

package/lib/model-router/health.mjs

@@ -0,0 +1,453 @@
+/**
+ * lib/model-router/health.mjs — per-key circuit breakers for the model router.
+ *
+ * WRAPS (does not replace) `lib/rate-guard.mjs`. rate-guard remains the single
+ * shared 429/overload breaker per *provider* with decorrelated-jitter backoff;
+ * health.mjs adds the router's finer-grained per-KEY breaker, where a key is
+ * either a backend (`anthropic`) or a backend:model-class (`anthropic:sonnet`).
+ * Model-scoped keys are the load-bearing primitive behind the disposition
+ * table's `backend:model` cooldown scope — Haiku keeps flowing while the Sonnet
+ * bucket is dry (SPEC §4.5 / §4.6).
+ *
+ * Breaker file shape (one JSON per key under state/rate-limits/, same dir/
+ * pattern as rate-guard so doctor dumps both):
+ *   { key, openedAt, cooldownUntil, reason, strikes, lastProbeAt, halfOpen }
+ *
+ * Two correctness properties over the audited rate-guard race (M4):
+ *   1. Writes go through an O_EXCL lock so two concurrent `open()` calls on the
+ *      same key converge to ONE coherent file instead of clobbering each other
+ *      mid read-modify-write. The loser of the lock retries-then-merges.
+ *   2. `strikes` is monotonic per key across the lock so the stepped rate_limit
+ *      ladder (30s→1m→5m) advances correctly even under concurrency.
+ *
+ * Half-open probing: at most ONE probe per backend per 30s, only within 2 min
+ * of cooldown expiry — lets a single in-flight failover run a cheap canary
+ * before the chain commits, without a probe storm. Counters reset on expiry.
+ *
+ * House style: ESM, Node built-ins only, never throws on I/O failure
+ * (fail-open-for-work — a breaker we can't read allows the spawn), clock + dir
+ * injectable so tests are hermetic and never touch the real fs/clock.
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  renameSync,
+  unlinkSync,
+  openSync,
+  closeSync,
+  readdirSync,
+} from "node:fs";
+import { join, resolve, dirname } from "node:path";
+import { randomBytes } from "node:crypto";
+
+import { sanitizeProvider } from "../rate-guard.mjs";
+import { DISPOSITION, cooldownMsFor } from "./taxonomy.mjs";
+
+const SECOND = 1000;
+const MINUTE = 60 * SECOND;
+
+/** Half-open: at most one probe per backend per this window. */
+export const HALF_OPEN_PROBE_INTERVAL_MS = 30 * SECOND;
+/** Half-open is eligible only within this window before cooldown expiry. */
+export const HALF_OPEN_WINDOW_MS = 2 * MINUTE;
+/** How long we busy-spin trying to grab the O_EXCL lock before giving up. */
+const LOCK_TIMEOUT_MS = 2 * SECOND;
+const LOCK_SPIN_MS = 5;
+
+// ---------------------------------------------------------------------------
+// Injected primitives + paths
+// ---------------------------------------------------------------------------
+
+function clock(deps) {
+  return deps && typeof deps.now === "function" ? deps.now : Date.now;
+}
+
+function resolveStateDir(deps) {
+  if (deps && deps.stateDir) return resolve(deps.stateDir);
+  const root =
+    (deps && deps.agentRoot) ||
+    process.env.AGENT_ROOT ||
+    process.env.AGENT_DIR ||
+    process.cwd();
+  return join(resolve(root), "state", "rate-limits");
+}
+
+/**
+ * Canonical breaker key. Accepts either:
+ *   keyFor("anthropic")            → "anthropic"
+ *   keyFor("anthropic", "sonnet")  → "anthropic:sonnet"
+ * and sanitizes both segments the same way rate-guard sanitizes providers.
+ *
+ * @param {string} backend
+ * @param {string} [model]
+ * @returns {string}
+ */
+export function keyFor(backend, model) {
+  const b = sanitizeProvider(backend);
+  if (model == null || model === "") return b;
+  // sanitizeProvider lowercases + strips unsafe chars; reuse it for the model
+  // class so the composite key stays filesystem-safe and collision-resistant.
+  const m = sanitizeProvider(model);
+  return `${b}:${m}`;
+}
+
+/** The backend segment of a key ("anthropic:sonnet" → "anthropic"). */
+export function backendOfKey(key) {
+  const i = String(key).indexOf(":");
+  return i === -1 ? String(key) : String(key).slice(0, i);
+}
+
+function keyPath(key, deps) {
+  // The composite "backend:model" must stay one file; ":" is fs-legal on the
+  // mac/linux targets but we normalize to "__" to avoid surprises and to keep
+  // doctor globbing simple.
+  const safe = String(key).replace(/:/g, "__");
+  return join(resolveStateDir(deps), `breaker-${safe}.json`);
+}
+
+// ---------------------------------------------------------------------------
+// State (de)serialization
+// ---------------------------------------------------------------------------
+
+function emptyState(key) {
+  return {
+    key,
+    openedAt: 0,
+    cooldownUntil: 0,
+    reason: null,
+    strikes: 0,
+    lastProbeAt: 0,
+    halfOpen: false,
+  };
+}
+
+function numOr0(v) {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : 0;
+}
+
+// JSON cannot represent Infinity (JSON.stringify emits null), but an
+// operator-held breaker (auth_permanent / model_not_found) MUST persist an
+// open-until-operator cooldown. Round-trip it through a string sentinel so a
+// reboot doesn't silently re-open a permanently-cooled backend.
+const INF_SENTINEL = "never";
+
+/** Parse a persisted cooldownUntil, honoring the Infinity sentinel. */
+function parseCooldown(v) {
+  if (v === INF_SENTINEL || v === "Infinity") return Infinity;
+  return numOr0(v);
+}
+
+/** Serialize a state for disk, encoding Infinity as the sentinel string. */
+function serializeState(state) {
+  return {
+    ...state,
+    cooldownUntil: state.cooldownUntil === Infinity ? INF_SENTINEL : state.cooldownUntil,
+  };
+}
+
+/**
+ * Read a breaker's state. Missing/corrupt → closed (empty) state. Never throws.
+ *
+ * @param {string} key
+ * @param {object} [deps]
+ * @returns {{key,openedAt,cooldownUntil,reason,strikes,lastProbeAt,halfOpen}}
+ */
+export function readBreaker(key, deps) {
+  const p = keyPath(key, deps);
+  if (!existsSync(p)) return emptyState(key);
+  try {
+    const raw = JSON.parse(readFileSync(p, "utf-8"));
+    return {
+      key,
+      openedAt: numOr0(raw.openedAt),
+      cooldownUntil: parseCooldown(raw.cooldownUntil),
+      reason: typeof raw.reason === "string" ? raw.reason : null,
+      strikes: numOr0(raw.strikes),
+      lastProbeAt: numOr0(raw.lastProbeAt),
+      halfOpen: raw.halfOpen === true,
+    };
+  } catch {
+    return emptyState(key);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// O_EXCL lock — the fix for the unlocked read-modify-write race (M4).
+//
+// withLock(key, deps, mutator) acquires `${file}.lock` via openSync(..,"wx")
+// (O_CREAT|O_EXCL), runs mutator(currentState) → nextState under the lock,
+// atomically writes nextState, releases the lock. Two concurrent opens on the
+// same key serialize: the loser spins on EEXIST until the winner releases,
+// then re-reads the now-updated state and applies its own mutation on top.
+// Result: monotonic `strikes`, no clobbered cooldownUntil.
+// ---------------------------------------------------------------------------
+
+function acquireLock(lockPath, deps) {
+  const now = clock(deps);
+  const deadline = now() + LOCK_TIMEOUT_MS;
+  // Synchronous spin — these critical sections are sub-millisecond writes.
+  for (;;) {
+    try {
+      const fd = openSync(lockPath, "wx"); // O_CREAT | O_EXCL | O_WRONLY
+      return fd;
+    } catch (err) {
+      if (err && err.code === "EEXIST") {
+        // Stale-lock break: if the lockfile is older than the timeout window,
+        // assume the holder died and steal it. Best-effort; never throw.
+        try {
+          const raw = readFileSync(lockPath, "utf-8");
+          const heldAt = Number(JSON.parse(raw).at);
+          if (Number.isFinite(heldAt) && now() - heldAt > LOCK_TIMEOUT_MS) {
+            try { unlinkSync(lockPath); } catch { /* */ }
+            continue;
+          }
+        } catch { /* unreadable lock — fall through to spin/timeout */ }
+        if (now() >= deadline) return null; // give up; caller degrades
+        busyWaitMs(LOCK_SPIN_MS);
+        continue;
+      }
+      // Any other error (e.g. ENOENT on a missing dir) — let caller mkdir+retry
+      return undefined;
+    }
+  }
+}
+
+function busyWaitMs(ms) {
+  const end = Date.now() + ms;
+  while (Date.now() < end) { /* spin */ }
+}
+
+/**
+ * Run `mutator(currentState) → nextState` under an exclusive per-key lock and
+ * atomically persist the result. Returns the persisted next state. On any
+ * unrecoverable I/O failure, applies the mutation in-memory and returns it
+ * (fail-open: a breaker we can't persist degrades to in-memory, never blocks).
+ */
+function withLock(key, deps, mutator) {
+  const file = keyPath(key, deps);
+  const lockPath = `${file}.lock`;
+  const now = clock(deps);
+  try {
+    mkdirSync(dirname(file), { recursive: true });
+  } catch { /* */ }
+
+  let fd = acquireLock(lockPath, deps);
+  if (fd === undefined) {
+    // Dir may not have existed; retry once after mkdir.
+    try { mkdirSync(dirname(file), { recursive: true }); } catch { /* */ }
+    fd = acquireLock(lockPath, deps);
+  }
+  if (fd == null) {
+    // Couldn't lock — degrade to a best-effort unlocked write so we still
+    // record the breaker (better a slightly racy file than no breaker at all).
+    const cur = readBreaker(key, deps);
+    const next = mutator(cur) || cur;
+    bestEffortWrite(file, next, deps);
+    return next;
+  }
+
+  try {
+    // Stamp the lockfile so a crashed holder can be detected + stolen.
+    try { writeFileSync(fd, JSON.stringify({ at: now(), pid: process.pid })); } catch { /* */ }
+    const cur = readBreaker(key, deps);
+    const next = mutator(cur) || cur;
+    bestEffortWrite(file, next, deps);
+    return next;
+  } finally {
+    try { closeSync(fd); } catch { /* */ }
+    try { unlinkSync(lockPath); } catch { /* */ }
+  }
+}
+
+function bestEffortWrite(file, state, deps) {
+  try {
+    mkdirSync(dirname(file), { recursive: true });
+    const tmp = `${file}.tmp.${process.pid}.${Date.now()}.${randomBytes(2).toString("hex")}`;
+    writeFileSync(tmp, JSON.stringify(serializeState(state), null, 2) + "\n");
+    try {
+      renameSync(tmp, file);
+    } catch (err) {
+      try { unlinkSync(tmp); } catch { /* */ }
+      throw err;
+    }
+  } catch {
+    /* fail-open: in-memory only */
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Is the breaker for `key` currently open (blocking work) at time `now`?
+ * A breaker is open while `cooldownUntil > now`. cooldownUntil === Infinity
+ * (auth_permanent / model_not_found) means open-until-operator. Never throws.
+ *
+ * @param {string} key
+ * @param {number} [now]
+ * @param {object} [deps]
+ * @returns {{open:boolean, until:number, reason:string|null, strikes:number}}
+ */
+export function isOpen(key, now, deps) {
+  const t = Number.isFinite(now) ? now : clock(deps)();
+  const s = readBreaker(key, deps);
+  const open = s.cooldownUntil > t;
+  return { open, until: s.cooldownUntil, reason: s.reason, strikes: s.strikes };
+}
+
+/**
+ * Open (or advance) the breaker for `key` with `reason`. If `cooldownMs` is
+ * omitted, it is derived from the disposition table using the post-increment
+ * strike count (so the stepped rate_limit ladder advances on each consecutive
+ * hit). cooldownMs === Infinity ⇒ cooldownUntil = Infinity (operator-cleared).
+ *
+ * Locked read-modify-write: concurrent opens converge — `strikes` is the prior
+ * value + 1 (monotonic), and `cooldownUntil` is the max of the existing and the
+ * newly computed window so a longer cooldown is never shortened by a racing
+ * shorter one.
+ *
+ * @param {string} key
+ * @param {string} reason   FailoverReason
+ * @param {number|null} [cooldownMs]  explicit ms; null/undefined → from table
+ * @param {number} [now]
+ * @param {object} [deps]
+ * @returns {{key,openedAt,cooldownUntil,reason,strikes,...}}
+ */
+export function open(key, reason, cooldownMs, now, deps) {
+  const t = Number.isFinite(now) ? now : clock(deps)();
+  return withLock(key, deps, (cur) => {
+    const strikes = numOr0(cur.strikes) + 1;
+    let ms = cooldownMs;
+    if (ms == null) {
+      // strike-1 indexes the ladder from 0 (first hit → 30s).
+      ms = cooldownMsFor(reason, strikes - 1);
+    }
+    const until = ms === Infinity ? Infinity : t + Math.max(0, ms);
+    const cooldownUntil =
+      cur.cooldownUntil === Infinity
+        ? Infinity
+        : Math.max(numOr0(cur.cooldownUntil), until);
+    return {
+      key,
+      openedAt: cur.openedAt > 0 && cur.cooldownUntil > t ? cur.openedAt : t,
+      cooldownUntil,
+      reason,
+      strikes,
+      lastProbeAt: numOr0(cur.lastProbeAt),
+      halfOpen: false, // a fresh open clears any half-open probe state
+    };
+  });
+}
+
+/**
+ * Record a clean success for `key` — closes the breaker and resets strikes so
+ * the next incident starts from the base step of the ladder. Idempotent: a
+ * no-op (no write) when already closed and unstruck, to avoid churning the file
+ * on every successful spawn.
+ *
+ * @returns {object} the (closed) state
+ */
+export function close(key, now, deps) {
+  const cur = readBreaker(key, deps);
+  if (cur.cooldownUntil === 0 && cur.strikes === 0 && cur.openedAt === 0 && !cur.halfOpen) {
+    return cur; // already closed — don't churn
+  }
+  return withLock(key, deps, () => emptyState(key));
+}
+
+/** Alias matching the rate-guard vocabulary. */
+export const recordSuccess = close;
+
+/**
+ * Operator clear — force a breaker closed regardless of cooldownUntil
+ * (including auth_permanent / model_not_found Infinity holds). For the
+ * `until operator` and `permanent` dispositions.
+ */
+export function clear(key, deps) {
+  return withLock(key, deps, () => emptyState(key));
+}
+
+/**
+ * Half-open probe gate. Returns true (and stamps `lastProbeAt`) iff a single
+ * canary probe is permitted right now for `key`'s backend:
+ *   - the breaker is currently open, AND
+ *   - we are within HALF_OPEN_WINDOW_MS of cooldown expiry (so a probe is
+ *     worthwhile — we're about to retry anyway), AND
+ *   - no probe has run for this BACKEND within HALF_OPEN_PROBE_INTERVAL_MS
+ *     (1 probe/backend/30s — across all its model-class keys).
+ * Infinity cooldowns (auth_permanent / model_not_found) are never probed —
+ * they require an operator, not a canary.
+ *
+ * The stamp is written under the lock so two concurrent failover runs don't
+ * both think they "own" the probe.
+ *
+ * @param {string} key
+ * @param {number} [now]
+ * @param {object} [deps]
+ * @returns {boolean} true ⇒ caller may run exactly one probe
+ */
+export function halfOpenProbe(key, now, deps) {
+  const t = Number.isFinite(now) ? now : clock(deps)();
+  const s = readBreaker(key, deps);
+  // Closed, or not within the pre-expiry window, or operator-held → no probe.
+  if (s.cooldownUntil <= t) return false; // already closed; just spawn normally
+  if (s.cooldownUntil === Infinity) return false; // operator-held
+  if (s.cooldownUntil - t > HALF_OPEN_WINDOW_MS) return false; // too early
+
+  // Backend-level probe rate limit: scan this key's own lastProbeAt (and, for a
+  // backend:model key, the backend key too) so it's truly 1/backend/30s.
+  const backend = backendOfKey(key);
+  const recent = (st) => st.lastProbeAt && t - st.lastProbeAt < HALF_OPEN_PROBE_INTERVAL_MS;
+  if (recent(s)) return false;
+  if (key !== backend) {
+    const backendState = readBreaker(backend, deps);
+    if (recent(backendState)) return false;
+  }
+
+  // Stamp under the lock and mark half-open.
+  withLock(key, deps, (cur) => ({
+    ...cur,
+    key,
+    lastProbeAt: t,
+    halfOpen: true,
+  }));
+  return true;
+}
+
+/**
+ * Snapshot every breaker file under the state dir (for `maestro doctor`'s
+ * breaker dump). Best-effort, never throws; returns [] when the dir is absent.
+ */
+export function listBreakers(now, deps) {
+  const dir = resolveStateDir(deps);
+  if (!existsSync(dir)) return [];
+  let names = [];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const t = Number.isFinite(now) ? now : clock(deps)();
+  const out = [];
+  for (const name of names) {
+    if (!name.startsWith("breaker-") || !name.endsWith(".json")) continue;
+    const key = name.slice("breaker-".length, -".json".length).replace(/__/g, ":");
+    const s = readBreaker(key, deps);
+    out.push({ ...s, open: s.cooldownUntil > t });
+  }
+  return out;
+}
+
+export const _internals = {
+  emptyState,
+  keyPath,
+  withLock,
+  HALF_OPEN_PROBE_INTERVAL_MS,
+  HALF_OPEN_WINDOW_MS,
+  DISPOSITION,
+};