@cohortapp/agent-sdk 2.3.0

package/lib/util/reconnect.test.mjs

@@ -0,0 +1,369 @@
+/**
+ * lib/util/reconnect.test.mjs
+ *
+ * node:test coverage for the bounded-backoff reconnect supervisor. Everything is
+ * driven by an injected fake clock + timer queue + RNG, so there are no real
+ * sockets and no real time — the backoff-and-recover lifecycle is asserted
+ * deterministically.
+ *
+ * Covered:
+ *  - first connect surfaces connecting → up
+ *  - a drop (notifyDown) drives down → connecting → up, recovering the connection
+ *  - backoff grows decorrelated-jittered between attempts and is bounded by max
+ *  - the attempt budget is honoured: exhausting it surfaces giving-up (never silent)
+ *  - a failing connect() is treated as a drop and retried
+ *  - isHealthy polling trips a reconnect when it returns false / throws
+ *  - notifyDown is idempotent while a reconnect is already in flight
+ *  - stop() cancels pending backoff + health poll and prevents further reconnects
+ *  - decorrelatedBackoff bounds with an injected rng
+ *  - a throwing onState sink can't crash the supervisor
+ *
+ * Run: cd /Users/layla/maestro && node --test lib/util/reconnect.test.mjs
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  superviseConnection,
+  decorrelatedBackoff,
+  CONN_STATE,
+  DEFAULT_BASE_BACKOFF_MS,
+  DEFAULT_MAX_BACKOFF_MS,
+} from "./reconnect.mjs";
+
+/**
+ * A deterministic timer harness: setTimeout enqueues a callback with its delay;
+ * `tick()` fires the earliest-due timer (advancing a logical clock), and
+ * `flush()` runs everything queued. Returns the injectables to pass as cfg.
+ */
+function fakeTimers() {
+  let nowMs = 0;
+  let seq = 0;
+  const timers = new Map(); // id -> { at, fn }
+  const api = {
+    now: () => nowMs,
+    setTimeout(fn, ms) {
+      const id = ++seq;
+      timers.set(id, { at: nowMs + Math.max(0, Number(ms) || 0), fn });
+      return { id, unref() { return this; } };
+    },
+    clearTimeout(handle) {
+      if (handle && timers.has(handle.id)) timers.delete(handle.id);
+    },
+    /** Fire the single earliest-due timer; advances the clock to its `at`. */
+    async tick() {
+      let next = null;
+      for (const [id, t] of timers) {
+        if (!next || t.at < next.t.at) next = { id, t };
+      }
+      if (!next) return false;
+      timers.delete(next.id);
+      nowMs = next.t.at;
+      // The fn may return a promise (the async connect attempt); await it so a
+      // failed attempt's freshly-armed backoff timer is visible to the next loop.
+      await next.t.fn();
+      // Yield a microtask turn so any .then/.catch scheduled by the connect
+      // settles before we pick the next timer.
+      await Promise.resolve();
+      return true;
+    },
+    /** Drain all currently-pending timers (and any they enqueue), to a cap. */
+    async flush(maxIters = 1000) {
+      let n = 0;
+      // Let any already-in-flight async attempt arm its first timer before we
+      // start draining.
+      await Promise.resolve();
+      while (timers.size && n < maxIters) {
+        await api.tick();
+        n++;
+      }
+    },
+    pending() { return timers.size; },
+  };
+  return api;
+}
+
+/** A connect() that resolves/rejects on demand, recording call order. */
+function scriptedConnect(plan) {
+  let i = 0;
+  const calls = [];
+  const connect = async () => {
+    const step = plan[Math.min(i, plan.length - 1)];
+    calls.push(step);
+    i++;
+    if (step && step.fail) throw new Error(step.error || "connect failed");
+    return step && step.handle !== undefined ? step.handle : `conn-${i}`;
+  };
+  connect.calls = calls;
+  connect.count = () => i;
+  return connect;
+}
+
+/** Collect onState transitions for assertions. */
+function recorder() {
+  const states = [];
+  const fn = (state, info) => states.push({ state, info });
+  fn.states = states;
+  fn.names = () => states.map((s) => s.state);
+  return fn;
+}
+
+test("first connect surfaces connecting → up and exposes the connection", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "socket-A" }]);
+  const sup = superviseConnection({ connect, onState, ...T });
+  await sup.started;
+  assert.deepEqual(onState.names(), [CONN_STATE.CONNECTING, CONN_STATE.UP]);
+  assert.equal(sup.state, CONN_STATE.UP);
+  assert.equal(sup.connection, "socket-A");
+  const up = onState.states.find((s) => s.state === CONN_STATE.UP);
+  assert.equal(up.info.reconnected, false);
+});
+
+test("a drop reconnects: down → connecting(backoff) → up, connection recovered", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "first" }, { handle: "second" }]);
+  const sup = superviseConnection({
+    connect,
+    onState,
+    rng: () => 0, // pin jitter to the floor for a predictable delay
+    baseBackoffMs: 100,
+    ...T,
+  });
+  await sup.started;
+  assert.equal(sup.state, CONN_STATE.UP);
+
+  // The owner observes a socket close.
+  sup.notifyDown("socket-closed");
+  assert.equal(sup.state, CONN_STATE.DOWN);
+  assert.equal(connect.count(), 1, "should not reconnect until the backoff fires");
+
+  // Fire the backoff timer → reconnect attempt → up again.
+  await T.flush();
+  assert.equal(sup.state, CONN_STATE.UP);
+  assert.equal(sup.connection, "second");
+  assert.equal(connect.count(), 2);
+
+  const names = onState.names();
+  // The reconnected "up" carries reconnected:true.
+  const lastUp = onState.states.filter((s) => s.state === CONN_STATE.UP).at(-1);
+  assert.equal(lastUp.info.reconnected, true);
+  assert.ok(names.includes(CONN_STATE.DOWN));
+  // The failure budget reset after recovery.
+  assert.equal(sup.attempts, 0);
+});
+
+test("backoff grows between attempts and is bounded by maxBackoffMs", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  // Always fail so we can watch successive backoff windows grow.
+  const connect = scriptedConnect([{ fail: true }]);
+  const sup = superviseConnection({
+    connect,
+    onState,
+    rng: () => 0.999999, // pin jitter to the TOP of each range
+    baseBackoffMs: 100,
+    maxBackoffMs: 1000,
+    backoffFactor: 3,
+    maxAttempts: 10,
+    ...T,
+  });
+  await sup.started; // first (failing) attempt completes and arms the first backoff
+  // Drive a handful of attempts and collect the scheduled backoff delays.
+  await T.flush(20);
+
+  // While a backoff timer is pending the connection is DOWN, and that event
+  // carries delayMs (the window until the next attempt).
+  const delays = onState.states
+    .filter((s) => s.state === CONN_STATE.DOWN && typeof s.info.delayMs === "number")
+    .map((s) => s.info.delayMs);
+
+  assert.ok(delays.length >= 3, `expected several backoff windows, got ${delays.length}`);
+  // Decorrelated growth with rng pinned to the top of each range:
+  //   prev=0   → [100,100] → 100
+  //   prev=100 → [100,300] → ~300
+  //   prev=300 → [100,900] → ~900
+  //   prev=900 → [100,2700] capped at 1000
+  assert.ok(delays[1] >= delays[0], `windows should trend upward: ${delays.slice(0, 3)}`);
+  // Every window respects the ceiling.
+  for (const d of delays) assert.ok(d <= 1000, `window ${d} exceeded max`);
+  // And none drops below the floor.
+  for (const d of delays) assert.ok(d >= 100, `window ${d} below base`);
+});
+
+test("exhausting the attempt budget surfaces giving-up (never silent)", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ fail: true }]); // always fails
+  const sup = superviseConnection({
+    connect,
+    onState,
+    maxAttempts: 3,
+    baseBackoffMs: 10,
+    rng: () => 0,
+    ...T,
+  });
+  await sup.started; // first failing attempt completes and arms the first backoff
+  await T.flush();
+  assert.equal(sup.state, CONN_STATE.GIVING_UP, "must reach the terminal state, not die silently");
+  const giveUp = onState.states.find((s) => s.state === CONN_STATE.GIVING_UP);
+  assert.ok(giveUp, "a giving-up transition must be emitted");
+  assert.equal(giveUp.info.attempts, 3);
+  // No more reconnects after giving up.
+  const before = connect.count();
+  sup.notifyDown("late drop");
+  await T.flush();
+  assert.equal(connect.count(), before, "no reconnect attempts after giving-up");
+});
+
+test("a failing connect is treated as a drop and retried until it succeeds", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  // Fail twice, then succeed.
+  const connect = scriptedConnect([{ fail: true }, { fail: true }, { handle: "live" }]);
+  const sup = superviseConnection({
+    connect,
+    onState,
+    maxAttempts: 5,
+    baseBackoffMs: 10,
+    rng: () => 0,
+    ...T,
+  });
+  await sup.started; // first failing attempt completes and arms the first backoff
+  await T.flush();
+  assert.equal(sup.state, CONN_STATE.UP);
+  assert.equal(sup.connection, "live");
+  assert.equal(connect.count(), 3);
+});
+
+test("isHealthy polling trips a reconnect when it returns false", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "a" }, { handle: "b" }]);
+  let healthy = true;
+  const sup = superviseConnection({
+    connect,
+    onState,
+    isHealthy: () => healthy,
+    healthIntervalMs: 1000,
+    baseBackoffMs: 10,
+    rng: () => 0,
+    ...T,
+  });
+  await sup.started;
+  assert.equal(sup.state, CONN_STATE.UP);
+
+  // Mark the connection unhealthy; the next poll should see it and reconnect.
+  healthy = false;
+  await T.tick(); // fire the health poll → reportDown
+  // After the poll trips, make subsequent connects healthy again.
+  healthy = true;
+  await T.flush(); // backoff + reconnect + re-arm poll
+  assert.equal(sup.state, CONN_STATE.UP);
+  assert.equal(connect.count(), 2, "reconnected after a failed health poll");
+});
+
+test("a throwing isHealthy counts as unhealthy and reconnects", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "a" }, { handle: "b" }]);
+  let mode = "ok";
+  const sup = superviseConnection({
+    connect,
+    onState,
+    isHealthy: () => {
+      if (mode === "throw") throw new Error("health probe blew up");
+      return true;
+    },
+    healthIntervalMs: 1000,
+    baseBackoffMs: 10,
+    rng: () => 0,
+    ...T,
+  });
+  await sup.started;
+  mode = "throw";
+  await T.tick(); // health poll throws → treated as down
+  mode = "ok";
+  await T.flush();
+  assert.equal(sup.state, CONN_STATE.UP);
+  assert.equal(connect.count(), 2);
+});
+
+test("notifyDown is idempotent while a reconnect is already in flight", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "a" }, { handle: "b" }]);
+  const sup = superviseConnection({ connect, onState, baseBackoffMs: 50, rng: () => 0, ...T });
+  await sup.started;
+
+  // Several rapid close events should collapse into ONE reconnect cycle.
+  sup.notifyDown("close-1");
+  sup.notifyDown("close-2");
+  sup.notifyDown("close-3");
+  assert.equal(T.pending(), 1, "only one backoff timer should be armed");
+  await T.flush();
+  assert.equal(connect.count(), 2, "exactly one reconnect, not three");
+});
+
+test("stop() cancels pending backoff + health poll and blocks further reconnects", async () => {
+  const T = fakeTimers();
+  const onState = recorder();
+  const connect = scriptedConnect([{ handle: "a" }]);
+  const sup = superviseConnection({
+    connect,
+    onState,
+    isHealthy: () => true,
+    healthIntervalMs: 1000,
+    baseBackoffMs: 50,
+    rng: () => 0,
+    ...T,
+  });
+  await sup.started;
+  assert.ok(T.pending() >= 1, "health poll timer should be armed while up");
+
+  sup.notifyDown("drop"); // arms a backoff timer
+  sup.stop("shutting down");
+  assert.equal(T.pending(), 0, "stop() must clear all pending timers");
+
+  const before = connect.count();
+  sup.notifyDown("late drop");
+  await T.flush();
+  assert.equal(connect.count(), before, "no reconnect after stop()");
+});
+
+test("a throwing onState sink cannot crash the supervisor", async () => {
+  const T = fakeTimers();
+  const connect = scriptedConnect([{ handle: "a" }]);
+  const sup = superviseConnection({
+    connect,
+    onState: () => { throw new Error("bad sink"); },
+    ...T,
+  });
+  await assert.doesNotReject(() => sup.started);
+  assert.equal(sup.state, CONN_STATE.UP);
+});
+
+test("decorrelatedBackoff bounds the window with an injected rng", () => {
+  // rng=0 → floor; rng→1 → top of [base, prev*factor] capped at max.
+  assert.equal(decorrelatedBackoff(0, { base: 100, max: 5000, factor: 3, rng: () => 0 }), 100);
+  const top = decorrelatedBackoff(100, { base: 100, max: 5000, factor: 3, rng: () => 0.999999 });
+  assert.ok(top <= 300 && top >= 100, `expected within [100,300], got ${top}`);
+  // The cap holds even with a large prev.
+  assert.equal(
+    decorrelatedBackoff(100000, { base: 100, max: 1000, factor: 3, rng: () => 0.999999 }),
+    1000
+  );
+});
+
+test("exposes sane default backoff constants", () => {
+  assert.equal(typeof DEFAULT_BASE_BACKOFF_MS, "number");
+  assert.equal(typeof DEFAULT_MAX_BACKOFF_MS, "number");
+  assert.ok(DEFAULT_MAX_BACKOFF_MS > DEFAULT_BASE_BACKOFF_MS);
+});
+
+test("superviseConnection requires a connect function", () => {
+  assert.throws(() => superviseConnection({}), /connect must be a function/);
+});

package/lib/util/unhandled.mjs

@@ -0,0 +1,205 @@
+/**
+ * lib/util/unhandled.mjs — process-level unhandled-failure handlers that LOG
+ * instead of dying silently.
+ *
+ * The audit (gaps-product-quality P0-6) found "no unhandledRejection handler in
+ * any daemon entrypoint (one stray rejection from an async third-party listener
+ * crashes the daemon)". Node's defaults make this a silent killer: an
+ * unhandledRejection prints a warning and (since Node 15) exits with a non-zero
+ * code with no structured record, and an uncaughtException tears the process down
+ * the same way. On a 24/7 Mac mini that is a dead agent with nothing in the logs
+ * a human or doctor reads.
+ *
+ * `installUnhandledHandlers({ onEvent })` installs the two process listeners with
+ * a SAFE default policy that matches what a resilient daemon wants:
+ *   - unhandledRejection → LOG and KEEP RUNNING. A stray rejection from a
+ *     third-party listener should not take down a healthy daemon; it should be
+ *     counted and surfaced. (The pattern already exists in scripts/poller/.)
+ *   - uncaughtException   → LOG, run the caller's flush, then exit. An uncaught
+ *     exception means corrupted in-process state; the correct move is a clean,
+ *     LOUD exit (so launchd restarts the process) rather than limping on. The
+ *     exit is overridable so tests — and callers with their own supervisor —
+ *     can observe the event without the test runner dying.
+ *
+ * Every event is routed through the injected `onEvent` (and, by default, the
+ * loud diagnostics counters) so the failure is counted somewhere doctor looks.
+ *
+ * The installer is IDEMPOTENT — calling it twice does not double-register — and
+ * RETURNS an `uninstall()` so tests (and a clean daemon shutdown) can remove the
+ * listeners. The handlers themselves NEVER throw; a failure inside the handler
+ * must not become a second uncaught exception.
+ *
+ * Constraints (CLAUDE.md): ESM, Node built-ins + lib/diagnostics/counters only,
+ * everything (process, onEvent, exit, counter) injectable for hermetic tests.
+ *
+ * @module lib/util/unhandled
+ */
+
+import { bump as defaultBump } from "../diagnostics/counters.mjs";
+
+/** Counter names so doctor and tests can reference the exact keys. Frozen. */
+export const UNHANDLED_COUNTERS = Object.freeze({
+  REJECTION: "process.unhandled_rejection",
+  EXCEPTION: "process.uncaught_exception",
+});
+
+/**
+ * Module-level guard tracking the live installation on a given process object,
+ * so a second install on the same process is a no-op (idempotent) and uninstall
+ * removes exactly the listeners we added. Keyed by the process object so a test
+ * driving a fake process and the real daemon don't collide.
+ * @type {WeakMap<object, { onRejection: Function, onException: Function }>}
+ */
+const INSTALLED = new WeakMap();
+
+/**
+ * Coerce a thrown/rejected value into a structured, serializable shape — we must
+ * never let formatting the error throw a second time inside the handler.
+ * @param {unknown} value
+ * @returns {{ name?: string, message: string, stack?: string }}
+ */
+function describe(value) {
+  try {
+    if (value instanceof Error) {
+      return {
+        name: value.name,
+        message: value.message,
+        stack: typeof value.stack === "string" ? value.stack : undefined,
+      };
+    }
+    if (value == null) return { message: String(value) };
+    if (typeof value === "object") {
+      // A rejected non-Error (e.g. `reject("nope")` or `reject({code})`).
+      const msg = "message" in value ? String(value.message) : safeJson(value);
+      return { message: msg };
+    }
+    return { message: String(value) };
+  } catch {
+    return { message: "<undescribable>" };
+  }
+}
+
+function safeJson(v) {
+  try {
+    return JSON.stringify(v);
+  } catch {
+    return String(v);
+  }
+}
+
+/**
+ * Install process-level unhandledRejection + uncaughtException handlers that log
+ * (never silently die). Idempotent and removable.
+ *
+ * @param {object} [opts]
+ * @param {(event: {kind:string, error:object, fatal:boolean}) => void} [opts.onEvent]
+ *        Called for every captured failure. `kind` is "unhandledRejection" |
+ *        "uncaughtException"; `error` is the structured {name,message,stack};
+ *        `fatal` is true for the uncaught-exception path (process will exit).
+ *        A throwing onEvent is swallowed — the handler must stay safe.
+ * @param {object} [opts.process]  the process to attach to (default global process).
+ *        Injectable so tests drive a fake EventEmitter-shaped object.
+ * @param {(code: number) => void} [opts.exit]  called to terminate on an uncaught
+ *        exception (default process.exit). Injectable so tests don't kill the
+ *        runner; pass a no-op to log-only.
+ * @param {boolean} [opts.exitOnUncaught]  exit the process on uncaughtException
+ *        (default true). Set false to log-and-continue (a caller with its own
+ *        supervisor). unhandledRejection NEVER exits regardless.
+ * @param {number} [opts.exitCode]  exit code for the uncaught path (default 1).
+ * @param {(name:string, attrs?:object) => void} [opts.bump]  counter (default the
+ *        loud diagnostics counter). Injectable for tests.
+ * @returns {{ uninstall: () => void, installed: boolean }}
+ *          `installed` is false when a prior install was already live (idempotent).
+ */
+export function installUnhandledHandlers(opts = {}) {
+  const proc = opts.process || process;
+  const onEvent = typeof opts.onEvent === "function" ? opts.onEvent : null;
+  const exit =
+    typeof opts.exit === "function"
+      ? opts.exit
+      : (code) => proc.exit?.(code);
+  const exitOnUncaught = opts.exitOnUncaught !== false;
+  const exitCode = Number.isInteger(opts.exitCode) ? opts.exitCode : 1;
+  const bump = typeof opts.bump === "function" ? opts.bump : defaultBump;
+
+  // Idempotent: if we already installed on this process, do nothing and report
+  // it so a double-install (e.g. daemon re-entry) is a safe no-op.
+  if (INSTALLED.has(proc)) {
+    return { uninstall: () => uninstallFor(proc), installed: false };
+  }
+
+  /** Route one failure through the counter, the sink, and (if fatal) the exit. */
+  function dispatch(kind, value, fatal) {
+    const error = describe(value);
+    // 1. Count it — loud, durable, where doctor looks. Never throws.
+    try {
+      bump(
+        kind === "uncaughtException"
+          ? UNHANDLED_COUNTERS.EXCEPTION
+          : UNHANDLED_COUNTERS.REJECTION,
+        { name: error.name || null, message: error.message, fatal: !!fatal }
+      );
+    } catch {
+      /* counter must never crash the handler */
+    }
+    // 2. Tell the caller. A throwing sink is isolated.
+    if (onEvent) {
+      try {
+        onEvent({ kind, error, fatal: !!fatal });
+      } catch {
+        /* a buggy sink must not become a second uncaught exception */
+      }
+    }
+  }
+
+  function onRejection(reason) {
+    // Log + keep running. A stray rejection should never down a healthy daemon.
+    dispatch("unhandledRejection", reason, false);
+  }
+
+  function onException(err) {
+    // Log + flush via onEvent, then exit LOUDLY (default) so launchd restarts a
+    // clean process rather than limping on with corrupted state.
+    dispatch("uncaughtException", err, true);
+    if (exitOnUncaught) {
+      try {
+        exit(exitCode);
+      } catch {
+        /* if even exit throws, there's nothing more we can safely do */
+      }
+    }
+  }
+
+  proc.on?.("unhandledRejection", onRejection);
+  proc.on?.("uncaughtException", onException);
+  INSTALLED.set(proc, { onRejection, onException });
+
+  return { uninstall: () => uninstallFor(proc), installed: true };
+}
+
+/**
+ * Remove the handlers this module installed on `proc`, if any. Safe to call
+ * multiple times. Never throws.
+ * @param {object} proc
+ * @returns {boolean}  true if listeners were removed
+ */
+function uninstallFor(proc) {
+  const live = INSTALLED.get(proc);
+  if (!live) return false;
+  try { proc.removeListener?.("unhandledRejection", live.onRejection); } catch { /* */ }
+  try { proc.removeListener?.("uncaughtException", live.onException); } catch { /* */ }
+  INSTALLED.delete(proc);
+  return true;
+}
+
+/**
+ * True if this module currently has handlers installed on `proc` (default global
+ * process). Lets a caller/doctor check without re-installing.
+ * @param {object} [proc]
+ * @returns {boolean}
+ */
+export function isInstalled(proc = process) {
+  return INSTALLED.has(proc);
+}
+
+export default { installUnhandledHandlers, isInstalled, UNHANDLED_COUNTERS };