@cohortapp/agent-sdk 2.3.0

package/lib/util/fetch-timeout.test.mjs

@@ -0,0 +1,202 @@
+/**
+ * lib/util/fetch-timeout.test.mjs
+ *
+ * node:test coverage for fetchWithTimeout:
+ *  - a fast request passes through and returns the Response
+ *  - a slow request aborts after the timeout with a typed FetchTimeoutError
+ *  - a caller-supplied AbortSignal abort surfaces as the caller's error, not a
+ *    timeout, and the timer is cleared
+ *  - timeoutMs <= 0 disables the timeout (pass-through, no internal signal)
+ *  - init is forwarded; the internal abort signal is injected
+ *
+ * Run: cd /Users/layla/maestro && node --test lib/util/fetch-timeout.test.mjs
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  fetchWithTimeout,
+  FetchTimeoutError,
+  DEFAULT_TIMEOUT_MS,
+} from "./fetch-timeout.mjs";
+
+/**
+ * A fetch stub that resolves after `delayMs`, but rejects with an AbortError if
+ * the provided signal aborts first — mirroring real `fetch` abort semantics.
+ *
+ * @param {number} delayMs
+ * @param {*} [response]
+ */
+function slowFetch(delayMs, response = { ok: true, status: 200 }) {
+  return (url, init) =>
+    new Promise((resolve, reject) => {
+      const signal = init && init.signal;
+      const t = setTimeout(() => resolve(response), delayMs);
+      if (signal) {
+        if (signal.aborted) {
+          clearTimeout(t);
+          const e = new Error("aborted");
+          e.name = "AbortError";
+          reject(e);
+          return;
+        }
+        signal.addEventListener(
+          "abort",
+          () => {
+            clearTimeout(t);
+            const e = new Error("aborted");
+            e.name = "AbortError";
+            reject(e);
+          },
+          { once: true }
+        );
+      }
+    });
+}
+
+/* ------------------------------- pass-through ----------------------------- */
+
+test("fast request passes through and returns the response", async () => {
+  const resp = { ok: true, status: 200, body: "hi" };
+  const out = await fetchWithTimeout("https://x/", {}, 1000, {
+    fetchImpl: slowFetch(0, resp),
+  });
+  assert.equal(out, resp);
+});
+
+test("init is forwarded and an abort signal is injected", async () => {
+  let seenInit = null;
+  const fetchImpl = (url, init) => {
+    seenInit = init;
+    return Promise.resolve({ ok: true });
+  };
+  await fetchWithTimeout(
+    "https://x/",
+    { method: "POST", headers: { "x-test": "1" } },
+    1000,
+    { fetchImpl }
+  );
+  assert.equal(seenInit.method, "POST");
+  assert.equal(seenInit.headers["x-test"], "1");
+  assert.ok(seenInit.signal, "internal AbortSignal should be injected into init");
+  assert.equal(seenInit.signal.aborted, false);
+});
+
+/* --------------------------------- timeout -------------------------------- */
+
+test("slow request aborts after the timeout with a typed error", async () => {
+  await assert.rejects(
+    () =>
+      fetchWithTimeout("https://slow/", {}, 20, {
+        fetchImpl: slowFetch(10_000),
+      }),
+    (err) => {
+      assert.ok(err instanceof FetchTimeoutError, `wrong type: ${err && err.name}`);
+      assert.equal(err.name, "FetchTimeoutError");
+      assert.equal(err.code, "ETIMEDOUT");
+      assert.equal(err.timeoutMs, 20);
+      assert.ok(String(err.message).includes("timed out"), err.message);
+      assert.ok(String(err.message).includes("slow"), "url should appear in message");
+      return true;
+    }
+  );
+});
+
+/* --------------------------- caller abort vs timeout ---------------------- */
+
+test("caller-supplied abort surfaces as the caller error, not a timeout", async () => {
+  const ac = new AbortController();
+  // Abort well before the (long) timeout would fire.
+  setTimeout(() => ac.abort(new Error("caller-cancelled")), 5);
+
+  await assert.rejects(
+    () =>
+      fetchWithTimeout("https://x/", { signal: ac.signal }, 10_000, {
+        fetchImpl: slowFetch(10_000),
+      }),
+    (err) => {
+      assert.ok(!(err instanceof FetchTimeoutError), "must not be a timeout error");
+      assert.notEqual(err.code, "ETIMEDOUT");
+      return true;
+    }
+  );
+});
+
+test("an already-aborted caller signal aborts immediately", async () => {
+  const ac = new AbortController();
+  ac.abort(new Error("pre-aborted"));
+  await assert.rejects(
+    () =>
+      fetchWithTimeout("https://x/", { signal: ac.signal }, 10_000, {
+        fetchImpl: slowFetch(10_000),
+      }),
+    (err) => {
+      assert.ok(!(err instanceof FetchTimeoutError));
+      return true;
+    }
+  );
+});
+
+/* ------------------------------ disabled timeout -------------------------- */
+
+test("non-positive timeout disables timing out (plain pass-through)", async () => {
+  let seenInit = null;
+  const fetchImpl = (url, init) => {
+    seenInit = init;
+    return Promise.resolve({ ok: true });
+  };
+  const out = await fetchWithTimeout("https://x/", { method: "GET" }, 0, {
+    fetchImpl,
+  });
+  assert.equal(out.ok, true);
+  // Pass-through leaves init untouched (no injected signal).
+  assert.equal(seenInit.method, "GET");
+  assert.equal(seenInit.signal, undefined);
+});
+
+/* -------------------------------- exports --------------------------------- */
+
+test("exposes a sane default timeout constant", () => {
+  assert.equal(typeof DEFAULT_TIMEOUT_MS, "number");
+  assert.ok(DEFAULT_TIMEOUT_MS > 0);
+});
+
+test("a non-function fetchImpl falls back to the global fetch", async () => {
+  // resolveFetch ignores a non-function fetchImpl and uses the global. We stub
+  // the global so this is deterministic and never hits the network.
+  const original = globalThis.fetch;
+  let used = false;
+  globalThis.fetch = () => {
+    used = true;
+    return Promise.resolve({ ok: true });
+  };
+  try {
+    const out = await fetchWithTimeout("https://x/", {}, 1000, { fetchImpl: 123 });
+    assert.equal(out.ok, true);
+    assert.equal(used, true);
+  } finally {
+    globalThis.fetch = original;
+  }
+});
+
+test("throws a clear error when no fetch implementation is available", async () => {
+  const original = globalThis.fetch;
+  // Remove the global so neither an injected nor a global fetch exists.
+  delete globalThis.fetch;
+  try {
+    await assert.rejects(
+      () => fetchWithTimeout("https://x/", {}, 1000),
+      (err) => {
+        assert.ok(err instanceof TypeError, `expected TypeError, got ${err && err.name}`);
+        assert.ok(
+          String(err.message).includes("no fetch implementation"),
+          err.message
+        );
+        return true;
+      }
+    );
+  } finally {
+    globalThis.fetch = original;
+  }
+});

package/lib/util/reconnect.mjs

@@ -0,0 +1,343 @@
+/**
+ * lib/util/reconnect.mjs — a shared bounded-backoff reconnect supervisor.
+ *
+ * The audit's silent-failure epidemic (gaps-product-quality P0-6) flagged the
+ * channel layer's reconnect logic as a one-line death: "Baileys reconnect is one
+ * swallowed attempt then permanent death; Telegram long-poll death is one warn
+ * with healthCheck() green." A live connection that dies and never comes back —
+ * with nothing surfacing the transition — is the worst kind of silent failure:
+ * the agent looks healthy and is deaf.
+ *
+ * The correct pattern already exists in lib/voice/openai-realtime.mjs (bounded
+ * reconnect, deterministic teardown on exhaustion, never goes silent). This
+ * module lifts that pattern into a standalone, transport-agnostic supervisor the
+ * daemon and every channel adapter can adopt:
+ *
+ *   superviseConnection({ connect, isHealthy, onState, ... })
+ *
+ * It owns the loop: open the connection, watch its health, and on a drop
+ * reconnect with DECORRELATED jittered backoff (the same AWS-blog algorithm as
+ * lib/rate-guard.mjs — each delay is a uniform pick in [base, prev*factor],
+ * capped at maxBackoffMs) until it recovers or the attempt budget is exhausted.
+ * Every transition is surfaced through `onState` (connecting | up | down |
+ * giving-up) so a human/doctor/counter always sees the state — it NEVER silently
+ * dies.
+ *
+ * Everything is injectable — `connect`, `isHealthy`, the clock (`now`),
+ * `setTimeout`/`clearTimeout`, and the RNG — so the whole backoff-and-recover
+ * lifecycle is testable with no real sockets and no real time.
+ *
+ * Constraints (CLAUDE.md): ESM, Node built-ins only, fail-open, injectable seams.
+ *
+ * @module lib/util/reconnect
+ */
+
+/** Connection lifecycle states surfaced through `onState`. Frozen as a contract. */
+export const CONN_STATE = Object.freeze({
+  CONNECTING: "connecting",
+  UP: "up",
+  DOWN: "down",
+  GIVING_UP: "giving-up",
+});
+
+/** Default floor of every backoff window (ms). */
+export const DEFAULT_BASE_BACKOFF_MS = 500;
+/** Default ceiling on any single backoff window (ms). */
+export const DEFAULT_MAX_BACKOFF_MS = 30_000;
+/** Default multiplier applied to the previous backoff to size the next jitter range. */
+export const DEFAULT_BACKOFF_FACTOR = 3;
+
+/**
+ * Default RNG → uniform float in [0, 1). Tests inject a deterministic one; the
+ * default is fine for production jitter. We avoid Math.random only for parity
+ * with the rest of the codebase's "deterministic-in-tests" discipline.
+ * @returns {number}
+ */
+function defaultRng() {
+  return Math.random();
+}
+
+/**
+ * Decorrelated-jitter backoff (AWS architecture blog), identical in shape to
+ * lib/rate-guard.mjs:decorrelatedBackoff so the fleet backs off the same way
+ * everywhere. The next delay is a uniform random pick in
+ *   [base, max(base, prevBackoff * factor)]
+ * capped at `max`. With prev=0 the range collapses to [base, base].
+ *
+ * Exported so callers/tests can assert the bounds with an injected rng.
+ *
+ * @param {number} prevBackoffMs  the previous backoff window (0 on first attempt)
+ * @param {object} [opts]
+ * @param {number} [opts.base]    floor (default DEFAULT_BASE_BACKOFF_MS)
+ * @param {number} [opts.max]     ceiling (default DEFAULT_MAX_BACKOFF_MS)
+ * @param {number} [opts.factor]  growth multiplier (default DEFAULT_BACKOFF_FACTOR)
+ * @param {() => number} [opts.rng]  uniform [0,1) source (default Math.random)
+ * @returns {number}  the next backoff in ms (integer, within [base, max])
+ */
+export function decorrelatedBackoff(prevBackoffMs, opts = {}) {
+  const base = posNum(opts.base, DEFAULT_BASE_BACKOFF_MS);
+  const max = posNum(opts.max, DEFAULT_MAX_BACKOFF_MS);
+  const factor = posNum(opts.factor, DEFAULT_BACKOFF_FACTOR);
+  const prev = Number.isFinite(Number(prevBackoffMs)) ? Math.max(0, Number(prevBackoffMs)) : 0;
+  const rng = typeof opts.rng === "function" ? opts.rng : defaultRng;
+
+  const hi = Math.max(base, prev * factor);
+  const r = rng();
+  const clampedR = r < 0 ? 0 : r >= 1 ? 0.999999 : r;
+  const pick = base + clampedR * (hi - base);
+  return Math.min(max, Math.max(base, Math.round(pick)));
+}
+
+function posNum(v, dflt) {
+  const n = Number(v);
+  return Number.isFinite(n) && n > 0 ? n : dflt;
+}
+
+/**
+ * Supervise a connection: open it, and whenever it drops, reconnect with
+ * bounded decorrelated-jitter backoff — surfacing every state transition so the
+ * connection can never silently die.
+ *
+ * Lifecycle, expressed through `onState(state, info)`:
+ *   - `connecting` — about to (re)open. `info` = { attempt, delayMs?, reason? }
+ *   - `up`         — connect() resolved; the connection is live. `info` =
+ *                    { attempt, reconnected }
+ *   - `down`       — a live connection dropped (reported via `notifyDown`, an
+ *                    `isHealthy` poll returning false, or a failed connect).
+ *                    `info` = { attempt, reason }
+ *   - `giving-up`  — the attempt budget was exhausted; the supervisor has torn
+ *                    down deterministically. `info` = { attempts, reason }
+ *
+ * `connect()` is called for each (re)connect attempt and must resolve when the
+ * connection is established (it may return a handle; whatever it returns is
+ * passed back as `controller.connection`). A rejection is treated as a failed
+ * attempt → backoff → retry, exactly like a drop.
+ *
+ * Health is watched two ways, either or both optional:
+ *   - the caller drives it explicitly by calling `controller.notifyDown(reason)`
+ *     when its own listener observes a socket close (the channel-adapter path);
+ *   - and/or this supervisor polls `isHealthy()` every `healthIntervalMs` and
+ *     treats a falsey result as a drop (the daemon health-sweep path).
+ *
+ * The returned controller lets the owner stop the supervisor cleanly:
+ *   - `controller.notifyDown(reason)` — report an observed drop (idempotent while
+ *     a reconnect is already in flight).
+ *   - `controller.stop(reason)`       — deliberate shutdown: cancels any pending
+ *     backoff timer and the health poll; a subsequent drop does NOT reconnect.
+ *   - `controller.state`              — the current CONN_STATE.
+ *   - `controller.connection`         — the most recent value `connect` resolved.
+ *
+ * @param {object} cfg
+ * @param {() => Promise<any>|any} cfg.connect  open the connection; resolves when live.
+ * @param {() => Promise<boolean>|boolean} [cfg.isHealthy]  optional liveness poll.
+ * @param {(state: string, info: object) => void} [cfg.onState]  transition sink.
+ * @param {number} [cfg.maxAttempts]  consecutive failed attempts before giving up
+ *        (default 8). A successful connect resets the counter.
+ * @param {number} [cfg.baseBackoffMs]  backoff floor (default DEFAULT_BASE_BACKOFF_MS).
+ * @param {number} [cfg.maxBackoffMs]   backoff ceiling (default DEFAULT_MAX_BACKOFF_MS).
+ * @param {number} [cfg.backoffFactor]  growth multiplier (default DEFAULT_BACKOFF_FACTOR).
+ * @param {boolean} [cfg.jitter]  apply decorrelated jitter (default true). When
+ *        false, the delay is the deterministic `prev*factor` (capped) — useful
+ *        for tests / strict ordering.
+ * @param {number} [cfg.healthIntervalMs]  poll `isHealthy` this often (0/absent →
+ *        no polling; rely on notifyDown).
+ * @param {() => number} [cfg.now]  clock (default Date.now) — for info timestamps.
+ * @param {(fn: Function, ms: number) => any} [cfg.setTimeout]  default global.
+ * @param {(handle: any) => void} [cfg.clearTimeout]  default global.
+ * @param {() => number} [cfg.rng]  uniform [0,1) for jitter (default Math.random).
+ * @returns {{ notifyDown:(reason?:string)=>void, stop:(reason?:string)=>void,
+ *            readonly state:string, readonly connection:any, readonly attempts:number,
+ *            started:Promise<void> }}
+ */
+export function superviseConnection(cfg = {}) {
+  if (!cfg || typeof cfg.connect !== "function") {
+    throw new TypeError("superviseConnection: cfg.connect must be a function");
+  }
+
+  const connect = cfg.connect;
+  const isHealthy = typeof cfg.isHealthy === "function" ? cfg.isHealthy : null;
+  const onState = typeof cfg.onState === "function" ? cfg.onState : () => {};
+  const maxAttempts = posNum(cfg.maxAttempts, 8);
+  const base = posNum(cfg.baseBackoffMs, DEFAULT_BASE_BACKOFF_MS);
+  const max = posNum(cfg.maxBackoffMs, DEFAULT_MAX_BACKOFF_MS);
+  const factor = posNum(cfg.backoffFactor, DEFAULT_BACKOFF_FACTOR);
+  const jitter = cfg.jitter !== false;
+  const healthIntervalMs = Number(cfg.healthIntervalMs) > 0 ? Number(cfg.healthIntervalMs) : 0;
+  const now = typeof cfg.now === "function" ? cfg.now : Date.now;
+  const setT = typeof cfg.setTimeout === "function" ? cfg.setTimeout : setTimeout;
+  const clearT = typeof cfg.clearTimeout === "function" ? cfg.clearTimeout : clearTimeout;
+  const rng = typeof cfg.rng === "function" ? cfg.rng : defaultRng;
+
+  // Mutable supervisor state.
+  let state = CONN_STATE.CONNECTING;
+  let connection = null;
+  let failedAttempts = 0;       // consecutive failed connect attempts
+  let lastBackoffMs = 0;        // feeds the decorrelated-jitter recurrence
+  let reconnecting = false;     // a connect attempt is in flight
+  let stopped = false;          // deliberate shutdown — no further reconnects
+  let backoffTimer = null;
+  let healthTimer = null;
+
+  /** Emit a transition, isolating a throwing sink so it can't break the loop. */
+  function emit(next, info) {
+    state = next;
+    try {
+      onState(next, { ...info, ts: new Date(now()).toISOString(), state: next });
+    } catch {
+      /* a buggy state sink must never crash the supervisor */
+    }
+  }
+
+  function clearBackoff() {
+    if (backoffTimer != null) {
+      try { clearT(backoffTimer); } catch { /* */ }
+      backoffTimer = null;
+    }
+  }
+
+  function clearHealth() {
+    if (healthTimer != null) {
+      try { clearT(healthTimer); } catch { /* */ }
+      healthTimer = null;
+    }
+  }
+
+  /** Next backoff window: decorrelated jitter, or deterministic if jitter off. */
+  function nextDelay() {
+    if (jitter) {
+      lastBackoffMs = decorrelatedBackoff(lastBackoffMs, { base, max, factor, rng });
+    } else {
+      const grown = lastBackoffMs > 0 ? lastBackoffMs * factor : base;
+      lastBackoffMs = Math.min(max, Math.max(base, Math.round(grown)));
+    }
+    return lastBackoffMs;
+  }
+
+  /** Schedule the periodic health poll (no-op if polling disabled / stopped). */
+  function scheduleHealthPoll() {
+    if (!isHealthy || healthIntervalMs <= 0 || stopped) return;
+    clearHealth();
+    healthTimer = setT(runHealthPoll, healthIntervalMs);
+    if (healthTimer && typeof healthTimer.unref === "function") healthTimer.unref();
+  }
+
+  async function runHealthPoll() {
+    healthTimer = null;
+    if (stopped || state !== CONN_STATE.UP) return;
+    let healthy = true;
+    try {
+      healthy = await isHealthy();
+    } catch {
+      healthy = false; // a throwing health check is a failing health check
+    }
+    if (stopped) return;
+    if (!healthy) {
+      // An unhealthy live connection is a drop — kick the reconnect loop.
+      reportDown("health-check-failed");
+    } else {
+      scheduleHealthPoll();
+    }
+  }
+
+  /** One connect attempt. On success → up; on failure → backoff/give-up. */
+  async function attemptConnect(reason) {
+    if (stopped) return;
+    reconnecting = true;
+    const attempt = failedAttempts + 1;
+    // We're actively opening the socket now → connecting.
+    emit(CONN_STATE.CONNECTING, { attempt, reason: reason || null });
+    try {
+      connection = await connect();
+      if (stopped) return; // stopped mid-connect → leave the just-opened handle to the caller's stop path
+      // Live again: reset the failure budget + backoff recurrence.
+      failedAttempts = 0;
+      lastBackoffMs = 0;
+      reconnecting = false;
+      const reconnected = reason != null;
+      emit(CONN_STATE.UP, { attempt, reconnected });
+      scheduleHealthPoll();
+    } catch (err) {
+      reconnecting = false;
+      if (stopped) return;
+      failedAttempts += 1;
+      const why = errMsg(err) || reason || "connect-failed";
+      // A failed attempt is a drop: schedule the next backoff (or give up). The
+      // down/giving-up transition is emitted by scheduleBackoff so there's one
+      // owner of that decision.
+      scheduleBackoff(why);
+    }
+  }
+
+  /**
+   * Arm the backoff timer for the next attempt, or give up if exhausted. This
+   * is the single owner of the `down` and `giving-up` transitions: while the
+   * backoff timer is pending the connection is DOWN (state stays `down`,
+   * carrying the delayMs so an observer knows when the retry fires), and an
+   * exhausted budget is the only path to `giving-up`.
+   */
+  function scheduleBackoff(reason) {
+    if (stopped) return;
+    if (failedAttempts >= maxAttempts) {
+      giveUp(reason);
+      return;
+    }
+    const delayMs = nextDelay();
+    emit(CONN_STATE.DOWN, { attempt: failedAttempts + 1, delayMs, reason });
+    clearBackoff();
+    backoffTimer = setT(() => {
+      backoffTimer = null;
+      return attemptConnect(reason); // return the promise so an awaiting timer driver chains the connect
+    }, delayMs);
+    if (backoffTimer && typeof backoffTimer.unref === "function") backoffTimer.unref();
+  }
+
+  /** Deterministic teardown after exhausting the attempt budget. */
+  function giveUp(reason) {
+    clearBackoff();
+    clearHealth();
+    emit(CONN_STATE.GIVING_UP, { attempts: failedAttempts, reason: reason || "exhausted" });
+  }
+
+  /**
+   * Handle an observed drop of a live connection. Idempotent while a reconnect
+   * is already in flight (a flurry of close events collapses to one cycle).
+   */
+  function reportDown(reason) {
+    if (stopped) return;
+    if (reconnecting) return;           // already cycling — don't stack attempts
+    if (state === CONN_STATE.GIVING_UP) return; // already terminal
+    if (state === CONN_STATE.DOWN && backoffTimer != null) return; // backoff already pending
+    clearHealth();
+    scheduleBackoff(reason || "connection-dropped");
+  }
+
+  // Kick off the first connect immediately.
+  const started = attemptConnect(null);
+
+  return {
+    notifyDown(reason) { reportDown(reason); },
+    stop(reason) {
+      if (stopped) return;
+      stopped = true;
+      reconnecting = false;
+      clearBackoff();
+      clearHealth();
+      // We don't emit a CONN_STATE here — stop is the OWNER's deliberate action,
+      // not a connection transition. The owner already knows it stopped.
+      void reason;
+    },
+    get state() { return state; },
+    get connection() { return connection; },
+    get attempts() { return failedAttempts; },
+    started,
+  };
+}
+
+/** Coerce any thrown value to a short message string. */
+function errMsg(err) {
+  if (!err) return "";
+  if (typeof err === "string") return err;
+  return err.message || String(err);
+}
+
+export default { superviseConnection, decorrelatedBackoff, CONN_STATE };