@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/model-router/health.test.mjs

@@ -1,338 +0,0 @@
-/**
- * health.test.mjs — per-key breaker files, half-open probes, expiry, and the
- * O_EXCL race (two concurrent opens converge to one coherent file).
- *
- * All tests inject a temp `stateDir` + a controllable clock so they never touch
- * the real state dir or wall clock.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { promises as fsp } from "node:fs";
-import { existsSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import {
-  keyFor,
-  backendOfKey,
-  readBreaker,
-  isOpen,
-  open,
-  close,
-  clear,
-  recordSuccess,
-  halfOpenProbe,
-  listBreakers,
-  HALF_OPEN_PROBE_INTERVAL_MS,
-  HALF_OPEN_WINDOW_MS,
-} from "./health.mjs";
-
-async function tmpDir() {
-  const p = join(
-    tmpdir(),
-    `health-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
-  );
-  await fsp.mkdir(p, { recursive: true });
-  return p;
-}
-async function rm(p) {
-  try { await fsp.rm(p, { recursive: true, force: true }); } catch { /* */ }
-}
-/** Mutable clock helper. */
-function fakeClock(start = 1_000_000) {
-  let t = start;
-  const now = () => t;
-  now.set = (v) => { t = v; };
-  now.advance = (ms) => { t += ms; };
-  return now;
-}
-
-// ---------------------------------------------------------------------------
-// Keys
-// ---------------------------------------------------------------------------
-
-test("keyFor builds backend and backend:model keys, sanitized", () => {
-  assert.equal(keyFor("anthropic"), "anthropic");
-  assert.equal(keyFor("anthropic", "sonnet"), "anthropic:sonnet");
-  assert.equal(keyFor("Moon Shot", "Kimi K2.6"), "moon-shot:kimi-k2.6");
-  assert.equal(backendOfKey("anthropic:sonnet"), "anthropic");
-  assert.equal(backendOfKey("anthropic"), "anthropic");
-});
-
-// ---------------------------------------------------------------------------
-// Open / isOpen / expiry
-// ---------------------------------------------------------------------------
-
-test("open() opens the breaker; isOpen reflects cooldownUntil; expires on clock advance", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("anthropic", "sonnet");
-    assert.equal(isOpen(key, now(), deps).open, false);
-
-    // First rate_limit hit → 30s (step 0).
-    const s = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s.reason, "rate_limit");
-    assert.equal(s.strikes, 1);
-    assert.equal(s.cooldownUntil, now() + 30_000);
-    assert.equal(isOpen(key, now(), deps).open, true);
-
-    // Just before expiry → still open; at expiry → closed.
-    now.advance(29_999);
-    assert.equal(isOpen(key, now(), deps).open, true);
-    now.advance(2);
-    assert.equal(isOpen(key, now(), deps).open, false);
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("stepped rate_limit ladder advances 30s→1m→5m across consecutive opens", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("anthropic", "sonnet");
-    const s1 = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s1.cooldownUntil - now(), 30_000);
-    const s2 = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s2.strikes, 2);
-    assert.equal(s2.cooldownUntil - now(), 60_000);
-    const s3 = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s3.strikes, 3);
-    assert.equal(s3.cooldownUntil - now(), 300_000);
-    const s4 = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s4.cooldownUntil - now(), 300_000); // saturates
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("auth_permanent opens an Infinity (operator-held) cooldown; clear() reopens", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("moonshot");
-    const s = open(key, "auth_permanent", null, now(), deps);
-    assert.equal(s.cooldownUntil, Infinity);
-    now.advance(10 * 365 * 24 * 60 * 60 * 1000); // a decade
-    assert.equal(isOpen(key, now(), deps).open, true); // still open
-    clear(key, deps);
-    assert.equal(isOpen(key, now(), deps).open, false);
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("explicit cooldownMs overrides the table; longer cooldown never shortened by a racing shorter one", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("deepseek", "v4");
-    open(key, "rate_limit", 600_000, now(), deps); // explicit 10m
-    // A subsequent shorter (30s) open must not pull cooldownUntil back in.
-    const s = open(key, "rate_limit", 30_000, now(), deps);
-    assert.equal(s.cooldownUntil, now() + 600_000);
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("close()/recordSuccess() resets the breaker; idempotent no-op when already closed", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("anthropic", "haiku");
-    open(key, "rate_limit", null, now(), deps);
-    assert.equal(isOpen(key, now(), deps).open, true);
-    const cleared = close(key, now(), deps);
-    assert.equal(cleared.cooldownUntil, 0);
-    assert.equal(cleared.strikes, 0);
-    // Next incident starts from step 0 again.
-    const s = open(key, "rate_limit", null, now(), deps);
-    assert.equal(s.cooldownUntil - now(), 30_000);
-    assert.equal(recordSuccess, close); // alias
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("missing/corrupt breaker file reads as closed (fail-open)", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("nonexistent");
-    assert.equal(isOpen(key, now(), deps).open, false);
-    // Write a corrupt file directly under the breaker filename.
-    await fsp.writeFile(join(dir, "breaker-nonexistent.json"), "{not json");
-    const s = readBreaker(key, deps);
-    assert.equal(s.cooldownUntil, 0);
-    assert.equal(s.strikes, 0);
-  } finally {
-    await rm(dir);
-  }
-});
-
-// ---------------------------------------------------------------------------
-// Half-open probing
-// ---------------------------------------------------------------------------
-
-test("halfOpenProbe: only within the pre-expiry window, 1/backend/30s", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("anthropic", "sonnet");
-    // Open for 5 minutes.
-    open(key, "rate_limit", 5 * 60_000, now(), deps);
-
-    // Far from expiry → no probe.
-    assert.equal(halfOpenProbe(key, now(), deps), false);
-
-    // Move to within the 2-min window before expiry → probe allowed once.
-    now.set(now() + 5 * 60_000 - (HALF_OPEN_WINDOW_MS - 1_000));
-    assert.equal(halfOpenProbe(key, now(), deps), true);
-
-    // Immediately again → rate-limited (1/backend/30s).
-    assert.equal(halfOpenProbe(key, now(), deps), false);
-
-    // After the 30s probe interval → allowed again (still in window).
-    now.advance(HALF_OPEN_PROBE_INTERVAL_MS + 1);
-    assert.equal(halfOpenProbe(key, now(), deps), true);
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("halfOpenProbe: closed breaker and Infinity (operator) holds are never probed", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const closedKey = keyFor("anthropic");
-    assert.equal(halfOpenProbe(closedKey, now(), deps), false); // closed
-
-    const permKey = keyFor("moonshot");
-    open(permKey, "auth_permanent", null, now(), deps);
-    assert.equal(halfOpenProbe(permKey, now(), deps), false); // operator-held
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("a backend probe suppresses its model-class probes within the interval", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const backend = keyFor("anthropic");
-    const modelKey = keyFor("anthropic", "sonnet");
-    // Open both, each within the pre-expiry window.
-    open(backend, "auth", 90_000, now(), deps); // auth → backend scope; 90s
-    open(modelKey, "rate_limit", 90_000, now(), deps);
-    now.set(now() + 90_000 - 1_000); // inside the 2-min window for both
-
-    // Probe the backend first.
-    assert.equal(halfOpenProbe(backend, now(), deps), true);
-    // The model-class probe is suppressed because its backend just probed.
-    assert.equal(halfOpenProbe(modelKey, now(), deps), false);
-  } finally {
-    await rm(dir);
-  }
-});
-
-// ---------------------------------------------------------------------------
-// listBreakers (doctor dump)
-// ---------------------------------------------------------------------------
-
-test("listBreakers returns every breaker with an open flag", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    open(keyFor("anthropic", "sonnet"), "rate_limit", null, now(), deps);
-    open(keyFor("deepseek"), "billing", null, now(), deps);
-    const list = listBreakers(now(), deps);
-    assert.equal(list.length, 2);
-    const byKey = Object.fromEntries(list.map((b) => [b.key, b]));
-    assert.ok(byKey["anthropic:sonnet"].open);
-    assert.ok(byKey["deepseek"].open);
-    assert.equal(listBreakers(now(), { stateDir: join(dir, "nope"), now }).length, 0);
-  } finally {
-    await rm(dir);
-  }
-});
-
-// ---------------------------------------------------------------------------
-// O_EXCL race: two concurrent opens converge to ONE coherent file
-// ---------------------------------------------------------------------------
-
-test("two concurrent opens on the same key converge (strikes monotonic, file coherent)", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const key = keyFor("anthropic", "sonnet");
-    // Fire many opens "concurrently". The O_EXCL lock serializes them; because
-    // each is synchronous under the lock, the final strikes count must equal
-    // the number of opens with NO lost updates (the unlocked-RMW race would
-    // drop some). Promise.all just schedules them; the lock does the rest.
-    const N = 12;
-    await Promise.all(
-      Array.from({ length: N }, () =>
-        Promise.resolve().then(() => open(key, "rate_limit", null, now(), deps))
-      )
-    );
-    const s = readBreaker(key, deps);
-    assert.equal(s.strikes, N, "no lost increments under concurrency");
-
-    // The persisted file is valid JSON with the full shape.
-    const raw = JSON.parse(
-      await fsp.readFile(join(dir, "breaker-anthropic__sonnet.json"), "utf-8")
-    );
-    assert.equal(raw.key, key);
-    assert.equal(raw.strikes, N);
-    assert.equal(typeof raw.cooldownUntil, "number");
-    assert.ok(!existsSync(join(dir, "breaker-anthropic__sonnet.json.lock")), "lock released");
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("concurrent open() on distinct keys do not interfere", async () => {
-  const dir = await tmpDir();
-  const now = fakeClock();
-  const deps = { stateDir: dir, now };
-  try {
-    const a = keyFor("anthropic", "sonnet");
-    const b = keyFor("deepseek", "v4");
-    await Promise.all([
-      Promise.resolve().then(() => open(a, "rate_limit", null, now(), deps)),
-      Promise.resolve().then(() => open(b, "overloaded", null, now(), deps)),
-      Promise.resolve().then(() => open(a, "rate_limit", null, now(), deps)),
-    ]);
-    assert.equal(readBreaker(a, deps).strikes, 2);
-    assert.equal(readBreaker(b, deps).strikes, 1);
-    assert.equal(readBreaker(b, deps).reason, "overloaded");
-  } finally {
-    await rm(dir);
-  }
-});
-
-test("breaker writes never throw even when the state dir is unwritable", () => {
-  // Point at a path under a regular file so mkdir/write fail; open() must
-  // degrade in-memory and still return a coherent object.
-  const bogus = join(tmpdir(), "health-test-bogus", "\0invalid");
-  const now = fakeClock();
-  assert.doesNotThrow(() => {
-    const s = open("anthropic", "rate_limit", null, now(), { stateDir: bogus, now });
-    assert.equal(s.reason, "rate_limit");
-    assert.equal(s.strikes, 1);
-  });
-});