@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/model-router/failover.test.mjs

@@ -1,439 +0,0 @@
-/**
- * failover.test.mjs — stepChain walks the chain applying DISPOSITION.
- *
- * Uses an injected fake attempt fn (a scripted queue of outcomes), a fake
- * clock, and an in-memory fake breaker so the test is hermetic and pure.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-
-import { stepChain, summarizeExhaustion, _internals } from "./failover.mjs";
-
-// ---------------------------------------------------------------------------
-// Test doubles
-// ---------------------------------------------------------------------------
-
-function fakeClock(start = 1_000_000) {
-  let t = start;
-  const now = () => t;
-  now.advance = (ms) => { t += ms; };
-  now.set = (v) => { t = v; };
-  return now;
-}
-
-/** In-memory breaker honoring open/isOpen/close with stepped rate_limit. */
-function memBreaker() {
-  const map = new Map();
-  const STEPS = [30_000, 60_000, 300_000];
-  return {
-    map,
-    isOpen(key, now) {
-      const s = map.get(key);
-      if (!s) return { open: false, until: 0, reason: null, strikes: 0 };
-      return { open: s.cooldownUntil > now, until: s.cooldownUntil, reason: s.reason, strikes: s.strikes };
-    },
-    open(key, reason, ms, now) {
-      const prev = map.get(key) || { strikes: 0, cooldownUntil: 0 };
-      const strikes = prev.strikes + 1;
-      let cd = ms;
-      if (cd == null) {
-        if (reason === "rate_limit") cd = STEPS[Math.min(strikes - 1, STEPS.length - 1)];
-        else if (reason === "auth_permanent" || reason === "model_not_found") cd = Infinity;
-        else if (reason === "billing") cd = 6 * 60 * 60 * 1000;
-        else if (reason === "overloaded" || reason === "server_error") cd = 60_000;
-        else if (reason === "auth") cd = 5 * 60_000;
-        else if (reason === "provider_policy") cd = 60 * 60_000;
-        else cd = 0;
-      }
-      const until = cd === Infinity ? Infinity : Math.max(prev.cooldownUntil, now + cd);
-      const next = { reason, strikes, cooldownUntil: until };
-      map.set(key, next);
-      return { ...next, cooldownUntil: until };
-    },
-    close(key) { map.delete(key); },
-  };
-}
-
-/** Scripted attempt fn: feeds outcomes per (ref, attempt) in order. */
-function scriptedAttempt(script) {
-  // script: array of {ref, ...outcome}. Each call shifts the next matching ref.
-  const calls = [];
-  const fn = (candidate) => {
-    calls.push(candidate.ref);
-    const idx = script.findIndex((s) => !s._used && s.ref === candidate.ref);
-    if (idx === -1) return { ok: false, reason: "unknown", meta: { unscripted: true } };
-    script[idx]._used = true;
-    const { ref, _used, ...rest } = script[idx];
-    return rest;
-  };
-  fn.calls = calls;
-  return fn;
-}
-
-const chainOf = (...refs) =>
-  refs.map((ref) => ({ ref, breakerKey: `${ref.split("/")[0]}:${ref.split("/")[1] || "default"}`, backendKey: ref.split("/")[0] }));
-
-// ---------------------------------------------------------------------------
-// Happy path
-// ---------------------------------------------------------------------------
-
-test("first candidate succeeds → ok, chosen, closes its breaker", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([{ ref: "anthropic/sonnet", ok: true, value: 42 }]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), {
-    attempt, now, breaker,
-  });
-  assert.equal(r.ok, true);
-  assert.equal(r.value, 42);
-  assert.equal(r.chosen.ref, "anthropic/sonnet");
-  assert.equal(r.reason, null);
-  assert.equal(r.attempts, 1);
-  assert.equal(attempt.calls.length, 1);
-});
-
-// ---------------------------------------------------------------------------
-// rate_limit: backend:model scope, stepped, then failover
-// ---------------------------------------------------------------------------
-
-test("rate_limit fails over to next candidate and cools backend:model with stepped 30s", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "rate_limit" },
-    { ref: "anthropic/haiku", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), {
-    attempt, now, breaker,
-  });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/haiku");
-  // Sonnet's model-scoped breaker is open for 30s; Haiku's is not.
-  assert.equal(breaker.isOpen("anthropic:sonnet", now()).open, true);
-  assert.equal(breaker.isOpen("anthropic:sonnet", now()).until, now() + 30_000);
-  assert.equal(breaker.isOpen("anthropic:haiku", now()).open, false);
-  // tried trail records the rate_limit then the success.
-  assert.equal(r.tried[0].reason, "rate_limit");
-  assert.equal(r.tried[1].reason, "ok");
-});
-
-test("rate_limit on every candidate → exhaustion summary with soonest cooldown expiry", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "rate_limit" },
-    { ref: "anthropic/haiku", ok: false, reason: "rate_limit" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), {
-    attempt, now, breaker,
-  });
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "rate_limit");
-  assert.equal(r.tried.length, 2);
-  // Both cooled 30s from the same clock → soonest is now+30s.
-  assert.equal(r.soonestCooldownExpiry, now() + 30_000);
-  assert.match(summarizeExhaustion(r), /chain exhausted \(final=rate_limit\)/);
-});
-
-// ---------------------------------------------------------------------------
-// context_overflow: NEVER fails over
-// ---------------------------------------------------------------------------
-
-test("context_overflow stops the walk immediately — never fails over, no cooldown", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "context_overflow" },
-    { ref: "anthropic/haiku", ok: true, value: "should-not-reach" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), {
-    attempt, now, breaker,
-  });
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "context_overflow");
-  assert.equal(attempt.calls.length, 1, "second candidate never attempted");
-  assert.equal(r.soonestCooldownExpiry, null);
-  // No breaker opened.
-  assert.equal(breaker.map.size, 0);
-});
-
-// ---------------------------------------------------------------------------
-// payload_too_large: no failover, no cooldown, no breaker
-// ---------------------------------------------------------------------------
-
-test("payload_too_large stops the walk and opens no breaker", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([{ ref: "anthropic/sonnet", ok: false, reason: "payload_too_large" }]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "payload_too_large");
-  assert.equal(attempt.calls.length, 1);
-  assert.equal(breaker.map.size, 0);
-});
-
-// ---------------------------------------------------------------------------
-// network: NEVER fails over, opens NO breaker, burns no retry budget (DEFER)
-// ---------------------------------------------------------------------------
-
-test("network stops the walk, opens no breaker, leaves chain intact (DEFER)", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "network" },
-    { ref: "anthropic/haiku", ok: true, value: "x" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "network");
-  assert.equal(attempt.calls.length, 1, "no failover on network");
-  assert.equal(breaker.map.size, 0, "network opens no breaker");
-  assert.equal(r.soonestCooldownExpiry, null, "no cooldown → re-queue immediately/DEFER");
-});
-
-// ---------------------------------------------------------------------------
-// content_policy: sticky by default; one re-route when opted in
-// ---------------------------------------------------------------------------
-
-test("content_policy is sticky by default — no failover on same content", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "content_policy" },
-    { ref: "anthropic/opus", ok: true, value: "nope" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/opus"), { attempt, now, breaker });
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "content_policy");
-  assert.equal(attempt.calls.length, 1);
-});
-
-test("content_policy with allowContentReroute advances ONCE", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "content_policy" },
-    { ref: "anthropic/opus", ok: true, value: "rerouted" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/opus"), {
-    attempt, now, breaker, allowContentReroute: true,
-  });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/opus");
-  assert.equal(attempt.calls.length, 2);
-});
-
-test("content_policy re-route is permitted only once across the chain", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "content_policy" },
-    { ref: "anthropic/opus", ok: false, reason: "content_policy" },
-    { ref: "anthropic/haiku", ok: true, value: "third" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/opus", "anthropic/haiku"), {
-    attempt, now, breaker, allowContentReroute: true,
-  });
-  // First content_policy re-routes; the second is sticky → walk stops.
-  assert.equal(r.ok, false);
-  assert.equal(r.reason, "content_policy");
-  assert.equal(attempt.calls.length, 2);
-});
-
-// ---------------------------------------------------------------------------
-// retry-same dispositions
-// ---------------------------------------------------------------------------
-
-test("timeout retries the SAME candidate once, then succeeds", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "timeout" },
-    { ref: "anthropic/sonnet", ok: true, value: "second-try" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/sonnet");
-  assert.equal(attempt.calls.length, 2);
-  assert.deepEqual(attempt.calls, ["anthropic/sonnet", "anthropic/sonnet"]);
-  assert.equal(breaker.map.size, 0, "timeout opens no breaker");
-});
-
-test("timeout twice → retry once then failover to next candidate", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "timeout" },
-    { ref: "anthropic/sonnet", ok: false, reason: "timeout" },
-    { ref: "anthropic/haiku", ok: true, value: "failed-over" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/haiku");
-  // sonnet attempted twice (initial + 1 retry), then haiku.
-  assert.deepEqual(attempt.calls, ["anthropic/sonnet", "anthropic/sonnet", "anthropic/haiku"]);
-});
-
-test("overloaded retries once jittered then fails over; cools backend:model 60s", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "overloaded" },
-    { ref: "anthropic/sonnet", ok: false, reason: "overloaded" },
-    { ref: "anthropic/haiku", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  // sonnet retried once before failover.
-  assert.equal(attempt.calls.filter((c) => c === "anthropic/sonnet").length, 2);
-  assert.equal(breaker.isOpen("anthropic:sonnet", now()).open, true);
-  assert.equal(breaker.isOpen("anthropic:sonnet", now()).until, now() + 60_000);
-});
-
-test("server_error (CLI-retried) does NOT retry same; fails over immediately", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "server_error" },
-    { ref: "anthropic/haiku", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/haiku");
-  // sonnet attempted exactly once — the CLI already retried internally.
-  assert.equal(attempt.calls.filter((c) => c === "anthropic/sonnet").length, 1);
-});
-
-// ---------------------------------------------------------------------------
-// Skip-on-cooldown
-// ---------------------------------------------------------------------------
-
-test("a candidate whose breaker is already open is skipped (not attempted)", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  // Pre-open sonnet's model breaker.
-  breaker.open("anthropic:sonnet", "rate_limit", 120_000, now());
-  const attempt = scriptedAttempt([{ ref: "anthropic/haiku", ok: true, value: "ok" }]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/haiku");
-  assert.equal(attempt.calls.includes("anthropic/sonnet"), false, "sonnet skipped");
-  assert.ok(r.tried[0].skipped);
-  assert.match(r.tried[0].reason, /^breaker_open:/);
-});
-
-test("backend-scoped open breaker skips all that backend's model candidates", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  // Open the coarse backend key.
-  breaker.open("anthropic", "billing", null, now());
-  const chain = [
-    { ref: "anthropic/sonnet", breakerKey: "anthropic:sonnet", backendKey: "anthropic" },
-    { ref: "deepseek/v4", breakerKey: "deepseek:v4", backendKey: "deepseek" },
-  ];
-  const attempt = scriptedAttempt([{ ref: "deepseek/v4", ok: true, value: "ok" }]);
-  const r = await stepChain(chain, { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "deepseek/v4");
-  assert.ok(r.tried[0].skipped);
-});
-
-// ---------------------------------------------------------------------------
-// Robustness
-// ---------------------------------------------------------------------------
-
-test("an attempt fn that throws is caught and classified unknown (never throws)", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = () => { throw new Error("boom"); };
-  let result;
-  await assert.doesNotReject(async () => {
-    result = await stepChain(chainOf("anthropic/sonnet"), { attempt, now, breaker });
-  });
-  // unknown retries once (backoff) then exhausts.
-  assert.equal(result.ok, false);
-  assert.equal(result.reason, "unknown");
-});
-
-test("empty chain / missing attempt fn produce a safe summary, never throw", async () => {
-  const r1 = await stepChain([], { attempt: () => ({ ok: true }) });
-  assert.equal(r1.ok, false);
-  const r2 = await stepChain(chainOf("anthropic/sonnet"), {});
-  assert.equal(r2.ok, false);
-  assert.equal(r2.tried[0].reason, "no_attempt_fn");
-});
-
-test("onAttempt side-channel is called per attempt and cannot break the walk", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const seen = [];
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "rate_limit" },
-    { ref: "anthropic/haiku", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), {
-    attempt, now, breaker,
-    onAttempt: (rec) => { seen.push(rec.reason); throw new Error("side-channel boom"); },
-  });
-  assert.equal(r.ok, true);
-  assert.deepEqual(seen, ["rate_limit", "ok"]);
-});
-
-test("unknown retries once with backoff then fails over", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "anthropic/sonnet", ok: false, reason: "unknown" },
-    { ref: "anthropic/sonnet", ok: false, reason: "unknown" },
-    { ref: "anthropic/haiku", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(chainOf("anthropic/sonnet", "anthropic/haiku"), { attempt, now, breaker });
-  assert.equal(r.ok, true);
-  assert.equal(r.chosen.ref, "anthropic/haiku");
-  assert.equal(attempt.calls.filter((c) => c === "anthropic/sonnet").length, 2);
-});
-
-test("model_not_found fails over and opens a model-id (Infinity) breaker", async () => {
-  const breaker = memBreaker();
-  const now = fakeClock();
-  const attempt = scriptedAttempt([
-    { ref: "moonshot/kimi-k99", ok: false, reason: "model_not_found" },
-    { ref: "anthropic/sonnet", ok: true, value: "ok" },
-  ]);
-  const r = await stepChain(
-    [
-      { ref: "moonshot/kimi-k99", breakerKey: "moonshot:kimi-k99", backendKey: "moonshot" },
-      { ref: "anthropic/sonnet", breakerKey: "anthropic:sonnet", backendKey: "anthropic" },
-    ],
-    { attempt, now, breaker }
-  );
-  assert.equal(r.ok, true);
-  assert.equal(breaker.isOpen("moonshot:kimi-k99", now()).until, Infinity);
-  // Infinity cooldown is not a wake time → not folded into soonestCooldownExpiry.
-  assert.equal(r.soonestCooldownExpiry, null);
-});
-
-test("cooldownKey maps scopes to the right breaker key", () => {
-  const c = { ref: "anthropic/sonnet", breakerKey: "anthropic:sonnet", backendKey: "anthropic" };
-  assert.equal(_internals.cooldownKey(c, "none"), null);
-  assert.equal(_internals.cooldownKey(c, "backend"), "anthropic");
-  assert.equal(_internals.cooldownKey(c, "operator"), "anthropic");
-  assert.equal(_internals.cooldownKey(c, "backend:model"), "anthropic:sonnet");
-  assert.equal(_internals.cooldownKey(c, "model-id"), "anthropic:sonnet");
-});
-
-test("summarizeExhaustion renders success and exhaustion strings", () => {
-  assert.match(
-    summarizeExhaustion({ ok: true, chosen: { ref: "anthropic/haiku" }, attempts: 2 }),
-    /succeeded on anthropic\/haiku after 2/
-  );
-  const ex = summarizeExhaustion({
-    ok: false,
-    reason: "rate_limit",
-    tried: [{ ref: "a/x", reason: "rate_limit" }, { ref: "a/y", reason: "rate_limit" }],
-    soonestCooldownExpiry: null,
-  });
-  assert.match(ex, /a\/x=rate_limit → a\/y=rate_limit/);
-  assert.match(ex, /no cooldown/);
-});