@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/lib/model-router/economics.test.mjs

@@ -1,438 +0,0 @@
-/**
- * Tests for lib/model-router/economics.mjs — the router economics levers:
- *   - budgetLadder transitions at 100 / 125 / 150 (a ladder, not a cliff) —
- *     the SAME rungs lib/budget-guard bands the seat envelope on
- *   - degradeChain tightens a candidate chain per band
- *   - batchLaneFor classifies deferrable vs realtime work
- *   - cacheAffinity pin / lookupPin / resolvePin override (+ pin_overridden audit)
- *   - spawnKnobsFor task-class defaults incl. the cheap-subagent map
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-
-import {
-  budgetLadder,
-  degradeChain,
-  batchLaneFor,
-  pin,
-  lookupPin,
-  resolvePin,
-  spawnKnobsFor,
-  ttlMs,
-  hashSessionKey,
-  _internals,
-} from "./economics.mjs";
-
-// ---------------------------------------------------------------------------
-// budgetLadder — transitions at 100/125/150 (the governance ladder's rungs)
-// ---------------------------------------------------------------------------
-
-test("budgetLadder: below 100% is band 0, no degradation", () => {
-  for (const spent of [0, 10, 37, 80, 99.9]) {
-    const r = budgetLadder(spent, 100);
-    assert.equal(r.band, 0, `spent ${spent} → band 0`);
-    assert.equal(r.degrade.preferCheaper, false);
-    assert.equal(r.degrade.cheapOnly, false);
-    assert.equal(r.degrade.essentialOnly, false);
-  }
-});
-
-test("budgetLadder: the 80% supervisor notice does NOT degrade routing (band 0)", () => {
-  // budget-guard notifies the SUPERVISOR at 80%, but a notice is not a routing
-  // change — the first degradation is at 100%, where the seat has spent what the
-  // org actually funded.
-  const r = budgetLadder(80, 100);
-  assert.equal(r.band, 0);
-  assert.equal(r.degrade.downgradeTiers, 0);
-  assert.equal(r.degrade.selfDirected, true);
-});
-
-test("budgetLadder: 100% → cheapest class, fan-out 1, no self-directed work", () => {
-  const r = budgetLadder(100, 100);
-  assert.equal(r.band, 100);
-  assert.equal(r.degrade.preferCheaper, true);
-  assert.equal(r.degrade.cheapOnly, true);
-  assert.equal(r.degrade.batchDeferrable, true);
-  assert.equal(r.degrade.maxFanout, 1);
-  assert.equal(r.degrade.selfDirected, false, "D3 stops here");
-  assert.equal(r.degrade.essentialOnly, false, "inbox + obligations still run — this rung is not a cliff");
-});
-
-test("budgetLadder: 125% → essential-only on top of the degrade rung", () => {
-  const r = budgetLadder(125, 100);
-  assert.equal(r.band, 125);
-  assert.equal(r.degrade.essentialOnly, true);
-  assert.equal(r.degrade.cheapOnly, true);
-});
-
-test("budgetLadder: past 150% stays band 150", () => {
-  assert.equal(budgetLadder(150, 100).band, 150);
-  assert.equal(budgetLadder(250, 100).band, 150);
-  assert.equal(budgetLadder(250, 100).degrade.essentialOnly, true);
-});
-
-test("budgetLadder: a cap of ZERO with money spent is EXHAUSTED, not uncapped", () => {
-  // This test used to assert band 0 for every cap<=0, and it was correct when
-  // capUSD could only be a local config number > 0. `dailyAllowance` now returns
-  // 0 for a seat whose MONTHLY envelope is spent out — so cap===0 became a
-  // reachable production value meaning "there is no room at all", and collapsing
-  // it to band 0 inverted the ladder: the further past its envelope a seat was,
-  // the less the router degraded.
-  assert.equal(budgetLadder(500, 0).band, 150, "$500 spent against a $0 allowance is fully over");
-  assert.equal(budgetLadder(500, -1).band, 150);
-  assert.equal(budgetLadder(500, NaN).band, 150);
-  // Nothing spent and no cap is genuinely unknown — do not degrade on it.
-  assert.equal(budgetLadder(0, 0).band, 0);
-  assert.equal(budgetLadder(0, NaN).band, 0);
-});
-
-test("budgetLadder: the governor's band wins over any re-derivation", () => {
-  // The dispatcher and the cadence consumer pass `dailyStatus().band`, which has
-  // already reconciled day vs month vs blindness. Re-deriving from (spent, cap)
-  // is what let a seat at 109% of its MONTH route the full frontier chain
-  // because today happened to look quiet.
-  const r = budgetLadder(3, 30, { band: 125 });
-  assert.equal(r.band, 125);
-  assert.equal(r.degrade.cheapOnly, true);
-  assert.equal(r.degrade.essentialOnly, true);
-  assert.equal(budgetLadder(3, 30).band, 0, "and without it, the local reading still applies");
-});
-
-test("budgetLadder: pct is rounded and exposed", () => {
-  const r = budgetLadder(33.333, 100);
-  assert.equal(r.pct, 33.33);
-  assert.equal(r.capUSD, 100);
-});
-
-// ---------------------------------------------------------------------------
-// degradeChain — tighten per band
-// ---------------------------------------------------------------------------
-
-const ALIASES = {
-  frontier: "anthropic/claude-opus-4-8",
-  default: "anthropic/claude-sonnet-4-6",
-  fast: "anthropic/claude-haiku-4-5",
-  cheap: "deepseek/deepseek-v4-flash",
-};
-
-function aliasChain(...names) {
-  return names.map((n) => ({ ref: ALIASES[n] || n, alias: ALIASES[n] ? n : null }));
-}
-
-test("degradeChain: band < 100 returns the chain unchanged", () => {
-  const chain = aliasChain("frontier", "default");
-  const out = degradeChain(chain, 0, { aliases: ALIASES });
-  assert.deepEqual(out, chain);
-  assert.notEqual(out, chain, "returns a copy, not the same reference");
-});
-
-test("degradeChain: the 80% notice rung changes no routing at all", () => {
-  // There is deliberately no one-rung "prefer cheaper" step any more: the band
-  // below the drop rung is a supervisor NOTICE, and a notice must not silently
-  // re-route the seat onto a worse model.
-  const chain = aliasChain("frontier", "default");
-  assert.deepEqual(degradeChain(chain, 80, { aliases: ALIASES }), chain);
-});
-
-test("degradeChain: band 100 drops frontier/default head, keeps cheap/fast", () => {
-  const chain = aliasChain("frontier", "default", "fast", "cheap");
-  const out = degradeChain(chain, 100, { aliases: ALIASES });
-  const aliasesOut = out.map((m) => m.alias);
-  assert.ok(!aliasesOut.includes("frontier"));
-  assert.ok(!aliasesOut.includes("default"));
-  assert.deepEqual(aliasesOut, ["fast", "cheap"]);
-});
-
-test("degradeChain: band 100 with an all-frontier chain keeps the single cheapest as a safety net", () => {
-  const chain = aliasChain("frontier", "default");
-  const out = degradeChain(chain, 100, { aliases: ALIASES });
-  assert.equal(out.length, 1);
-  // 'default' is cheaper than 'frontier', so it's the safety-net survivor.
-  assert.equal(out[0].alias, "default");
-});
-
-test("degradeChain: exempt (pin/critical) bypasses degradation", () => {
-  const chain = aliasChain("frontier", "default");
-  const out = degradeChain(chain, 100, { aliases: ALIASES, exempt: true });
-  assert.deepEqual(out.map((m) => m.alias), ["frontier", "default"]);
-});
-
-test("degradeChain: preserves a rules sentinel as a tail at any band", () => {
-  const chain = [...aliasChain("frontier"), { ref: "rules", isRules: true }];
-  const out90 = degradeChain(chain, 100, { aliases: ALIASES });
-  assert.ok(out90.some((m) => m.ref === "rules"));
-});
-
-test("degradeChain: infers tier from literal refs without alias tags", () => {
-  const chain = [{ ref: "anthropic/claude-opus-4-8" }, { ref: "anthropic/claude-haiku-4-5" }];
-  const out = degradeChain(chain, 100, { aliases: ALIASES });
-  // opus (frontier) dropped; haiku (fast) kept.
-  assert.equal(out.length, 1);
-  assert.match(out[0].ref, /haiku/);
-});
-
-test("degradeChain: de-dupes refs that collapse onto the same model", () => {
-  const chain = aliasChain("fast", "fast");
-  const out = degradeChain(chain, 100, { aliases: ALIASES });
-  assert.equal(out.length, 1);
-});
-
-// ---------------------------------------------------------------------------
-// inferTier — the untagged-ref heuristic behind degradeChain
-// ---------------------------------------------------------------------------
-
-const { inferTier } = _internals;
-
-test("inferTier: a cheap PROVIDER beats a 'flash'-shaped model name", () => {
-  // The regression this locks: the fast check (haiku|flash|mini) used to run
-  // BEFORE the cheap check, so the two cheapest rows in the bundled catalog were
-  // filed on the Haiku rung — deepseek-v4-flash is $0.14/$0.28 against Haiku's
-  // $1/$5, a ~7x mis-read of the ladder's own bottom rung.
-  assert.equal(inferTier({ ref: "deepseek/deepseek-v4-flash" }), "cheap");
-  assert.equal(inferTier({ ref: "qwen/qwen3.5-flash" }), "cheap");
-  // …while a genuinely fast-tier Anthropic row is untouched.
-  assert.equal(inferTier({ ref: "anthropic/claude-haiku-4-5" }), "fast");
-  assert.equal(inferTier({ ref: "anthropic/claude-opus-5" }), "frontier");
-  assert.equal(inferTier({ ref: "anthropic/claude-sonnet-5" }), "default");
-});
-
-test("inferTier: every cheap provider the economics layer NAMES is classified", () => {
-  // economics.mjs has always claimed deepseek/kimi/glm/qwen as the cheap tier;
-  // xai/grok is added so the classifier is ready the moment a Grok row lands
-  // (an agent can add one today via catalog_overrides — see model-router.test.mjs).
-  for (const ref of [
-    "deepseek/deepseek-v4-flash",
-    "moonshot/kimi-k2.6",
-    "qwen/qwen3-coder-plus",
-    "xai/grok-4-fast",
-    "zhipu/glm-5",
-    "kimi-k2.6",   // bare, no provider segment
-    "grok-4-fast", // bare, no provider segment
-  ]) {
-    assert.equal(inferTier({ ref }), "cheap", `${ref} should classify as cheap-tier`);
-  }
-});
-
-test("inferTier: a cheap provider's WHOLE line is cheap, not just its famous model", () => {
-  // The model-name hints ("kimi", "grok", "glm") would carry the refs above on
-  // their own, which makes them a trap: the next model each of these providers
-  // ships will not be called "grok-4" or "kimi-k2". The provider segment is the
-  // durable signal, so these refs — none of which contain a hint word — are the
-  // ones that actually hold CHEAP_PROVIDERS in place.
-  assert.equal(inferTier({ ref: "xai/sonic-2" }), "cheap");
-  assert.equal(inferTier({ ref: "moonshot/k3-preview" }), "cheap");
-  assert.equal(inferTier({ ref: "deepseek/v5" }), "cheap");
-});
-
-test("inferTier: an unrecognised ref stays null so degradeChain fails OPEN", () => {
-  // A model we cannot classify must be KEPT, never dropped: a budget heuristic
-  // that silently deletes candidates it doesn't recognise strands the caller
-  // with an empty chain. Asserted against a MIXED chain on purpose — with an
-  // unknown row alone, the "keep the single cheapest" safety net returns it
-  // either way, so a fail-closed filter would slip through unnoticed.
-  assert.equal(inferTier({ ref: "acme/some-new-model" }), null);
-  const out = degradeChain(
-    [{ ref: "anthropic/claude-opus-5" }, { ref: "acme/some-new-model" }],
-    150,
-    {}
-  );
-  assert.deepEqual(
-    out.map((m) => m.ref),
-    ["acme/some-new-model"],
-    "frontier dropped, the unclassifiable row survives"
-  );
-});
-
-test("degradeChain: a cheap third-party row survives band 150 alongside fast rows", () => {
-  const chain = [
-    { ref: "anthropic/claude-opus-5" },
-    { ref: "anthropic/claude-haiku-4-5" },
-    { ref: "xai/grok-4-fast" },
-  ];
-  const out = degradeChain(chain, 150, {}).map((m) => m.ref);
-  assert.deepEqual(out, ["anthropic/claude-haiku-4-5", "xai/grok-4-fast"]);
-});
-
-// ---------------------------------------------------------------------------
-// batchLaneFor — classify deferrable vs realtime
-// ---------------------------------------------------------------------------
-
-test("batchLaneFor: explicit lane wins", () => {
-  assert.equal(batchLaneFor({ lane: "batch", task_class: "classify.inbox" }), true);
-  assert.equal(batchLaneFor({ lane: "realtime", task_class: "learning.reflect" }), false);
-});
-
-test("batchLaneFor: deferrable task classes default to batch", () => {
-  assert.equal(batchLaneFor({ task_class: "learning.reflect" }), true);
-  assert.equal(batchLaneFor({ task_class: "learning.curate" }), true);
-  assert.equal(batchLaneFor({ task_class: "digest.daily" }), true);
-  assert.equal(batchLaneFor({ task_class: "index.sessions" }), true);
-  assert.equal(batchLaneFor({ task_class: "curate.skills" }), true);
-  assert.equal(batchLaneFor({ task_class: "enrich" }), true);
-});
-
-test("batchLaneFor: realtime classes never defer", () => {
-  assert.equal(batchLaneFor({ task_class: "classify.inbox" }), false);
-  assert.equal(batchLaneFor({ task_class: "session.responder" }), false);
-  assert.equal(batchLaneFor({ task_class: "huddle" }), false);
-  assert.equal(batchLaneFor({ task_class: "voice.brief" }), false);
-});
-
-test("batchLaneFor: high/critical priority never defers even a learning task", () => {
-  assert.equal(batchLaneFor({ task_class: "learning.reflect", priority: "critical" }), false);
-  assert.equal(batchLaneFor({ task_class: "learning.reflect", priority: "high" }), false);
-});
-
-test("batchLaneFor: unknown class defaults to realtime (never silently defers user work)", () => {
-  assert.equal(batchLaneFor({ task_class: "lookup.gmail" }), false);
-  assert.equal(batchLaneFor({}), false);
-});
-
-// ---------------------------------------------------------------------------
-// cacheAffinity — pin / lookupPin / resolvePin override
-// ---------------------------------------------------------------------------
-
-test("ttlMs: parses lane cache_ttl strings", () => {
-  assert.equal(ttlMs("1h"), 3600_000);
-  assert.equal(ttlMs("5m"), 300_000);
-  assert.equal(ttlMs("30s"), 30_000);
-  assert.equal(ttlMs("0"), 0);
-  assert.equal(ttlMs("none"), 0);
-  assert.equal(ttlMs(60_000), 60_000);
-  assert.equal(ttlMs(undefined), 3600_000);
-});
-
-test("cacheAffinity: pin then lookupPin within TTL", () => {
-  const store = {};
-  let clock = 1_000_000;
-  const deps = { store, now: () => clock };
-  const r = pin("slack:C1:thread", "anthropic/claude-sonnet-4-6", "1h", { ...deps, decisionId: "d1" });
-  assert.equal(r.pinned, true);
-  assert.equal(r.expires_at, 1_000_000 + 3600_000);
-
-  const live = lookupPin("slack:C1:thread", deps);
-  assert.ok(live);
-  assert.equal(live.ref, "anthropic/claude-sonnet-4-6");
-  assert.equal(live.decision_id, "d1");
-});
-
-test("cacheAffinity: pin expires after TTL", () => {
-  const store = {};
-  let clock = 0;
-  const deps = { store, now: () => clock };
-  pin("k", "anthropic/claude-haiku-4-5", "5m", deps);
-  clock = 4 * 60_000;
-  assert.ok(lookupPin("k", deps), "live at 4m");
-  clock = 6 * 60_000;
-  assert.equal(lookupPin("k", deps), null, "expired at 6m");
-});
-
-test("cacheAffinity: ttl 0 does not pin", () => {
-  const store = {};
-  const r = pin("k", "anthropic/x", "0", { store, now: () => 0 });
-  assert.equal(r.pinned, false);
-  assert.equal(lookupPin("k", { store, now: () => 0 }), null);
-});
-
-test("cacheAffinity: on-disk store hashes the session_key (no raw PII)", () => {
-  const store = {};
-  pin("dm:+15551234567", "anthropic/x", "1h", { store, now: () => 0 });
-  const keys = Object.keys(store);
-  assert.equal(keys.length, 1);
-  assert.equal(keys[0], hashSessionKey("dm:+15551234567"));
-  assert.ok(!keys[0].includes("5551234567"));
-});
-
-test("resolvePin: honors a live pin that passes the gate", () => {
-  const store = {};
-  const deps = { store, now: () => 0 };
-  pin("k", "anthropic/claude-sonnet-4-6", "1h", deps);
-  const r = resolvePin("k", () => ({ ok: true }), deps);
-  assert.equal(r.ref, "anthropic/claude-sonnet-4-6");
-  assert.equal(r.overridden, false);
-});
-
-test("resolvePin: drops a pin the gate rejects and emits pin_overridden", () => {
-  const store = {};
-  const audit = [];
-  const deps = { store, now: () => 0, audit: (row) => audit.push(row), decisionId: "d9" };
-  pin("k", "moonshot/kimi-k2.6", "1h", deps);
-
-  const r = resolvePin("k", () => ({ ok: false, reason: "breaker_open" }), deps);
-  assert.equal(r.ref, null);
-  assert.equal(r.overridden, true);
-  assert.equal(r.reason, "breaker_open");
-
-  assert.equal(audit.length, 1);
-  assert.equal(audit[0].kind, "pin_overridden");
-  assert.equal(audit[0].ref, "moonshot/kimi-k2.6");
-  assert.equal(audit[0].reason, "breaker_open");
-  // The raw session_key is hashed in the audit row, not stored verbatim.
-  assert.equal(audit[0].session_key_hash, hashSessionKey("k"));
-});
-
-test("resolvePin: no live pin returns a clean null result", () => {
-  const r = resolvePin("absent", () => ({ ok: true }), { store: {}, now: () => 0 });
-  assert.equal(r.ref, null);
-  assert.equal(r.overridden, false);
-});
-
-test("resolvePin: a throwing gate is treated as a rejection (fail-closed for the pin)", () => {
-  const store = {};
-  const audit = [];
-  const deps = { store, now: () => 0, audit: (row) => audit.push(row) };
-  pin("k", "anthropic/x", "1h", deps);
-  const r = resolvePin("k", () => { throw new Error("boom"); }, deps);
-  assert.equal(r.overridden, true);
-  assert.equal(r.reason, "gate_error");
-  assert.equal(audit.length, 1);
-});
-
-// ---------------------------------------------------------------------------
-// spawnKnobsFor — per-task-class defaults incl. cheap-subagent map
-// ---------------------------------------------------------------------------
-
-test("spawnKnobsFor: heavy session classes get the Haiku-Explore subagent map", () => {
-  const k = spawnKnobsFor("session.responder");
-  assert.equal(k.maxTurns, 40);
-  assert.ok(k.agentsJson);
-  assert.ok(k.agentsJson["Haiku-Explore"]);
-  assert.match(k.agentsJson["Haiku-Explore"].model, /haiku/);
-});
-
-test("spawnKnobsFor: cheap one-shot classes get no subagent map", () => {
-  assert.equal(spawnKnobsFor("classify.inbox").agentsJson, null);
-  assert.equal(spawnKnobsFor("classify.inbox").maxTurns, 1);
-  assert.equal(spawnKnobsFor("lookup.gmail").maxTurns, 2); // prefix match
-  assert.equal(spawnKnobsFor("lookup.gmail").agentsJson, null);
-});
-
-test("spawnKnobsFor: learning.reflect uses medium effort", () => {
-  assert.equal(spawnKnobsFor("learning.reflect").effort, "medium");
-});
-
-test("spawnKnobsFor: prefix match for cadence.* ", () => {
-  const k = spawnKnobsFor("cadence.daily-digest");
-  assert.equal(k.maxTurns, 20);
-  assert.ok(k.agentsJson);
-});
-
-test("spawnKnobsFor: unknown class falls back to the generic ceiling", () => {
-  const k = spawnKnobsFor("totally.unknown");
-  assert.equal(k.maxTurns, 40);
-  assert.equal(k.agentsJson, null);
-});
-
-test("spawnKnobsFor: returns a fresh agentsJson clone (safe to mutate)", () => {
-  const a = spawnKnobsFor("session.responder");
-  const b = spawnKnobsFor("session.responder");
-  assert.notEqual(a.agentsJson, b.agentsJson);
-  a.agentsJson["Haiku-Explore"].model = "tampered";
-  assert.notEqual(b.agentsJson["Haiku-Explore"].model, "tampered");
-});
-
-test("internals exposed for cross-layer integration", () => {
-  assert.ok(Array.isArray(_internals.DEFERRABLE_PREFIXES));
-  assert.ok(_internals.TIER_RANK.frontier < _internals.TIER_RANK.cheap);
-});