@cohortapp/agent-sdk 2.17.0 → 2.18.4

package/lib/model-router/ledger.test.mjs

@@ -1,415 +0,0 @@
-/**
- * Tests for lib/model-router/ledger.mjs — estimateCost (cache discount, volatile
- * steady-state, tiered-by-prompt-size, cache-blind), and the ledger v2 row
- * writer's budget-guard compatibility (`estimated_usd`).
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, readFileSync, existsSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { loadCatalog } from "./catalog.mjs";
-import { billableUsd } from "../cost/ledger-row.mjs";
-import {
-  estimateCost,
-  buildLedgerRow,
-  writeLedgerRow,
-  sumLedgerToday,
-} from "./ledger.mjs";
-
-const CATALOG_YAML = {
-  anthropic: `
-provider: anthropic
-auth_env: ANTHROPIC_API_KEY
-endpoints: { anthropic: https://api.anthropic.com }
-data_residency: us
-models:
-  - id: claude-opus-4-8
-    status: available
-    context_window: 1000000
-    context_tokens: 950000
-    max_tokens: 128000
-    cost: { input: 5.0, output: 25.0, cache_read: 0.5, cache_write: 10.0 }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
-    compat: { cache_control: anthropic }
-    tool_reliability: A
-    harness: { session: true, direct: true, batch: true }
-`,
-  deepseek: `
-provider: deepseek
-auth_env: DEEPSEEK_API_KEY
-endpoints: { anthropic: https://api.deepseek.com/anthropic }
-data_residency: prc
-models:
-  - id: deepseek-v4-flash
-    status: available
-    context_window: 1000000
-    context_tokens: 900000
-    max_tokens: 384000
-    cost: { input: 0.14, output: 0.28, cache_read: 0.0028, cache_write: null }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
-    compat: { cache_control: implicit }
-    tool_reliability: B
-    harness: { session: true, direct: true, batch: false }
-  - id: deepseek-v4-pro
-    status: available
-    context_window: 1000000
-    context_tokens: 900000
-    max_tokens: 384000
-    cost:
-      input: 0.435
-      output: 0.87
-      cache_read: 0.003625
-      cache_write: null
-      tiered:
-        - { kind: steady_state, input: 1.74, output: 3.48, cache_read: 0.145 }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
-    compat: { cache_control: implicit }
-    tool_reliability: B
-    harness: { session: true, direct: true, batch: false }
-`,
-  qwen: `
-provider: qwen
-auth_env: DASHSCOPE_API_KEY
-endpoints: { openai: https://dashscope-intl.aliyuncs.com/compatible-mode/v1 }
-data_residency: sg
-models:
-  - id: qwen3-coder-plus
-    status: available
-    context_window: 1000000
-    context_tokens: 900000
-    max_tokens: 65536
-    cost:
-      input: 1.00
-      output: 5.00
-      tiered:
-        - { kind: input_range, min_input_tokens: 0,      max_input_tokens: 32000,  input: 1.00, output: 5.00 }
-        - { kind: input_range, min_input_tokens: 32000,  max_input_tokens: 128000, input: 1.80, output: 9.00 }
-        - { kind: input_range, min_input_tokens: 128000, max_input_tokens: null,   input: 6.00, output: 60.00 }
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
-    compat: { cache_control: implicit }
-    tool_reliability: B
-    harness: { session: false, direct: true, batch: false }
-  - id: text-only-none
-    status: available
-    cost: { input: 2.0, output: 4.0 }   # no cache_read; cache_control none
-    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
-    compat: { cache_control: none }
-    tool_reliability: C
-    harness: { direct: true }
-`,
-};
-
-function cat() {
-  return loadCatalog("/x", { catalogText: CATALOG_YAML });
-}
-
-// ---------------------------------------------------------------------------
-// estimateCost — basic + cache discount
-// ---------------------------------------------------------------------------
-
-test("estimateCost prices output + input at the per-model rate", () => {
-  const c = cat();
-  // Opus: 1M input (no cache), 100K output. input 5/MTok, output 25/MTok.
-  const r = estimateCost(c, {
-    ref: "anthropic/claude-opus-4-8",
-    inputTokens: 1_000_000,
-    outputTokens: 100_000,
-  });
-  // input 1e6/1e6*5 = 5.0 ; output 1e5/1e6*25 = 2.5
-  assert.equal(r.usd, 7.5);
-  assert.equal(r.breakdown.inputUsd, 5.0);
-  assert.equal(r.breakdown.outputUsd, 2.5);
-});
-
-test("estimateCost applies the cache-read discount (cache portion at cache_read rate)", () => {
-  const c = cat();
-  // 1M input of which 800K are cache hits. miss=200K @ 5/MTok = 1.0;
-  // cache 800K @ 0.5/MTok = 0.4 ; output 0 -> total 1.4
-  const r = estimateCost(c, {
-    ref: "anthropic/claude-opus-4-8",
-    inputTokens: 1_000_000,
-    cacheReadTokens: 800_000,
-    outputTokens: 0,
-  });
-  assert.equal(r.breakdown.inputUsd, 1.0);
-  assert.equal(r.breakdown.cacheReadUsd, 0.4);
-  assert.equal(r.usd, 1.4);
-});
-
-test("estimateCost prices cache-write (creation) tokens at cache_write rate", () => {
-  const c = cat();
-  // 100K cache-write @ 10/MTok = 1.0
-  const r = estimateCost(c, {
-    ref: "anthropic/claude-opus-4-8",
-    inputTokens: 0,
-    cacheWriteTokens: 100_000,
-  });
-  assert.equal(r.breakdown.cacheWriteUsd, 1.0);
-  assert.equal(r.usd, 1.0);
-});
-
-test("cacheReadTokens is clamped to inputTokens", () => {
-  const c = cat();
-  // cacheRead 2M but input only 1M -> all 1M priced as cache, 0 miss.
-  const r = estimateCost(c, {
-    ref: "anthropic/claude-opus-4-8",
-    inputTokens: 1_000_000,
-    cacheReadTokens: 2_000_000,
-  });
-  assert.equal(r.breakdown.inputUsd, 0);
-  assert.equal(r.breakdown.cacheReadUsd, 0.5); // 1M @ 0.5/MTok
-});
-
-// ---------------------------------------------------------------------------
-// Cache-blind ("none") rows
-// ---------------------------------------------------------------------------
-
-test("a cache_control:none row prices cache reads at the FULL input rate (cache-blind)", () => {
-  const c = cat();
-  // text-only-none: input 2/MTok, no cache_read. 1M input incl 1M 'cache' reads.
-  const r = estimateCost(c, {
-    ref: "qwen/text-only-none",
-    inputTokens: 1_000_000,
-    cacheReadTokens: 1_000_000,
-  });
-  // cache-blind: the 1M 'cache' tokens are priced at the input rate 2/MTok = 2.0
-  assert.equal(r.breakdown.cacheReadUsd, 2.0);
-  assert.equal(r.usd, 2.0);
-});
-
-// ---------------------------------------------------------------------------
-// Volatile -> steady-state budget projection
-// ---------------------------------------------------------------------------
-
-test("volatile row with steady_state tier budgets at the HIGHER steady price", () => {
-  const c = cat();
-  // deepseek-v4-pro budget:true -> steady_state input 1.74, output 3.48.
-  const r = estimateCost(c, {
-    ref: "deepseek/deepseek-v4-pro",
-    inputTokens: 1_000_000,
-    outputTokens: 1_000_000,
-    budget: true,
-  });
-  assert.equal(r.basis, "steady_state");
-  assert.equal(r.volatile, true);
-  // 1.74 + 3.48 = 5.22
-  assert.equal(r.usd, 5.22);
-});
-
-test("volatile row priced as ACTUALS (budget:false) uses the sticker price", () => {
-  const c = cat();
-  const r = estimateCost(c, {
-    ref: "deepseek/deepseek-v4-pro",
-    inputTokens: 1_000_000,
-    outputTokens: 1_000_000,
-    budget: false,
-  });
-  // sticker 0.435 + 0.87 = 1.305
-  assert.equal(r.usd, 1.305);
-  assert.notEqual(r.basis, "steady_state");
-});
-
-test("volatile row WITHOUT a steady_state tier keeps sticker but flags the basis", () => {
-  const c = cat();
-  // deepseek-v4-flash is volatile but has no steady_state tier.
-  const r = estimateCost(c, {
-    ref: "deepseek/deepseek-v4-flash",
-    inputTokens: 1_000_000,
-    outputTokens: 0,
-    budget: true,
-  });
-  assert.equal(r.volatile, true);
-  assert.equal(r.usd, 0.14); // sticker input rate
-  assert.match(r.basis, /volatile/);
-});
-
-// ---------------------------------------------------------------------------
-// Tiered-by-prompt-size (Qwen hazard)
-// ---------------------------------------------------------------------------
-
-test("tiered-by-prompt-size picks the bracket containing the input token count", () => {
-  const c = cat();
-  // 20K input -> first tier (1.00 in / 5.00 out)
-  const small = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 20_000, outputTokens: 10_000, budget: false });
-  // 20K/1e6*1.00 = 0.02 ; 10K/1e6*5.00 = 0.05 -> 0.07
-  assert.equal(small.basis, "input_range");
-  assert.equal(small.usd, 0.07);
-
-  // 100K input -> second tier (1.80 in / 9.00 out)
-  const mid = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 100_000, outputTokens: 0, budget: false });
-  // 100K/1e6*1.80 = 0.18
-  assert.equal(mid.usd, 0.18);
-
-  // 300K input -> top tier (6.00 in / 60.00 out)
-  const big = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 300_000, outputTokens: 0, budget: false });
-  // 300K/1e6*6.00 = 1.8
-  assert.equal(big.usd, 1.8);
-});
-
-test("input beyond the top bracket max uses the highest tier", () => {
-  const c = cat();
-  const huge = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 5_000_000, outputTokens: 0, budget: false });
-  // 5M/1e6*6.00 = 30
-  assert.equal(huge.usd, 30);
-});
-
-// ---------------------------------------------------------------------------
-// Missing rows / never-throw
-// ---------------------------------------------------------------------------
-
-test("estimateCost returns usd:null for unknown ref, never throws", () => {
-  const c = cat();
-  const r = estimateCost(c, { ref: "ghost/model", inputTokens: 100 });
-  assert.equal(r.usd, null);
-  assert.equal(r.basis, "no-row");
-});
-
-test("estimateCost with provider+model assembles the ref", () => {
-  const c = cat();
-  const r = estimateCost(c, { provider: "anthropic", model: "claude-opus-4-8", inputTokens: 1_000_000 });
-  assert.equal(r.ref, "anthropic/claude-opus-4-8");
-  assert.equal(r.usd, 5.0);
-});
-
-// ---------------------------------------------------------------------------
-// Ledger row — budget-guard compatibility
-// ---------------------------------------------------------------------------
-
-test("buildLedgerRow emits the load-bearing estimated_usd field (budget-guard contract)", () => {
-  const c = cat();
-  const row = buildLedgerRow(c, {
-    agent: "alex",
-    provider: "anthropic",
-    model: "claude-opus-4-8",
-    inputTokens: 1_000_000,
-    outputTokens: 100_000,
-    task_class: "session.responder",
-    decision_id: "01ABC",
-  });
-  assert.equal(typeof row.estimated_usd, "number");
-  assert.equal(row.estimated_usd, 7.5); // matches estimateCost
-  assert.equal(row.total_cost_usd, 7.5);
-  assert.equal(row.estimated, true);
-  assert.equal(row.backend, "anthropic");
-  assert.equal(row.model, "claude-opus-4-8");
-  assert.equal(row.decision_id, "01ABC");
-  assert.equal(row.exit_reason, "ok");
-});
-
-test("buildLedgerRow uses the authoritative totalCostUSD when present (estimated:false)", () => {
-  const c = cat();
-  const row = buildLedgerRow(c, {
-    agent: "alex",
-    provider: "anthropic",
-    model: "claude-opus-4-8",
-    inputTokens: 1_000_000,
-    outputTokens: 100_000,
-    totalCostUSD: 0.525, // from --output-format json result
-  });
-  assert.equal(row.total_cost_usd, 0.525);
-  assert.equal(row.estimated_usd, 0.525);
-  assert.equal(row.estimated, false);
-});
-
-test("buildLedgerRow: an UNPRICEABLE row is null, not 0 — a $0 here reads as a free session", () => {
-  // This asserted `estimated_usd === 0` for a model with no catalog row, on the
-  // theory that budget-guard needed a finite number on every line. It does not:
-  // readers bill through lib/cost/ledger-row.mjs, which treats a 0 on a row that
-  // NAMED a model as "could not price", imputes it, and turns doctor red. A
-  // literal 0 was the "a zero that means unmeasured" defect, re-created by the
-  // v2 writer the module was written to replace.
-  const c = cat();
-  const row = buildLedgerRow(c, { agent: "x", provider: "ghost", model: "model", inputTokens: 10 });
-  assert.equal(row.estimated_usd, null);
-  assert.equal(row.total_cost_usd, null);
-  assert.equal(row.measurement, "measured", "usage WAS supplied; only the price is missing");
-  assert.equal(billableUsd(row).basis, "unpriced");
-  assert.equal(billableUsd(row).usd, null);
-});
-
-test("buildLedgerRow: usage nobody supplied is `unknown`, never a measured zero", () => {
-  // llm-task.mjs#writeRow passes `usage.inputTokens` straight through, so an
-  // unreadable usage envelope arrives here as undefined. `nonNeg` mapped that to
-  // 0 and the row was stamped `measurement:"measured"` billing $0 — no
-  // degradation, no unmeasured count, doctor green.
-  const c = cat();
-  const row = buildLedgerRow(c, { agent: "x", ref: "anthropic/claude-opus-4-6", provider: "anthropic", model: "claude-opus-4-6" });
-  assert.equal(row.measurement, "unknown");
-  assert.equal(row.input_tokens, null);
-  assert.equal(row.output_tokens, null);
-  assert.equal(row.estimated_usd, null);
-  assert.match(row.unmeasured_reason, /no usage was supplied/);
-  assert.equal(billableUsd(row).usd, null, "unknown cost, not free");
-});
-
-test("buildLedgerRow: a zero-model attribution row is still a real $0", () => {
-  // The one case where 0 is a FACT: a message send, no model, no tokens.
-  const row = buildLedgerRow(cat(), { agent: "x", source: "messaging", task_class: "messaging.send" });
-  assert.equal(row.measurement, "n/a");
-  assert.equal(billableUsd(row).usd, 0);
-  assert.equal(billableUsd(row).basis, "non_llm");
-});
-
-test("buildLedgerRow projects volatile spend at steady-state for budgeting", () => {
-  const c = cat();
-  const row = buildLedgerRow(c, {
-    agent: "x",
-    provider: "deepseek",
-    model: "deepseek-v4-pro",
-    inputTokens: 1_000_000,
-    outputTokens: 1_000_000,
-  });
-  assert.equal(row.estimated_usd, 5.22); // steady-state, not 1.305
-  assert.equal(row.volatile, true);
-  assert.equal(row.cost_basis, "steady_state");
-});
-
-test("writeLedgerRow appends to state/cost-tracking/<date>.jsonl and sumLedgerToday reads it", () => {
-  const c = cat();
-  const root = mkdtempSync(join(tmpdir(), "maestro-ledger-"));
-  const fixedNow = Date.UTC(2026, 5, 11, 12, 0, 0); // 2026-06-11
-  const deps = { agentRoot: root, now: () => fixedNow };
-
-  const w1 = writeLedgerRow(c, { agent: "x", provider: "anthropic", model: "claude-opus-4-8", inputTokens: 1_000_000, outputTokens: 100_000 }, deps);
-  const w2 = writeLedgerRow(c, { agent: "x", provider: "deepseek", model: "deepseek-v4-flash", inputTokens: 1_000_000, outputTokens: 0 }, deps);
-  assert.ok(w1.written);
-  assert.ok(w2.written);
-
-  const file = join(root, "state", "cost-tracking", "2026-06-11.jsonl");
-  assert.ok(existsSync(file));
-  const lines = readFileSync(file, "utf-8").trim().split("\n");
-  assert.equal(lines.length, 2);
-
-  const sum = sumLedgerToday(deps);
-  assert.equal(sum.sessions, 2);
-  // 7.5 (opus) + 0.14 (deepseek flash) = 7.64
-  assert.equal(sum.spentUSD, 7.64);
-  assert.equal(sum.byModel["claude-opus-4-8"], 7.5);
-  assert.equal(sum.byModel["deepseek-v4-flash"], 0.14);
-});
-
-test("writeLedgerRow rows are summable by a budget-guard-style estimated_usd reducer", () => {
-  const c = cat();
-  const root = mkdtempSync(join(tmpdir(), "maestro-ledger-bg-"));
-  const fixedNow = Date.UTC(2026, 5, 11, 1, 0, 0);
-  const deps = { agentRoot: root, now: () => fixedNow };
-  writeLedgerRow(c, { agent: "x", provider: "anthropic", model: "claude-opus-4-8", inputTokens: 500_000, outputTokens: 50_000 }, deps);
-
-  // Replicate budget-guard's sumToday exactly: sum Number(row.estimated_usd).
-  const file = join(root, "state", "cost-tracking", "2026-06-11.jsonl");
-  const body = readFileSync(file, "utf-8");
-  let spent = 0;
-  let sessions = 0;
-  for (const line of body.split("\n")) {
-    if (!line.trim()) continue;
-    const row = JSON.parse(line);
-    const usd = Number(row.estimated_usd);
-    if (Number.isFinite(usd)) spent += usd;
-    sessions += 1;
-  }
-  assert.equal(sessions, 1);
-  assert.ok(spent > 0, "budget-guard would see a non-zero spend");
-});