@cohortapp/agent-sdk 2.3.0

package/lib/model-router/ledger.test.mjs

@@ -0,0 +1,382 @@
+/**
+ * Tests for lib/model-router/ledger.mjs — estimateCost (cache discount, volatile
+ * steady-state, tiered-by-prompt-size, cache-blind), and the ledger v2 row
+ * writer's budget-guard compatibility (`estimated_usd`).
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { loadCatalog } from "./catalog.mjs";
+import {
+  estimateCost,
+  buildLedgerRow,
+  writeLedgerRow,
+  sumLedgerToday,
+} from "./ledger.mjs";
+
+const CATALOG_YAML = {
+  anthropic: `
+provider: anthropic
+auth_env: ANTHROPIC_API_KEY
+endpoints: { anthropic: https://api.anthropic.com }
+data_residency: us
+models:
+  - id: claude-opus-4-8
+    status: available
+    context_window: 1000000
+    context_tokens: 950000
+    max_tokens: 128000
+    cost: { input: 5.0, output: 25.0, cache_read: 0.5, cache_write: 10.0 }
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
+    compat: { cache_control: anthropic }
+    tool_reliability: A
+    harness: { session: true, direct: true, batch: true }
+`,
+  deepseek: `
+provider: deepseek
+auth_env: DEEPSEEK_API_KEY
+endpoints: { anthropic: https://api.deepseek.com/anthropic }
+data_residency: prc
+models:
+  - id: deepseek-v4-flash
+    status: available
+    context_window: 1000000
+    context_tokens: 900000
+    max_tokens: 384000
+    cost: { input: 0.14, output: 0.28, cache_read: 0.0028, cache_write: null }
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
+    compat: { cache_control: implicit }
+    tool_reliability: B
+    harness: { session: true, direct: true, batch: false }
+  - id: deepseek-v4-pro
+    status: available
+    context_window: 1000000
+    context_tokens: 900000
+    max_tokens: 384000
+    cost:
+      input: 0.435
+      output: 0.87
+      cache_read: 0.003625
+      cache_write: null
+      tiered:
+        - { kind: steady_state, input: 1.74, output: 3.48, cache_read: 0.145 }
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
+    compat: { cache_control: implicit }
+    tool_reliability: B
+    harness: { session: true, direct: true, batch: false }
+`,
+  qwen: `
+provider: qwen
+auth_env: DASHSCOPE_API_KEY
+endpoints: { openai: https://dashscope-intl.aliyuncs.com/compatible-mode/v1 }
+data_residency: sg
+models:
+  - id: qwen3-coder-plus
+    status: available
+    context_window: 1000000
+    context_tokens: 900000
+    max_tokens: 65536
+    cost:
+      input: 1.00
+      output: 5.00
+      tiered:
+        - { kind: input_range, min_input_tokens: 0,      max_input_tokens: 32000,  input: 1.00, output: 5.00 }
+        - { kind: input_range, min_input_tokens: 32000,  max_input_tokens: 128000, input: 1.80, output: 9.00 }
+        - { kind: input_range, min_input_tokens: 128000, max_input_tokens: null,   input: 6.00, output: 60.00 }
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: true }
+    compat: { cache_control: implicit }
+    tool_reliability: B
+    harness: { session: false, direct: true, batch: false }
+  - id: text-only-none
+    status: available
+    cost: { input: 2.0, output: 4.0 }   # no cache_read; cache_control none
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
+    compat: { cache_control: none }
+    tool_reliability: C
+    harness: { direct: true }
+`,
+};
+
+function cat() {
+  return loadCatalog("/x", { catalogText: CATALOG_YAML });
+}
+
+// ---------------------------------------------------------------------------
+// estimateCost — basic + cache discount
+// ---------------------------------------------------------------------------
+
+test("estimateCost prices output + input at the per-model rate", () => {
+  const c = cat();
+  // Opus: 1M input (no cache), 100K output. input 5/MTok, output 25/MTok.
+  const r = estimateCost(c, {
+    ref: "anthropic/claude-opus-4-8",
+    inputTokens: 1_000_000,
+    outputTokens: 100_000,
+  });
+  // input 1e6/1e6*5 = 5.0 ; output 1e5/1e6*25 = 2.5
+  assert.equal(r.usd, 7.5);
+  assert.equal(r.breakdown.inputUsd, 5.0);
+  assert.equal(r.breakdown.outputUsd, 2.5);
+});
+
+test("estimateCost applies the cache-read discount (cache portion at cache_read rate)", () => {
+  const c = cat();
+  // 1M input of which 800K are cache hits. miss=200K @ 5/MTok = 1.0;
+  // cache 800K @ 0.5/MTok = 0.4 ; output 0 -> total 1.4
+  const r = estimateCost(c, {
+    ref: "anthropic/claude-opus-4-8",
+    inputTokens: 1_000_000,
+    cacheReadTokens: 800_000,
+    outputTokens: 0,
+  });
+  assert.equal(r.breakdown.inputUsd, 1.0);
+  assert.equal(r.breakdown.cacheReadUsd, 0.4);
+  assert.equal(r.usd, 1.4);
+});
+
+test("estimateCost prices cache-write (creation) tokens at cache_write rate", () => {
+  const c = cat();
+  // 100K cache-write @ 10/MTok = 1.0
+  const r = estimateCost(c, {
+    ref: "anthropic/claude-opus-4-8",
+    inputTokens: 0,
+    cacheWriteTokens: 100_000,
+  });
+  assert.equal(r.breakdown.cacheWriteUsd, 1.0);
+  assert.equal(r.usd, 1.0);
+});
+
+test("cacheReadTokens is clamped to inputTokens", () => {
+  const c = cat();
+  // cacheRead 2M but input only 1M -> all 1M priced as cache, 0 miss.
+  const r = estimateCost(c, {
+    ref: "anthropic/claude-opus-4-8",
+    inputTokens: 1_000_000,
+    cacheReadTokens: 2_000_000,
+  });
+  assert.equal(r.breakdown.inputUsd, 0);
+  assert.equal(r.breakdown.cacheReadUsd, 0.5); // 1M @ 0.5/MTok
+});
+
+// ---------------------------------------------------------------------------
+// Cache-blind ("none") rows
+// ---------------------------------------------------------------------------
+
+test("a cache_control:none row prices cache reads at the FULL input rate (cache-blind)", () => {
+  const c = cat();
+  // text-only-none: input 2/MTok, no cache_read. 1M input incl 1M 'cache' reads.
+  const r = estimateCost(c, {
+    ref: "qwen/text-only-none",
+    inputTokens: 1_000_000,
+    cacheReadTokens: 1_000_000,
+  });
+  // cache-blind: the 1M 'cache' tokens are priced at the input rate 2/MTok = 2.0
+  assert.equal(r.breakdown.cacheReadUsd, 2.0);
+  assert.equal(r.usd, 2.0);
+});
+
+// ---------------------------------------------------------------------------
+// Volatile -> steady-state budget projection
+// ---------------------------------------------------------------------------
+
+test("volatile row with steady_state tier budgets at the HIGHER steady price", () => {
+  const c = cat();
+  // deepseek-v4-pro budget:true -> steady_state input 1.74, output 3.48.
+  const r = estimateCost(c, {
+    ref: "deepseek/deepseek-v4-pro",
+    inputTokens: 1_000_000,
+    outputTokens: 1_000_000,
+    budget: true,
+  });
+  assert.equal(r.basis, "steady_state");
+  assert.equal(r.volatile, true);
+  // 1.74 + 3.48 = 5.22
+  assert.equal(r.usd, 5.22);
+});
+
+test("volatile row priced as ACTUALS (budget:false) uses the sticker price", () => {
+  const c = cat();
+  const r = estimateCost(c, {
+    ref: "deepseek/deepseek-v4-pro",
+    inputTokens: 1_000_000,
+    outputTokens: 1_000_000,
+    budget: false,
+  });
+  // sticker 0.435 + 0.87 = 1.305
+  assert.equal(r.usd, 1.305);
+  assert.notEqual(r.basis, "steady_state");
+});
+
+test("volatile row WITHOUT a steady_state tier keeps sticker but flags the basis", () => {
+  const c = cat();
+  // deepseek-v4-flash is volatile but has no steady_state tier.
+  const r = estimateCost(c, {
+    ref: "deepseek/deepseek-v4-flash",
+    inputTokens: 1_000_000,
+    outputTokens: 0,
+    budget: true,
+  });
+  assert.equal(r.volatile, true);
+  assert.equal(r.usd, 0.14); // sticker input rate
+  assert.match(r.basis, /volatile/);
+});
+
+// ---------------------------------------------------------------------------
+// Tiered-by-prompt-size (Qwen hazard)
+// ---------------------------------------------------------------------------
+
+test("tiered-by-prompt-size picks the bracket containing the input token count", () => {
+  const c = cat();
+  // 20K input -> first tier (1.00 in / 5.00 out)
+  const small = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 20_000, outputTokens: 10_000, budget: false });
+  // 20K/1e6*1.00 = 0.02 ; 10K/1e6*5.00 = 0.05 -> 0.07
+  assert.equal(small.basis, "input_range");
+  assert.equal(small.usd, 0.07);
+
+  // 100K input -> second tier (1.80 in / 9.00 out)
+  const mid = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 100_000, outputTokens: 0, budget: false });
+  // 100K/1e6*1.80 = 0.18
+  assert.equal(mid.usd, 0.18);
+
+  // 300K input -> top tier (6.00 in / 60.00 out)
+  const big = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 300_000, outputTokens: 0, budget: false });
+  // 300K/1e6*6.00 = 1.8
+  assert.equal(big.usd, 1.8);
+});
+
+test("input beyond the top bracket max uses the highest tier", () => {
+  const c = cat();
+  const huge = estimateCost(c, { ref: "qwen/qwen3-coder-plus", inputTokens: 5_000_000, outputTokens: 0, budget: false });
+  // 5M/1e6*6.00 = 30
+  assert.equal(huge.usd, 30);
+});
+
+// ---------------------------------------------------------------------------
+// Missing rows / never-throw
+// ---------------------------------------------------------------------------
+
+test("estimateCost returns usd:null for unknown ref, never throws", () => {
+  const c = cat();
+  const r = estimateCost(c, { ref: "ghost/model", inputTokens: 100 });
+  assert.equal(r.usd, null);
+  assert.equal(r.basis, "no-row");
+});
+
+test("estimateCost with provider+model assembles the ref", () => {
+  const c = cat();
+  const r = estimateCost(c, { provider: "anthropic", model: "claude-opus-4-8", inputTokens: 1_000_000 });
+  assert.equal(r.ref, "anthropic/claude-opus-4-8");
+  assert.equal(r.usd, 5.0);
+});
+
+// ---------------------------------------------------------------------------
+// Ledger row — budget-guard compatibility
+// ---------------------------------------------------------------------------
+
+test("buildLedgerRow emits the load-bearing estimated_usd field (budget-guard contract)", () => {
+  const c = cat();
+  const row = buildLedgerRow(c, {
+    agent: "alex",
+    provider: "anthropic",
+    model: "claude-opus-4-8",
+    inputTokens: 1_000_000,
+    outputTokens: 100_000,
+    task_class: "session.responder",
+    decision_id: "01ABC",
+  });
+  assert.equal(typeof row.estimated_usd, "number");
+  assert.equal(row.estimated_usd, 7.5); // matches estimateCost
+  assert.equal(row.total_cost_usd, 7.5);
+  assert.equal(row.estimated, true);
+  assert.equal(row.backend, "anthropic");
+  assert.equal(row.model, "claude-opus-4-8");
+  assert.equal(row.decision_id, "01ABC");
+  assert.equal(row.exit_reason, "ok");
+});
+
+test("buildLedgerRow uses the authoritative totalCostUSD when present (estimated:false)", () => {
+  const c = cat();
+  const row = buildLedgerRow(c, {
+    agent: "alex",
+    provider: "anthropic",
+    model: "claude-opus-4-8",
+    inputTokens: 1_000_000,
+    outputTokens: 100_000,
+    totalCostUSD: 0.525, // from --output-format json result
+  });
+  assert.equal(row.total_cost_usd, 0.525);
+  assert.equal(row.estimated_usd, 0.525);
+  assert.equal(row.estimated, false);
+});
+
+test("buildLedgerRow estimated_usd is never null (budget-guard counts every row)", () => {
+  const c = cat();
+  const row = buildLedgerRow(c, { agent: "x", provider: "ghost", model: "model", inputTokens: 10 });
+  assert.equal(row.estimated_usd, 0);
+  assert.equal(row.total_cost_usd, null);
+});
+
+test("buildLedgerRow projects volatile spend at steady-state for budgeting", () => {
+  const c = cat();
+  const row = buildLedgerRow(c, {
+    agent: "x",
+    provider: "deepseek",
+    model: "deepseek-v4-pro",
+    inputTokens: 1_000_000,
+    outputTokens: 1_000_000,
+  });
+  assert.equal(row.estimated_usd, 5.22); // steady-state, not 1.305
+  assert.equal(row.volatile, true);
+  assert.equal(row.cost_basis, "steady_state");
+});
+
+test("writeLedgerRow appends to state/cost-tracking/<date>.jsonl and sumLedgerToday reads it", () => {
+  const c = cat();
+  const root = mkdtempSync(join(tmpdir(), "maestro-ledger-"));
+  const fixedNow = Date.UTC(2026, 5, 11, 12, 0, 0); // 2026-06-11
+  const deps = { agentRoot: root, now: () => fixedNow };
+
+  const w1 = writeLedgerRow(c, { agent: "x", provider: "anthropic", model: "claude-opus-4-8", inputTokens: 1_000_000, outputTokens: 100_000 }, deps);
+  const w2 = writeLedgerRow(c, { agent: "x", provider: "deepseek", model: "deepseek-v4-flash", inputTokens: 1_000_000, outputTokens: 0 }, deps);
+  assert.ok(w1.written);
+  assert.ok(w2.written);
+
+  const file = join(root, "state", "cost-tracking", "2026-06-11.jsonl");
+  assert.ok(existsSync(file));
+  const lines = readFileSync(file, "utf-8").trim().split("\n");
+  assert.equal(lines.length, 2);
+
+  const sum = sumLedgerToday(deps);
+  assert.equal(sum.sessions, 2);
+  // 7.5 (opus) + 0.14 (deepseek flash) = 7.64
+  assert.equal(sum.spentUSD, 7.64);
+  assert.equal(sum.byModel["claude-opus-4-8"], 7.5);
+  assert.equal(sum.byModel["deepseek-v4-flash"], 0.14);
+});
+
+test("writeLedgerRow rows are summable by a budget-guard-style estimated_usd reducer", () => {
+  const c = cat();
+  const root = mkdtempSync(join(tmpdir(), "maestro-ledger-bg-"));
+  const fixedNow = Date.UTC(2026, 5, 11, 1, 0, 0);
+  const deps = { agentRoot: root, now: () => fixedNow };
+  writeLedgerRow(c, { agent: "x", provider: "anthropic", model: "claude-opus-4-8", inputTokens: 500_000, outputTokens: 50_000 }, deps);
+
+  // Replicate budget-guard's sumToday exactly: sum Number(row.estimated_usd).
+  const file = join(root, "state", "cost-tracking", "2026-06-11.jsonl");
+  const body = readFileSync(file, "utf-8");
+  let spent = 0;
+  let sessions = 0;
+  for (const line of body.split("\n")) {
+    if (!line.trim()) continue;
+    const row = JSON.parse(line);
+    const usd = Number(row.estimated_usd);
+    if (Number.isFinite(usd)) spent += usd;
+    sessions += 1;
+  }
+  assert.equal(sessions, 1);
+  assert.ok(spent > 0, "budget-guard would see a non-zero spend");
+});