auto-model-router 0.31.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +32 -2
  3. package/omp-extension/router-configure.ts +9 -7
  4. package/package.json +1 -1
  5. package/src/cli/config-cmd.ts +8 -7
  6. package/src/cli/explain.ts +10 -5
  7. package/src/cli/export.ts +6 -5
  8. package/src/cli/models.ts +10 -7
  9. package/src/cli/report.ts +6 -1
  10. package/src/cli/stats.ts +7 -7
  11. package/src/config/load.ts +10 -1
  12. package/src/config/types.ts +10 -1
  13. package/src/context/bridge.ts +7 -7
  14. package/src/context/index.ts +3 -3
  15. package/src/context/store.ts +39 -56
  16. package/src/context/types.ts +7 -6
  17. package/src/cost/blended.ts +28 -7
  18. package/src/cost/feedback.ts +33 -37
  19. package/src/cost/ledger-sql.ts +547 -0
  20. package/src/cost/ledger.ts +30 -459
  21. package/src/cost/report.ts +171 -129
  22. package/src/cost/retention.ts +10 -10
  23. package/src/cost/summary.ts +15 -10
  24. package/src/cost/types.ts +43 -62
  25. package/src/cost/views.ts +79 -49
  26. package/src/lib.ts +6 -2
  27. package/src/router/candidates.ts +7 -15
  28. package/src/router/classify.ts +6 -4
  29. package/src/router/index.ts +95 -9
  30. package/src/router/select.ts +38 -21
  31. package/src/router/state.ts +90 -102
  32. package/src/router/types.ts +11 -5
  33. package/src/server/advise.ts +6 -4
  34. package/src/server/compaction-digest.ts +1 -1
  35. package/src/server/digest.ts +9 -10
  36. package/src/server/http.ts +101 -41
  37. package/src/server/providers.ts +18 -4
  38. package/src/server/turn.ts +32 -9
  39. package/src/tokens/estimate.ts +16 -6
  40. package/src/upstream/ollama-usage.ts +21 -11
  41. package/src/util/schema.ts +201 -0
  42. package/src/util/sql.ts +246 -0
  43. package/src/wire/anthropic/messages.ts +3 -4
  44. package/src/wire/openai/request.ts +1 -0
  45. package/src/wire/types.ts +7 -0
  46. package/test/anthropic-wire.test.ts +9 -9
  47. package/test/benchmark-feeds.test.ts +7 -7
  48. package/test/cache-control.test.ts +7 -7
  49. package/test/cache-estimate.test.ts +5 -5
  50. package/test/catalog-view.test.ts +4 -4
  51. package/test/catalog.test.ts +11 -11
  52. package/test/classify.test.ts +24 -24
  53. package/test/compaction.test.ts +20 -20
  54. package/test/config-wizard.test.ts +32 -32
  55. package/test/config.test.ts +10 -10
  56. package/test/connect-harnesses.test.ts +11 -11
  57. package/test/context-bridge.test.ts +40 -30
  58. package/test/context-prune.test.ts +43 -36
  59. package/test/context-query.test.ts +8 -8
  60. package/test/controls.test.ts +54 -27
  61. package/test/cost.test.ts +12 -12
  62. package/test/digest.test.ts +55 -44
  63. package/test/embed-lifecycle.test.ts +5 -5
  64. package/test/embed-logic.test.ts +26 -26
  65. package/test/escalate.test.ts +17 -17
  66. package/test/eval.test.ts +13 -13
  67. package/test/executable.test.ts +6 -6
  68. package/test/exploration.test.ts +19 -20
  69. package/test/failover.test.ts +22 -21
  70. package/test/fakes.ts +105 -0
  71. package/test/features.test.ts +21 -21
  72. package/test/harness-requests.test.ts +3 -3
  73. package/test/harness-switch.test.ts +5 -5
  74. package/test/hold-exploration.test.ts +13 -13
  75. package/test/hot-reload.test.ts +5 -5
  76. package/test/learned.test.ts +5 -5
  77. package/test/ledger-sql.test.ts +342 -0
  78. package/test/mcp-entry.test.ts +5 -5
  79. package/test/migrations.test.ts +28 -22
  80. package/test/models-yml.test.ts +18 -18
  81. package/test/ollama.test.ts +40 -34
  82. package/test/omp-credentials.test.ts +16 -16
  83. package/test/policy.test.ts +3 -3
  84. package/test/reconfigure.test.ts +4 -4
  85. package/test/redaction.test.ts +41 -35
  86. package/test/remote.test.ts +12 -12
  87. package/test/report-logic.test.ts +8 -8
  88. package/test/report.test.ts +95 -87
  89. package/test/retention.test.ts +79 -66
  90. package/test/schema.test.ts +123 -0
  91. package/test/scope.test.ts +8 -8
  92. package/test/select.test.ts +216 -257
  93. package/test/skills.test.ts +3 -3
  94. package/test/sql-shim.test.ts +154 -0
  95. package/test/state.test.ts +43 -36
  96. package/test/summary.test.ts +38 -27
  97. package/test/tier-plan.test.ts +45 -62
  98. package/test/toast-logic.test.ts +31 -31
  99. package/test/tokens.test.ts +95 -80
  100. package/test/trust-attribution.test.ts +217 -187
  101. package/test/trust-window.test.ts +37 -32
  102. package/test/turn.test.ts +55 -23
  103. package/test/upstreams.test.ts +13 -13
  104. package/test/views.test.ts +81 -59
  105. package/test/wire-request.test.ts +17 -17
  106. package/test/wire-responses.test.ts +4 -4
  107. package/tools/agentdox-e2e.ts +5 -2
  108. package/tools/export-benchmarks.ts +5 -5
  109. package/tools/ledger-parity.ts +266 -0
  110. package/tools/replay.ts +16 -8
@@ -1,10 +1,15 @@
1
1
  import { describe, expect, test } from "bun:test";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+
2
5
 
3
6
  import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
4
- import { createLedger } from "../src/cost/ledger.ts";
7
+ import { createSqlLedger } from "../src/cost/ledger-sql.ts";
8
+ import { migrateStore } from "../src/util/schema.ts";
9
+ import type { AsyncLedger } from "../src/cost/types.ts";
5
10
  import { buildUsageReport, renderUsageReport } from "../src/cost/report.ts";
6
11
  import type { LedgerEntry } from "../src/cost/types.ts";
7
- import { openDb } from "../src/util/sqlite.ts";
12
+ import { openSqlDb, type SqlDb } from "../src/util/sql.ts";
8
13
 
9
14
  /**
10
15
  * `buildUsageReport` is what `/router report`, the `report` CLI and
@@ -55,21 +60,24 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
55
60
  } as LedgerEntry;
56
61
  }
57
62
 
58
- function seeded() {
63
+ async function seeded(): Promise<{ db: SqlDb; ledger: AsyncLedger }> {
59
64
  const cfg = structuredClone(DEFAULT_CONFIG);
60
- cfg.ledger.path = ":memory:";
61
- const db = openDb(":memory:");
62
- const ledger = createLedger(db, cfg);
63
- return { db, ledger };
65
+ // A file rather than `:memory:`: the report reads through its own handle on
66
+ // the store, which an in-memory database cannot share.
67
+ const path = join(tmpdir(), `report-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`);
68
+ cfg.ledger.path = path;
69
+ const db = openSqlDb(path);
70
+ await migrateStore(db);
71
+ return { db, ledger: createSqlLedger(db, cfg, { findModel: () => null }) };
64
72
  }
65
73
 
66
74
  describe("buildUsageReport", () => {
67
- test("totals, providers, models and tiers over a mixed window", () => {
68
- const { db, ledger } = seeded();
75
+ test("totals, providers, models and tiers over a mixed window", async () => {
76
+ const { db, ledger } = await seeded();
69
77
  // Two OpenRouter turns on one model (one with a warm cache), one Ollama
70
78
  // turn served by a different slug than decided, one escalation, one error.
71
- ledger.record(entry({ conversationKey: "a", turn: 1, reportedUsd: 0.01 }));
72
- ledger.record(
79
+ await ledger.record(entry({ conversationKey: "a", turn: 1, reportedUsd: 0.01 }));
80
+ await ledger.record(
73
81
  entry({
74
82
  conversationKey: "a",
75
83
  turn: 2,
@@ -77,7 +85,7 @@ describe("buildUsageReport", () => {
77
85
  usage: { promptTokens: 1000, cachedTokens: 800, cacheWriteTokens: 0, completionTokens: 100, reasoningTokens: 0, images: 0 },
78
86
  }),
79
87
  );
80
- ledger.record(
88
+ await ledger.record(
81
89
  entry({
82
90
  conversationKey: "b",
83
91
  slug: "ollama/glm-5.3-flash",
@@ -89,10 +97,10 @@ describe("buildUsageReport", () => {
89
97
  ttftMs: 100,
90
98
  }),
91
99
  );
92
- ledger.record(entry({ conversationKey: "c", tier: "hard", escalationSignal: "refusal", wasted: true, reportedUsd: 0.002 }));
93
- ledger.record(entry({ conversationKey: "c", tier: "hard", error: "request aborted", reportedUsd: 0, ttftMs: null }));
100
+ await ledger.record(entry({ conversationKey: "c", tier: "hard", escalationSignal: "refusal", wasted: true, reportedUsd: 0.002 }));
101
+ await ledger.record(entry({ conversationKey: "c", tier: "hard", error: "request aborted", reportedUsd: 0, ttftMs: null }));
94
102
 
95
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
103
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
96
104
  expect(r.totals.dispatches).toBe(5);
97
105
  expect(r.totals.conversations).toBe(3);
98
106
  // 0.01 + 0.005 + 0.02 (predicted: no reported) + 0.002 + 0
@@ -129,54 +137,54 @@ describe("buildUsageReport", () => {
129
137
  db.close();
130
138
  });
131
139
 
132
- test("counts a model switch only between consecutive kept rows of one conversation", () => {
133
- const { db, ledger } = seeded();
134
- ledger.record(entry({ conversationKey: "a", turn: 1, slug: "x/one", createdAtMs: NOW - 3 * HOUR }));
135
- ledger.record(entry({ conversationKey: "a", turn: 2, slug: "x/two", createdAtMs: NOW - 2 * HOUR }));
136
- ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/two", createdAtMs: NOW - 1 * HOUR }));
140
+ test("counts a model switch only between consecutive kept rows of one conversation", async () => {
141
+ const { db, ledger } = await seeded();
142
+ await ledger.record(entry({ conversationKey: "a", turn: 1, slug: "x/one", createdAtMs: NOW - 3 * HOUR }));
143
+ await ledger.record(entry({ conversationKey: "a", turn: 2, slug: "x/two", createdAtMs: NOW - 2 * HOUR }));
144
+ await ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/two", createdAtMs: NOW - 1 * HOUR }));
137
145
  // A wasted probe on another slug is not a switch.
138
- ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/three", wasted: true, createdAtMs: NOW - 1 * HOUR + 1 }));
146
+ await ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/three", wasted: true, createdAtMs: NOW - 1 * HOUR + 1 }));
139
147
  // A different conversation starting on another model is not a switch either.
140
- ledger.record(entry({ conversationKey: "b", turn: 1, slug: "x/three", createdAtMs: NOW - HOUR }));
141
- const r = buildUsageReport(db, { windowDays: 1, nowMs: NOW });
148
+ await ledger.record(entry({ conversationKey: "b", turn: 1, slug: "x/three", createdAtMs: NOW - HOUR }));
149
+ const r = await buildUsageReport(db, { windowDays: 1, nowMs: NOW });
142
150
  expect(r.totals.modelSwitches).toBe(1);
143
151
  db.close();
144
152
  });
145
153
 
146
- test("window and harness scope exclude rows", () => {
147
- const { db, ledger } = seeded();
148
- ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - HOUR }));
149
- ledger.record(entry({ harnessId: "hermes", createdAtMs: NOW - HOUR }));
150
- ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - 10 * 24 * HOUR }));
151
- expect(buildUsageReport(db, { windowDays: 7, nowMs: NOW }).totals.dispatches).toBe(2);
152
- expect(buildUsageReport(db, { windowDays: 30, nowMs: NOW }).totals.dispatches).toBe(3);
153
- const scoped = buildUsageReport(db, { windowDays: 30, harnessId: "omp", nowMs: NOW });
154
+ test("window and harness scope exclude rows", async () => {
155
+ const { db, ledger } = await seeded();
156
+ await ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - HOUR }));
157
+ await ledger.record(entry({ harnessId: "hermes", createdAtMs: NOW - HOUR }));
158
+ await ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - 10 * 24 * HOUR }));
159
+ expect((await buildUsageReport(db, { windowDays: 7, nowMs: NOW })).totals.dispatches).toBe(2);
160
+ expect((await buildUsageReport(db, { windowDays: 30, nowMs: NOW })).totals.dispatches).toBe(3);
161
+ const scoped = await buildUsageReport(db, { windowDays: 30, harnessId: "omp", nowMs: NOW });
154
162
  expect(scoped.totals.dispatches).toBe(2);
155
163
  expect(scoped.harnessId).toBe("omp");
156
164
  db.close();
157
165
  });
158
166
 
159
- test("a comma-separated harness list reports the union (a team group)", () => {
160
- const { db, ledger } = seeded();
167
+ test("a comma-separated harness list reports the union (a team group)", async () => {
168
+ const { db, ledger } = await seeded();
161
169
  try {
162
- ledger.record(entry({ harnessId: "u_a", reportedUsd: 1 }));
163
- ledger.record(entry({ harnessId: "u_b", reportedUsd: 2 }));
164
- ledger.record(entry({ harnessId: "u_c", reportedUsd: 4 }));
165
- expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_a,u_b" }).totals.spendUsd).toBeCloseTo(3, 6);
166
- expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: " u_c , u_a " }).totals.spendUsd).toBeCloseTo(5, 6);
167
- expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_b" }).totals.spendUsd).toBeCloseTo(2, 6);
170
+ await ledger.record(entry({ harnessId: "u_a", reportedUsd: 1 }));
171
+ await ledger.record(entry({ harnessId: "u_b", reportedUsd: 2 }));
172
+ await ledger.record(entry({ harnessId: "u_c", reportedUsd: 4 }));
173
+ expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_a,u_b" })).totals.spendUsd).toBeCloseTo(3, 6);
174
+ expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: " u_c , u_a " })).totals.spendUsd).toBeCloseTo(5, 6);
175
+ expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_b" })).totals.spendUsd).toBeCloseTo(2, 6);
168
176
  } finally {
169
177
  db.close();
170
178
  }
171
179
  });
172
180
 
173
- test("prompt anatomy averages the recorded byte shares", () => {
174
- const { db, ledger } = seeded();
181
+ test("prompt anatomy averages the recorded byte shares", async () => {
182
+ const { db, ledger } = await seeded();
175
183
  const feat = (tool: number, older: number, stale: number) => ({ toolSchemaBytes: 1000, anatomy: { messages: 30, systemBytes: 1000, userBytes: 500, assistantBytes: 500, toolBytes: tool, olderHalfBytes: older, staleToolBytes: stale } });
176
- ledger.record(entry({ features: feat(8000, 5000, 4000) }));
177
- ledger.record(entry({ features: feat(6000, 4000, 2000) }));
178
- ledger.record(entry({ features: null })); // pre-anatomy row: ignored
179
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
184
+ await ledger.record(entry({ features: feat(8000, 5000, 4000) }));
185
+ await ledger.record(entry({ features: feat(6000, 4000, 2000) }));
186
+ await ledger.record(entry({ features: null })); // pre-anatomy row: ignored
187
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
180
188
  const a = r.anatomy!;
181
189
  expect(a.rows).toBe(2);
182
190
  // mean bytes: system 1000, user 500, assistant 500, tool 7000 ⇒ total 9000
@@ -189,10 +197,10 @@ describe("buildUsageReport", () => {
189
197
  db.close();
190
198
  });
191
199
 
192
- test("baselines price the window on one model with its own cache hit rate", () => {
193
- const { db, ledger } = seeded();
194
- ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
195
- const r = buildUsageReport(db, {
200
+ test("baselines price the window on one model with its own cache hit rate", async () => {
201
+ const { db, ledger } = await seeded();
202
+ await ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
203
+ const r = await buildUsageReport(db, {
196
204
  windowDays: 7,
197
205
  nowMs: NOW,
198
206
  baselines: [
@@ -211,21 +219,21 @@ describe("buildUsageReport", () => {
211
219
  db.close();
212
220
  });
213
221
 
214
- test("subagent turns are counted with their spend", () => {
215
- const { db, ledger } = seeded();
216
- ledger.record(entry({ reportedUsd: 0.01, features: { isSubagent: true } }));
217
- ledger.record(entry({ reportedUsd: 0.03, features: { isSubagent: false } }));
218
- ledger.record(entry({ reportedUsd: 0.06 }));
219
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
222
+ test("subagent turns are counted with their spend", async () => {
223
+ const { db, ledger } = await seeded();
224
+ await ledger.record(entry({ reportedUsd: 0.01, features: { isSubagent: true } }));
225
+ await ledger.record(entry({ reportedUsd: 0.03, features: { isSubagent: false } }));
226
+ await ledger.record(entry({ reportedUsd: 0.06 }));
227
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
220
228
  expect(r.totals.subagentDispatches).toBe(1);
221
229
  expect(r.totals.subagentSpendUsd).toBeCloseTo(0.01, 6);
222
230
  expect(renderUsageReport(r)).toContain("subagents: 1 dispatches, $0.0100 (10% of spend)");
223
231
  db.close();
224
232
  });
225
233
 
226
- test("empty ledger yields zeroed totals and null speeds", () => {
227
- const { db } = seeded();
228
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
234
+ test("empty ledger yields zeroed totals and null speeds", async () => {
235
+ const { db } = await seeded();
236
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
229
237
  expect(r.totals).toEqual({
230
238
  dispatches: 0,
231
239
  conversations: 0,
@@ -254,11 +262,11 @@ describe("buildUsageReport", () => {
254
262
  db.close();
255
263
  });
256
264
 
257
- test("speed ignores errored and non-streamed rows", () => {
258
- const { db, ledger } = seeded();
259
- ledger.record(entry({ ttftMs: null }));
260
- ledger.record(entry({ error: "boom" }));
261
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
265
+ test("speed ignores errored and non-streamed rows", async () => {
266
+ const { db, ledger } = await seeded();
267
+ await ledger.record(entry({ ttftMs: null }));
268
+ await ledger.record(entry({ error: "boom" }));
269
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
262
270
  expect(r.providers[0]!.avgTtftMs).toBeNull();
263
271
  expect(r.providers[0]!.tokensPerSec).toBeNull();
264
272
  db.close();
@@ -267,10 +275,10 @@ describe("buildUsageReport", () => {
267
275
 
268
276
  describe("renderUsageReport", () => {
269
277
  test("router-estimated cache counts render as an estimate", async () => {
270
- const { db, ledger } = seeded();
271
- ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
272
- ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
273
- const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
278
+ const { db, ledger } = await seeded();
279
+ await ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
280
+ await ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
281
+ const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
274
282
  expect(r.totals.cacheEstimated).toBe(true);
275
283
  expect(r.providers.find((p) => p.key === "ollama")!.cacheEstimated).toBe(true);
276
284
  expect(r.providers.find((p) => p.key === "openrouter")!.cacheEstimated).toBe(false);
@@ -281,11 +289,11 @@ describe("renderUsageReport", () => {
281
289
  db.close();
282
290
  });
283
291
 
284
- test("renders every section as plain fixed-width text", () => {
285
- const { db, ledger } = seeded();
286
- ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
287
- ledger.record(entry({ slug: "ollama/kimi", servedSlug: "ollama/kimi", tier: "hard", reportedUsd: 0.5, createdAtMs: NOW - 30 * HOUR }));
288
- const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }));
292
+ test("renders every section as plain fixed-width text", async () => {
293
+ const { db, ledger } = await seeded();
294
+ await ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
295
+ await ledger.record(entry({ slug: "ollama/kimi", servedSlug: "ollama/kimi", tier: "hard", reportedUsd: 0.5, createdAtMs: NOW - 30 * HOUR }));
296
+ const text = renderUsageReport(await buildUsageReport(db, { windowDays: 7, nowMs: NOW }));
289
297
  expect(text).toContain("last 7d");
290
298
  expect(text).toContain("spend $1.75 over 2 dispatches");
291
299
  expect(text).toContain("providers");
@@ -299,35 +307,35 @@ describe("renderUsageReport", () => {
299
307
  db.close();
300
308
  });
301
309
 
302
- test("caps the model table and says so", () => {
303
- const { db, ledger } = seeded();
304
- for (let i = 0; i < 5; i++) ledger.record(entry({ slug: `v/m${i}`, servedSlug: `v/m${i}` }));
305
- const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }), { maxModels: 2 });
310
+ test("caps the model table and says so", async () => {
311
+ const { db, ledger } = await seeded();
312
+ for (let i = 0; i < 5; i++) await ledger.record(entry({ slug: `v/m${i}`, servedSlug: `v/m${i}` }));
313
+ const text = renderUsageReport(await buildUsageReport(db, { windowDays: 7, nowMs: NOW }), { maxModels: 2 });
306
314
  expect(text).toContain("models (top 2 of 5 by spend)");
307
315
  db.close();
308
316
  });
309
317
  });
310
318
 
311
319
  describe("digest re-runs and forecast accuracy", () => {
312
- test("wasted digest rows count as re-runs; forecast error is judged on clean kept rows only", () => {
313
- const { db, ledger } = seeded();
320
+ test("wasted digest rows count as re-runs; forecast error is judged on clean kept rows only", async () => {
321
+ const { db, ledger } = await seeded();
314
322
  try {
315
- ledger.record(entry({ requestedModel: "digest", conversationKey: "d1", reportedUsd: 0.001, predictedUsd: 0.001 }));
316
- ledger.record(entry({ requestedModel: "digest", conversationKey: "d2", reportedUsd: 0.001, predictedUsd: 0.001, wasted: true }));
323
+ await ledger.record(entry({ requestedModel: "digest", conversationKey: "d1", reportedUsd: 0.001, predictedUsd: 0.001 }));
324
+ await ledger.record(entry({ requestedModel: "digest", conversationKey: "d2", reportedUsd: 0.001, predictedUsd: 0.001, wasted: true }));
317
325
  // Two clean turns: one predicted double, one predicted half.
318
- ledger.record(entry({ predictedUsd: 0.02, reportedUsd: 0.01 }));
319
- ledger.record(entry({ predictedUsd: 0.005, reportedUsd: 0.01 }));
326
+ await ledger.record(entry({ predictedUsd: 0.02, reportedUsd: 0.01 }));
327
+ await ledger.record(entry({ predictedUsd: 0.005, reportedUsd: 0.01 }));
320
328
  // Excluded from the forecast judgement: wasted, errored, no reported cost.
321
- ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, wasted: true }));
322
- ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, error: "upstream_error: 500" }));
323
- ledger.record(entry({ predictedUsd: 1, reportedUsd: null }));
324
- const t = buildUsageReport(db, { windowDays: 1, nowMs: NOW }).totals;
329
+ await ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, wasted: true }));
330
+ await ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, error: "upstream_error: 500" }));
331
+ await ledger.record(entry({ predictedUsd: 1, reportedUsd: null }));
332
+ const t = (await buildUsageReport(db, { windowDays: 1, nowMs: NOW })).totals;
325
333
  expect(t.digests).toBe(2);
326
334
  expect(t.digestReruns).toBe(1);
327
335
  expect(t.forecastSamples).toBe(2);
328
336
  expect(t.forecastMeanError).toBeCloseTo((1 + 0.5) / 2, 6);
329
337
  expect(t.forecastOverShare).toBeCloseTo(0.5, 6);
330
- const text = renderUsageReport(buildUsageReport(db, { windowDays: 1, nowMs: NOW }));
338
+ const text = renderUsageReport(await buildUsageReport(db, { windowDays: 1, nowMs: NOW }));
331
339
  expect(text).toContain("re-run rate 50% (1 fetched again in full)");
332
340
  expect(text).toContain("forecast: mean error 75% of reported cost over 2 turns · 50% over-predicted");
333
341
  } finally {
@@ -1,4 +1,6 @@
1
1
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2
+ import { migrateStore } from "../src/util/schema.ts";
3
+ import { num, openSqlDb, type SqlDb } from "../src/util/sql.ts";
2
4
  import { mkdtempSync, rmSync } from "node:fs";
3
5
  import { tmpdir } from "node:os";
4
6
  import { join } from "node:path";
@@ -6,14 +8,14 @@ import { join } from "node:path";
6
8
  import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
7
9
  import type { RouterConfig } from "../src/config/types.ts";
8
10
  import { createFeedbackStore } from "../src/cost/feedback.ts";
9
- import { createLedger } from "../src/cost/ledger.ts";
11
+ import { fakeLedger } from "./fakes.ts";
12
+ import { createSqlLedger } from "../src/cost/ledger-sql.ts";
10
13
  import { createRetentionRunner, RETENTION_INTERVAL_MS } from "../src/cost/retention.ts";
11
- import { EMPTY_USAGE, type Ledger, type LedgerEntry, type PruneResult } from "../src/cost/types.ts";
14
+ import { EMPTY_USAGE, type AsyncLedger, type LedgerEntry, type PruneResult } from "../src/cost/types.ts";
12
15
  import { startServer, type StartedServer } from "../src/server/http.ts";
13
- import { openDb } from "../src/util/sqlite.ts";
14
16
 
15
17
  /**
16
- * Ledger retention: how long turns are kept, what goes with them, and the
18
+ * AsyncLedger retention: how long turns are kept, what goes with them, and the
17
19
  * route a front door asks through — the team edition holds a read-only handle
18
20
  * on the ledger and must never delete from it itself.
19
21
  */
@@ -59,124 +61,133 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
59
61
  };
60
62
  }
61
63
 
62
- function seeded(path = ":memory:", base = NOW): { db: ReturnType<typeof openDb>; ledger: Ledger; ids: string[] } {
63
- const db = openDb(path);
64
+ async function seeded(
65
+ path = join(tmpdir(), `retention-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`),
66
+ base = NOW,
67
+ ): Promise<{ db: SqlDb; ledger: AsyncLedger; ids: string[] }> {
68
+ const db = openSqlDb(path);
69
+ await migrateStore(db);
64
70
  const cfg: RouterConfig = { ...DEFAULT_CONFIG, ledger: { ...DEFAULT_CONFIG.ledger, path } };
65
- const ledger = createLedger(db, cfg);
71
+ const ledger = createSqlLedger(db, cfg, { findModel: () => null });
66
72
  // 400, 200 and 1 days old.
67
- const ids = [400, 200, 1].map((age) => {
73
+ const ids: string[] = [];
74
+ for (const age of [400, 200, 1]) {
68
75
  const e = entry({ createdAtMs: base - age * DAY });
69
- ledger.record(e);
70
- return e.id;
71
- });
76
+ await ledger.record(e);
77
+ ids.push(e.id);
78
+ }
72
79
  return { db, ledger, ids };
73
80
  }
74
81
 
75
82
  describe("prune", () => {
76
- test("rows past the window go, newer ones stay, and the oldest kept is reported", () => {
77
- const { db, ledger } = seeded();
83
+ test("rows past the window go, newer ones stay, and the oldest kept is reported", async () => {
84
+ const { db, ledger } = await seeded();
78
85
  try {
79
- expect(ledger.prune?.(365, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - 200 * DAY });
80
- expect(ledger.recentEntries(10)).toHaveLength(2);
81
- expect(ledger.prune?.(30, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - DAY });
82
- expect(ledger.recentEntries(10)).toHaveLength(1);
86
+ expect(await ledger.prune(365, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - 200 * DAY });
87
+ expect(await ledger.recentEntries(10)).toHaveLength(2);
88
+ expect(await ledger.prune(30, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - DAY });
89
+ expect(await ledger.recentEntries(10)).toHaveLength(1);
83
90
  } finally {
84
- db.close();
91
+ await db.close();
85
92
  }
86
93
  });
87
94
 
88
- test("a null window (the default) deletes nothing, and neither does 0", () => {
89
- const { db, ledger } = seeded();
95
+ test("a null window (the default) deletes nothing, and neither does 0", async () => {
96
+ const { db, ledger } = await seeded();
90
97
  try {
91
98
  expect(DEFAULT_CONFIG.ledger.retentionDays).toBeNull();
92
- expect(ledger.prune?.(null, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
93
- expect(ledger.prune?.(0, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
94
- expect(ledger.recentEntries(10)).toHaveLength(3);
99
+ expect(await ledger.prune(null, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
100
+ expect(await ledger.prune(0, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
101
+ expect(await ledger.recentEntries(10)).toHaveLength(3);
95
102
  } finally {
96
- db.close();
103
+ await db.close();
97
104
  }
98
105
  });
99
106
 
100
- test("feedback keyed to a deleted turn goes with it; a verdict on a kept turn stays", () => {
101
- const { db, ledger, ids } = seeded();
107
+ test("feedback keyed to a deleted turn goes with it; a verdict on a kept turn stays", async () => {
108
+ const { db, ledger, ids } = await seeded();
102
109
  try {
103
110
  const feedback = createFeedbackStore(db);
104
111
  const [old, mid, recent] = ids as [string, string, string];
105
- feedback.record({ ledgerId: old, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "bad", note: "" }, NOW - 400 * DAY);
106
- feedback.record({ ledgerId: mid, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - 200 * DAY);
107
- feedback.record({ ledgerId: recent, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - DAY);
108
- expect(ledger.prune?.(365, NOW)?.deleted).toBe(1);
109
- expect(feedback.forLedgerId(old)).toHaveLength(0);
110
- expect(feedback.forLedgerId(mid)).toHaveLength(1);
111
- expect(feedback.forLedgerId(recent)).toHaveLength(1);
112
+ await feedback.record({ ledgerId: old, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "bad", note: "" }, NOW - 400 * DAY);
113
+ await feedback.record({ ledgerId: mid, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - 200 * DAY);
114
+ await feedback.record({ ledgerId: recent, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - DAY);
115
+ expect((await ledger.prune(365, NOW))?.deleted).toBe(1);
116
+ expect(await feedback.forLedgerId(old)).toHaveLength(0);
117
+ expect(await feedback.forLedgerId(mid)).toHaveLength(1);
118
+ expect(await feedback.forLedgerId(recent)).toHaveLength(1);
112
119
  } finally {
113
- db.close();
120
+ await db.close();
114
121
  }
115
122
  });
116
123
 
117
- test("ollama meter samples age out with the rows they calibrate", () => {
118
- const { db, ledger } = seeded();
124
+ test("ollama meter samples age out with the rows they calibrate", async () => {
125
+ const { db, ledger } = await seeded();
119
126
  try {
120
- db.run("INSERT INTO ollama_meter_samples (at_ms, meter_usd, ledger_usd) VALUES (?, 1, 1), (?, 2, 2)", [NOW - 400 * DAY, NOW - DAY]);
121
- ledger.prune?.(365, NOW);
122
- expect((db.query("SELECT COUNT(*) AS n FROM ollama_meter_samples").get() as { n: number }).n).toBe(1);
127
+ for (const atMs of [NOW - 400 * DAY, NOW - DAY]) {
128
+ await db.sql`INSERT INTO ollama_meter_samples (at_ms, meter_usd, ledger_usd) VALUES (${atMs}, 1, 1)`;
129
+ }
130
+ await ledger.prune(365, NOW);
131
+ const kept = await db.one<{ n: unknown }>("SELECT COUNT(*) AS n FROM ollama_meter_samples");
132
+ expect(num(kept?.n)).toBe(1);
123
133
  } finally {
124
- db.close();
134
+ await db.close();
125
135
  }
126
136
  });
127
137
 
128
- test("an empty ledger reports no oldest row rather than a cutoff of its own", () => {
129
- const db = openDb(":memory:");
138
+ test("an empty ledger reports no oldest row rather than a cutoff of its own", async () => {
139
+ const db = openSqlDb(join(tmpdir(), `t-retention.test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`));
140
+ await migrateStore(db);
130
141
  try {
131
- const ledger = createLedger(db, { ...DEFAULT_CONFIG, ledger: { ...DEFAULT_CONFIG.ledger, path: ":memory:" } });
132
- expect(ledger.prune?.(30, NOW)).toEqual({ deleted: 0, oldestKeptMs: null });
142
+ const ledger = createSqlLedger(db, DEFAULT_CONFIG, { findModel: () => null });
143
+ expect(await ledger.prune(30, NOW)).toEqual({ deleted: 0, oldestKeptMs: null });
133
144
  } finally {
134
- db.close();
145
+ await db.close();
135
146
  }
136
147
  });
137
148
  });
138
149
 
139
150
  describe("the schedule", () => {
140
- function counting(): { ledger: Ledger; runs: number[] } {
151
+ function counting(): { ledger: AsyncLedger; runs: number[] } {
141
152
  const runs: number[] = [];
142
- const ledger = {
143
- prune: (_days: number | null, nowMs?: number): PruneResult => {
153
+ const ledger = fakeLedger({
154
+ prune: async (_days: number | null, nowMs?: number): Promise<PruneResult> => {
144
155
  runs.push(nowMs ?? 0);
145
156
  return { deleted: 1, oldestKeptMs: null };
146
157
  },
147
- } as unknown as Ledger;
158
+ });
148
159
  return { ledger, runs };
149
160
  }
150
161
 
151
- test("the scheduled path runs at most once an hour, however often it is asked", () => {
162
+ test("the scheduled path runs at most once an hour, however often it is asked", async () => {
152
163
  const { ledger, runs } = counting();
153
164
  const runner = createRetentionRunner({ ledger, retentionDays: () => 30 });
154
165
  // The first call is always due, so a lowered window applies at boot.
155
- expect(runner.maybeRun(NOW)?.deleted).toBe(1);
166
+ expect((await runner.maybeRun(NOW))?.deleted).toBe(1);
156
167
  // A minute timer ticking for an hour must not re-run it.
157
- for (let t = 60_000; t < RETENTION_INTERVAL_MS; t += 60_000) expect(runner.maybeRun(NOW + t)).toBeNull();
168
+ for (let t = 60_000; t < RETENTION_INTERVAL_MS; t += 60_000) expect(await runner.maybeRun(NOW + t)).toBeNull();
158
169
  expect(runs).toEqual([NOW]);
159
- expect(runner.maybeRun(NOW + RETENTION_INTERVAL_MS)?.deleted).toBe(1);
170
+ expect((await runner.maybeRun(NOW + RETENTION_INTERVAL_MS))?.deleted).toBe(1);
160
171
  expect(runs).toEqual([NOW, NOW + RETENTION_INTERVAL_MS]);
161
172
  });
162
173
 
163
- test("an explicit run always happens, and satisfies the schedule for the next hour", () => {
174
+ test("an explicit run always happens, and satisfies the schedule for the next hour", async () => {
164
175
  const { ledger, runs } = counting();
165
176
  const runner = createRetentionRunner({ ledger, retentionDays: () => 30 });
166
- runner.runNow(NOW);
167
- runner.runNow(NOW + 1_000);
177
+ await runner.runNow(NOW);
178
+ await runner.runNow(NOW + 1_000);
168
179
  expect(runs).toEqual([NOW, NOW + 1_000]);
169
- expect(runner.maybeRun(NOW + 2_000)).toBeNull();
170
- expect(runner.maybeRun(NOW + 1_000 + RETENTION_INTERVAL_MS)).not.toBeNull();
180
+ expect(await runner.maybeRun(NOW + 2_000)).toBeNull();
181
+ expect(await runner.maybeRun(NOW + 1_000 + RETENTION_INTERVAL_MS)).not.toBeNull();
171
182
  });
172
183
 
173
- test("the window is read live, so a hot reload applies on the next run", () => {
184
+ test("the window is read live, so a hot reload applies on the next run", async () => {
174
185
  let days: number | null = null;
175
186
  const { ledger, runs } = counting();
176
187
  const runner = createRetentionRunner({ ledger, retentionDays: () => days });
177
188
  expect(runner.retentionDays()).toBeNull();
178
189
  days = 7;
179
- runner.runNow(NOW);
190
+ await runner.runNow(NOW);
180
191
  expect(runner.retentionDays()).toBe(7);
181
192
  expect(runs).toHaveLength(1);
182
193
  });
@@ -191,11 +202,11 @@ describe("POST /v1/router/prune", () => {
191
202
  // relative to it rather than to the fixed NOW the unit tests use.
192
203
  const realNow = Date.now();
193
204
 
194
- beforeAll(() => {
205
+ beforeAll(async () => {
195
206
  dir = mkdtempSync(join(tmpdir(), "amr-retention-"));
196
207
  dbPath = join(dir, "router.db");
197
- const { db } = seeded(dbPath, realNow);
198
- db.close();
208
+ const { db } = await seeded(dbPath, realNow);
209
+ await db.close();
199
210
  const cfg: RouterConfig = {
200
211
  ...structuredClone(DEFAULT_CONFIG),
201
212
  server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
@@ -223,11 +234,13 @@ describe("POST /v1/router/prune", () => {
223
234
  expect(body.oldestKeptMs).toBe(realNow - 200 * DAY);
224
235
  expect(body.retentionDays).toBe(365);
225
236
  // The rows are really gone from the file, not just from a view.
226
- const db = openDb(dbPath);
237
+ const db = openSqlDb(dbPath);
238
+ await migrateStore(db);
227
239
  try {
228
- expect((db.query("SELECT COUNT(*) AS n FROM ledger").get() as { n: number }).n).toBe(2);
240
+ const left = await db.one<{ n: unknown }>("SELECT COUNT(*) AS n FROM ledger");
241
+ expect(num(left?.n)).toBe(2);
229
242
  } finally {
230
- db.close();
243
+ await db.close();
231
244
  }
232
245
  });
233
246