auto-model-router 0.2.32 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +225 -29
  3. package/docs/review-2026-09-05.md +267 -0
  4. package/omp-extension/configure-logic.ts +71 -15
  5. package/omp-extension/pi-coding-agent.d.ts +79 -2
  6. package/omp-extension/report-hub.ts +376 -0
  7. package/omp-extension/report-logic.ts +117 -0
  8. package/omp-extension/router-configure.ts +203 -51
  9. package/omp-extension/router-url.ts +52 -0
  10. package/omp-extension/toast-logic.ts +14 -2
  11. package/package.json +1 -1
  12. package/src/catalog/composite.ts +97 -0
  13. package/src/catalog/ollama-catalog.ts +309 -0
  14. package/src/catalog/ollama-prices.ts +85 -0
  15. package/src/catalog/openrouter-catalog.ts +39 -1
  16. package/src/catalog/types.ts +31 -1
  17. package/src/cli/args.ts +1 -0
  18. package/src/cli/config-wizard.ts +190 -28
  19. package/src/cli/explain.ts +2 -4
  20. package/src/cli/models.ts +2 -4
  21. package/src/cli/report.ts +37 -0
  22. package/src/config/defaults.ts +46 -2
  23. package/src/config/load.ts +25 -1
  24. package/src/config/omp-credentials.ts +31 -7
  25. package/src/config/schema.ts +28 -0
  26. package/src/config/types.ts +120 -2
  27. package/src/cost/cache-estimate.ts +52 -0
  28. package/src/cost/ledger.ts +73 -4
  29. package/src/cost/report.ts +351 -0
  30. package/src/cost/types.ts +39 -1
  31. package/src/index.ts +5 -8
  32. package/src/router/candidates.ts +52 -4
  33. package/src/router/classify.ts +33 -6
  34. package/src/router/features.ts +13 -1
  35. package/src/router/select.ts +55 -8
  36. package/src/router/state.ts +6 -2
  37. package/src/router/tier-plan.ts +49 -11
  38. package/src/router/types.ts +10 -0
  39. package/src/server/http.ts +50 -6
  40. package/src/server/providers.ts +54 -0
  41. package/src/server/turn.ts +138 -34
  42. package/src/tokens/estimate.ts +16 -0
  43. package/src/upstream/multi.ts +26 -0
  44. package/src/upstream/ollama-usage.ts +163 -0
  45. package/src/upstream/ollama.ts +275 -0
  46. package/src/upstream/openrouter.ts +19 -1
  47. package/src/upstream/types.ts +2 -0
  48. package/src/util/sqlite.ts +25 -1
  49. package/test/cache-estimate.test.ts +48 -0
  50. package/test/catalog.test.ts +44 -0
  51. package/test/classify.test.ts +41 -5
  52. package/test/compaction.test.ts +1 -0
  53. package/test/config-wizard.test.ts +77 -1
  54. package/test/configure-logic.test.ts +129 -33
  55. package/test/embed-lifecycle.test.ts +1 -0
  56. package/test/failover.test.ts +148 -3
  57. package/test/features.test.ts +35 -0
  58. package/test/http-resilience.test.ts +24 -0
  59. package/test/ollama.test.ts +521 -0
  60. package/test/omp-credentials.test.ts +43 -1
  61. package/test/report-hub.test.ts +343 -0
  62. package/test/report-logic.test.ts +93 -0
  63. package/test/report.test.ts +233 -0
  64. package/test/select.test.ts +151 -1
  65. package/test/tier-plan.test.ts +159 -1
  66. package/test/toast-logic.test.ts +11 -2
  67. package/test/tokens.test.ts +71 -1
  68. package/test/trust-attribution.test.ts +2 -2
  69. package/test/turn.test.ts +173 -7
  70. package/tools/recompute-ollama-cache.ts +129 -0
@@ -0,0 +1,343 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { type HubKeys, type HubTheme, ReportHub, WINDOWS } from "../omp-extension/report-hub.ts";
4
+ import type { UsageReport } from "../src/cost/report.ts";
5
+
6
+ /**
7
+ * The hub is a pure component: identity styling, ASCII box glyphs and raw
8
+ * key names stand in for omp's theme, pi-tui and keybindings. These pin the
9
+ * frame geometry, navigation, window/scope cycling and the async load path.
10
+ */
11
+
12
+ const theme: HubTheme = {
13
+ fg: (_c, t) => t,
14
+ bg: (_c, t) => t,
15
+ bold: (t) => t,
16
+ boxRound: {
17
+ topLeft: "+",
18
+ topRight: "+",
19
+ bottomLeft: "+",
20
+ bottomRight: "+",
21
+ horizontal: "-",
22
+ vertical: "|",
23
+ teeDown: "T",
24
+ teeUp: "U",
25
+ teeLeft: "<",
26
+ teeRight: ">",
27
+ },
28
+ nav: { cursor: ">" },
29
+ };
30
+
31
+ const text = {
32
+ visibleWidth: (s: string) => s.length,
33
+ truncateToWidth: (s: string, w: number) => (s.length <= w ? s : s.slice(0, w)),
34
+ };
35
+
36
+ const keys: HubKeys = {
37
+ up: (d) => d === "UP",
38
+ down: (d) => d === "DOWN",
39
+ left: (d) => d === "LEFT",
40
+ right: (d) => d === "RIGHT",
41
+ pageUp: (d) => d === "PGUP",
42
+ pageDown: (d) => d === "PGDN",
43
+ cancel: (d) => d === "ESC",
44
+ confirm: (d) => d === "ENTER",
45
+ };
46
+
47
+ function report(over: Partial<UsageReport> = {}): UsageReport {
48
+ const row = (key: string, spend: number) => ({
49
+ key,
50
+ dispatches: 10,
51
+ spendUsd: spend,
52
+ share: 0.5,
53
+ cacheHitRate: 0.8,
54
+ cacheEstimated: false,
55
+ avgPromptTokens: 1000,
56
+ avgTtftMs: 900,
57
+ tokensPerSec: 120,
58
+ escalations: 1,
59
+ errors: 0,
60
+ });
61
+ return {
62
+ generatedAtMs: Date.UTC(2026, 8, 6, 12),
63
+ windowDays: 7,
64
+ sinceMs: 0,
65
+ harnessId: "",
66
+ totals: {
67
+ dispatches: 20,
68
+ conversations: 2,
69
+ spendUsd: 3,
70
+ cacheHitRate: 0.8,
71
+ promptTokens: 20000,
72
+ completionTokens: 2000,
73
+ escalations: 2,
74
+ failovers: 0,
75
+ errors: 0,
76
+ aborted: 0,
77
+ modelSwitches: 1,
78
+ cacheEstimated: false,
79
+ },
80
+ providers: [row("openrouter", 2), row("ollama", 1)],
81
+ models: [
82
+ { ...row("z-ai/glm", 2), provider: "openrouter", tiers: { simple: 6, moderate: 4 } },
83
+ { ...row("ollama/kimi", 1), provider: "ollama", tiers: { hard: 10 } },
84
+ ],
85
+ tiers: [row("simple", 2), row("hard", 1)],
86
+ days: [
87
+ { day: "2026-09-05", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
88
+ { day: "2026-09-06", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
89
+ ],
90
+ ...over,
91
+ };
92
+ }
93
+
94
+ interface Harness {
95
+ hub: ReportHub;
96
+ renders: number;
97
+ closed: boolean;
98
+ requests: { windowDays: number; harnessId: string }[];
99
+ settle(): Promise<void>;
100
+ }
101
+
102
+ function mount(opts: { harnessId?: string; initialHarness?: string; fail?: boolean; rows?: number } = {}): Harness {
103
+ const h: Harness = {
104
+ hub: undefined as unknown as ReportHub,
105
+ renders: 0,
106
+ closed: false,
107
+ requests: [],
108
+ settle: async () => {
109
+ await new Promise((r) => setTimeout(r, 0));
110
+ await new Promise((r) => setTimeout(r, 0));
111
+ },
112
+ };
113
+ h.hub = new ReportHub({
114
+ theme,
115
+ text,
116
+ keys,
117
+ source: {
118
+ report: async (req) => {
119
+ h.requests.push({ ...req });
120
+ if (opts.fail === true) throw new Error("boom");
121
+ return report({ windowDays: req.windowDays, harnessId: req.harnessId });
122
+ },
123
+ status: async () => "auto-model-router at http://h: ok\nopenrouter: key configured (omp)",
124
+ },
125
+ rows: () => opts.rows ?? 30,
126
+ requestRender: () => {
127
+ h.renders++;
128
+ },
129
+ close: () => {
130
+ h.closed = true;
131
+ },
132
+ initial: { windowDays: 7, harnessId: opts.initialHarness ?? "" },
133
+ harnessId: opts.harnessId ?? "",
134
+ });
135
+ return h;
136
+ }
137
+
138
+ describe("ReportHub frame", () => {
139
+ test("draws a titled two-column frame sized to the terminal", async () => {
140
+ const h = mount({ rows: 30 });
141
+ await h.settle();
142
+ const lines = h.hub.render(100);
143
+ // 1 top + (rows-4) content + divider + footer + bottom
144
+ expect(lines).toHaveLength(30);
145
+ expect(lines[0]).toMatch(/^\+- Router report -+T-+\+$/);
146
+ expect(lines[lines.length - 1]).toMatch(/^\+-+\+$/);
147
+ expect(lines[lines.length - 3]).toMatch(/^>-+U-+<$/);
148
+ for (const l of lines) expect(l.length).toBe(100);
149
+ // Every content row is | sidebar | body |.
150
+ expect(lines[1]).toMatch(/^\| .{18} \| .* \|$/);
151
+ });
152
+
153
+ test("overview shows the summary, provider and model tables, and the status row", async () => {
154
+ const h = mount();
155
+ await h.settle();
156
+ const body = h.hub.render(120).join("\n");
157
+ expect(body).toContain("> ◎ Overview");
158
+ expect(body).toContain("last 7d");
159
+ expect(body).toContain("all harnesses");
160
+ expect(body).toContain("spend $3.00 over 20 dispatches");
161
+ expect(body).toContain("providers");
162
+ expect(body).toContain("openrouter");
163
+ expect(body).toContain("z-ai/glm");
164
+ expect(body).toContain("window (7d)");
165
+ });
166
+
167
+ test("shows loading before data arrives and the error when the load fails", async () => {
168
+ const h = mount({ fail: true });
169
+ expect(h.hub.render(100).join("\n")).toContain("loading…");
170
+ await h.settle();
171
+ const out = h.hub.render(100).join("\n");
172
+ expect(out).toContain("could not load: boom");
173
+ expect(h.renders).toBeGreaterThanOrEqual(2);
174
+ });
175
+ });
176
+
177
+ describe("ReportHub navigation", () => {
178
+ test("up/down move through views, window entries and status, skipping rules and labels, and wrap", async () => {
179
+ const h = mount();
180
+ await h.settle();
181
+ expect(h.hub.activeView).toBe("overview");
182
+ h.hub.handleInput("DOWN");
183
+ expect(h.hub.activeView).toBe("providers");
184
+ for (let i = 0; i < 3; i++) h.hub.handleInput("DOWN");
185
+ expect(h.hub.activeView).toBe("days");
186
+ // Over the rule and the "Window" label onto the first window entry:
187
+ // the view stays put until Enter picks something.
188
+ h.hub.handleInput("DOWN");
189
+ expect(h.hub.cursorEntry).toEqual({ kind: "window", days: 1, label: "24 hours" });
190
+ expect(h.hub.activeView).toBe("days");
191
+ for (let i = 0; i < 3; i++) h.hub.handleInput("DOWN");
192
+ expect(h.hub.cursorEntry).toEqual({ kind: "window", days: 90, label: "90 days" });
193
+ h.hub.handleInput("DOWN"); // over the rule to Status (no scope entry without a harness id)
194
+ expect(h.hub.activeView).toBe("status");
195
+ expect(h.hub.render(100).join("\n")).toContain("openrouter: key configured (omp)");
196
+ h.hub.handleInput("DOWN"); // wraps
197
+ expect(h.hub.activeView).toBe("overview");
198
+ h.hub.handleInput("UP");
199
+ expect(h.hub.activeView).toBe("status");
200
+ });
201
+
202
+ test("the sidebar shows the window selector with the active window marked", async () => {
203
+ const h = mount();
204
+ await h.settle();
205
+ const side = h.hub.render(100).join("\n");
206
+ expect(side).toContain(" Window");
207
+ expect(side).toContain("○ 24 hours");
208
+ expect(side).toContain("● 7 days");
209
+ expect(side).toContain("○ 30 days");
210
+ expect(side).toContain("○ 90 days");
211
+ expect(side).not.toContain("Scope");
212
+ });
213
+
214
+ test("enter on a window entry applies it and reloads; the view is unchanged", async () => {
215
+ const h = mount();
216
+ await h.settle();
217
+ // Moving down passes every view (each shows as the cursor lands on it)
218
+ // and stops on the 30-day entry; Enter there changes the window only.
219
+ for (let i = 0; i < 7; i++) h.hub.handleInput("j");
220
+ expect(h.hub.cursorEntry).toEqual({ kind: "window", days: 30, label: "30 days" });
221
+ expect(h.hub.activeView).toBe("days");
222
+ h.hub.handleInput("ENTER");
223
+ expect(h.hub.request.windowDays).toBe(30);
224
+ expect(h.hub.activeView).toBe("days");
225
+ h.hub.handleInput("ENTER"); // same window again: no reload
226
+ await h.settle();
227
+ expect(h.requests.map((r) => r.windowDays)).toEqual([7, 30]);
228
+ expect(h.hub.render(100).join("\n")).toContain("● 30 days");
229
+ expect(h.hub.render(100).join("\n")).toContain("enter set window");
230
+ });
231
+
232
+ test("j/k are aliases and a chosen view renders only its table", async () => {
233
+ const h = mount();
234
+ await h.settle();
235
+ h.hub.handleInput("j");
236
+ h.hub.handleInput("j");
237
+ expect(h.hub.activeView).toBe("models");
238
+ const out = h.hub.render(120).join("\n");
239
+ expect(out).toContain("models (top 2 of 2 by spend)");
240
+ expect(out).toContain("simple:6 moderate:4");
241
+ expect(out).not.toContain("spend $3.00 over");
242
+ h.hub.handleInput("k");
243
+ expect(h.hub.activeView).toBe("providers");
244
+ });
245
+
246
+ test("left/right cycle the window and reload", async () => {
247
+ const h = mount();
248
+ await h.settle();
249
+ h.hub.handleInput("RIGHT");
250
+ expect(h.hub.request.windowDays).toBe(30);
251
+ h.hub.handleInput("RIGHT");
252
+ expect(h.hub.request.windowDays).toBe(90);
253
+ h.hub.handleInput("RIGHT");
254
+ expect(h.hub.request.windowDays).toBe(WINDOWS[0]!);
255
+ h.hub.handleInput("LEFT");
256
+ expect(h.hub.request.windowDays).toBe(90);
257
+ await h.settle();
258
+ expect(h.requests.map((r) => r.windowDays)).toEqual([7, 30, 90, 1, 90]);
259
+ expect(h.hub.render(100).join("\n")).toContain("last 90d");
260
+ });
261
+
262
+ test("the scope entry exists only with a harness id; enter (or a) toggles it", async () => {
263
+ const none = mount();
264
+ await none.settle();
265
+ none.hub.handleInput("a");
266
+ expect(none.hub.request.harnessId).toBe("");
267
+
268
+ const h = mount({ harnessId: "omp", initialHarness: "omp" });
269
+ await h.settle();
270
+ let side = h.hub.render(100).join("\n");
271
+ expect(side).toContain(" Scope");
272
+ expect(side).toContain("◉ this harness");
273
+ expect(side).toContain("harness omp");
274
+ // views(5) + window(4) → the scope entry is the 10th selectable.
275
+ for (let i = 0; i < 9; i++) h.hub.handleInput("DOWN");
276
+ expect(h.hub.cursorEntry).toEqual({ kind: "scope" });
277
+ h.hub.handleInput("ENTER");
278
+ expect(h.hub.request.harnessId).toBe("");
279
+ side = h.hub.render(100).join("\n");
280
+ expect(side).toContain("◎ all harnesses");
281
+ expect(side).toContain("enter toggle scope");
282
+ h.hub.handleInput("a");
283
+ expect(h.hub.request.harnessId).toBe("omp");
284
+ await h.settle();
285
+ expect(h.requests.map((r) => r.harnessId)).toEqual(["omp", "", "omp"]);
286
+ });
287
+
288
+ test("esc and q close; r reloads", async () => {
289
+ const h = mount();
290
+ await h.settle();
291
+ h.hub.handleInput("r");
292
+ await h.settle();
293
+ expect(h.requests).toHaveLength(2);
294
+ h.hub.handleInput("ESC");
295
+ expect(h.closed).toBe(true);
296
+ const h2 = mount();
297
+ h2.hub.handleInput("q");
298
+ expect(h2.closed).toBe(true);
299
+ });
300
+
301
+ test("page down scrolls a table longer than the pane and shows the remainder", async () => {
302
+ const h = mount({ rows: 16 });
303
+ await h.settle();
304
+ // 12 body rows; the overview is longer than that.
305
+ const first = h.hub.render(100).join("\n");
306
+ expect(first).toContain("more (pgdn)");
307
+ h.hub.handleInput("PGDN");
308
+ const second = h.hub.render(100).join("\n");
309
+ expect(second).not.toBe(first);
310
+ h.hub.handleInput("PGUP");
311
+ expect(h.hub.render(100).join("\n")).toBe(first);
312
+ });
313
+
314
+ test("a stale load never overwrites a newer one", async () => {
315
+ let resolveSlow: ((r: UsageReport) => void) | undefined;
316
+ const seen: number[] = [];
317
+ const hub = new ReportHub({
318
+ theme,
319
+ text,
320
+ keys,
321
+ source: {
322
+ report: (req) => {
323
+ seen.push(req.windowDays);
324
+ if (req.windowDays === 7) return new Promise<UsageReport>((r) => (resolveSlow = r));
325
+ return Promise.resolve(report({ windowDays: req.windowDays }));
326
+ },
327
+ status: async () => "ok",
328
+ },
329
+ rows: () => 30,
330
+ requestRender: () => {},
331
+ close: () => {},
332
+ initial: { windowDays: 7, harnessId: "" },
333
+ harnessId: "",
334
+ });
335
+ hub.handleInput("RIGHT"); // 30d, resolves immediately
336
+ await new Promise((r) => setTimeout(r, 0));
337
+ resolveSlow?.(report({ windowDays: 7, totals: { ...report().totals, dispatches: 999 } }));
338
+ await new Promise((r) => setTimeout(r, 0));
339
+ expect(seen).toEqual([7, 30]);
340
+ expect(hub.render(100).join("\n")).toContain("last 30d");
341
+ expect(hub.render(100).join("\n")).not.toContain("999");
342
+ });
343
+ });
@@ -0,0 +1,93 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { fetchReport, parseReportArgs, renderStatus, type HealthSnapshot } from "../omp-extension/report-logic.ts";
4
+
5
+ describe("parseReportArgs", () => {
6
+ test("defaults to 7 days scoped to the harness", () => {
7
+ expect(parseReportArgs("", "omp")).toEqual({ windowDays: 7, harnessId: "omp" });
8
+ });
9
+
10
+ test("accepts bare numbers and d/h/w suffixes", () => {
11
+ expect(parseReportArgs("30", "")).toEqual({ windowDays: 30, harnessId: "" });
12
+ expect(parseReportArgs("14d", "")).toEqual({ windowDays: 14, harnessId: "" });
13
+ expect(parseReportArgs("24h", "")).toEqual({ windowDays: 1, harnessId: "" });
14
+ expect(parseReportArgs("36h", "")).toEqual({ windowDays: 2, harnessId: "" });
15
+ expect(parseReportArgs("2w", "")).toEqual({ windowDays: 14, harnessId: "" });
16
+ });
17
+
18
+ test("--all drops the harness scope and --harness= sets one", () => {
19
+ expect(parseReportArgs("7d --all", "omp").harnessId).toBe("");
20
+ expect(parseReportArgs("--harness=hermes", "omp").harnessId).toBe("hermes");
21
+ });
22
+
23
+ test("ignores junk and clamps the window", () => {
24
+ expect(parseReportArgs("bogus 0 -3", "x")).toEqual({ windowDays: 7, harnessId: "x" });
25
+ expect(parseReportArgs("9999", "").windowDays).toBe(365);
26
+ });
27
+ });
28
+
29
+ describe("fetchReport", () => {
30
+ test("builds the query and returns the JSON body", async () => {
31
+ const seen: string[] = [];
32
+ const fake = async (url: string, init?: RequestInit): Promise<Response> => {
33
+ seen.push(url);
34
+ expect((init?.headers as Record<string, string>).authorization).toBe("Bearer k");
35
+ return new Response(JSON.stringify({ totals: { dispatches: 3 } }), { status: 200 });
36
+ };
37
+ const r = await fetchReport("http://127.0.0.1:1", { windowDays: 3, harnessId: "omp" }, { authorization: "Bearer k" }, fake);
38
+ expect(r.totals.dispatches).toBe(3);
39
+ expect(seen).toEqual(["http://127.0.0.1:1/v1/router/report?days=3&harness=omp"]);
40
+ });
41
+
42
+ test("omits the harness param when unscoped and throws on a non-2xx", async () => {
43
+ const seen: string[] = [];
44
+ const fake = async (url: string): Promise<Response> => {
45
+ seen.push(url);
46
+ return new Response("nope", { status: 503 });
47
+ };
48
+ await expect(fetchReport("http://h", { windowDays: 7, harnessId: "" }, {}, fake)).rejects.toThrow("503");
49
+ expect(seen).toEqual(["http://h/v1/router/report?days=7"]);
50
+ });
51
+ });
52
+
53
+ describe("renderStatus", () => {
54
+ test("summarises keys, catalog, ollama and agentdox", () => {
55
+ const now = 1_000_000_000;
56
+ const h: HealthSnapshot = {
57
+ status: "ok",
58
+ apiKeyConfigured: true,
59
+ apiKeySource: "omp",
60
+ agentdox: { url: "http://localhost:3003", defaultScope: "omp-router", recordTurns: true },
61
+ ollama: {
62
+ models: 12,
63
+ available: false,
64
+ cooldownUntilMs: now + 60_000,
65
+ apiKeySource: "omp",
66
+ lastTrip: { kind: "quota", atMs: now - 120_000, message: "402" },
67
+ usage: { monthlyUsedFraction: 0.42, activityCostUsd: 3.1, fetchedAtMs: now },
68
+ meter: { usedUsd: 25.2, creditsUsd: 60 },
69
+ costBias: { configured: 0.1, effective: 0.1, biasUntilUsage: 0.9 },
70
+ },
71
+ catalog: { models: 240, ageMs: 5 * 60_000, keyScoped: true, shrink: { fromModels: 300, toModels: 120, atMs: now } },
72
+ };
73
+ const text = renderStatus("http://127.0.0.1:8788", h, now);
74
+ expect(text).toContain("configured (omp)");
75
+ expect(text).toContain("240 models");
76
+ expect(text).toContain("refreshed 5m ago");
77
+ expect(text).toContain("SHRANK 300 -> 120");
78
+ expect(text).toContain("COOLING DOWN");
79
+ expect(text).toContain("plan usage 42.0% ($25.20 of $60)");
80
+ expect(text).toContain("cost bias ×0.1 (until 90%)");
81
+ expect(text).toContain("last trip quota 2m ago");
82
+ expect(text).toContain("scope omp-router");
83
+ expect(text).toContain("recording turns");
84
+ });
85
+
86
+ test("degrades cleanly when sections are absent", () => {
87
+ const text = renderStatus("http://h", { status: "ok", apiKeyConfigured: false });
88
+ expect(text).toContain("key MISSING");
89
+ expect(text).toContain("catalog: not fetched yet");
90
+ expect(text).toContain("ollama cloud: disabled");
91
+ expect(text).toContain("agentdox: off");
92
+ });
93
+ });
@@ -0,0 +1,233 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
4
+ import { createLedger } from "../src/cost/ledger.ts";
5
+ import { buildUsageReport, renderUsageReport } from "../src/cost/report.ts";
6
+ import type { LedgerEntry } from "../src/cost/types.ts";
7
+ import { openDb } from "../src/util/sqlite.ts";
8
+
9
+ /**
10
+ * `buildUsageReport` is what `/router report`, the `report` CLI and
11
+ * `GET /v1/router/report` all render. These pin the aggregation rules:
12
+ * spend follows the ledger's reported-else-predicted rule, cache hit rate is
13
+ * cached/prompt tokens, speed comes only from clean streamed rows, the
14
+ * provider is derived from the served slug, and harness scoping works.
15
+ */
16
+
17
+ const HOUR = 3_600_000;
18
+ const NOW = Date.UTC(2026, 8, 6, 12, 0, 0);
19
+
20
+ function entry(over: Partial<LedgerEntry>): LedgerEntry {
21
+ return {
22
+ id: crypto.randomUUID(),
23
+ createdAtMs: NOW - HOUR,
24
+ conversationKey: "k",
25
+ sessionId: "s",
26
+ turn: 1,
27
+ requestedModel: "auto",
28
+ harnessId: "",
29
+ ompSessionId: "",
30
+ slug: "vendor/model",
31
+ servedSlug: "vendor/model",
32
+ tier: "simple",
33
+ classificationSource: "heuristic",
34
+ reasons: [],
35
+ features: null,
36
+ score: null,
37
+ confidence: null,
38
+ task: null,
39
+ classifierReasons: null,
40
+ exploredFrom: null,
41
+ holdArm: null,
42
+ predictedUsd: 0.001,
43
+ reportedUsd: 0.001,
44
+ usage: { promptTokens: 1000, cachedTokens: 0, cacheWriteTokens: 0, completionTokens: 100, reasoningTokens: 0, images: 0 },
45
+ attempt: 0,
46
+ escalationSignal: null,
47
+ latencyMs: 1_100,
48
+ ttftMs: 100,
49
+ finishReason: "stop",
50
+ wasted: false,
51
+ upstreamGenerationId: null,
52
+ error: null,
53
+ promptTokensSaved: null,
54
+ ...over,
55
+ } as LedgerEntry;
56
+ }
57
+
58
+ function seeded() {
59
+ const cfg = structuredClone(DEFAULT_CONFIG);
60
+ cfg.ledger.path = ":memory:";
61
+ const db = openDb(":memory:");
62
+ const ledger = createLedger(db, cfg);
63
+ return { db, ledger };
64
+ }
65
+
66
+ describe("buildUsageReport", () => {
67
+ test("totals, providers, models and tiers over a mixed window", () => {
68
+ const { db, ledger } = seeded();
69
+ // Two OpenRouter turns on one model (one with a warm cache), one Ollama
70
+ // turn served by a different slug than decided, one escalation, one error.
71
+ ledger.record(entry({ conversationKey: "a", turn: 1, reportedUsd: 0.01 }));
72
+ ledger.record(
73
+ entry({
74
+ conversationKey: "a",
75
+ turn: 2,
76
+ reportedUsd: 0.005,
77
+ usage: { promptTokens: 1000, cachedTokens: 800, cacheWriteTokens: 0, completionTokens: 100, reasoningTokens: 0, images: 0 },
78
+ }),
79
+ );
80
+ ledger.record(
81
+ entry({
82
+ conversationKey: "b",
83
+ slug: "ollama/glm-5.3-flash",
84
+ servedSlug: "ollama/glm-5.3-flash",
85
+ tier: "moderate",
86
+ predictedUsd: 0.02,
87
+ reportedUsd: null,
88
+ latencyMs: 2_100,
89
+ ttftMs: 100,
90
+ }),
91
+ );
92
+ ledger.record(entry({ conversationKey: "c", tier: "hard", escalationSignal: "refusal", wasted: true, reportedUsd: 0.002 }));
93
+ ledger.record(entry({ conversationKey: "c", tier: "hard", error: "request aborted", reportedUsd: 0, ttftMs: null }));
94
+
95
+ const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
96
+ expect(r.totals.dispatches).toBe(5);
97
+ expect(r.totals.conversations).toBe(3);
98
+ // 0.01 + 0.005 + 0.02 (predicted: no reported) + 0.002 + 0
99
+ expect(r.totals.spendUsd).toBeCloseTo(0.037, 6);
100
+ expect(r.totals.cacheHitRate).toBeCloseTo(800 / 5000, 6);
101
+ expect(r.totals.escalations).toBe(1);
102
+ expect(r.totals.errors).toBe(1);
103
+ expect(r.totals.aborted).toBe(1);
104
+
105
+ // Ordered by spend: the single Ollama turn (0.02 predicted) outspends OpenRouter (0.017).
106
+ expect(r.providers.map((p) => p.key)).toEqual(["ollama", "openrouter"]);
107
+ const or = r.providers[1]!;
108
+ expect(or.dispatches).toBe(4);
109
+ expect(or.spendUsd).toBeCloseTo(0.017, 6);
110
+ expect(or.share).toBeCloseTo(0.017 / 0.037, 6);
111
+ // Speed: 3 clean streamed rows × 100 completion tokens over (1100-100) ms each.
112
+ expect(or.avgTtftMs).toBe(100);
113
+ expect(or.tokensPerSec).toBeCloseTo(100, 3);
114
+
115
+ const ollama = r.providers[0]!;
116
+ expect(ollama.dispatches).toBe(1);
117
+ expect(ollama.tokensPerSec).toBeCloseTo(50, 3);
118
+
119
+ expect(r.models.map((m) => m.key)).toEqual(["ollama/glm-5.3-flash", "vendor/model"]);
120
+ expect(r.models[0]!.provider).toBe("ollama");
121
+ expect(r.models[1]!.tiers).toEqual({ simple: 2, hard: 2 });
122
+
123
+ const tierKeys = r.tiers.map((t) => t.key).sort();
124
+ expect(tierKeys).toEqual(["hard", "moderate", "simple"]);
125
+ expect(r.tiers.find((t) => t.key === "hard")!.escalations).toBe(1);
126
+
127
+ expect(r.days).toHaveLength(1);
128
+ expect(r.days[0]!.day).toBe("2026-09-06");
129
+ db.close();
130
+ });
131
+
132
+ test("counts a model switch only between consecutive kept rows of one conversation", () => {
133
+ const { db, ledger } = seeded();
134
+ ledger.record(entry({ conversationKey: "a", turn: 1, slug: "x/one", createdAtMs: NOW - 3 * HOUR }));
135
+ ledger.record(entry({ conversationKey: "a", turn: 2, slug: "x/two", createdAtMs: NOW - 2 * HOUR }));
136
+ ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/two", createdAtMs: NOW - 1 * HOUR }));
137
+ // A wasted probe on another slug is not a switch.
138
+ ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/three", wasted: true, createdAtMs: NOW - 1 * HOUR + 1 }));
139
+ // A different conversation starting on another model is not a switch either.
140
+ ledger.record(entry({ conversationKey: "b", turn: 1, slug: "x/three", createdAtMs: NOW - HOUR }));
141
+ const r = buildUsageReport(db, { windowDays: 1, nowMs: NOW });
142
+ expect(r.totals.modelSwitches).toBe(1);
143
+ db.close();
144
+ });
145
+
146
+ test("window and harness scope exclude rows", () => {
147
+ const { db, ledger } = seeded();
148
+ ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - HOUR }));
149
+ ledger.record(entry({ harnessId: "hermes", createdAtMs: NOW - HOUR }));
150
+ ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - 10 * 24 * HOUR }));
151
+ expect(buildUsageReport(db, { windowDays: 7, nowMs: NOW }).totals.dispatches).toBe(2);
152
+ expect(buildUsageReport(db, { windowDays: 30, nowMs: NOW }).totals.dispatches).toBe(3);
153
+ const scoped = buildUsageReport(db, { windowDays: 30, harnessId: "omp", nowMs: NOW });
154
+ expect(scoped.totals.dispatches).toBe(2);
155
+ expect(scoped.harnessId).toBe("omp");
156
+ db.close();
157
+ });
158
+
159
+ test("empty ledger yields zeroed totals and null speeds", () => {
160
+ const { db } = seeded();
161
+ const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
162
+ expect(r.totals).toEqual({
163
+ dispatches: 0,
164
+ conversations: 0,
165
+ spendUsd: 0,
166
+ cacheHitRate: 0,
167
+ promptTokens: 0,
168
+ completionTokens: 0,
169
+ escalations: 0,
170
+ failovers: 0,
171
+ errors: 0,
172
+ aborted: 0,
173
+ modelSwitches: 0,
174
+ cacheEstimated: false,
175
+ });
176
+ expect(r.providers).toEqual([]);
177
+ expect(r.models).toEqual([]);
178
+ db.close();
179
+ });
180
+
181
+ test("speed ignores errored and non-streamed rows", () => {
182
+ const { db, ledger } = seeded();
183
+ ledger.record(entry({ ttftMs: null }));
184
+ ledger.record(entry({ error: "boom" }));
185
+ const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
186
+ expect(r.providers[0]!.avgTtftMs).toBeNull();
187
+ expect(r.providers[0]!.tokensPerSec).toBeNull();
188
+ db.close();
189
+ });
190
+ });
191
+
192
+ describe("renderUsageReport", () => {
193
+ test("router-estimated cache counts render as an estimate", async () => {
194
+ const { db, ledger } = seeded();
195
+ ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
196
+ ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
197
+ const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
198
+ expect(r.totals.cacheEstimated).toBe(true);
199
+ expect(r.providers.find((p) => p.key === "ollama")!.cacheEstimated).toBe(true);
200
+ expect(r.providers.find((p) => p.key === "openrouter")!.cacheEstimated).toBe(false);
201
+ const text = renderUsageReport(r);
202
+ expect(text).toContain("(cache hit ~70%)");
203
+ expect(text).toMatch(/ollama\s+1\s+\S+\s+\S+\s+~90%/);
204
+ expect(text).toMatch(/openrouter\s+1\s+\S+\s+\S+\s+50%/);
205
+ db.close();
206
+ });
207
+
208
+ test("renders every section as plain fixed-width text", () => {
209
+ const { db, ledger } = seeded();
210
+ ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
211
+ ledger.record(entry({ slug: "ollama/kimi", servedSlug: "ollama/kimi", tier: "hard", reportedUsd: 0.5, createdAtMs: NOW - 30 * HOUR }));
212
+ const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }));
213
+ expect(text).toContain("last 7d");
214
+ expect(text).toContain("spend $1.75 over 2 dispatches");
215
+ expect(text).toContain("providers");
216
+ expect(text).toContain("openrouter");
217
+ expect(text).toContain("ollama/kimi");
218
+ expect(text).toContain("tiers");
219
+ expect(text).toContain("by day (UTC)");
220
+ expect(text).toContain("2026-09-05");
221
+ // No markdown or ANSI: it goes into a code block as-is.
222
+ expect(text).not.toMatch(/[|*`]/);
223
+ db.close();
224
+ });
225
+
226
+ test("caps the model table and says so", () => {
227
+ const { db, ledger } = seeded();
228
+ for (let i = 0; i < 5; i++) ledger.record(entry({ slug: `v/m${i}`, servedSlug: `v/m${i}` }));
229
+ const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }), { maxModels: 2 });
230
+ expect(text).toContain("models (top 2 of 5 by spend)");
231
+ db.close();
232
+ });
233
+ });