auto-model-router 0.31.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +32 -2
- package/omp-extension/router-configure.ts +9 -7
- package/package.json +1 -1
- package/src/cli/config-cmd.ts +8 -7
- package/src/cli/explain.ts +10 -5
- package/src/cli/export.ts +6 -5
- package/src/cli/models.ts +10 -7
- package/src/cli/report.ts +6 -1
- package/src/cli/stats.ts +7 -7
- package/src/config/load.ts +10 -1
- package/src/config/types.ts +10 -1
- package/src/context/bridge.ts +7 -7
- package/src/context/index.ts +3 -3
- package/src/context/store.ts +39 -56
- package/src/context/types.ts +7 -6
- package/src/cost/blended.ts +28 -7
- package/src/cost/feedback.ts +33 -37
- package/src/cost/ledger-sql.ts +547 -0
- package/src/cost/ledger.ts +30 -459
- package/src/cost/report.ts +171 -129
- package/src/cost/retention.ts +10 -10
- package/src/cost/summary.ts +15 -10
- package/src/cost/types.ts +43 -62
- package/src/cost/views.ts +79 -49
- package/src/lib.ts +6 -2
- package/src/router/candidates.ts +7 -15
- package/src/router/classify.ts +6 -4
- package/src/router/index.ts +95 -9
- package/src/router/select.ts +38 -21
- package/src/router/state.ts +90 -102
- package/src/router/types.ts +11 -5
- package/src/server/advise.ts +6 -4
- package/src/server/compaction-digest.ts +1 -1
- package/src/server/digest.ts +9 -10
- package/src/server/http.ts +101 -41
- package/src/server/providers.ts +18 -4
- package/src/server/turn.ts +32 -9
- package/src/tokens/estimate.ts +16 -6
- package/src/upstream/ollama-usage.ts +21 -11
- package/src/util/schema.ts +201 -0
- package/src/util/sql.ts +246 -0
- package/src/wire/anthropic/messages.ts +3 -4
- package/src/wire/openai/request.ts +1 -0
- package/src/wire/types.ts +7 -0
- package/test/anthropic-wire.test.ts +9 -9
- package/test/benchmark-feeds.test.ts +7 -7
- package/test/cache-control.test.ts +7 -7
- package/test/cache-estimate.test.ts +5 -5
- package/test/catalog-view.test.ts +4 -4
- package/test/catalog.test.ts +11 -11
- package/test/classify.test.ts +24 -24
- package/test/compaction.test.ts +20 -20
- package/test/config-wizard.test.ts +32 -32
- package/test/config.test.ts +10 -10
- package/test/connect-harnesses.test.ts +11 -11
- package/test/context-bridge.test.ts +40 -30
- package/test/context-prune.test.ts +43 -36
- package/test/context-query.test.ts +8 -8
- package/test/controls.test.ts +54 -27
- package/test/cost.test.ts +12 -12
- package/test/digest.test.ts +55 -44
- package/test/embed-lifecycle.test.ts +5 -5
- package/test/embed-logic.test.ts +26 -26
- package/test/escalate.test.ts +17 -17
- package/test/eval.test.ts +13 -13
- package/test/executable.test.ts +6 -6
- package/test/exploration.test.ts +19 -20
- package/test/failover.test.ts +22 -21
- package/test/fakes.ts +105 -0
- package/test/features.test.ts +21 -21
- package/test/harness-requests.test.ts +3 -3
- package/test/harness-switch.test.ts +5 -5
- package/test/hold-exploration.test.ts +13 -13
- package/test/hot-reload.test.ts +5 -5
- package/test/learned.test.ts +5 -5
- package/test/ledger-sql.test.ts +342 -0
- package/test/mcp-entry.test.ts +5 -5
- package/test/migrations.test.ts +28 -22
- package/test/models-yml.test.ts +18 -18
- package/test/ollama.test.ts +40 -34
- package/test/omp-credentials.test.ts +16 -16
- package/test/policy.test.ts +3 -3
- package/test/reconfigure.test.ts +4 -4
- package/test/redaction.test.ts +41 -35
- package/test/remote.test.ts +12 -12
- package/test/report-logic.test.ts +8 -8
- package/test/report.test.ts +95 -87
- package/test/retention.test.ts +79 -66
- package/test/schema.test.ts +123 -0
- package/test/scope.test.ts +8 -8
- package/test/select.test.ts +216 -257
- package/test/skills.test.ts +3 -3
- package/test/sql-shim.test.ts +154 -0
- package/test/state.test.ts +43 -36
- package/test/summary.test.ts +38 -27
- package/test/tier-plan.test.ts +45 -62
- package/test/toast-logic.test.ts +31 -31
- package/test/tokens.test.ts +95 -80
- package/test/trust-attribution.test.ts +217 -187
- package/test/trust-window.test.ts +37 -32
- package/test/turn.test.ts +55 -23
- package/test/upstreams.test.ts +13 -13
- package/test/views.test.ts +81 -59
- package/test/wire-request.test.ts +17 -17
- package/test/wire-responses.test.ts +4 -4
- package/tools/agentdox-e2e.ts +5 -2
- package/tools/export-benchmarks.ts +5 -5
- package/tools/ledger-parity.ts +266 -0
- package/tools/replay.ts +16 -8
package/test/report.test.ts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
2
5
|
|
|
3
6
|
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
|
-
import {
|
|
7
|
+
import { createSqlLedger } from "../src/cost/ledger-sql.ts";
|
|
8
|
+
import { migrateStore } from "../src/util/schema.ts";
|
|
9
|
+
import type { AsyncLedger } from "../src/cost/types.ts";
|
|
5
10
|
import { buildUsageReport, renderUsageReport } from "../src/cost/report.ts";
|
|
6
11
|
import type { LedgerEntry } from "../src/cost/types.ts";
|
|
7
|
-
import {
|
|
12
|
+
import { openSqlDb, type SqlDb } from "../src/util/sql.ts";
|
|
8
13
|
|
|
9
14
|
/**
|
|
10
15
|
* `buildUsageReport` is what `/router report`, the `report` CLI and
|
|
@@ -55,21 +60,24 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
|
55
60
|
} as LedgerEntry;
|
|
56
61
|
}
|
|
57
62
|
|
|
58
|
-
function seeded() {
|
|
63
|
+
async function seeded(): Promise<{ db: SqlDb; ledger: AsyncLedger }> {
|
|
59
64
|
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
|
|
65
|
+
// A file rather than `:memory:`: the report reads through its own handle on
|
|
66
|
+
// the store, which an in-memory database cannot share.
|
|
67
|
+
const path = join(tmpdir(), `report-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`);
|
|
68
|
+
cfg.ledger.path = path;
|
|
69
|
+
const db = openSqlDb(path);
|
|
70
|
+
await migrateStore(db);
|
|
71
|
+
return { db, ledger: createSqlLedger(db, cfg, { findModel: () => null }) };
|
|
64
72
|
}
|
|
65
73
|
|
|
66
74
|
describe("buildUsageReport", () => {
|
|
67
|
-
test("totals, providers, models and tiers over a mixed window", () => {
|
|
68
|
-
const { db, ledger } = seeded();
|
|
75
|
+
test("totals, providers, models and tiers over a mixed window", async () => {
|
|
76
|
+
const { db, ledger } = await seeded();
|
|
69
77
|
// Two OpenRouter turns on one model (one with a warm cache), one Ollama
|
|
70
78
|
// turn served by a different slug than decided, one escalation, one error.
|
|
71
|
-
ledger.record(entry({ conversationKey: "a", turn: 1, reportedUsd: 0.01 }));
|
|
72
|
-
ledger.record(
|
|
79
|
+
await ledger.record(entry({ conversationKey: "a", turn: 1, reportedUsd: 0.01 }));
|
|
80
|
+
await ledger.record(
|
|
73
81
|
entry({
|
|
74
82
|
conversationKey: "a",
|
|
75
83
|
turn: 2,
|
|
@@ -77,7 +85,7 @@ describe("buildUsageReport", () => {
|
|
|
77
85
|
usage: { promptTokens: 1000, cachedTokens: 800, cacheWriteTokens: 0, completionTokens: 100, reasoningTokens: 0, images: 0 },
|
|
78
86
|
}),
|
|
79
87
|
);
|
|
80
|
-
ledger.record(
|
|
88
|
+
await ledger.record(
|
|
81
89
|
entry({
|
|
82
90
|
conversationKey: "b",
|
|
83
91
|
slug: "ollama/glm-5.3-flash",
|
|
@@ -89,10 +97,10 @@ describe("buildUsageReport", () => {
|
|
|
89
97
|
ttftMs: 100,
|
|
90
98
|
}),
|
|
91
99
|
);
|
|
92
|
-
ledger.record(entry({ conversationKey: "c", tier: "hard", escalationSignal: "refusal", wasted: true, reportedUsd: 0.002 }));
|
|
93
|
-
ledger.record(entry({ conversationKey: "c", tier: "hard", error: "request aborted", reportedUsd: 0, ttftMs: null }));
|
|
100
|
+
await ledger.record(entry({ conversationKey: "c", tier: "hard", escalationSignal: "refusal", wasted: true, reportedUsd: 0.002 }));
|
|
101
|
+
await ledger.record(entry({ conversationKey: "c", tier: "hard", error: "request aborted", reportedUsd: 0, ttftMs: null }));
|
|
94
102
|
|
|
95
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
103
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
96
104
|
expect(r.totals.dispatches).toBe(5);
|
|
97
105
|
expect(r.totals.conversations).toBe(3);
|
|
98
106
|
// 0.01 + 0.005 + 0.02 (predicted: no reported) + 0.002 + 0
|
|
@@ -129,54 +137,54 @@ describe("buildUsageReport", () => {
|
|
|
129
137
|
db.close();
|
|
130
138
|
});
|
|
131
139
|
|
|
132
|
-
test("counts a model switch only between consecutive kept rows of one conversation", () => {
|
|
133
|
-
const { db, ledger } = seeded();
|
|
134
|
-
ledger.record(entry({ conversationKey: "a", turn: 1, slug: "x/one", createdAtMs: NOW - 3 * HOUR }));
|
|
135
|
-
ledger.record(entry({ conversationKey: "a", turn: 2, slug: "x/two", createdAtMs: NOW - 2 * HOUR }));
|
|
136
|
-
ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/two", createdAtMs: NOW - 1 * HOUR }));
|
|
140
|
+
test("counts a model switch only between consecutive kept rows of one conversation", async () => {
|
|
141
|
+
const { db, ledger } = await seeded();
|
|
142
|
+
await ledger.record(entry({ conversationKey: "a", turn: 1, slug: "x/one", createdAtMs: NOW - 3 * HOUR }));
|
|
143
|
+
await ledger.record(entry({ conversationKey: "a", turn: 2, slug: "x/two", createdAtMs: NOW - 2 * HOUR }));
|
|
144
|
+
await ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/two", createdAtMs: NOW - 1 * HOUR }));
|
|
137
145
|
// A wasted probe on another slug is not a switch.
|
|
138
|
-
ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/three", wasted: true, createdAtMs: NOW - 1 * HOUR + 1 }));
|
|
146
|
+
await ledger.record(entry({ conversationKey: "a", turn: 3, slug: "x/three", wasted: true, createdAtMs: NOW - 1 * HOUR + 1 }));
|
|
139
147
|
// A different conversation starting on another model is not a switch either.
|
|
140
|
-
ledger.record(entry({ conversationKey: "b", turn: 1, slug: "x/three", createdAtMs: NOW - HOUR }));
|
|
141
|
-
const r = buildUsageReport(db, { windowDays: 1, nowMs: NOW });
|
|
148
|
+
await ledger.record(entry({ conversationKey: "b", turn: 1, slug: "x/three", createdAtMs: NOW - HOUR }));
|
|
149
|
+
const r = await buildUsageReport(db, { windowDays: 1, nowMs: NOW });
|
|
142
150
|
expect(r.totals.modelSwitches).toBe(1);
|
|
143
151
|
db.close();
|
|
144
152
|
});
|
|
145
153
|
|
|
146
|
-
test("window and harness scope exclude rows", () => {
|
|
147
|
-
const { db, ledger } = seeded();
|
|
148
|
-
ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - HOUR }));
|
|
149
|
-
ledger.record(entry({ harnessId: "hermes", createdAtMs: NOW - HOUR }));
|
|
150
|
-
ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - 10 * 24 * HOUR }));
|
|
151
|
-
expect(buildUsageReport(db, { windowDays: 7, nowMs: NOW }).totals.dispatches).toBe(2);
|
|
152
|
-
expect(buildUsageReport(db, { windowDays: 30, nowMs: NOW }).totals.dispatches).toBe(3);
|
|
153
|
-
const scoped = buildUsageReport(db, { windowDays: 30, harnessId: "omp", nowMs: NOW });
|
|
154
|
+
test("window and harness scope exclude rows", async () => {
|
|
155
|
+
const { db, ledger } = await seeded();
|
|
156
|
+
await ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - HOUR }));
|
|
157
|
+
await ledger.record(entry({ harnessId: "hermes", createdAtMs: NOW - HOUR }));
|
|
158
|
+
await ledger.record(entry({ harnessId: "omp", createdAtMs: NOW - 10 * 24 * HOUR }));
|
|
159
|
+
expect((await buildUsageReport(db, { windowDays: 7, nowMs: NOW })).totals.dispatches).toBe(2);
|
|
160
|
+
expect((await buildUsageReport(db, { windowDays: 30, nowMs: NOW })).totals.dispatches).toBe(3);
|
|
161
|
+
const scoped = await buildUsageReport(db, { windowDays: 30, harnessId: "omp", nowMs: NOW });
|
|
154
162
|
expect(scoped.totals.dispatches).toBe(2);
|
|
155
163
|
expect(scoped.harnessId).toBe("omp");
|
|
156
164
|
db.close();
|
|
157
165
|
});
|
|
158
166
|
|
|
159
|
-
test("a comma-separated harness list reports the union (a team group)", () => {
|
|
160
|
-
const { db, ledger } = seeded();
|
|
167
|
+
test("a comma-separated harness list reports the union (a team group)", async () => {
|
|
168
|
+
const { db, ledger } = await seeded();
|
|
161
169
|
try {
|
|
162
|
-
ledger.record(entry({ harnessId: "u_a", reportedUsd: 1 }));
|
|
163
|
-
ledger.record(entry({ harnessId: "u_b", reportedUsd: 2 }));
|
|
164
|
-
ledger.record(entry({ harnessId: "u_c", reportedUsd: 4 }));
|
|
165
|
-
expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_a,u_b" }).totals.spendUsd).toBeCloseTo(3, 6);
|
|
166
|
-
expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: " u_c , u_a " }).totals.spendUsd).toBeCloseTo(5, 6);
|
|
167
|
-
expect(buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_b" }).totals.spendUsd).toBeCloseTo(2, 6);
|
|
170
|
+
await ledger.record(entry({ harnessId: "u_a", reportedUsd: 1 }));
|
|
171
|
+
await ledger.record(entry({ harnessId: "u_b", reportedUsd: 2 }));
|
|
172
|
+
await ledger.record(entry({ harnessId: "u_c", reportedUsd: 4 }));
|
|
173
|
+
expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_a,u_b" })).totals.spendUsd).toBeCloseTo(3, 6);
|
|
174
|
+
expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: " u_c , u_a " })).totals.spendUsd).toBeCloseTo(5, 6);
|
|
175
|
+
expect((await buildUsageReport(db, { windowDays: 1, nowMs: NOW, harnessId: "u_b" })).totals.spendUsd).toBeCloseTo(2, 6);
|
|
168
176
|
} finally {
|
|
169
177
|
db.close();
|
|
170
178
|
}
|
|
171
179
|
});
|
|
172
180
|
|
|
173
|
-
test("prompt anatomy averages the recorded byte shares", () => {
|
|
174
|
-
const { db, ledger } = seeded();
|
|
181
|
+
test("prompt anatomy averages the recorded byte shares", async () => {
|
|
182
|
+
const { db, ledger } = await seeded();
|
|
175
183
|
const feat = (tool: number, older: number, stale: number) => ({ toolSchemaBytes: 1000, anatomy: { messages: 30, systemBytes: 1000, userBytes: 500, assistantBytes: 500, toolBytes: tool, olderHalfBytes: older, staleToolBytes: stale } });
|
|
176
|
-
ledger.record(entry({ features: feat(8000, 5000, 4000) }));
|
|
177
|
-
ledger.record(entry({ features: feat(6000, 4000, 2000) }));
|
|
178
|
-
ledger.record(entry({ features: null })); // pre-anatomy row: ignored
|
|
179
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
184
|
+
await ledger.record(entry({ features: feat(8000, 5000, 4000) }));
|
|
185
|
+
await ledger.record(entry({ features: feat(6000, 4000, 2000) }));
|
|
186
|
+
await ledger.record(entry({ features: null })); // pre-anatomy row: ignored
|
|
187
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
180
188
|
const a = r.anatomy!;
|
|
181
189
|
expect(a.rows).toBe(2);
|
|
182
190
|
// mean bytes: system 1000, user 500, assistant 500, tool 7000 ⇒ total 9000
|
|
@@ -189,10 +197,10 @@ describe("buildUsageReport", () => {
|
|
|
189
197
|
db.close();
|
|
190
198
|
});
|
|
191
199
|
|
|
192
|
-
test("baselines price the window on one model with its own cache hit rate", () => {
|
|
193
|
-
const { db, ledger } = seeded();
|
|
194
|
-
ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
|
|
195
|
-
const r = buildUsageReport(db, {
|
|
200
|
+
test("baselines price the window on one model with its own cache hit rate", async () => {
|
|
201
|
+
const { db, ledger } = await seeded();
|
|
202
|
+
await ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
|
|
203
|
+
const r = await buildUsageReport(db, {
|
|
196
204
|
windowDays: 7,
|
|
197
205
|
nowMs: NOW,
|
|
198
206
|
baselines: [
|
|
@@ -211,21 +219,21 @@ describe("buildUsageReport", () => {
|
|
|
211
219
|
db.close();
|
|
212
220
|
});
|
|
213
221
|
|
|
214
|
-
test("subagent turns are counted with their spend", () => {
|
|
215
|
-
const { db, ledger } = seeded();
|
|
216
|
-
ledger.record(entry({ reportedUsd: 0.01, features: { isSubagent: true } }));
|
|
217
|
-
ledger.record(entry({ reportedUsd: 0.03, features: { isSubagent: false } }));
|
|
218
|
-
ledger.record(entry({ reportedUsd: 0.06 }));
|
|
219
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
222
|
+
test("subagent turns are counted with their spend", async () => {
|
|
223
|
+
const { db, ledger } = await seeded();
|
|
224
|
+
await ledger.record(entry({ reportedUsd: 0.01, features: { isSubagent: true } }));
|
|
225
|
+
await ledger.record(entry({ reportedUsd: 0.03, features: { isSubagent: false } }));
|
|
226
|
+
await ledger.record(entry({ reportedUsd: 0.06 }));
|
|
227
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
220
228
|
expect(r.totals.subagentDispatches).toBe(1);
|
|
221
229
|
expect(r.totals.subagentSpendUsd).toBeCloseTo(0.01, 6);
|
|
222
230
|
expect(renderUsageReport(r)).toContain("subagents: 1 dispatches, $0.0100 (10% of spend)");
|
|
223
231
|
db.close();
|
|
224
232
|
});
|
|
225
233
|
|
|
226
|
-
test("empty ledger yields zeroed totals and null speeds", () => {
|
|
227
|
-
const { db } = seeded();
|
|
228
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
234
|
+
test("empty ledger yields zeroed totals and null speeds", async () => {
|
|
235
|
+
const { db } = await seeded();
|
|
236
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
229
237
|
expect(r.totals).toEqual({
|
|
230
238
|
dispatches: 0,
|
|
231
239
|
conversations: 0,
|
|
@@ -254,11 +262,11 @@ describe("buildUsageReport", () => {
|
|
|
254
262
|
db.close();
|
|
255
263
|
});
|
|
256
264
|
|
|
257
|
-
test("speed ignores errored and non-streamed rows", () => {
|
|
258
|
-
const { db, ledger } = seeded();
|
|
259
|
-
ledger.record(entry({ ttftMs: null }));
|
|
260
|
-
ledger.record(entry({ error: "boom" }));
|
|
261
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
265
|
+
test("speed ignores errored and non-streamed rows", async () => {
|
|
266
|
+
const { db, ledger } = await seeded();
|
|
267
|
+
await ledger.record(entry({ ttftMs: null }));
|
|
268
|
+
await ledger.record(entry({ error: "boom" }));
|
|
269
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
262
270
|
expect(r.providers[0]!.avgTtftMs).toBeNull();
|
|
263
271
|
expect(r.providers[0]!.tokensPerSec).toBeNull();
|
|
264
272
|
db.close();
|
|
@@ -267,10 +275,10 @@ describe("buildUsageReport", () => {
|
|
|
267
275
|
|
|
268
276
|
describe("renderUsageReport", () => {
|
|
269
277
|
test("router-estimated cache counts render as an estimate", async () => {
|
|
270
|
-
const { db, ledger } = seeded();
|
|
271
|
-
ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
|
|
272
|
-
ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
|
|
273
|
-
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
278
|
+
const { db, ledger } = await seeded();
|
|
279
|
+
await ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
|
|
280
|
+
await ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
|
|
281
|
+
const r = await buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
274
282
|
expect(r.totals.cacheEstimated).toBe(true);
|
|
275
283
|
expect(r.providers.find((p) => p.key === "ollama")!.cacheEstimated).toBe(true);
|
|
276
284
|
expect(r.providers.find((p) => p.key === "openrouter")!.cacheEstimated).toBe(false);
|
|
@@ -281,11 +289,11 @@ describe("renderUsageReport", () => {
|
|
|
281
289
|
db.close();
|
|
282
290
|
});
|
|
283
291
|
|
|
284
|
-
test("renders every section as plain fixed-width text", () => {
|
|
285
|
-
const { db, ledger } = seeded();
|
|
286
|
-
ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
|
|
287
|
-
ledger.record(entry({ slug: "ollama/kimi", servedSlug: "ollama/kimi", tier: "hard", reportedUsd: 0.5, createdAtMs: NOW - 30 * HOUR }));
|
|
288
|
-
const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }));
|
|
292
|
+
test("renders every section as plain fixed-width text", async () => {
|
|
293
|
+
const { db, ledger } = await seeded();
|
|
294
|
+
await ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
|
|
295
|
+
await ledger.record(entry({ slug: "ollama/kimi", servedSlug: "ollama/kimi", tier: "hard", reportedUsd: 0.5, createdAtMs: NOW - 30 * HOUR }));
|
|
296
|
+
const text = renderUsageReport(await buildUsageReport(db, { windowDays: 7, nowMs: NOW }));
|
|
289
297
|
expect(text).toContain("last 7d");
|
|
290
298
|
expect(text).toContain("spend $1.75 over 2 dispatches");
|
|
291
299
|
expect(text).toContain("providers");
|
|
@@ -299,35 +307,35 @@ describe("renderUsageReport", () => {
|
|
|
299
307
|
db.close();
|
|
300
308
|
});
|
|
301
309
|
|
|
302
|
-
test("caps the model table and says so", () => {
|
|
303
|
-
const { db, ledger } = seeded();
|
|
304
|
-
for (let i = 0; i < 5; i++) ledger.record(entry({ slug: `v/m${i}`, servedSlug: `v/m${i}` }));
|
|
305
|
-
const text = renderUsageReport(buildUsageReport(db, { windowDays: 7, nowMs: NOW }), { maxModels: 2 });
|
|
310
|
+
test("caps the model table and says so", async () => {
|
|
311
|
+
const { db, ledger } = await seeded();
|
|
312
|
+
for (let i = 0; i < 5; i++) await ledger.record(entry({ slug: `v/m${i}`, servedSlug: `v/m${i}` }));
|
|
313
|
+
const text = renderUsageReport(await buildUsageReport(db, { windowDays: 7, nowMs: NOW }), { maxModels: 2 });
|
|
306
314
|
expect(text).toContain("models (top 2 of 5 by spend)");
|
|
307
315
|
db.close();
|
|
308
316
|
});
|
|
309
317
|
});
|
|
310
318
|
|
|
311
319
|
describe("digest re-runs and forecast accuracy", () => {
|
|
312
|
-
test("wasted digest rows count as re-runs; forecast error is judged on clean kept rows only", () => {
|
|
313
|
-
const { db, ledger } = seeded();
|
|
320
|
+
test("wasted digest rows count as re-runs; forecast error is judged on clean kept rows only", async () => {
|
|
321
|
+
const { db, ledger } = await seeded();
|
|
314
322
|
try {
|
|
315
|
-
ledger.record(entry({ requestedModel: "digest", conversationKey: "d1", reportedUsd: 0.001, predictedUsd: 0.001 }));
|
|
316
|
-
ledger.record(entry({ requestedModel: "digest", conversationKey: "d2", reportedUsd: 0.001, predictedUsd: 0.001, wasted: true }));
|
|
323
|
+
await ledger.record(entry({ requestedModel: "digest", conversationKey: "d1", reportedUsd: 0.001, predictedUsd: 0.001 }));
|
|
324
|
+
await ledger.record(entry({ requestedModel: "digest", conversationKey: "d2", reportedUsd: 0.001, predictedUsd: 0.001, wasted: true }));
|
|
317
325
|
// Two clean turns: one predicted double, one predicted half.
|
|
318
|
-
ledger.record(entry({ predictedUsd: 0.02, reportedUsd: 0.01 }));
|
|
319
|
-
ledger.record(entry({ predictedUsd: 0.005, reportedUsd: 0.01 }));
|
|
326
|
+
await ledger.record(entry({ predictedUsd: 0.02, reportedUsd: 0.01 }));
|
|
327
|
+
await ledger.record(entry({ predictedUsd: 0.005, reportedUsd: 0.01 }));
|
|
320
328
|
// Excluded from the forecast judgement: wasted, errored, no reported cost.
|
|
321
|
-
ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, wasted: true }));
|
|
322
|
-
ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, error: "upstream_error: 500" }));
|
|
323
|
-
ledger.record(entry({ predictedUsd: 1, reportedUsd: null }));
|
|
324
|
-
const t = buildUsageReport(db, { windowDays: 1, nowMs: NOW }).totals;
|
|
329
|
+
await ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, wasted: true }));
|
|
330
|
+
await ledger.record(entry({ predictedUsd: 1, reportedUsd: 0.01, error: "upstream_error: 500" }));
|
|
331
|
+
await ledger.record(entry({ predictedUsd: 1, reportedUsd: null }));
|
|
332
|
+
const t = (await buildUsageReport(db, { windowDays: 1, nowMs: NOW })).totals;
|
|
325
333
|
expect(t.digests).toBe(2);
|
|
326
334
|
expect(t.digestReruns).toBe(1);
|
|
327
335
|
expect(t.forecastSamples).toBe(2);
|
|
328
336
|
expect(t.forecastMeanError).toBeCloseTo((1 + 0.5) / 2, 6);
|
|
329
337
|
expect(t.forecastOverShare).toBeCloseTo(0.5, 6);
|
|
330
|
-
const text = renderUsageReport(buildUsageReport(db, { windowDays: 1, nowMs: NOW }));
|
|
338
|
+
const text = renderUsageReport(await buildUsageReport(db, { windowDays: 1, nowMs: NOW }));
|
|
331
339
|
expect(text).toContain("re-run rate 50% (1 fetched again in full)");
|
|
332
340
|
expect(text).toContain("forecast: mean error 75% of reported cost over 2 turns · 50% over-predicted");
|
|
333
341
|
} finally {
|
package/test/retention.test.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
|
+
import { migrateStore } from "../src/util/schema.ts";
|
|
3
|
+
import { num, openSqlDb, type SqlDb } from "../src/util/sql.ts";
|
|
2
4
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
5
|
import { tmpdir } from "node:os";
|
|
4
6
|
import { join } from "node:path";
|
|
@@ -6,14 +8,14 @@ import { join } from "node:path";
|
|
|
6
8
|
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
7
9
|
import type { RouterConfig } from "../src/config/types.ts";
|
|
8
10
|
import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
9
|
-
import {
|
|
11
|
+
import { fakeLedger } from "./fakes.ts";
|
|
12
|
+
import { createSqlLedger } from "../src/cost/ledger-sql.ts";
|
|
10
13
|
import { createRetentionRunner, RETENTION_INTERVAL_MS } from "../src/cost/retention.ts";
|
|
11
|
-
import { EMPTY_USAGE, type
|
|
14
|
+
import { EMPTY_USAGE, type AsyncLedger, type LedgerEntry, type PruneResult } from "../src/cost/types.ts";
|
|
12
15
|
import { startServer, type StartedServer } from "../src/server/http.ts";
|
|
13
|
-
import { openDb } from "../src/util/sqlite.ts";
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
|
-
*
|
|
18
|
+
* AsyncLedger retention: how long turns are kept, what goes with them, and the
|
|
17
19
|
* route a front door asks through — the team edition holds a read-only handle
|
|
18
20
|
* on the ledger and must never delete from it itself.
|
|
19
21
|
*/
|
|
@@ -59,124 +61,133 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
|
59
61
|
};
|
|
60
62
|
}
|
|
61
63
|
|
|
62
|
-
function seeded(
|
|
63
|
-
|
|
64
|
+
async function seeded(
|
|
65
|
+
path = join(tmpdir(), `retention-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`),
|
|
66
|
+
base = NOW,
|
|
67
|
+
): Promise<{ db: SqlDb; ledger: AsyncLedger; ids: string[] }> {
|
|
68
|
+
const db = openSqlDb(path);
|
|
69
|
+
await migrateStore(db);
|
|
64
70
|
const cfg: RouterConfig = { ...DEFAULT_CONFIG, ledger: { ...DEFAULT_CONFIG.ledger, path } };
|
|
65
|
-
const ledger =
|
|
71
|
+
const ledger = createSqlLedger(db, cfg, { findModel: () => null });
|
|
66
72
|
// 400, 200 and 1 days old.
|
|
67
|
-
const ids = [
|
|
73
|
+
const ids: string[] = [];
|
|
74
|
+
for (const age of [400, 200, 1]) {
|
|
68
75
|
const e = entry({ createdAtMs: base - age * DAY });
|
|
69
|
-
ledger.record(e);
|
|
70
|
-
|
|
71
|
-
}
|
|
76
|
+
await ledger.record(e);
|
|
77
|
+
ids.push(e.id);
|
|
78
|
+
}
|
|
72
79
|
return { db, ledger, ids };
|
|
73
80
|
}
|
|
74
81
|
|
|
75
82
|
describe("prune", () => {
|
|
76
|
-
test("rows past the window go, newer ones stay, and the oldest kept is reported", () => {
|
|
77
|
-
const { db, ledger } = seeded();
|
|
83
|
+
test("rows past the window go, newer ones stay, and the oldest kept is reported", async () => {
|
|
84
|
+
const { db, ledger } = await seeded();
|
|
78
85
|
try {
|
|
79
|
-
expect(ledger.prune
|
|
80
|
-
expect(ledger.recentEntries(10)).toHaveLength(2);
|
|
81
|
-
expect(ledger.prune
|
|
82
|
-
expect(ledger.recentEntries(10)).toHaveLength(1);
|
|
86
|
+
expect(await ledger.prune(365, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - 200 * DAY });
|
|
87
|
+
expect(await ledger.recentEntries(10)).toHaveLength(2);
|
|
88
|
+
expect(await ledger.prune(30, NOW)).toEqual({ deleted: 1, oldestKeptMs: NOW - DAY });
|
|
89
|
+
expect(await ledger.recentEntries(10)).toHaveLength(1);
|
|
83
90
|
} finally {
|
|
84
|
-
db.close();
|
|
91
|
+
await db.close();
|
|
85
92
|
}
|
|
86
93
|
});
|
|
87
94
|
|
|
88
|
-
test("a null window (the default) deletes nothing, and neither does 0", () => {
|
|
89
|
-
const { db, ledger } = seeded();
|
|
95
|
+
test("a null window (the default) deletes nothing, and neither does 0", async () => {
|
|
96
|
+
const { db, ledger } = await seeded();
|
|
90
97
|
try {
|
|
91
98
|
expect(DEFAULT_CONFIG.ledger.retentionDays).toBeNull();
|
|
92
|
-
expect(ledger.prune
|
|
93
|
-
expect(ledger.prune
|
|
94
|
-
expect(ledger.recentEntries(10)).toHaveLength(3);
|
|
99
|
+
expect(await ledger.prune(null, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
|
|
100
|
+
expect(await ledger.prune(0, NOW)).toEqual({ deleted: 0, oldestKeptMs: NOW - 400 * DAY });
|
|
101
|
+
expect(await ledger.recentEntries(10)).toHaveLength(3);
|
|
95
102
|
} finally {
|
|
96
|
-
db.close();
|
|
103
|
+
await db.close();
|
|
97
104
|
}
|
|
98
105
|
});
|
|
99
106
|
|
|
100
|
-
test("feedback keyed to a deleted turn goes with it; a verdict on a kept turn stays", () => {
|
|
101
|
-
const { db, ledger, ids } = seeded();
|
|
107
|
+
test("feedback keyed to a deleted turn goes with it; a verdict on a kept turn stays", async () => {
|
|
108
|
+
const { db, ledger, ids } = await seeded();
|
|
102
109
|
try {
|
|
103
110
|
const feedback = createFeedbackStore(db);
|
|
104
111
|
const [old, mid, recent] = ids as [string, string, string];
|
|
105
|
-
feedback.record({ ledgerId: old, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "bad", note: "" }, NOW - 400 * DAY);
|
|
106
|
-
feedback.record({ ledgerId: mid, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - 200 * DAY);
|
|
107
|
-
feedback.record({ ledgerId: recent, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - DAY);
|
|
108
|
-
expect(ledger.prune
|
|
109
|
-
expect(feedback.forLedgerId(old)).toHaveLength(0);
|
|
110
|
-
expect(feedback.forLedgerId(mid)).toHaveLength(1);
|
|
111
|
-
expect(feedback.forLedgerId(recent)).toHaveLength(1);
|
|
112
|
+
await feedback.record({ ledgerId: old, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "bad", note: "" }, NOW - 400 * DAY);
|
|
113
|
+
await feedback.record({ ledgerId: mid, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - 200 * DAY);
|
|
114
|
+
await feedback.record({ ledgerId: recent, ompSessionId: "omp-1", slug: "a/b", tier: "simple", verdict: "good", note: "" }, NOW - DAY);
|
|
115
|
+
expect((await ledger.prune(365, NOW))?.deleted).toBe(1);
|
|
116
|
+
expect(await feedback.forLedgerId(old)).toHaveLength(0);
|
|
117
|
+
expect(await feedback.forLedgerId(mid)).toHaveLength(1);
|
|
118
|
+
expect(await feedback.forLedgerId(recent)).toHaveLength(1);
|
|
112
119
|
} finally {
|
|
113
|
-
db.close();
|
|
120
|
+
await db.close();
|
|
114
121
|
}
|
|
115
122
|
});
|
|
116
123
|
|
|
117
|
-
test("ollama meter samples age out with the rows they calibrate", () => {
|
|
118
|
-
const { db, ledger } = seeded();
|
|
124
|
+
test("ollama meter samples age out with the rows they calibrate", async () => {
|
|
125
|
+
const { db, ledger } = await seeded();
|
|
119
126
|
try {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
127
|
+
for (const atMs of [NOW - 400 * DAY, NOW - DAY]) {
|
|
128
|
+
await db.sql`INSERT INTO ollama_meter_samples (at_ms, meter_usd, ledger_usd) VALUES (${atMs}, 1, 1)`;
|
|
129
|
+
}
|
|
130
|
+
await ledger.prune(365, NOW);
|
|
131
|
+
const kept = await db.one<{ n: unknown }>("SELECT COUNT(*) AS n FROM ollama_meter_samples");
|
|
132
|
+
expect(num(kept?.n)).toBe(1);
|
|
123
133
|
} finally {
|
|
124
|
-
db.close();
|
|
134
|
+
await db.close();
|
|
125
135
|
}
|
|
126
136
|
});
|
|
127
137
|
|
|
128
|
-
test("an empty ledger reports no oldest row rather than a cutoff of its own", () => {
|
|
129
|
-
const db =
|
|
138
|
+
test("an empty ledger reports no oldest row rather than a cutoff of its own", async () => {
|
|
139
|
+
const db = openSqlDb(join(tmpdir(), `t-retention.test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`));
|
|
140
|
+
await migrateStore(db);
|
|
130
141
|
try {
|
|
131
|
-
const ledger =
|
|
132
|
-
expect(ledger.prune
|
|
142
|
+
const ledger = createSqlLedger(db, DEFAULT_CONFIG, { findModel: () => null });
|
|
143
|
+
expect(await ledger.prune(30, NOW)).toEqual({ deleted: 0, oldestKeptMs: null });
|
|
133
144
|
} finally {
|
|
134
|
-
db.close();
|
|
145
|
+
await db.close();
|
|
135
146
|
}
|
|
136
147
|
});
|
|
137
148
|
});
|
|
138
149
|
|
|
139
150
|
describe("the schedule", () => {
|
|
140
|
-
function counting(): { ledger:
|
|
151
|
+
function counting(): { ledger: AsyncLedger; runs: number[] } {
|
|
141
152
|
const runs: number[] = [];
|
|
142
|
-
const ledger = {
|
|
143
|
-
prune: (_days: number | null, nowMs?: number): PruneResult => {
|
|
153
|
+
const ledger = fakeLedger({
|
|
154
|
+
prune: async (_days: number | null, nowMs?: number): Promise<PruneResult> => {
|
|
144
155
|
runs.push(nowMs ?? 0);
|
|
145
156
|
return { deleted: 1, oldestKeptMs: null };
|
|
146
157
|
},
|
|
147
|
-
}
|
|
158
|
+
});
|
|
148
159
|
return { ledger, runs };
|
|
149
160
|
}
|
|
150
161
|
|
|
151
|
-
test("the scheduled path runs at most once an hour, however often it is asked", () => {
|
|
162
|
+
test("the scheduled path runs at most once an hour, however often it is asked", async () => {
|
|
152
163
|
const { ledger, runs } = counting();
|
|
153
164
|
const runner = createRetentionRunner({ ledger, retentionDays: () => 30 });
|
|
154
165
|
// The first call is always due, so a lowered window applies at boot.
|
|
155
|
-
expect(runner.maybeRun(NOW)?.deleted).toBe(1);
|
|
166
|
+
expect((await runner.maybeRun(NOW))?.deleted).toBe(1);
|
|
156
167
|
// A minute timer ticking for an hour must not re-run it.
|
|
157
|
-
for (let t = 60_000; t < RETENTION_INTERVAL_MS; t += 60_000) expect(runner.maybeRun(NOW + t)).toBeNull();
|
|
168
|
+
for (let t = 60_000; t < RETENTION_INTERVAL_MS; t += 60_000) expect(await runner.maybeRun(NOW + t)).toBeNull();
|
|
158
169
|
expect(runs).toEqual([NOW]);
|
|
159
|
-
expect(runner.maybeRun(NOW + RETENTION_INTERVAL_MS)?.deleted).toBe(1);
|
|
170
|
+
expect((await runner.maybeRun(NOW + RETENTION_INTERVAL_MS))?.deleted).toBe(1);
|
|
160
171
|
expect(runs).toEqual([NOW, NOW + RETENTION_INTERVAL_MS]);
|
|
161
172
|
});
|
|
162
173
|
|
|
163
|
-
test("an explicit run always happens, and satisfies the schedule for the next hour", () => {
|
|
174
|
+
test("an explicit run always happens, and satisfies the schedule for the next hour", async () => {
|
|
164
175
|
const { ledger, runs } = counting();
|
|
165
176
|
const runner = createRetentionRunner({ ledger, retentionDays: () => 30 });
|
|
166
|
-
runner.runNow(NOW);
|
|
167
|
-
runner.runNow(NOW + 1_000);
|
|
177
|
+
await runner.runNow(NOW);
|
|
178
|
+
await runner.runNow(NOW + 1_000);
|
|
168
179
|
expect(runs).toEqual([NOW, NOW + 1_000]);
|
|
169
|
-
expect(runner.maybeRun(NOW + 2_000)).toBeNull();
|
|
170
|
-
expect(runner.maybeRun(NOW + 1_000 + RETENTION_INTERVAL_MS)).not.toBeNull();
|
|
180
|
+
expect(await runner.maybeRun(NOW + 2_000)).toBeNull();
|
|
181
|
+
expect(await runner.maybeRun(NOW + 1_000 + RETENTION_INTERVAL_MS)).not.toBeNull();
|
|
171
182
|
});
|
|
172
183
|
|
|
173
|
-
test("the window is read live, so a hot reload applies on the next run", () => {
|
|
184
|
+
test("the window is read live, so a hot reload applies on the next run", async () => {
|
|
174
185
|
let days: number | null = null;
|
|
175
186
|
const { ledger, runs } = counting();
|
|
176
187
|
const runner = createRetentionRunner({ ledger, retentionDays: () => days });
|
|
177
188
|
expect(runner.retentionDays()).toBeNull();
|
|
178
189
|
days = 7;
|
|
179
|
-
runner.runNow(NOW);
|
|
190
|
+
await runner.runNow(NOW);
|
|
180
191
|
expect(runner.retentionDays()).toBe(7);
|
|
181
192
|
expect(runs).toHaveLength(1);
|
|
182
193
|
});
|
|
@@ -191,11 +202,11 @@ describe("POST /v1/router/prune", () => {
|
|
|
191
202
|
// relative to it rather than to the fixed NOW the unit tests use.
|
|
192
203
|
const realNow = Date.now();
|
|
193
204
|
|
|
194
|
-
beforeAll(() => {
|
|
205
|
+
beforeAll(async () => {
|
|
195
206
|
dir = mkdtempSync(join(tmpdir(), "amr-retention-"));
|
|
196
207
|
dbPath = join(dir, "router.db");
|
|
197
|
-
const { db } = seeded(dbPath, realNow);
|
|
198
|
-
db.close();
|
|
208
|
+
const { db } = await seeded(dbPath, realNow);
|
|
209
|
+
await db.close();
|
|
199
210
|
const cfg: RouterConfig = {
|
|
200
211
|
...structuredClone(DEFAULT_CONFIG),
|
|
201
212
|
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
@@ -223,11 +234,13 @@ describe("POST /v1/router/prune", () => {
|
|
|
223
234
|
expect(body.oldestKeptMs).toBe(realNow - 200 * DAY);
|
|
224
235
|
expect(body.retentionDays).toBe(365);
|
|
225
236
|
// The rows are really gone from the file, not just from a view.
|
|
226
|
-
const db =
|
|
237
|
+
const db = openSqlDb(dbPath);
|
|
238
|
+
await migrateStore(db);
|
|
227
239
|
try {
|
|
228
|
-
|
|
240
|
+
const left = await db.one<{ n: unknown }>("SELECT COUNT(*) AS n FROM ledger");
|
|
241
|
+
expect(num(left?.n)).toBe(2);
|
|
229
242
|
} finally {
|
|
230
|
-
db.close();
|
|
243
|
+
await db.close();
|
|
231
244
|
}
|
|
232
245
|
});
|
|
233
246
|
|