auto-model-router 0.2.27 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -7
- package/package.json +1 -1
- package/src/config/defaults.ts +5 -0
- package/src/config/schema.ts +1 -0
- package/src/config/types.ts +16 -0
- package/src/context/bridge.ts +5 -0
- package/src/context/store.ts +10 -1
- package/src/context/types.ts +13 -0
- package/src/cost/ledger.ts +15 -6
- package/src/server/http.ts +12 -0
- package/src/util/sqlite.ts +11 -1
- package/test/context-prune.test.ts +84 -0
- package/test/failover.test.ts +1 -1
- package/test/trust-attribution.test.ts +2 -2
- package/test/trust-window.test.ts +136 -0
- package/test/turn.test.ts +2 -1
package/README.md
CHANGED
|
@@ -102,6 +102,29 @@ score nearly as well. If your workload needs a frontier model on hard turns,
|
|
|
102
102
|
raise the tier price ceiling and `qualityExponent` — measured thresholds are in
|
|
103
103
|
[`docs/routing-benchmark-findings.md`](docs/routing-benchmark-findings.md).
|
|
104
104
|
|
|
105
|
+
### Real-world — a week on the live ledger
|
|
106
|
+
|
|
107
|
+
The suites above are small and clean. To measure the economics on *actual*
|
|
108
|
+
usage we replayed a week of real omp traffic from the router's own ledger —
|
|
109
|
+
**6 918 billed turns across 299 conversations, 7 days, 410:1 input-to-output,
|
|
110
|
+
68% cache hit** — and repriced the identical token stream against a single Opus 5
|
|
111
|
+
model with its own cache namespace.
|
|
112
|
+
|
|
113
|
+
| | auto-model-router | Claude Opus 5 (single-model) |
|
|
114
|
+
| --- | --- | --- |
|
|
115
|
+
| Spend over the week | **$61.69** | $921.20 |
|
|
116
|
+
| Per turn | **$0.0089** | $0.133 |
|
|
117
|
+
| Extrapolated / month | **$263** | $3 932 |
|
|
118
|
+
|
|
119
|
+
**≈15× cheaper, ~93% saved** — a four-figure monthly bill becomes a three-figure
|
|
120
|
+
one. This baseline is deliberately conservative: one cache namespace, with each
|
|
121
|
+
conversation's cache replayed on the real turn gaps. A naive like-for-like
|
|
122
|
+
repricing at Opus rates reports ~31×, but on a single model the replayed context
|
|
123
|
+
is cache reads at $0.50/MTok, so ≈15× is the number we stand behind. Unlike the
|
|
124
|
+
core suite, sustained work on a large codebase is dominated by the conversation
|
|
125
|
+
resent each turn rather than per-token price — exactly where a single frontier
|
|
126
|
+
model gets expensive and routing's per-turn cache awareness pays off.
|
|
127
|
+
|
|
105
128
|
### Scope
|
|
106
129
|
|
|
107
130
|
These are small, self-contained tasks of one to three files, solved in under 25
|
|
@@ -111,11 +134,6 @@ separates. The cost multiple varied between 14× and 32× across runs depending
|
|
|
111
134
|
which task the baseline stalled on — treat "well over an order of magnitude" as
|
|
112
135
|
the claim, not a specific figure.
|
|
113
136
|
|
|
114
|
-
For sustained work on a large codebase the economics differ: cost there is
|
|
115
|
-
dominated by the conversation being resent each turn rather than by per-token
|
|
116
|
-
price. Replaying a week of real omp traffic (6 918 billed turns, 410:1
|
|
117
|
-
input-to-output) against a single-model baseline gives **≈15×**.
|
|
118
|
-
|
|
119
137
|
Harness, tasks and raw per-turn data:
|
|
120
138
|
[`docs/routing-benchmark-findings.md`](docs/routing-benchmark-findings.md).
|
|
121
139
|
|
|
@@ -666,13 +684,13 @@ and brief:
|
|
|
666
684
|
```bash
|
|
667
685
|
export AGENTDOX_URL=http://localhost:3003
|
|
668
686
|
export AGENTDOX_TOKEN=<pat with read+write on the scope>
|
|
669
|
-
export AGENTDOX_SCOPE=
|
|
687
|
+
export AGENTDOX_SCOPE=myproject # fallback only; see below
|
|
670
688
|
```
|
|
671
689
|
|
|
672
690
|
Setting a URL and a token is enough to turn it on.
|
|
673
691
|
|
|
674
692
|
The scope is **derived per workspace** from the directory basename
|
|
675
|
-
(`E:/projects/
|
|
693
|
+
(`E:/projects/myproject` → `myproject`), and that derivation wins. `AGENTDOX_SCOPE` /
|
|
676
694
|
`context.defaultScope` is only a fallback for workspaces it cannot resolve, because one router
|
|
677
695
|
install serves every project on the machine — a slug pinned there would be sent for all of
|
|
678
696
|
them, injecting one project's context into another's work. A single configured token also
|
package/package.json
CHANGED
package/src/config/defaults.ts
CHANGED
|
@@ -71,6 +71,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
71
71
|
// Shared trust by default: more samples, demotion guard stays effective
|
|
72
72
|
// even with a tiny guardrail-narrowed catalog.
|
|
73
73
|
trustScopedByHarness: false,
|
|
74
|
+
// 0 = all-time, the shipped behaviour. Reliability is slow-moving, so a
|
|
75
|
+
// wide sample is right on the merits; a window exists to bound the
|
|
76
|
+
// per-slug scan once history is large, and it changes routing, so it is
|
|
77
|
+
// opt-in after a replay run prices it.
|
|
78
|
+
trustWindowDays: 0,
|
|
74
79
|
contextHeadroom: 1.25,
|
|
75
80
|
// Latency scoring is off by default (weight 0): opt in after establishing a
|
|
76
81
|
// baseline. Expected total wait (TTFT + expected completion / throughput)
|
package/src/config/schema.ts
CHANGED
|
@@ -67,6 +67,7 @@ const filters = z.strictObject({
|
|
|
67
67
|
minTrust: z.number().min(0).max(1).optional(),
|
|
68
68
|
minTrustSamples: z.number().int().nonnegative().optional(),
|
|
69
69
|
trustScopedByHarness: z.boolean().optional(),
|
|
70
|
+
trustWindowDays: z.number().nonnegative().optional(),
|
|
70
71
|
contextHeadroom: z.number().positive().optional(),
|
|
71
72
|
latencyWeight: z.number().nonnegative().optional(),
|
|
72
73
|
latencyReferenceMs: z.number().positive().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -167,6 +167,22 @@ export interface FilterConfig {
|
|
|
167
167
|
* to learn its own reliability.
|
|
168
168
|
*/
|
|
169
169
|
trustScopedByHarness: boolean;
|
|
170
|
+
/**
|
|
171
|
+
* Only count ledger rows from the last N days toward model trust. 0 (the
|
|
172
|
+
* default) keeps the all-time behaviour.
|
|
173
|
+
*
|
|
174
|
+
* Trust is deliberately all-time: reliability is slow-moving, and a wide
|
|
175
|
+
* sample keeps the demotion guard stable. The cost is that the per-slug
|
|
176
|
+
* trust aggregate scans every row a model ever had, and that runs for each
|
|
177
|
+
* candidate on every turn — measured on a real ledger it grows from 0.8 ms at
|
|
178
|
+
* 9k rows to 11.6 ms at 75k, i.e. it becomes a per-turn latency tax as
|
|
179
|
+
* history accumulates. A window bounds that scan.
|
|
180
|
+
*
|
|
181
|
+
* Setting it CHANGES ROUTING (smaller denominators move success rates), so
|
|
182
|
+
* price it on the ledger with `bun tools/replay.ts --set
|
|
183
|
+
* filters.trustWindowDays=N` before enabling.
|
|
184
|
+
*/
|
|
185
|
+
trustWindowDays: number;
|
|
170
186
|
/**
|
|
171
187
|
* Headroom multiplier applied to estimated prompt tokens when checking a
|
|
172
188
|
* model's context window, absorbing token-estimate error and the response.
|
package/src/context/bridge.ts
CHANGED
|
@@ -212,6 +212,10 @@ export function createContextBridge(opts: BridgeOptions): ContextBridge {
|
|
|
212
212
|
await queue;
|
|
213
213
|
},
|
|
214
214
|
|
|
215
|
+
pruneBlocks(maxAgeMs: number) {
|
|
216
|
+
return store.prune(maxAgeMs);
|
|
217
|
+
},
|
|
218
|
+
|
|
215
219
|
close() {
|
|
216
220
|
closed = true;
|
|
217
221
|
pending.clear();
|
|
@@ -226,6 +230,7 @@ export function createDisabledBridge(): ContextBridge {
|
|
|
226
230
|
resolve: async () => null,
|
|
227
231
|
recordTurn: () => {},
|
|
228
232
|
flush: async () => {},
|
|
233
|
+
pruneBlocks: () => 0,
|
|
229
234
|
close: () => {},
|
|
230
235
|
};
|
|
231
236
|
}
|
package/src/context/store.ts
CHANGED
|
@@ -42,7 +42,16 @@ export function createContextStore(db: Database): ContextBlockStore {
|
|
|
42
42
|
VALUES ($key, $scope, $sessionId, $createdAtMs)
|
|
43
43
|
ON CONFLICT(conversation_key) DO UPDATE SET session_id = excluded.session_id
|
|
44
44
|
`);
|
|
45
|
-
|
|
45
|
+
// Age alone is the wrong test: a block older than the staleness TTL may still
|
|
46
|
+
// be PINNED by a live conversation, and deleting it forces that conversation
|
|
47
|
+
// to refetch and re-inject different bytes — a prompt-cache miss caused by
|
|
48
|
+
// housekeeping. Blocks are content-addressed and shared, so the safe set is
|
|
49
|
+
// "old AND referenced by no conversation".
|
|
50
|
+
const deleteStale: Statement<unknown, [number]> = db.query(`
|
|
51
|
+
DELETE FROM context_blocks
|
|
52
|
+
WHERE fetched_at_ms < ?
|
|
53
|
+
AND version NOT IN (SELECT context_version FROM conversations WHERE context_version IS NOT NULL)
|
|
54
|
+
`);
|
|
46
55
|
|
|
47
56
|
return {
|
|
48
57
|
get(version) {
|
package/src/context/types.ts
CHANGED
|
@@ -77,6 +77,13 @@ export interface ContextBridge {
|
|
|
77
77
|
recordTurn(rec: TurnRecord): void;
|
|
78
78
|
/** Drains the write queue. For tests and shutdown. */
|
|
79
79
|
flush(): Promise<void>;
|
|
80
|
+
/**
|
|
81
|
+
* Housekeeping: drops stored blocks older than `maxAgeMs` that no
|
|
82
|
+
* conversation still pins, returning the count removed. Blocks are
|
|
83
|
+
* content-addressed and shared, so nothing else reclaims them — without this
|
|
84
|
+
* the table grows for the life of the install.
|
|
85
|
+
*/
|
|
86
|
+
pruneBlocks(maxAgeMs: number): number;
|
|
80
87
|
close(): void;
|
|
81
88
|
}
|
|
82
89
|
|
|
@@ -87,5 +94,11 @@ export interface ContextBlockStore {
|
|
|
87
94
|
/** agentdox session id previously opened for a conversation. */
|
|
88
95
|
sessionFor(conversationKey: string): string | null;
|
|
89
96
|
bindSession(conversationKey: string, scope: string, sessionId: string): void;
|
|
97
|
+
/**
|
|
98
|
+
* Drops blocks older than `maxAgeMs` that NO conversation still pins.
|
|
99
|
+
* Returns the number removed. Referenced blocks are kept regardless of age:
|
|
100
|
+
* deleting one would force a live conversation to refetch and re-inject
|
|
101
|
+
* different bytes, turning housekeeping into a prompt-cache miss.
|
|
102
|
+
*/
|
|
90
103
|
prune(maxAgeMs: number): number;
|
|
91
104
|
}
|
package/src/cost/ledger.ts
CHANGED
|
@@ -22,6 +22,7 @@ import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageC
|
|
|
22
22
|
|
|
23
23
|
/** Estimates below this many samples are noise; the default ratio is better. */
|
|
24
24
|
const MIN_CALIBRATION_SAMPLES = 20;
|
|
25
|
+
const DAY_MS = 86_400_000;
|
|
25
26
|
|
|
26
27
|
// Row shapes below are fixed by our own schema in util/sqlite.ts.
|
|
27
28
|
interface LedgerRow {
|
|
@@ -243,9 +244,12 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
243
244
|
const spendSinceHarnessStmt = db.query(
|
|
244
245
|
"SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND harness_id = ?",
|
|
245
246
|
);
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
247
|
+
// `created_at_ms > ?` is always present, with a cutoff of 0 meaning all-time.
|
|
248
|
+
// One statement shape rather than two keeps the plan (and the index it uses,
|
|
249
|
+
// idx_ledger_slug_created) identical whether or not a window is configured.
|
|
250
|
+
const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
|
|
251
|
+
const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
|
|
252
|
+
const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
|
|
249
253
|
const latencyStmt = db.query(
|
|
250
254
|
`SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
|
|
251
255
|
);
|
|
@@ -349,16 +353,21 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
349
353
|
},
|
|
350
354
|
|
|
351
355
|
trust(slug: string, harnessId?: string): ModelTrust | null {
|
|
356
|
+
// Read the window at CALL time, not at construction: hot reload mutates
|
|
357
|
+
// the shared config object in place, so a pinned value would ignore an
|
|
358
|
+
// edit until restart. 0 => cutoff 0 => every row qualifies.
|
|
359
|
+
const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
|
|
352
360
|
const row =
|
|
353
361
|
harnessId !== undefined && harnessId !== ""
|
|
354
|
-
? (trustHarnessStmt.get(slug, harnessId) as TrustRow | null)
|
|
355
|
-
: (trustStmt.get(slug) as TrustRow | null);
|
|
362
|
+
? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
|
|
363
|
+
: (trustStmt.get(slug, cutoff) as TrustRow | null);
|
|
356
364
|
if (row === null || row.attempts === 0) return null;
|
|
357
365
|
return toTrust(slug, row);
|
|
358
366
|
},
|
|
359
367
|
|
|
360
368
|
allTrust(): ModelTrust[] {
|
|
361
|
-
const
|
|
369
|
+
const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
|
|
370
|
+
const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
|
|
362
371
|
return rows.map((row) => toTrust(row.slug, row));
|
|
363
372
|
},
|
|
364
373
|
|
package/src/server/http.ts
CHANGED
|
@@ -218,6 +218,8 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
218
218
|
log.warn("initial catalog fetch failed", { error: err instanceof Error ? err.message : String(err) });
|
|
219
219
|
});
|
|
220
220
|
|
|
221
|
+
// One housekeeping timer for both tables. `unref`'d so it never holds the
|
|
222
|
+
// process open.
|
|
221
223
|
const pruneTimer = setInterval(() => {
|
|
222
224
|
try {
|
|
223
225
|
const dropped = conversations.prune(cfg.ledger.conversationTtlMs);
|
|
@@ -225,6 +227,16 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
225
227
|
} catch (err) {
|
|
226
228
|
log.warn("conversation prune failed", { error: err instanceof Error ? err.message : String(err) });
|
|
227
229
|
}
|
|
230
|
+
try {
|
|
231
|
+
// Past the staleness TTL every pin refreshes anyway, so an unreferenced
|
|
232
|
+
// block of that age has no future reader. Nothing else reclaims these:
|
|
233
|
+
// blocks are content-addressed and shared, so they accumulated for the
|
|
234
|
+
// life of the install (measured: 220 rows / 2.7 MB, 68 unreferenced).
|
|
235
|
+
const dropped = context.pruneBlocks(cfg.context.maxStalenessMs);
|
|
236
|
+
if (dropped > 0) log.debug("pruned unreferenced context blocks", { dropped });
|
|
237
|
+
} catch (err) {
|
|
238
|
+
log.warn("context block prune failed", { error: err instanceof Error ? err.message : String(err) });
|
|
239
|
+
}
|
|
228
240
|
}, 60_000);
|
|
229
241
|
pruneTimer.unref();
|
|
230
242
|
|
package/src/util/sqlite.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
|
|
|
18
18
|
import { dirname } from "node:path";
|
|
19
19
|
|
|
20
20
|
/** Bump when a migration is added; guarded below so reopening never regresses it. */
|
|
21
|
-
const USER_VERSION =
|
|
21
|
+
const USER_VERSION = 14;
|
|
22
22
|
|
|
23
23
|
const MIGRATIONS = `
|
|
24
24
|
CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
@@ -71,6 +71,11 @@ CREATE TABLE IF NOT EXISTS ledger (
|
|
|
71
71
|
CREATE INDEX IF NOT EXISTS idx_ledger_conversation ON ledger (conversation_key);
|
|
72
72
|
CREATE INDEX IF NOT EXISTS idx_ledger_created ON ledger (created_at_ms);
|
|
73
73
|
CREATE INDEX IF NOT EXISTS idx_ledger_slug ON ledger (slug);
|
|
74
|
+
-- Per-slug newest-first reads: the latency window (newest N rows for one slug)
|
|
75
|
+
-- and any windowed trust. With only idx_ledger_slug those sorted every row the
|
|
76
|
+
-- slug ever had; measured on a real ledger, the latency statement went from
|
|
77
|
+
-- 5-10ms and RISING with history to a flat 0.04-0.08ms.
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_ledger_slug_created ON ledger (slug, created_at_ms DESC);
|
|
74
79
|
|
|
75
80
|
CREATE TABLE IF NOT EXISTS token_calibration (
|
|
76
81
|
tokenizer TEXT PRIMARY KEY,
|
|
@@ -230,6 +235,11 @@ ALTER TABLE conversations ADD COLUMN compaction_plan TEXT;
|
|
|
230
235
|
// is on. Another new table via the idempotent MIGRATIONS block; version bump
|
|
231
236
|
// only, no ALTER guard.
|
|
232
237
|
|
|
238
|
+
// v14: idx_ledger_slug_created (slug, created_at_ms DESC) serves the per-slug
|
|
239
|
+
// newest-first reads — the latency window, and trust when filters.trustWindowDays
|
|
240
|
+
// is set. Created idempotently by the MIGRATIONS block above, so the version bump
|
|
241
|
+
// alone records it; no ALTER guard needed.
|
|
242
|
+
|
|
233
243
|
export function openDb(path: string): Database {
|
|
234
244
|
// ":memory:" has no parent directory to create.
|
|
235
245
|
if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { createContextStore } from "../src/context/store.ts";
|
|
4
|
+
import { createConversationStore } from "../src/router/state.ts";
|
|
5
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `context_blocks` is content-addressed and shared between conversations, so
|
|
9
|
+
* nothing reclaims a block when the conversation that fetched it goes away.
|
|
10
|
+
* Until the prune below was wired into the server's housekeeping timer the table
|
|
11
|
+
* grew for the life of the install (measured on a real install: 220 rows /
|
|
12
|
+
* 2.7 MB, 68 of them referenced by nothing).
|
|
13
|
+
*
|
|
14
|
+
* Age alone is the wrong test, though: a block past the staleness TTL may still
|
|
15
|
+
* be PINNED, and deleting it forces that conversation to refetch and inject
|
|
16
|
+
* different bytes — housekeeping causing a prompt-cache miss. So the safe set is
|
|
17
|
+
* "old AND unreferenced".
|
|
18
|
+
*/
|
|
19
|
+
describe("context block prune", () => {
|
|
20
|
+
const HOUR = 3_600_000;
|
|
21
|
+
|
|
22
|
+
function seed() {
|
|
23
|
+
const db = openDb(":memory:");
|
|
24
|
+
const blocks = createContextStore(db);
|
|
25
|
+
const conversations = createConversationStore(db);
|
|
26
|
+
const now = Date.now();
|
|
27
|
+
|
|
28
|
+
blocks.put("scope", { version: "old-pinned", block: "A", fetchedAtMs: now - 5 * HOUR });
|
|
29
|
+
blocks.put("scope", { version: "old-orphan", block: "B", fetchedAtMs: now - 5 * HOUR });
|
|
30
|
+
blocks.put("scope", { version: "fresh-orphan", block: "C", fetchedAtMs: now });
|
|
31
|
+
|
|
32
|
+
// One live conversation still pins `old-pinned`.
|
|
33
|
+
const state = conversations.load("conv-1");
|
|
34
|
+
state.contextVersion = "old-pinned";
|
|
35
|
+
state.contextFetchedAtMs = now - 5 * HOUR;
|
|
36
|
+
conversations.save(state);
|
|
37
|
+
|
|
38
|
+
return { db, blocks };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
test("drops an old block that nothing references", () => {
|
|
42
|
+
const { db, blocks } = seed();
|
|
43
|
+
expect(blocks.prune(HOUR)).toBe(1);
|
|
44
|
+
expect(blocks.get("old-orphan")).toBeNull();
|
|
45
|
+
db.close();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("keeps an old block a conversation still pins", () => {
|
|
49
|
+
const { db, blocks } = seed();
|
|
50
|
+
blocks.prune(HOUR);
|
|
51
|
+
// Deleting this one would cost that conversation its warm prefix.
|
|
52
|
+
expect(blocks.get("old-pinned")?.block).toBe("A");
|
|
53
|
+
db.close();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("keeps a block younger than the age cutoff", () => {
|
|
57
|
+
const { db, blocks } = seed();
|
|
58
|
+
blocks.prune(HOUR);
|
|
59
|
+
expect(blocks.get("fresh-orphan")?.block).toBe("C");
|
|
60
|
+
db.close();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("is a no-op once the unreferenced blocks are gone", () => {
|
|
64
|
+
const { db, blocks } = seed();
|
|
65
|
+
expect(blocks.prune(HOUR)).toBe(1);
|
|
66
|
+
expect(blocks.prune(HOUR)).toBe(0);
|
|
67
|
+
db.close();
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("reclaims a block as soon as its last pin is dropped", () => {
|
|
71
|
+
const { db, blocks } = seed();
|
|
72
|
+
const conversations = createConversationStore(db);
|
|
73
|
+
// The conversation moves to a new context version (a refresh), which is
|
|
74
|
+
// what leaves the old block orphaned in production.
|
|
75
|
+
const state = conversations.load("conv-1");
|
|
76
|
+
state.contextVersion = "fresh-orphan";
|
|
77
|
+
conversations.save(state);
|
|
78
|
+
|
|
79
|
+
expect(blocks.prune(HOUR)).toBe(2); // old-pinned is now unreferenced too
|
|
80
|
+
expect(blocks.get("old-pinned")).toBeNull();
|
|
81
|
+
expect(blocks.get("fresh-orphan")?.block).toBe("C");
|
|
82
|
+
db.close();
|
|
83
|
+
});
|
|
84
|
+
});
|
package/test/failover.test.ts
CHANGED
|
@@ -44,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
44
44
|
data: { axis: "intelligence", minQuality: 0 },
|
|
45
45
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
46
46
|
},
|
|
47
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
47
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
48
48
|
classifier: {
|
|
49
49
|
ambiguityThreshold: 0,
|
|
50
50
|
model: "test/adjudicator",
|
|
@@ -244,11 +244,11 @@ describe("v4 migration", () => {
|
|
|
244
244
|
}
|
|
245
245
|
});
|
|
246
246
|
|
|
247
|
-
test("schema is at user_version
|
|
247
|
+
test("schema is at user_version 14", () => {
|
|
248
248
|
const db = openDb(":memory:");
|
|
249
249
|
try {
|
|
250
250
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
251
|
-
expect(row.user_version).toBe(
|
|
251
|
+
expect(row.user_version).toBe(14);
|
|
252
252
|
} finally {
|
|
253
253
|
db.close();
|
|
254
254
|
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
|
+
import { createLedger } from "../src/cost/ledger.ts";
|
|
5
|
+
import type { LedgerEntry } from "../src/cost/types.ts";
|
|
6
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* `filters.trustWindowDays` bounds the per-slug trust aggregate, which otherwise
|
|
10
|
+
* scans every row a model ever had — on every candidate, on every turn. Measured
|
|
11
|
+
* on a real ledger it grows from 0.8 ms at 9k rows to 11.6 ms at 75k, so it
|
|
12
|
+
* becomes a per-turn latency tax as history accumulates.
|
|
13
|
+
*
|
|
14
|
+
* It defaults to 0 (all-time) because narrowing it CHANGES ROUTING: smaller
|
|
15
|
+
* denominators move success rates, which moves the demotion guard. These tests
|
|
16
|
+
* pin both halves of that contract — off is byte-identical to the old behaviour,
|
|
17
|
+
* and on genuinely excludes old rows.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const DAY = 86_400_000;
|
|
21
|
+
|
|
22
|
+
function cfgWith(trustWindowDays: number) {
|
|
23
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
24
|
+
cfg.filters.trustWindowDays = trustWindowDays;
|
|
25
|
+
cfg.ledger.path = ":memory:";
|
|
26
|
+
return cfg;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
30
|
+
return {
|
|
31
|
+
id: crypto.randomUUID(),
|
|
32
|
+
createdAtMs: Date.now(),
|
|
33
|
+
conversationKey: "k",
|
|
34
|
+
sessionId: "s",
|
|
35
|
+
turn: 1,
|
|
36
|
+
requestedModel: "auto",
|
|
37
|
+
harnessId: "",
|
|
38
|
+
ompSessionId: "",
|
|
39
|
+
slug: "vendor/model",
|
|
40
|
+
servedSlug: "vendor/model",
|
|
41
|
+
tier: "simple",
|
|
42
|
+
classificationSource: "heuristic",
|
|
43
|
+
reasons: [],
|
|
44
|
+
features: null,
|
|
45
|
+
score: null,
|
|
46
|
+
confidence: null,
|
|
47
|
+
task: null,
|
|
48
|
+
classifierReasons: null,
|
|
49
|
+
exploredFrom: null,
|
|
50
|
+
holdArm: null,
|
|
51
|
+
predictedUsd: 0.001,
|
|
52
|
+
reportedUsd: 0.001,
|
|
53
|
+
usage: { promptTokens: 10, cachedTokens: 0, cacheWriteTokens: 0, completionTokens: 5, reasoningTokens: 0, images: 0 },
|
|
54
|
+
attempt: 0,
|
|
55
|
+
escalationSignal: null,
|
|
56
|
+
latencyMs: 100,
|
|
57
|
+
ttftMs: 50,
|
|
58
|
+
finishReason: "stop",
|
|
59
|
+
wasted: false,
|
|
60
|
+
upstreamGenerationId: null,
|
|
61
|
+
error: null,
|
|
62
|
+
errorKind: null,
|
|
63
|
+
promptTokensSaved: null,
|
|
64
|
+
...over,
|
|
65
|
+
} as LedgerEntry;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Old rows: half of them failures. Recent rows: all clean. */
|
|
69
|
+
function seed(windowDays: number) {
|
|
70
|
+
const cfg = cfgWith(windowDays);
|
|
71
|
+
const db = openDb(":memory:");
|
|
72
|
+
const ledger = createLedger(db, cfg);
|
|
73
|
+
const now = Date.now();
|
|
74
|
+
for (let i = 0; i < 10; i++) {
|
|
75
|
+
ledger.record(
|
|
76
|
+
entry({
|
|
77
|
+
createdAtMs: now - 30 * DAY,
|
|
78
|
+
...(i % 2 === 0 ? { error: "server_error: boom", errorKind: "server_error" } : {}),
|
|
79
|
+
}),
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
for (let i = 0; i < 10; i++) ledger.record(entry({ createdAtMs: now - 1 * DAY }));
|
|
83
|
+
return { db, ledger, cfg };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
describe("filters.trustWindowDays", () => {
|
|
87
|
+
test("0 means all-time: every row counts", () => {
|
|
88
|
+
const { db, ledger } = seed(0);
|
|
89
|
+
const trust = ledger.trust("vendor/model");
|
|
90
|
+
expect(trust?.attempts).toBe(20);
|
|
91
|
+
expect(trust?.errors).toBe(5);
|
|
92
|
+
db.close();
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("a window excludes rows older than it", () => {
|
|
96
|
+
const { db, ledger } = seed(7);
|
|
97
|
+
const trust = ledger.trust("vendor/model");
|
|
98
|
+
// Only the 10 recent, clean rows remain.
|
|
99
|
+
expect(trust?.attempts).toBe(10);
|
|
100
|
+
expect(trust?.errors).toBe(0);
|
|
101
|
+
db.close();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("the window moves the success rate, which is why it is opt-in", () => {
|
|
105
|
+
const all = seed(0);
|
|
106
|
+
const windowed = seed(7);
|
|
107
|
+
const allTrust = all.ledger.trust("vendor/model");
|
|
108
|
+
const winTrust = windowed.ledger.trust("vendor/model");
|
|
109
|
+
expect(allTrust?.successRate).toBeLessThan(winTrust?.successRate ?? 0);
|
|
110
|
+
all.db.close();
|
|
111
|
+
windowed.db.close();
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("is read per call, so a hot-reloaded edit takes effect immediately", () => {
|
|
115
|
+
const { db, ledger, cfg } = seed(0);
|
|
116
|
+
expect(ledger.trust("vendor/model")?.attempts).toBe(20);
|
|
117
|
+
// Hot reload mutates the shared config object in place.
|
|
118
|
+
cfg.filters.trustWindowDays = 7;
|
|
119
|
+
expect(ledger.trust("vendor/model")?.attempts).toBe(10);
|
|
120
|
+
db.close();
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("allTrust honours the same window", () => {
|
|
124
|
+
const { db, ledger } = seed(7);
|
|
125
|
+
const rows = ledger.allTrust();
|
|
126
|
+
expect(rows).toHaveLength(1);
|
|
127
|
+
expect(rows[0]?.attempts).toBe(10);
|
|
128
|
+
db.close();
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("ships disabled, so the default install is unchanged", () => {
|
|
132
|
+
// DEFAULT_CONFIG, not loadConfig: loadConfig reads the machine's real
|
|
133
|
+
// config.yml, which has broken this suite before.
|
|
134
|
+
expect(DEFAULT_CONFIG.filters.trustWindowDays).toBe(0);
|
|
135
|
+
});
|
|
136
|
+
});
|
package/test/turn.test.ts
CHANGED
|
@@ -45,7 +45,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
45
45
|
data: { axis: "intelligence", minQuality: 0 },
|
|
46
46
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
},
|
|
48
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
48
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
49
49
|
classifier: {
|
|
50
50
|
ambiguityThreshold: 0,
|
|
51
51
|
model: "test/adjudicator",
|
|
@@ -583,6 +583,7 @@ describe("agentdox write-back sees the shape of the turn", () => {
|
|
|
583
583
|
records.push(rec);
|
|
584
584
|
},
|
|
585
585
|
flush: () => Promise.resolve(),
|
|
586
|
+
pruneBlocks: () => 0,
|
|
586
587
|
close: () => {},
|
|
587
588
|
},
|
|
588
589
|
};
|