auto-model-router 0.2.27 → 0.2.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -102,6 +102,29 @@ score nearly as well. If your workload needs a frontier model on hard turns,
102
102
  raise the tier price ceiling and `qualityExponent` — measured thresholds are in
103
103
  [`docs/routing-benchmark-findings.md`](docs/routing-benchmark-findings.md).
104
104
 
105
+ ### Real-world — a week on the live ledger
106
+
107
+ The suites above are small and clean. To measure the economics on *actual*
108
+ usage we replayed a week of real omp traffic from the router's own ledger —
109
+ **6 918 billed turns across 299 conversations, 7 days, 410:1 input-to-output,
110
+ 68% cache hit** — and repriced the identical token stream against a single Opus 5
111
+ model with its own cache namespace.
112
+
113
+ | | auto-model-router | Claude Opus 5 (single-model) |
114
+ | --- | --- | --- |
115
+ | Spend over the week | **$61.69** | $921.20 |
116
+ | Per turn | **$0.0089** | $0.133 |
117
+ | Extrapolated / month | **$263** | $3 932 |
118
+
119
+ **≈15× cheaper, ~93% saved** — a four-figure monthly bill becomes a three-figure
120
+ one. This baseline is deliberately conservative: one cache namespace, with each
121
+ conversation's cache replayed on the real turn gaps. A naive like-for-like
122
+ repricing at Opus rates reports ~31×, but on a single model the replayed context
123
+ is cache reads at $0.50/MTok, so ≈15× is the number we stand behind. Unlike the
124
+ core suite, sustained work on a large codebase is dominated by the conversation
125
+ resent each turn rather than per-token price — exactly where a single frontier
126
+ model gets expensive and routing's per-turn cache awareness pays off.
127
+
105
128
  ### Scope
106
129
 
107
130
  These are small, self-contained tasks of one to three files, solved in under 25
@@ -111,11 +134,6 @@ separates. The cost multiple varied between 14× and 32× across runs depending
111
134
  which task the baseline stalled on — treat "well over an order of magnitude" as
112
135
  the claim, not a specific figure.
113
136
 
114
- For sustained work on a large codebase the economics differ: cost there is
115
- dominated by the conversation being resent each turn rather than by per-token
116
- price. Replaying a week of real omp traffic (6 918 billed turns, 410:1
117
- input-to-output) against a single-model baseline gives **≈15×**.
118
-
119
137
  Harness, tasks and raw per-turn data:
120
138
  [`docs/routing-benchmark-findings.md`](docs/routing-benchmark-findings.md).
121
139
 
@@ -666,13 +684,13 @@ and brief:
666
684
  ```bash
667
685
  export AGENTDOX_URL=http://localhost:3003
668
686
  export AGENTDOX_TOKEN=<pat with read+write on the scope>
669
- export AGENTDOX_SCOPE=ashlands # fallback only; see below
687
+ export AGENTDOX_SCOPE=myproject # fallback only; see below
670
688
  ```
671
689
 
672
690
  Setting a URL and a token is enough to turn it on.
673
691
 
674
692
  The scope is **derived per workspace** from the directory basename
675
- (`E:/projects/ashlands` → `ashlands`), and that derivation wins. `AGENTDOX_SCOPE` /
693
+ (`E:/projects/myproject` → `myproject`), and that derivation wins. `AGENTDOX_SCOPE` /
676
694
  `context.defaultScope` is only a fallback for workspaces it cannot resolve, because one router
677
695
  install serves every project on the machine — a slug pinned there would be sent for all of
678
696
  them, injecting one project's context into another's work. A single configured token also
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "auto-model-router",
3
- "version": "0.2.27",
3
+ "version": "0.2.28",
4
4
  "private": false,
5
5
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
6
6
  "type": "module",
@@ -71,6 +71,11 @@ export const DEFAULT_CONFIG: RouterConfig = {
71
71
  // Shared trust by default: more samples, demotion guard stays effective
72
72
  // even with a tiny guardrail-narrowed catalog.
73
73
  trustScopedByHarness: false,
74
+ // 0 = all-time, the shipped behaviour. Reliability is slow-moving, so a
75
+ // wide sample is right on the merits; a window exists to bound the
76
+ // per-slug scan once history is large, and it changes routing, so it is
77
+ // opt-in after a replay run prices it.
78
+ trustWindowDays: 0,
74
79
  contextHeadroom: 1.25,
75
80
  // Latency scoring is off by default (weight 0): opt in after establishing a
76
81
  // baseline. Expected total wait (TTFT + expected completion / throughput)
@@ -67,6 +67,7 @@ const filters = z.strictObject({
67
67
  minTrust: z.number().min(0).max(1).optional(),
68
68
  minTrustSamples: z.number().int().nonnegative().optional(),
69
69
  trustScopedByHarness: z.boolean().optional(),
70
+ trustWindowDays: z.number().nonnegative().optional(),
70
71
  contextHeadroom: z.number().positive().optional(),
71
72
  latencyWeight: z.number().nonnegative().optional(),
72
73
  latencyReferenceMs: z.number().positive().optional(),
@@ -167,6 +167,22 @@ export interface FilterConfig {
167
167
  * to learn its own reliability.
168
168
  */
169
169
  trustScopedByHarness: boolean;
170
+ /**
171
+ * Only count ledger rows from the last N days toward model trust. 0 (the
172
+ * default) keeps the all-time behaviour.
173
+ *
174
+ * Trust is deliberately all-time: reliability is slow-moving, and a wide
175
+ * sample keeps the demotion guard stable. The cost is that the per-slug
176
+ * trust aggregate scans every row a model ever had, and that runs for each
177
+ * candidate on every turn — measured on a real ledger it grows from 0.8 ms at
178
+ * 9k rows to 11.6 ms at 75k, i.e. it becomes a per-turn latency tax as
179
+ * history accumulates. A window bounds that scan.
180
+ *
181
+ * Setting it CHANGES ROUTING (smaller denominators move success rates), so
182
+ * price it on the ledger with `bun tools/replay.ts --set
183
+ * filters.trustWindowDays=N` before enabling.
184
+ */
185
+ trustWindowDays: number;
170
186
  /**
171
187
  * Headroom multiplier applied to estimated prompt tokens when checking a
172
188
  * model's context window, absorbing token-estimate error and the response.
@@ -212,6 +212,10 @@ export function createContextBridge(opts: BridgeOptions): ContextBridge {
212
212
  await queue;
213
213
  },
214
214
 
215
+ pruneBlocks(maxAgeMs: number) {
216
+ return store.prune(maxAgeMs);
217
+ },
218
+
215
219
  close() {
216
220
  closed = true;
217
221
  pending.clear();
@@ -226,6 +230,7 @@ export function createDisabledBridge(): ContextBridge {
226
230
  resolve: async () => null,
227
231
  recordTurn: () => {},
228
232
  flush: async () => {},
233
+ pruneBlocks: () => 0,
229
234
  close: () => {},
230
235
  };
231
236
  }
@@ -42,7 +42,16 @@ export function createContextStore(db: Database): ContextBlockStore {
42
42
  VALUES ($key, $scope, $sessionId, $createdAtMs)
43
43
  ON CONFLICT(conversation_key) DO UPDATE SET session_id = excluded.session_id
44
44
  `);
45
- const deleteStale: Statement<unknown, [number]> = db.query("DELETE FROM context_blocks WHERE fetched_at_ms < ?");
45
+ // Age alone is the wrong test: a block older than the staleness TTL may still
46
+ // be PINNED by a live conversation, and deleting it forces that conversation
47
+ // to refetch and re-inject different bytes — a prompt-cache miss caused by
48
+ // housekeeping. Blocks are content-addressed and shared, so the safe set is
49
+ // "old AND referenced by no conversation".
50
+ const deleteStale: Statement<unknown, [number]> = db.query(`
51
+ DELETE FROM context_blocks
52
+ WHERE fetched_at_ms < ?
53
+ AND version NOT IN (SELECT context_version FROM conversations WHERE context_version IS NOT NULL)
54
+ `);
46
55
 
47
56
  return {
48
57
  get(version) {
@@ -77,6 +77,13 @@ export interface ContextBridge {
77
77
  recordTurn(rec: TurnRecord): void;
78
78
  /** Drains the write queue. For tests and shutdown. */
79
79
  flush(): Promise<void>;
80
+ /**
81
+ * Housekeeping: drops stored blocks older than `maxAgeMs` that no
82
+ * conversation still pins, returning the count removed. Blocks are
83
+ * content-addressed and shared, so nothing else reclaims them — without this
84
+ * the table grows for the life of the install.
85
+ */
86
+ pruneBlocks(maxAgeMs: number): number;
80
87
  close(): void;
81
88
  }
82
89
 
@@ -87,5 +94,11 @@ export interface ContextBlockStore {
87
94
  /** agentdox session id previously opened for a conversation. */
88
95
  sessionFor(conversationKey: string): string | null;
89
96
  bindSession(conversationKey: string, scope: string, sessionId: string): void;
97
+ /**
98
+ * Drops blocks older than `maxAgeMs` that NO conversation still pins.
99
+ * Returns the number removed. Referenced blocks are kept regardless of age:
100
+ * deleting one would force a live conversation to refetch and re-inject
101
+ * different bytes, turning housekeeping into a prompt-cache miss.
102
+ */
90
103
  prune(maxAgeMs: number): number;
91
104
  }
@@ -22,6 +22,7 @@ import type { BlendedRate, Ledger, LedgerEntry, ModelLatency, ModelTrust, UsageC
22
22
 
23
23
  /** Estimates below this many samples are noise; the default ratio is better. */
24
24
  const MIN_CALIBRATION_SAMPLES = 20;
25
+ const DAY_MS = 86_400_000;
25
26
 
26
27
  // Row shapes below are fixed by our own schema in util/sqlite.ts.
27
28
  interface LedgerRow {
@@ -243,9 +244,12 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
243
244
  const spendSinceHarnessStmt = db.query(
244
245
  "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND harness_id = ?",
245
246
  );
246
- const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ?`);
247
- const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ?`);
248
- const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger GROUP BY slug`);
247
+ // `created_at_ms > ?` is always present, with a cutoff of 0 meaning all-time.
248
+ // One statement shape rather than two keeps the plan (and the index it uses,
249
+ // idx_ledger_slug_created) identical whether or not a window is configured.
250
+ const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
251
+ const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
252
+ const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
249
253
  const latencyStmt = db.query(
250
254
  `SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
251
255
  );
@@ -349,16 +353,21 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
349
353
  },
350
354
 
351
355
  trust(slug: string, harnessId?: string): ModelTrust | null {
356
+ // Read the window at CALL time, not at construction: hot reload mutates
357
+ // the shared config object in place, so a pinned value would ignore an
358
+ // edit until restart. 0 => cutoff 0 => every row qualifies.
359
+ const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
352
360
  const row =
353
361
  harnessId !== undefined && harnessId !== ""
354
- ? (trustHarnessStmt.get(slug, harnessId) as TrustRow | null)
355
- : (trustStmt.get(slug) as TrustRow | null);
362
+ ? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
363
+ : (trustStmt.get(slug, cutoff) as TrustRow | null);
356
364
  if (row === null || row.attempts === 0) return null;
357
365
  return toTrust(slug, row);
358
366
  },
359
367
 
360
368
  allTrust(): ModelTrust[] {
361
- const rows = allTrustStmt.all() as (TrustRow & { slug: string })[];
369
+ const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
370
+ const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
362
371
  return rows.map((row) => toTrust(row.slug, row));
363
372
  },
364
373
 
@@ -218,6 +218,8 @@ export function startServer(cfg: RouterConfig): StartedServer {
218
218
  log.warn("initial catalog fetch failed", { error: err instanceof Error ? err.message : String(err) });
219
219
  });
220
220
 
221
+ // One housekeeping timer for both tables. `unref`'d so it never holds the
222
+ // process open.
221
223
  const pruneTimer = setInterval(() => {
222
224
  try {
223
225
  const dropped = conversations.prune(cfg.ledger.conversationTtlMs);
@@ -225,6 +227,16 @@ export function startServer(cfg: RouterConfig): StartedServer {
225
227
  } catch (err) {
226
228
  log.warn("conversation prune failed", { error: err instanceof Error ? err.message : String(err) });
227
229
  }
230
+ try {
231
+ // Past the staleness TTL every pin refreshes anyway, so an unreferenced
232
+ // block of that age has no future reader. Nothing else reclaims these:
233
+ // blocks are content-addressed and shared, so they accumulated for the
234
+ // life of the install (measured: 220 rows / 2.7 MB, 68 unreferenced).
235
+ const dropped = context.pruneBlocks(cfg.context.maxStalenessMs);
236
+ if (dropped > 0) log.debug("pruned unreferenced context blocks", { dropped });
237
+ } catch (err) {
238
+ log.warn("context block prune failed", { error: err instanceof Error ? err.message : String(err) });
239
+ }
228
240
  }, 60_000);
229
241
  pruneTimer.unref();
230
242
 
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
18
18
  import { dirname } from "node:path";
19
19
 
20
20
  /** Bump when a migration is added; guarded below so reopening never regresses it. */
21
- const USER_VERSION = 13;
21
+ const USER_VERSION = 14;
22
22
 
23
23
  const MIGRATIONS = `
24
24
  CREATE TABLE IF NOT EXISTS catalog_cache (
@@ -71,6 +71,11 @@ CREATE TABLE IF NOT EXISTS ledger (
71
71
  CREATE INDEX IF NOT EXISTS idx_ledger_conversation ON ledger (conversation_key);
72
72
  CREATE INDEX IF NOT EXISTS idx_ledger_created ON ledger (created_at_ms);
73
73
  CREATE INDEX IF NOT EXISTS idx_ledger_slug ON ledger (slug);
74
+ -- Per-slug newest-first reads: the latency window (newest N rows for one slug)
75
+ -- and any windowed trust. With only idx_ledger_slug those sorted every row the
76
+ -- slug ever had; measured on a real ledger, the latency statement went from
77
+ -- 5-10ms and RISING with history to a flat 0.04-0.08ms.
78
+ CREATE INDEX IF NOT EXISTS idx_ledger_slug_created ON ledger (slug, created_at_ms DESC);
74
79
 
75
80
  CREATE TABLE IF NOT EXISTS token_calibration (
76
81
  tokenizer TEXT PRIMARY KEY,
@@ -230,6 +235,11 @@ ALTER TABLE conversations ADD COLUMN compaction_plan TEXT;
230
235
  // is on. Another new table via the idempotent MIGRATIONS block; version bump
231
236
  // only, no ALTER guard.
232
237
 
238
+ // v14: idx_ledger_slug_created (slug, created_at_ms DESC) serves the per-slug
239
+ // newest-first reads — the latency window, and trust when filters.trustWindowDays
240
+ // is set. Created idempotently by the MIGRATIONS block above, so the version bump
241
+ // alone records it; no ALTER guard needed.
242
+
233
243
  export function openDb(path: string): Database {
234
244
  // ":memory:" has no parent directory to create.
235
245
  if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
@@ -0,0 +1,84 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { createContextStore } from "../src/context/store.ts";
4
+ import { createConversationStore } from "../src/router/state.ts";
5
+ import { openDb } from "../src/util/sqlite.ts";
6
+
7
+ /**
8
+ * `context_blocks` is content-addressed and shared between conversations, so
9
+ * nothing reclaims a block when the conversation that fetched it goes away.
10
+ * Until the prune below was wired into the server's housekeeping timer the table
11
+ * grew for the life of the install (measured on a real install: 220 rows /
12
+ * 2.7 MB, 68 of them referenced by nothing).
13
+ *
14
+ * Age alone is the wrong test, though: a block past the staleness TTL may still
15
+ * be PINNED, and deleting it forces that conversation to refetch and inject
16
+ * different bytes — housekeeping causing a prompt-cache miss. So the safe set is
17
+ * "old AND unreferenced".
18
+ */
19
+ describe("context block prune", () => {
20
+ const HOUR = 3_600_000;
21
+
22
+ function seed() {
23
+ const db = openDb(":memory:");
24
+ const blocks = createContextStore(db);
25
+ const conversations = createConversationStore(db);
26
+ const now = Date.now();
27
+
28
+ blocks.put("scope", { version: "old-pinned", block: "A", fetchedAtMs: now - 5 * HOUR });
29
+ blocks.put("scope", { version: "old-orphan", block: "B", fetchedAtMs: now - 5 * HOUR });
30
+ blocks.put("scope", { version: "fresh-orphan", block: "C", fetchedAtMs: now });
31
+
32
+ // One live conversation still pins `old-pinned`.
33
+ const state = conversations.load("conv-1");
34
+ state.contextVersion = "old-pinned";
35
+ state.contextFetchedAtMs = now - 5 * HOUR;
36
+ conversations.save(state);
37
+
38
+ return { db, blocks };
39
+ }
40
+
41
+ test("drops an old block that nothing references", () => {
42
+ const { db, blocks } = seed();
43
+ expect(blocks.prune(HOUR)).toBe(1);
44
+ expect(blocks.get("old-orphan")).toBeNull();
45
+ db.close();
46
+ });
47
+
48
+ test("keeps an old block a conversation still pins", () => {
49
+ const { db, blocks } = seed();
50
+ blocks.prune(HOUR);
51
+ // Deleting this one would cost that conversation its warm prefix.
52
+ expect(blocks.get("old-pinned")?.block).toBe("A");
53
+ db.close();
54
+ });
55
+
56
+ test("keeps a block younger than the age cutoff", () => {
57
+ const { db, blocks } = seed();
58
+ blocks.prune(HOUR);
59
+ expect(blocks.get("fresh-orphan")?.block).toBe("C");
60
+ db.close();
61
+ });
62
+
63
+ test("is a no-op once the unreferenced blocks are gone", () => {
64
+ const { db, blocks } = seed();
65
+ expect(blocks.prune(HOUR)).toBe(1);
66
+ expect(blocks.prune(HOUR)).toBe(0);
67
+ db.close();
68
+ });
69
+
70
+ test("reclaims a block as soon as its last pin is dropped", () => {
71
+ const { db, blocks } = seed();
72
+ const conversations = createConversationStore(db);
73
+ // The conversation moves to a new context version (a refresh), which is
74
+ // what leaves the old block orphaned in production.
75
+ const state = conversations.load("conv-1");
76
+ state.contextVersion = "fresh-orphan";
77
+ conversations.save(state);
78
+
79
+ expect(blocks.prune(HOUR)).toBe(2); // old-pinned is now unreferenced too
80
+ expect(blocks.get("old-pinned")).toBeNull();
81
+ expect(blocks.get("fresh-orphan")?.block).toBe("C");
82
+ db.close();
83
+ });
84
+ });
@@ -44,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
44
44
  data: { axis: "intelligence", minQuality: 0 },
45
45
  chat: { axis: "intelligence", minQuality: 0 },
46
46
  },
47
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
47
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
48
48
  classifier: {
49
49
  ambiguityThreshold: 0,
50
50
  model: "test/adjudicator",
@@ -244,11 +244,11 @@ describe("v4 migration", () => {
244
244
  }
245
245
  });
246
246
 
247
- test("schema is at user_version 13", () => {
247
+ test("schema is at user_version 14", () => {
248
248
  const db = openDb(":memory:");
249
249
  try {
250
250
  const row = db.query("PRAGMA user_version").get() as { user_version: number };
251
- expect(row.user_version).toBe(13);
251
+ expect(row.user_version).toBe(14);
252
252
  } finally {
253
253
  db.close();
254
254
  }
@@ -0,0 +1,136 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
4
+ import { createLedger } from "../src/cost/ledger.ts";
5
+ import type { LedgerEntry } from "../src/cost/types.ts";
6
+ import { openDb } from "../src/util/sqlite.ts";
7
+
8
+ /**
9
+ * `filters.trustWindowDays` bounds the per-slug trust aggregate, which otherwise
10
+ * scans every row a model ever had — on every candidate, on every turn. Measured
11
+ * on a real ledger it grows from 0.8 ms at 9k rows to 11.6 ms at 75k, so it
12
+ * becomes a per-turn latency tax as history accumulates.
13
+ *
14
+ * It defaults to 0 (all-time) because narrowing it CHANGES ROUTING: smaller
15
+ * denominators move success rates, which moves the demotion guard. These tests
16
+ * pin both halves of that contract — off is byte-identical to the old behaviour,
17
+ * and on genuinely excludes old rows.
18
+ */
19
+
20
+ const DAY = 86_400_000;
21
+
22
+ function cfgWith(trustWindowDays: number) {
23
+ const cfg = structuredClone(DEFAULT_CONFIG);
24
+ cfg.filters.trustWindowDays = trustWindowDays;
25
+ cfg.ledger.path = ":memory:";
26
+ return cfg;
27
+ }
28
+
29
+ function entry(over: Partial<LedgerEntry>): LedgerEntry {
30
+ return {
31
+ id: crypto.randomUUID(),
32
+ createdAtMs: Date.now(),
33
+ conversationKey: "k",
34
+ sessionId: "s",
35
+ turn: 1,
36
+ requestedModel: "auto",
37
+ harnessId: "",
38
+ ompSessionId: "",
39
+ slug: "vendor/model",
40
+ servedSlug: "vendor/model",
41
+ tier: "simple",
42
+ classificationSource: "heuristic",
43
+ reasons: [],
44
+ features: null,
45
+ score: null,
46
+ confidence: null,
47
+ task: null,
48
+ classifierReasons: null,
49
+ exploredFrom: null,
50
+ holdArm: null,
51
+ predictedUsd: 0.001,
52
+ reportedUsd: 0.001,
53
+ usage: { promptTokens: 10, cachedTokens: 0, cacheWriteTokens: 0, completionTokens: 5, reasoningTokens: 0, images: 0 },
54
+ attempt: 0,
55
+ escalationSignal: null,
56
+ latencyMs: 100,
57
+ ttftMs: 50,
58
+ finishReason: "stop",
59
+ wasted: false,
60
+ upstreamGenerationId: null,
61
+ error: null,
62
+ errorKind: null,
63
+ promptTokensSaved: null,
64
+ ...over,
65
+ } as LedgerEntry;
66
+ }
67
+
68
+ /** Old rows: half of them failures. Recent rows: all clean. */
69
+ function seed(windowDays: number) {
70
+ const cfg = cfgWith(windowDays);
71
+ const db = openDb(":memory:");
72
+ const ledger = createLedger(db, cfg);
73
+ const now = Date.now();
74
+ for (let i = 0; i < 10; i++) {
75
+ ledger.record(
76
+ entry({
77
+ createdAtMs: now - 30 * DAY,
78
+ ...(i % 2 === 0 ? { error: "server_error: boom", errorKind: "server_error" } : {}),
79
+ }),
80
+ );
81
+ }
82
+ for (let i = 0; i < 10; i++) ledger.record(entry({ createdAtMs: now - 1 * DAY }));
83
+ return { db, ledger, cfg };
84
+ }
85
+
86
+ describe("filters.trustWindowDays", () => {
87
+ test("0 means all-time: every row counts", () => {
88
+ const { db, ledger } = seed(0);
89
+ const trust = ledger.trust("vendor/model");
90
+ expect(trust?.attempts).toBe(20);
91
+ expect(trust?.errors).toBe(5);
92
+ db.close();
93
+ });
94
+
95
+ test("a window excludes rows older than it", () => {
96
+ const { db, ledger } = seed(7);
97
+ const trust = ledger.trust("vendor/model");
98
+ // Only the 10 recent, clean rows remain.
99
+ expect(trust?.attempts).toBe(10);
100
+ expect(trust?.errors).toBe(0);
101
+ db.close();
102
+ });
103
+
104
+ test("the window moves the success rate, which is why it is opt-in", () => {
105
+ const all = seed(0);
106
+ const windowed = seed(7);
107
+ const allTrust = all.ledger.trust("vendor/model");
108
+ const winTrust = windowed.ledger.trust("vendor/model");
109
+ expect(allTrust?.successRate).toBeLessThan(winTrust?.successRate ?? 0);
110
+ all.db.close();
111
+ windowed.db.close();
112
+ });
113
+
114
+ test("is read per call, so a hot-reloaded edit takes effect immediately", () => {
115
+ const { db, ledger, cfg } = seed(0);
116
+ expect(ledger.trust("vendor/model")?.attempts).toBe(20);
117
+ // Hot reload mutates the shared config object in place.
118
+ cfg.filters.trustWindowDays = 7;
119
+ expect(ledger.trust("vendor/model")?.attempts).toBe(10);
120
+ db.close();
121
+ });
122
+
123
+ test("allTrust honours the same window", () => {
124
+ const { db, ledger } = seed(7);
125
+ const rows = ledger.allTrust();
126
+ expect(rows).toHaveLength(1);
127
+ expect(rows[0]?.attempts).toBe(10);
128
+ db.close();
129
+ });
130
+
131
+ test("ships disabled, so the default install is unchanged", () => {
132
+ // DEFAULT_CONFIG, not loadConfig: loadConfig reads the machine's real
133
+ // config.yml, which has broken this suite before.
134
+ expect(DEFAULT_CONFIG.filters.trustWindowDays).toBe(0);
135
+ });
136
+ });
package/test/turn.test.ts CHANGED
@@ -45,7 +45,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
45
45
  data: { axis: "intelligence", minQuality: 0 },
46
46
  chat: { axis: "intelligence", minQuality: 0 },
47
47
  },
48
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
48
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
49
49
  classifier: {
50
50
  ambiguityThreshold: 0,
51
51
  model: "test/adjudicator",
@@ -583,6 +583,7 @@ describe("agentdox write-back sees the shape of the turn", () => {
583
583
  records.push(rec);
584
584
  },
585
585
  flush: () => Promise.resolve(),
586
+ pruneBlocks: () => 0,
586
587
  close: () => {},
587
588
  },
588
589
  };