auto-model-router 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +24 -2
  3. package/omp-extension/report-logic.ts +93 -0
  4. package/omp-extension/router-configure.ts +128 -2
  5. package/package.json +1 -1
  6. package/src/catalog/ollama-catalog.ts +30 -2
  7. package/src/cli/config-wizard.ts +8 -0
  8. package/src/cli/report.ts +7 -2
  9. package/src/config/defaults.ts +9 -0
  10. package/src/config/schema.ts +5 -0
  11. package/src/config/types.ts +41 -0
  12. package/src/cost/feedback.ts +81 -0
  13. package/src/cost/ledger.ts +73 -0
  14. package/src/cost/report.ts +106 -2
  15. package/src/cost/types.ts +28 -0
  16. package/src/router/candidates.ts +13 -4
  17. package/src/router/classify.ts +10 -0
  18. package/src/router/features.ts +20 -1
  19. package/src/router/index.ts +12 -1
  20. package/src/router/learned.ts +202 -0
  21. package/src/router/select.ts +64 -8
  22. package/src/router/types.ts +31 -1
  23. package/src/server/http.ts +82 -5
  24. package/src/server/overrides.ts +83 -0
  25. package/src/server/providers.ts +10 -2
  26. package/src/server/turn.ts +16 -1
  27. package/src/upstream/ollama-usage.ts +79 -2
  28. package/src/util/sqlite.ts +30 -0
  29. package/test/config-wizard.test.ts +2 -1
  30. package/test/controls.test.ts +223 -0
  31. package/test/failover.test.ts +3 -2
  32. package/test/features.test.ts +31 -0
  33. package/test/learned.test.ts +61 -0
  34. package/test/ollama.test.ts +74 -2
  35. package/test/report-hub.test.ts +4 -2
  36. package/test/report-logic.test.ts +3 -0
  37. package/test/report.test.ts +43 -0
  38. package/test/select.test.ts +126 -1
  39. package/test/trust-attribution.test.ts +42 -0
  40. package/test/turn.test.ts +33 -2
  41. package/tools/replay.ts +266 -156
  42. package/tools/train-classifier.ts +111 -0
@@ -12,17 +12,30 @@ import {
12
12
  parseOllamaListing,
13
13
  parseOllamaShow,
14
14
  type OllamaListing,
15
+ loadOllamaCatalogCache,
15
16
  } from "../src/catalog/ollama-catalog.ts";
16
17
  import { bareCloudName, ollamaRateFor } from "../src/catalog/ollama-prices.ts";
17
18
  import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
18
19
  import type { CatalogModel, CatalogSnapshot, CatalogSource } from "../src/catalog/types.ts";
19
20
  import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
21
+ import { openDb } from "../src/util/sqlite.ts";
20
22
  import type { OllamaConfig, RouterConfig } from "../src/config/types.ts";
21
23
  import { buildCandidates } from "../src/router/candidates.ts";
22
24
  import { extractFeatures } from "../src/router/features.ts";
23
25
  import { createMultiUpstream } from "../src/upstream/multi.ts";
24
26
  import { classifyOllamaStatus, createOllamaClient, toOllamaBody } from "../src/upstream/ollama.ts";
25
- import { createOllamaUsageSource, effectiveOllamaBias, NO_USAGE, ollamaMeter, parseOllamaPlan, parseOllamaUsage, usageFraction } from "../src/upstream/ollama-usage.ts";
27
+ import {
28
+ CALIBRATION_MAX_FACTOR,
29
+ CALIBRATION_MIN_FACTOR,
30
+ calibrationFrom,
31
+ createOllamaUsageSource,
32
+ effectiveOllamaBias,
33
+ NO_USAGE,
34
+ ollamaMeter,
35
+ parseOllamaPlan,
36
+ parseOllamaUsage,
37
+ usageFraction,
38
+ } from "../src/upstream/ollama-usage.ts";
26
39
  import type { Dispatch, DispatchOptions, UpstreamClient } from "../src/upstream/types.ts";
27
40
  import { createLogger } from "../src/util/log.ts";
28
41
  import { parseChatRequest } from "../src/wire/openai/request.ts";
@@ -195,6 +208,20 @@ describe("createOllamaCatalog", () => {
195
208
  expect(src.peek()).toHaveLength(1);
196
209
  });
197
210
 
211
+ test("a built set is persisted, and a fresh source hydrates it before its first listing", async () => {
212
+ const db = openDb(":memory:");
213
+ const fetchImpl = async (url: string): Promise<Response> => Response.json({ models: url.endsWith("/api/tags") ? DAEMON_TAGS : [] });
214
+ const src = createOllamaCatalog(OLLAMA, log, fetchImpl, db);
215
+ const built = await src.get(OR_MODELS);
216
+ expect(built.length).toBeGreaterThan(0);
217
+ expect(loadOllamaCatalogCache(db).models.map((m) => m.slug)).toEqual(built.map((m) => m.slug));
218
+ // A new process: peek() answers from disk with no network at all.
219
+ const dead = async (): Promise<Response> => { throw new Error("offline"); };
220
+ const again = createOllamaCatalog(OLLAMA, log, dead, db);
221
+ expect(again.peek().map((m) => m.slug)).toEqual(built.map((m) => m.slug));
222
+ db.close();
223
+ });
224
+
198
225
  test("a failed listing keeps the previous set", async () => {
199
226
  let fail = false;
200
227
  const fetchImpl = async (url: string): Promise<Response> => {
@@ -488,7 +515,7 @@ describe("ollama plan usage (credit-aware bias)", () => {
488
515
  const source = { get: async () => ollamaModels, peek: () => ollamaModels, invalidate: () => {} };
489
516
  const breaker = { available: () => true, cooldownUntilMs: () => null, lastTrip: () => null };
490
517
  let used = 0.2;
491
- const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }) };
518
+ const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), calibration: () => null };
492
519
  const catalog = createCompositeCatalog(openrouter, source, breaker, { costBias: 0.1, biasUntilUsage: 0.9, usage });
493
520
 
494
521
  const a = await catalog.get();
@@ -538,3 +565,48 @@ describe("ollamaMeter", () => {
538
565
  expect(parseOllamaPlan("nope")).toBeNull();
539
566
  });
540
567
  });
568
+
569
+ describe("ollama calibration", () => {
570
+ const s = (h: number, meterUsd: number, ledgerUsd: number) => ({ atMs: h * 3_600_000, meterUsd, ledgerUsd });
571
+
572
+ test("compares the newest reading with the oldest since the last meter reset", () => {
573
+ // Ledger estimated $4 over the span; the meter moved $5 ⇒ estimates run 20% low.
574
+ const c = calibrationFrom([s(0, 10, 20), s(12, 12.5, 22), s(24, 15, 24)])!;
575
+ expect(c.factor).toBeCloseTo(1.25, 6);
576
+ expect(c.spanHours).toBe(24);
577
+ expect(c.samples).toBe(3);
578
+ // A billing-cycle reset (meter drops) ends the span.
579
+ const reset = calibrationFrom([s(0, 50, 40), s(12, 0.5, 41), s(24, 1.5, 42)])!;
580
+ expect(reset.samples).toBe(2);
581
+ expect(reset.factor).toBeCloseTo(1, 6);
582
+ });
583
+
584
+ test("needs enough metered spend to mean anything, and clamps to 0.5–2×", () => {
585
+ expect(calibrationFrom([s(0, 1, 1), s(1, 1.03, 1.2)])).toBeNull(); // meter moved less than its resolution
586
+ expect(calibrationFrom([s(0, 1, 1), s(1, 2, 1.3)])).toBeNull(); // ledger moved less than $0.50
587
+ expect(calibrationFrom([s(0, 0, 0), s(1, 10, 1)])!.factor).toBe(CALIBRATION_MAX_FACTOR);
588
+ expect(calibrationFrom([s(0, 0, 0), s(1, 0.1, 10)])!.factor).toBe(CALIBRATION_MIN_FACTOR);
589
+ expect(calibrationFrom([s(0, 1, 1)])).toBeNull();
590
+ });
591
+
592
+ test("the source records a sample per poll and exposes the calibration", async () => {
593
+ const db = openDb(":memory:");
594
+ let ledgerUsd = 1;
595
+ let frac = 0.1;
596
+ const fetchImpl = async (url: string): Promise<Response> => {
597
+ if (url.endsWith("/api/me")) return Response.json({ Plan: "pro" });
598
+ return Response.json({ limits: { monthly: { usage: frac, models: [] } } });
599
+ };
600
+ const src = createOllamaUsageSource({ apiKey: "k", pollMs: 5, timeoutMs: 1000, log, fetchImpl, calibration: { db, ledgerUsd: () => ledgerUsd, planCreditsOverrideUsd: 0 } });
601
+ await src.get(); // meter $6 (10% of $60), ledger $1
602
+ expect(src.calibration()).toBeNull(); // one sample
603
+ await new Promise((r) => setTimeout(r, 20));
604
+ ledgerUsd = 3; // ledger +$2
605
+ frac = 0.15; // meter +$3
606
+ await src.get();
607
+ const c = src.calibration()!;
608
+ expect(c.factor).toBeCloseTo(1.5, 6);
609
+ expect((db.query("SELECT COUNT(*) n FROM ollama_meter_samples").get() as { n: number }).n).toBe(2);
610
+ db.close();
611
+ });
612
+ });
@@ -79,14 +79,16 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
79
79
  },
80
80
  providers: [row("openrouter", 2), row("ollama", 1)],
81
81
  models: [
82
- { ...row("z-ai/glm", 2), provider: "openrouter", tiers: { simple: 6, moderate: 4 } },
83
- { ...row("ollama/kimi", 1), provider: "ollama", tiers: { hard: 10 } },
82
+ { ...row("z-ai/glm", 2), provider: "openrouter", tiers: { simple: 6, moderate: 4 }, feedback: { good: 0, bad: 0 } },
83
+ { ...row("ollama/kimi", 1), provider: "ollama", tiers: { hard: 10 }, feedback: { good: 0, bad: 0 } },
84
84
  ],
85
85
  tiers: [row("simple", 2), row("hard", 1)],
86
86
  days: [
87
87
  { day: "2026-09-05", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
88
88
  { day: "2026-09-06", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
89
89
  ],
90
+ anatomy: null,
91
+ baselines: [],
90
92
  ...over,
91
93
  };
92
94
  }
@@ -66,6 +66,8 @@ describe("renderStatus", () => {
66
66
  lastTrip: { kind: "quota", atMs: now - 120_000, message: "402" },
67
67
  usage: { monthlyUsedFraction: 0.42, activityCostUsd: 3.1, fetchedAtMs: now },
68
68
  meter: { usedUsd: 25.2, creditsUsd: 60, plan: "pro" },
69
+ calibration: { factor: 1.25, meterDeltaUsd: 5, ledgerDeltaUsd: 4, spanHours: 48 },
70
+ runway: { dailyBurnUsd: 1.5, creditsLeftUsd: 34.8, days: 23.2 },
69
71
  costBias: { configured: 0.1, effective: 0.1, biasUntilUsage: 0.9 },
70
72
  },
71
73
  catalog: { models: 240, ageMs: 5 * 60_000, keyScoped: true, shrink: { fromModels: 300, toModels: 120, atMs: now } },
@@ -77,6 +79,7 @@ describe("renderStatus", () => {
77
79
  expect(text).toContain("SHRANK 300 -> 120");
78
80
  expect(text).toContain("COOLING DOWN");
79
81
  expect(text).toContain("pro plan usage 42.0% ($25.20 of $60)");
82
+ expect(text).toContain("ollama billing: ledger estimate ×1.25 to match the meter (48h span) · burn $1.50/day · ~23 days of credits left");
80
83
  expect(text).toContain("cost bias ×0.1 (until 90%)");
81
84
  expect(text).toContain("last trip quota 2m ago");
82
85
  expect(text).toContain("scope omp-router");
@@ -156,6 +156,47 @@ describe("buildUsageReport", () => {
156
156
  db.close();
157
157
  });
158
158
 
159
+ test("prompt anatomy averages the recorded byte shares", () => {
160
+ const { db, ledger } = seeded();
161
+ const feat = (tool: number, older: number, stale: number) => ({ toolSchemaBytes: 1000, anatomy: { messages: 30, systemBytes: 1000, userBytes: 500, assistantBytes: 500, toolBytes: tool, olderHalfBytes: older, staleToolBytes: stale } });
162
+ ledger.record(entry({ features: feat(8000, 5000, 4000) }));
163
+ ledger.record(entry({ features: feat(6000, 4000, 2000) }));
164
+ ledger.record(entry({ features: null })); // pre-anatomy row: ignored
165
+ const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
166
+ const a = r.anatomy!;
167
+ expect(a.rows).toBe(2);
168
+ // mean bytes: system 1000, user 500, assistant 500, tool 7000 ⇒ total 9000
169
+ expect(a.tool).toBeCloseTo(7000 / 9000, 6);
170
+ expect(a.system).toBeCloseTo(1000 / 9000, 6);
171
+ expect(a.schemas).toBeCloseTo(1000 / 9000, 6);
172
+ expect(a.olderHalf).toBeCloseTo(4500 / 9000, 6);
173
+ expect(a.staleTool).toBeCloseTo(3000 / 9000, 6);
174
+ expect(renderUsageReport(r)).toContain("prompt anatomy (mean of 2): tool results 78%");
175
+ db.close();
176
+ });
177
+
178
+ test("baselines price the window on one model with its own cache hit rate", () => {
179
+ const { db, ledger } = seeded();
180
+ ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
181
+ const r = buildUsageReport(db, {
182
+ windowDays: 7,
183
+ nowMs: NOW,
184
+ baselines: [
185
+ { slug: "big/model", prompt: 15 / 1e6, completion: 75 / 1e6, cacheRead: 1.5 / 1e6 },
186
+ { slug: "nocache/model", prompt: 3 / 1e6, completion: 15 / 1e6 },
187
+ ],
188
+ });
189
+ // 20k fresh × $15/M + 80k cached × $1.5/M + 1k completion × $75/M = 0.30 + 0.12 + 0.075
190
+ expect(r.baselines[0]!.usd).toBeCloseTo(0.495, 6);
191
+ expect(r.baselines[0]!.savedShare).toBeCloseTo(1 - 0.5 / 0.495, 6);
192
+ // No cache rate published: every prompt token at list price.
193
+ expect(r.baselines[1]!.usd).toBeCloseTo(0.3 + 0.015, 6);
194
+ const text = renderUsageReport(r);
195
+ expect(text).toContain("same traffic on one model: big/model $0.4950 (router cost extra 1%)");
196
+ expect(text).toContain("nocache/model $0.3150 (router cost extra 59%)");
197
+ db.close();
198
+ });
199
+
159
200
  test("empty ledger yields zeroed totals and null speeds", () => {
160
201
  const { db } = seeded();
161
202
  const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
@@ -175,6 +216,8 @@ describe("buildUsageReport", () => {
175
216
  });
176
217
  expect(r.providers).toEqual([]);
177
218
  expect(r.models).toEqual([]);
219
+ expect(r.anatomy).toBeNull();
220
+ expect(r.baselines).toEqual([]);
178
221
  db.close();
179
222
  });
180
223
 
@@ -8,7 +8,8 @@ import type { ProfileConfig, RouterConfig } from "../src/config/types.ts";
8
8
  import type { Ledger } from "../src/cost/types.ts";
9
9
  import { extractFeatures } from "../src/router/features.ts";
10
10
  import { scoreHeuristic } from "../src/router/classify.ts";
11
- import { BudgetExceededError, select } from "../src/router/select.ts";
11
+ import { latencyWeightFor } from "../src/router/candidates.ts";
12
+ import { BudgetExceededError, monthPace, monthStartMs, select } from "../src/router/select.ts";
12
13
  import type { ConversationState, Tier } from "../src/router/types.ts";
13
14
  import { parseChatRequest } from "../src/wire/openai/request.ts";
14
15
  import type { NormRequest } from "../src/wire/types.ts";
@@ -1050,3 +1051,127 @@ describe("hysteresis.confirmUpgradesBelowConfidence", () => {
1050
1051
  expect(d.upgradeDeferred).toBeNull();
1051
1052
  });
1052
1053
  });
1054
+
1055
+ describe("cache reliability in the stay/switch comparison", () => {
1056
+ const warmSlug = "x-ai/grok-4.6";
1057
+ function ledgerWithReliability(rate: number | null, samples = 50): Ledger {
1058
+ return {
1059
+ record: () => {},
1060
+ conversationSpend: () => 0,
1061
+ spendSince: () => 0,
1062
+ blendedRate: () => null,
1063
+ trust: () => null,
1064
+ allTrust: () => [],
1065
+ latency: () => null,
1066
+ tokenRatio: () => null,
1067
+ recentEntries: () => [],
1068
+ cacheReliability: (slug) => (rate === null || slug !== warmSlug ? null : { slug, samples, hitRate: rate }),
1069
+ };
1070
+ }
1071
+ const stayCostOf = (ledger: Ledger, cfg: RouterConfig = BASE): number => {
1072
+ const d = run({
1073
+ tier: "hard",
1074
+ promptTokens: 80_000,
1075
+ cfg,
1076
+ ledger,
1077
+ st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }),
1078
+ });
1079
+ const m = /cache: (?:keeping warm|switch) .*?stay \$([0-9.]+)/.exec(d.reasons.join("\n"));
1080
+ if (m === null) throw new Error(`no stay/switch reason in ${d.reasons.join(" | ")}`);
1081
+ return Number(m[1]);
1082
+ };
1083
+
1084
+ test("an unreliable cache prices staying at the fresh rate, a reliable one at the cached rate", () => {
1085
+ const reliable = stayCostOf(ledgerWithReliability(1));
1086
+ const flaky = stayCostOf(ledgerWithReliability(0));
1087
+ const unknown = stayCostOf(ledgerWithReliability(null));
1088
+ expect(flaky).toBeGreaterThan(reliable);
1089
+ expect(unknown).toBeCloseTo(reliable, 6);
1090
+ });
1091
+
1092
+ test("too few samples, or the feature off, assume a reliable cache", () => {
1093
+ const reliable = stayCostOf(ledgerWithReliability(1));
1094
+ expect(stayCostOf(ledgerWithReliability(0, 3))).toBeCloseTo(reliable, 6);
1095
+ const off: RouterConfig = { ...BASE, filters: { ...BASE.filters, cacheReliabilityMinSamples: 0 } };
1096
+ expect(stayCostOf(ledgerWithReliability(0), off)).toBeCloseTo(reliable, 6);
1097
+ });
1098
+
1099
+ test("the reason names the measured hit rate", () => {
1100
+ const d = run({
1101
+ tier: "hard",
1102
+ promptTokens: 80_000,
1103
+ ledger: ledgerWithReliability(0.5, 40),
1104
+ st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }),
1105
+ });
1106
+ expect(d.reasons.some((r) => r.includes("warm hit 50% over 40"))).toBe(true);
1107
+ });
1108
+ });
1109
+
1110
+ describe("session pin (forceSlug)", () => {
1111
+ test("a pinned catalog model wins over ranking and the warm model; an unknown pin is ignored with a reason", () => {
1112
+ const warmSlug = run({ tier: "moderate" }).slug;
1113
+ const pinSlug = run({ tier: "hard" }).slug; // a real, differently-ranked model
1114
+ const req = request("tidy the retry helper");
1115
+ const features = extractFeatures(req, 50_000);
1116
+ const base = {
1117
+ req,
1118
+ features,
1119
+ classification: scoreHeuristic(features, BASE),
1120
+ profile: PROFILE,
1121
+ state: state({ currentSlug: warmSlug, currentTier: "moderate", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 50_000 }),
1122
+ snapshot: SNAPSHOT,
1123
+ ledger: null,
1124
+ cfg: { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } },
1125
+ nowMs: Date.now(),
1126
+ };
1127
+ const pinned = select({ ...base, forceSlug: pinSlug });
1128
+ expect(pinned.slug).toBe(pinSlug);
1129
+ expect(pinned.sticky).toBe(false);
1130
+ expect(pinned.reasons.some((r) => r.includes(`pinned to ${pinSlug} by session override`))).toBe(true);
1131
+ const unknown = select({ ...base, forceSlug: "nope/model" });
1132
+ expect(unknown.slug).toBe(warmSlug); // the huge switch margin keeps the warm model
1133
+ expect(unknown.reasons.some((r) => r.includes("pin nope/model ignored"))).toBe(true);
1134
+ });
1135
+ });
1136
+
1137
+ describe("budget.perMonthUsd pacing", () => {
1138
+ test("monthPace spreads what is left over the days left, today included", () => {
1139
+ const sep7 = Date.UTC(2026, 8, 7, 12);
1140
+ expect(monthStartMs(sep7)).toBe(Date.UTC(2026, 8, 1));
1141
+ const p = monthPace(sep7, 60, 30);
1142
+ expect(p.daysLeft).toBe(24); // 7th..30th
1143
+ expect(p.dailyCapUsd).toBeCloseTo(30 / 24, 6);
1144
+ expect(monthPace(sep7, 60, 70).dailyCapUsd).toBe(0);
1145
+ expect(monthPace(Date.UTC(2026, 8, 30, 12), 60, 0).daysLeft).toBe(1);
1146
+ });
1147
+
1148
+ test("a month running ahead of pace tightens the daily cap and says so", () => {
1149
+ const ledger: Ledger = {
1150
+ record: () => {},
1151
+ conversationSpend: () => 0,
1152
+ spendSince: (sinceMs) => (sinceMs <= monthStartMs(Date.now()) + 1 ? 59.99 : 0), // month-to-date $59.99, last 24h $0
1153
+ blendedRate: () => null,
1154
+ trust: () => null,
1155
+ allTrust: () => [],
1156
+ latency: () => null,
1157
+ tokenRatio: () => null,
1158
+ recentEntries: () => [],
1159
+ };
1160
+ const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perMonthUsd: 60, onExceeded: "reject" } };
1161
+ expect(() => run({ tier: "hard", promptTokens: 50_000, cfg, ledger })).toThrow(/month pacing: \$59\.99 of \$60 spent/);
1162
+ // Under pace: the cap is generous and nothing breaches.
1163
+ const easy: Ledger = { ...ledger, spendSince: () => 1 };
1164
+ expect(run({ tier: "hard", promptTokens: 50_000, cfg, ledger: easy }).budgetDowngraded).toBe(false);
1165
+ });
1166
+ });
1167
+
1168
+ describe("filters.latencyWeightContinuation", () => {
1169
+ test("applies only to tool-result continuations, and only when set", () => {
1170
+ const f = { ...BASE.filters, latencyWeight: 0.75 };
1171
+ expect(latencyWeightFor(f, false)).toBe(0.75);
1172
+ expect(latencyWeightFor(f, true)).toBe(0.75);
1173
+ const g = { ...f, latencyWeightContinuation: 0.1 };
1174
+ expect(latencyWeightFor(g, false)).toBe(0.75);
1175
+ expect(latencyWeightFor(g, true)).toBe(0.1);
1176
+ });
1177
+ });
@@ -368,3 +368,45 @@ describe("v6 classifier instrumentation", () => {
368
368
  }
369
369
  });
370
370
  });
371
+
372
+ describe("cache reliability signal", () => {
373
+ // Observed hit rate when a warm cache was expected: the previous kept turn
374
+ // of the conversation was on the same model within the warm TTL.
375
+ function seed(rows: Array<Partial<LedgerEntry>>) {
376
+ const db = openDb(":memory:");
377
+ const ledger = createLedger(db, cfg);
378
+ for (const r of rows) ledger.record(entry(r));
379
+ return { db, ledger };
380
+ }
381
+ const t0 = Date.UTC(2026, 8, 7, 12);
382
+ const usage = (prompt: number, cached: number, estimated = false) => ({ ...EMPTY_USAGE, promptTokens: prompt, cachedTokens: cached, ...(estimated ? { cachedEstimated: true } : {}) });
383
+
384
+ test("a model that hits when warm scores 1; one that misses scores 0; the first turn never counts", () => {
385
+ const { db, ledger } = seed([
386
+ { conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0, usage: usage(50_000, 0) }, // first turn: no expectation
387
+ { conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 50_000) },
388
+ { conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0 + 120_000, usage: usage(70_000, 60_000) },
389
+ { conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0, usage: usage(50_000, 0) },
390
+ { conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 0) },
391
+ { conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0 + 120_000, usage: usage(70_000, 30_000) },
392
+ ]);
393
+ expect(ledger.cacheReliability?.("good/m")).toEqual({ slug: "good/m", samples: 2, hitRate: 1 });
394
+ const flaky = ledger.cacheReliability?.("flaky/m");
395
+ expect(flaky?.samples).toBe(2);
396
+ expect(flaky?.hitRate).toBeCloseTo(0.25, 6); // (0 + 30k/60k) / 2
397
+ expect(ledger.cacheReliability?.("never/m")).toBeNull();
398
+ db.close();
399
+ });
400
+
401
+ test("a switch, an idle gap past the TTL, or a router-estimated count is not a warm-expected sample", () => {
402
+ const { db, ledger } = seed([
403
+ { conversationKey: "a", slug: "x/m", servedSlug: "x/m", createdAtMs: t0, usage: usage(50_000, 0) },
404
+ { conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 0) }, // switch
405
+ { conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000 + cfg.hysteresis.cacheWarmTtlMs + 1, usage: usage(70_000, 0) }, // gap
406
+ { conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000 + cfg.hysteresis.cacheWarmTtlMs + 2, usage: usage(80_000, 70_000, true) }, // estimated
407
+ ]);
408
+ expect(ledger.cacheReliability?.("x/m")).toBeNull();
409
+ expect(ledger.cacheReliability?.("y/m")).toBeNull();
410
+ db.close();
411
+ });
412
+ });
package/test/turn.test.ts CHANGED
@@ -46,10 +46,10 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
46
46
  data: { axis: "intelligence", minQuality: 0 },
47
47
  chat: { axis: "intelligence", minQuality: 0 },
48
48
  },
49
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20, escalationCostWeight: 0 },
49
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
50
50
  classifier: {
51
51
  ambiguityThreshold: 0,
52
- model: "test/adjudicator",
52
+ model: "test/adjudicator", learnedModelPath: "",
53
53
  maxCostFraction: 0.1,
54
54
  maxCostUsd: 0.01,
55
55
  timeoutMs: 5000,
@@ -76,6 +76,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
76
76
  context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
77
77
  compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
78
78
  budget: { onExceeded: "downgrade" },
79
+ report: { baselines: [] },
79
80
  profiles: [],
80
81
  ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
81
82
  adaptiveTierFloors: true,
@@ -896,4 +897,34 @@ describe("latency measurement covers the work the router actually does", () => {
896
897
  expect(map.get("conv-test")!.cacheWarmSlug).toBe(ollamaModel.slug);
897
898
  });
898
899
 
900
+ test("an Ollama estimate is scaled by the ledger-vs-meter calibration", async () => {
901
+ const ollamaModel: CatalogModel = {
902
+ slug: "ollama/glm-5.3-flash",
903
+ provider: "ollama",
904
+ canonicalSlug: "ollama/glm-5.3-flash",
905
+ name: "glm",
906
+ contextLength: 1_000_000,
907
+ supportsTools: true,
908
+ supportsReasoning: true,
909
+ reasoningMandatory: false,
910
+ supportsToolChoice: false,
911
+ inputModalities: ["text"],
912
+ price: { prompt: 0.15 / 1e6, cacheRead: 0.03 / 1e6, completion: 0.5 / 1e6 },
913
+ priceTiers: [],
914
+ quality: {},
915
+ tokenizer: "Other",
916
+ isFree: false,
917
+ createdAtMs: 0,
918
+ author: "ollama",
919
+ };
920
+ const priced = { ...catalog, find: (slug: string) => (slug === ollamaModel.slug ? ollamaModel : undefined) };
921
+ const { router } = mkRouter([mkDecision("simple", ollamaModel.slug)]);
922
+ const { upstream } = mkUpstream([{ kind: "chunks", chunks: [startChunk(ollamaModel.slug), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 100_000, completionTokens: 0 }, null)] }]);
923
+ const { ledger, entries } = mkLedger();
924
+ const { store } = mkConversations();
925
+ await runTurn(mkReq(), mkSink().sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog: priced, context: createDisabledBridge(), ollamaCostScale: () => 1.25 }, new AbortController().signal);
926
+ // 100k × $0.15/M = $0.015, scaled ×1.25.
927
+ expect(entries[0]!.reportedUsd).toBeCloseTo(0.01875, 6);
928
+ });
929
+
899
930
  });