auto-model-router 0.3.4 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +29 -4
  3. package/omp-extension/report-logic.ts +93 -0
  4. package/omp-extension/router-configure.ts +128 -2
  5. package/omp-extension/router-embed.ts +9 -6
  6. package/package.json +1 -1
  7. package/src/catalog/ollama-catalog.ts +30 -2
  8. package/src/cli/config-wizard.ts +11 -0
  9. package/src/cli/report.ts +7 -2
  10. package/src/config/defaults.ts +17 -0
  11. package/src/config/schema.ts +8 -0
  12. package/src/config/types.ts +65 -0
  13. package/src/cost/feedback.ts +81 -0
  14. package/src/cost/ledger.ts +110 -5
  15. package/src/cost/report.ts +118 -2
  16. package/src/cost/types.ts +32 -1
  17. package/src/router/candidates.ts +13 -4
  18. package/src/router/classify.ts +13 -0
  19. package/src/router/features.ts +59 -1
  20. package/src/router/index.ts +23 -5
  21. package/src/router/learned.ts +202 -0
  22. package/src/router/select.ts +64 -8
  23. package/src/router/types.ts +39 -1
  24. package/src/server/http.ts +82 -5
  25. package/src/server/overrides.ts +83 -0
  26. package/src/server/providers.ts +10 -2
  27. package/src/server/turn.ts +16 -1
  28. package/src/upstream/ollama-usage.ts +79 -2
  29. package/src/util/sqlite.ts +30 -0
  30. package/src/wire/openai/request.ts +4 -0
  31. package/src/wire/types.ts +2 -0
  32. package/test/classify.test.ts +13 -0
  33. package/test/config-wizard.test.ts +8 -6
  34. package/test/controls.test.ts +238 -0
  35. package/test/escalate.test.ts +1 -0
  36. package/test/failover.test.ts +6 -4
  37. package/test/features.test.ts +63 -0
  38. package/test/http-resilience.test.ts +1 -1
  39. package/test/learned.test.ts +61 -0
  40. package/test/ollama.test.ts +74 -2
  41. package/test/report-hub.test.ts +6 -2
  42. package/test/report-logic.test.ts +3 -0
  43. package/test/report.test.ts +57 -0
  44. package/test/select.test.ts +126 -1
  45. package/test/trust-attribution.test.ts +95 -0
  46. package/test/turn.test.ts +36 -4
  47. package/tools/replay.ts +267 -156
  48. package/tools/train-classifier.ts +111 -0
@@ -15,6 +15,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
15
15
  // is deterministic, so peers reuse it), so this covers N sessions plus
16
16
  // their subagents. Was effectively 8 per session when each bound its own.
17
17
  maxConcurrentTurns: 24,
18
+ // omp subagents (no UI) route under this profile: delegated work, capped at moderate.
19
+ subagentProfile: "auto-sub",
18
20
  },
19
21
  openrouter: {
20
22
  baseUrl: "https://openrouter.ai/api/v1",
@@ -99,6 +101,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
99
101
  includeFree: false,
100
102
  requireToolSupport: true,
101
103
  minTrust: 0.7,
104
+ // Verdicts are recorded and reported first; weigh them once there are some.
105
+ feedbackWeight: 0,
102
106
  minTrustSamples: 12,
103
107
  // Shared trust by default: more samples, demotion guard stays effective
104
108
  // even with a tiny guardrail-narrowed catalog.
@@ -118,6 +122,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
118
122
  // slow models (e.g. deepseek-v4-flash ~20 tok/s) fall under it.
119
123
  latencyReferenceTokensPerSec: 30,
120
124
  latencyMinSamples: 20,
125
+ // Discount a warm model's stay price by its observed hit rate once this
126
+ // many warm-expected samples exist. See FilterConfig.cacheReliabilityMinSamples.
127
+ cacheReliabilityMinSamples: 10,
121
128
  // Off: pricing a model's measured escalation rate at what an escalated
122
129
  // retry actually costs changes rankings, so it is opt-in after a replay
123
130
  // run prices it. See FilterConfig.escalationCostWeight.
@@ -127,6 +134,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
127
134
  ambiguityThreshold: 0.6,
128
135
  // Cheapest competent slug in the catalog; adjudication prompts are tiny.
129
136
  model: "qwen/qwen3.7-flash",
137
+ // Off until a model is trained; see tools/train-classifier.ts.
138
+ learnedModelPath: "",
130
139
  maxCostFraction: 0.02,
131
140
  maxCostUsd: 0.002,
132
141
  timeoutMs: 4_000,
@@ -137,6 +146,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
137
146
  // A mechanical retry (failed tool call + tool-result continuation) keeps
138
147
  // only a fifth of the +0.26; a user-visible failure keeps the full weight.
139
148
  mechanicalRetryFactor: 0.2,
149
+ // Recorded, not acted on, until replay prices it. See ClassifierConfig.readOnlyToolWeight.
150
+ readOnlyToolWeight: 0,
140
151
  // Shipped reasoning values, unchanged. See ClassifierConfig.reasoningWeights:
141
152
  // a harness that pins the level for a whole session turns these into a
142
153
  // constant tier offset, in which case `medium` belongs near 0.
@@ -281,6 +292,10 @@ export const DEFAULT_CONFIG: RouterConfig = {
281
292
  elideSupersededReads: true,
282
293
  collapseDuplicateResults: true,
283
294
  },
295
+ report: {
296
+ // The frontier pair most omp users would otherwise run on.
297
+ baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
298
+ },
284
299
  budget: {
285
300
  // No caps by default; at a configured ceiling, downgrade rather than fail.
286
301
  onExceeded: "downgrade",
@@ -289,6 +304,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
289
304
  { id: "auto", name: "Auto (auto-model-router)", minTier: "trivial", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
290
305
  { id: "auto-cheap", name: "Auto Cheap (auto-model-router)", minTier: "trivial", maxTier: "simple", contextWindow: 400_000, maxTokens: 32_000 },
291
306
  { id: "auto-max", name: "Auto Max (auto-model-router)", minTier: "moderate", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
307
+ // Subagent envelope (server.subagentProfile): never the top tier for delegated work.
308
+ { id: "auto-sub", name: "Auto Subagent (auto-model-router)", minTier: "trivial", maxTier: "moderate", contextWindow: 400_000, maxTokens: 32_000 },
292
309
  ],
293
310
  ledger: {
294
311
  // Resolved by loadConfig: empty ⇒ `$AUTO_MODEL_ROUTER_HOME/router.db`.
@@ -20,6 +20,7 @@ const server = z.strictObject({
20
20
  port: z.number().int().min(0).max(65_535).optional(),
21
21
  apiKey: z.string().optional(),
22
22
  harnessId: z.string().optional(),
23
+ subagentProfile: z.string().optional(),
23
24
  maxConcurrentTurns: z.number().int().positive().max(1_000).optional(),
24
25
  });
25
26
 
@@ -88,6 +89,7 @@ const filters = z.strictObject({
88
89
  includeFree: z.boolean().optional(),
89
90
  requireToolSupport: z.boolean().optional(),
90
91
  minTrust: z.number().min(0).max(1).optional(),
92
+ feedbackWeight: z.number().nonnegative().optional(),
91
93
  minTrustSamples: z.number().int().nonnegative().optional(),
92
94
  trustScopedByHarness: z.boolean().optional(),
93
95
  trustWindowDays: z.number().nonnegative().optional(),
@@ -96,13 +98,16 @@ const filters = z.strictObject({
96
98
  latencyReferenceMs: z.number().positive().optional(),
97
99
  latencyReferenceTokensPerSec: z.number().positive().optional(),
98
100
  latencyMinSamples: z.number().int().nonnegative().optional(),
101
+ cacheReliabilityMinSamples: z.number().int().nonnegative().optional(),
99
102
  maxExpectedWaitMs: z.number().positive().optional(),
103
+ latencyWeightContinuation: z.number().nonnegative().optional(),
100
104
  escalationCostWeight: z.number().min(0).max(1).optional(),
101
105
  });
102
106
 
103
107
  const classifier = z.strictObject({
104
108
  ambiguityThreshold: z.number().min(0).max(1).optional(),
105
109
  model: z.string().min(1).optional(),
110
+ learnedModelPath: z.string().optional(),
106
111
  maxCostFraction: z.number().min(0).max(1).optional(),
107
112
  maxCostUsd: z.number().nonnegative().optional(),
108
113
  timeoutMs: z.number().positive().optional(),
@@ -111,6 +116,7 @@ const classifier = z.strictObject({
111
116
  chatAxis: qualityAxis.optional(),
112
117
  agenticLoopDepth: z.number().int().nonnegative().optional(),
113
118
  mechanicalRetryFactor: z.number().min(0).max(1).optional(),
119
+ readOnlyToolWeight: z.number().nonnegative().optional(),
114
120
  reasoningWeights: z
115
121
  .strictObject({
116
122
  medium: z.number().nonnegative().optional(),
@@ -205,6 +211,7 @@ const budget = z.strictObject({
205
211
  perTurnUsd: z.number().nonnegative().optional(),
206
212
  perConversationUsd: z.number().nonnegative().optional(),
207
213
  perDayUsd: z.number().nonnegative().optional(),
214
+ perMonthUsd: z.number().nonnegative().optional(),
208
215
  onExceeded: z.enum(["downgrade", "reject"]).optional(),
209
216
  });
210
217
 
@@ -265,6 +272,7 @@ export const configInputSchema = z.strictObject({
265
272
  compaction: compaction.optional(),
266
273
  budget: budget.optional(),
267
274
  profiles: z.array(profile).optional(),
275
+ report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
268
276
  ledger: ledger.optional(),
269
277
  adaptiveTierFloors: z.boolean().optional(),
270
278
  adaptivePriceCeilings: z.boolean().optional(),
@@ -38,6 +38,14 @@ export interface ServerConfig {
38
38
  * ⇒ no header (single-harness default).
39
39
  */
40
40
  harnessId?: string;
41
+ /**
42
+ * Profile that requests from omp subagents (`X-Omp-Subagent: 1`, set by
43
+ * the embed extension for sessions without a UI) are routed under when they
44
+ * ask for the default profile. Subagents do delegated, bounded work — file
45
+ * reads, searches, summaries — that rarely needs the top tier. Empty
46
+ * disables the remap; a name with no matching profile is ignored.
47
+ */
48
+ subagentProfile: string;
41
49
  /**
42
50
  * Concurrent in-flight turns this router process will accept; excess gets a
43
51
  * 429 rather than being queued, so a local flood cannot pile up unbounded
@@ -215,6 +223,15 @@ export interface FilterConfig {
215
223
  requireToolSupport: boolean;
216
224
  /** Drop models whose ledger success rate is below this, once `minTrustSamples` is met. */
217
225
  minTrust: number;
226
+ /**
227
+ * How much a user verdict (/router good|bad) weighs in a model's trust
228
+ * rate: each bad verdict counts as this many failures and each good one as
229
+ * this many successes, beside escalations and errors. 0 (default) records
230
+ * verdicts without acting on them. A person judging an answer wrong is a
231
+ * stronger signal than a probe rejection, so values of 2-5 are sensible
232
+ * once a week of verdicts is in the report.
233
+ */
234
+ feedbackWeight: number;
218
235
  /** Attempts required before `minTrust` is enforced against a model. */
219
236
  minTrustSamples: number;
220
237
  /**
@@ -264,6 +281,14 @@ export interface FilterConfig {
264
281
  latencyReferenceTokensPerSec: number;
265
282
  /** Streamed samples required before latency is scored against a model. */
266
283
  latencyMinSamples: number;
284
+ /**
285
+ * Warm-expected samples a model needs before its observed cache hit rate
286
+ * (ledger `cacheReliability`) discounts the "stay warm" price in the
287
+ * stay/switch comparison. Below it, and when 0, a cache is assumed fully
288
+ * reliable. Measured 2026-09-06: same-model short-gap turns still ran cold
289
+ * 5-6% on glm/gemini, 11% on ling and 50% on nex.
290
+ */
291
+ cacheReliabilityMinSamples: number;
267
292
  /**
268
293
  * Absolute expected-wait ceiling (ms). A hard drop, mirroring the price
269
294
  * ceiling: any model whose expected total wait (TTFT + streaming the expected
@@ -277,6 +302,13 @@ export interface FilterConfig {
277
302
  * Undefined ⇒ off (the default).
278
303
  */
279
304
  maxExpectedWaitMs?: number;
305
+ /**
306
+ * Latency weight for tool-result continuations (the agent loop's own
307
+ * follow-ups, where no person is waiting on first token). Unset ⇒
308
+ * `latencyWeight` applies to every turn. Lower it to spend speed only on
309
+ * user-facing turns.
310
+ */
311
+ latencyWeightContinuation?: number;
280
312
  /**
281
313
  * How much of a model's measured escalation risk to price into its effective
282
314
  * cost, 0-1. 0 (the default) disables the term.
@@ -305,6 +337,13 @@ export interface ClassifierConfig {
305
337
  ambiguityThreshold: number;
306
338
  /** Slug used for adjudication. Must be cheap and fast. */
307
339
  model: string;
340
+ /**
341
+ * Path of a model written by `tools/train-classifier.ts`. When set, every
342
+ * heuristic classification also carries the learned P(escalate) in its
343
+ * reasons (`learned: p(escalate)=…`) and `Classification.learnedRisk`.
344
+ * Advisory: it never moves a tier. Empty ⇒ off.
345
+ */
346
+ learnedModelPath: string;
308
347
  /** Skip adjudication when it would exceed this fraction of the forecast turn cost. */
309
348
  maxCostFraction: number;
310
349
  /** Absolute per-call ceiling, USD. */
@@ -325,6 +364,13 @@ export interface ClassifierConfig {
325
364
  * loops buy the hard tier. 1 preserves the shipped behaviour.
326
365
  */
327
366
  mechanicalRetryFactor: number;
367
+ /**
368
+ * Score subtracted when the newest assistant turn issued only read-only
369
+ * tools (read, grep, glob, ls, lsp…) and this is the tool-result
370
+ * continuation: the model is looking, not deciding. 0 (default) records
371
+ * the feature without acting on it — enable after a replay prices it.
372
+ */
373
+ readOnlyToolWeight: number;
328
374
  /**
329
375
  * Score added when the CLIENT asks for a reasoning effort, per level. The
330
376
  * premise is that asking for reasoning states expected difficulty directly.
@@ -508,6 +554,17 @@ export interface CacheConfig {
508
554
  milestoneTokens: number;
509
555
  }
510
556
 
557
+ /** Usage-report options. */
558
+ export interface ReportConfig {
559
+ /**
560
+ * Models to price the window's traffic on as if every turn had used that
561
+ * one model, at its list price with the window's own cache hit rate: the
562
+ * "what the router saved" counterfactual. Slugs missing from the catalog
563
+ * are skipped.
564
+ */
565
+ baselines: string[];
566
+ }
567
+
511
568
  export interface BudgetConfig {
512
569
  /** Reject or downgrade when a turn's cold forecast exceeds this, USD. */
513
570
  perTurnUsd?: number;
@@ -515,6 +572,13 @@ export interface BudgetConfig {
515
572
  perConversationUsd?: number;
516
573
  /** Rolling 24h ceiling, USD. */
517
574
  perDayUsd?: number;
575
+ /**
576
+ * Calendar-month (UTC) target, USD. Paced: the per-day ceiling becomes
577
+ * min(perDayUsd, remaining ÷ days left in the month), so a month that runs
578
+ * ahead of pace tightens automatically instead of failing on its last day.
579
+ * Scoped per harness like perDayUsd.
580
+ */
581
+ perMonthUsd?: number;
518
582
  /** At the ceiling: drop to the cheapest viable model, or fail the request outright. */
519
583
  onExceeded: "downgrade" | "reject";
520
584
  }
@@ -676,6 +740,7 @@ export interface RouterConfig {
676
740
  context: ContextConfig;
677
741
  compaction: CompactionConfig;
678
742
  budget: BudgetConfig;
743
+ report: ReportConfig;
679
744
  profiles: ProfileConfig[];
680
745
  ledger: LedgerConfig;
681
746
  /**
@@ -0,0 +1,81 @@
1
+ /**
2
+ * User verdicts on routed turns, from omp (`/router feedback good|bad`).
3
+ *
4
+ * The router otherwise learns only from escalation signals. A person saying
5
+ * a cheap model's answer was wrong — or that it was fine — is the label the
6
+ * de-escalation question needs. Each verdict is tied to the ledger row it
7
+ * judged, so it aggregates by served model, tier and task.
8
+ */
9
+
10
+ import type { Database } from "bun:sqlite";
11
+
12
+ export type Verdict = "good" | "bad";
13
+
14
+ export interface FeedbackRecord {
15
+ ledgerId: string;
16
+ ompSessionId: string;
17
+ slug: string;
18
+ tier: string;
19
+ verdict: Verdict;
20
+ note: string;
21
+ }
22
+
23
+ export interface FeedbackCounts {
24
+ good: number;
25
+ bad: number;
26
+ }
27
+
28
+ export interface FeedbackStore {
29
+ record(rec: FeedbackRecord, nowMs?: number): string;
30
+ /** Verdict counts per served slug since `sinceMs`. */
31
+ countsBySlug(sinceMs: number, harnessId?: string): Map<string, FeedbackCounts>;
32
+ /** Verdicts for one ledger row (a user may re-judge). */
33
+ forLedgerId(ledgerId: string): Array<{ verdict: Verdict; note: string; createdAtMs: number }>;
34
+ }
35
+
36
+ export function createFeedbackStore(db: Database): FeedbackStore {
37
+ const insert = db.query(
38
+ `INSERT INTO feedback (id, ledger_id, omp_session_id, slug, tier, verdict, note, created_at_ms)
39
+ VALUES ($id, $ledgerId, $ompSessionId, $slug, $tier, $verdict, $note, $createdAtMs)`,
40
+ );
41
+ const bySlug = db.query(
42
+ `SELECT f.slug, f.verdict, COUNT(*) AS n FROM feedback f
43
+ LEFT JOIN ledger l ON l.id = f.ledger_id
44
+ WHERE f.created_at_ms >= $since AND ($harness = '' OR l.harness_id = $harness)
45
+ GROUP BY f.slug, f.verdict`,
46
+ );
47
+ const forRow = db.query("SELECT verdict, note, created_at_ms FROM feedback WHERE ledger_id = ? ORDER BY created_at_ms DESC");
48
+ return {
49
+ record(rec, nowMs = Date.now()) {
50
+ const id = crypto.randomUUID();
51
+ insert.run({
52
+ $id: id,
53
+ $ledgerId: rec.ledgerId,
54
+ $ompSessionId: rec.ompSessionId,
55
+ $slug: rec.slug,
56
+ $tier: rec.tier,
57
+ $verdict: rec.verdict,
58
+ $note: rec.note.slice(0, 500),
59
+ $createdAtMs: nowMs,
60
+ });
61
+ return id;
62
+ },
63
+ countsBySlug(sinceMs, harnessId = "") {
64
+ const out = new Map<string, FeedbackCounts>();
65
+ for (const r of bySlug.all({ $since: sinceMs, $harness: harnessId }) as { slug: string; verdict: string; n: number }[]) {
66
+ const c = out.get(r.slug) ?? { good: 0, bad: 0 };
67
+ if (r.verdict === "good") c.good += r.n;
68
+ else c.bad += r.n;
69
+ out.set(r.slug, c);
70
+ }
71
+ return out;
72
+ },
73
+ forLedgerId(ledgerId) {
74
+ return (forRow.all(ledgerId) as { verdict: Verdict; note: string; created_at_ms: number }[]).map((r) => ({
75
+ verdict: r.verdict,
76
+ note: r.note,
77
+ createdAtMs: r.created_at_ms,
78
+ }));
79
+ },
80
+ };
81
+ }
@@ -24,6 +24,7 @@ import type {
24
24
  Ledger,
25
25
  LedgerEntry,
26
26
  LedgerSignals,
27
+ ModelCacheReliability,
27
28
  ModelLatency,
28
29
  ModelTrust,
29
30
  UsageCounts,
@@ -38,6 +39,14 @@ const MAX_SANE_BYTES_PER_TOKEN = 8;
38
39
  const MIN_ESCALATION_SAMPLES = 10;
39
40
  /** The escalation-cost aggregate scans a window of rows; memoised for this long. */
40
41
  const ESCALATION_COST_MEMO_MS = 60_000;
42
+
43
+ /**
44
+ * Cache reliability is one window-function pass over the newest rows, memoised
45
+ * for a minute: the previous kept turn of each conversation is found with LAG,
46
+ * and a row counts when that turn was on the same model within the warm TTL.
47
+ */
48
+ const CACHE_RELIABILITY_ROWS = 6_000;
49
+ const CACHE_RELIABILITY_MEMO_MS = 60_000;
41
50
  const DAY_MS = 86_400_000;
42
51
 
43
52
  // Row shapes below are fixed by our own schema in util/sqlite.ts.
@@ -85,6 +94,15 @@ interface TrustRow {
85
94
  mean_cost_error: number | null;
86
95
  }
87
96
 
97
+ interface FeedbackRow {
98
+ good: number | null;
99
+ bad: number | null;
100
+ }
101
+
102
+ /** Verdict counts per served slug since a cutoff (optionally one harness). */
103
+ const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
104
+ COALESCE(SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END), 0) AS bad`;
105
+
88
106
  interface LatencyRow {
89
107
  samples: number;
90
108
  ttft_ms: number | null;
@@ -177,15 +195,23 @@ function errorKindOf(error: string | null): string | null {
177
195
  return error.slice(0, sep);
178
196
  }
179
197
 
180
- function toTrust(slug: string, row: TrustRow): ModelTrust {
198
+ function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
181
199
  // Laplace smoothing: an untried model scores a neutral 1/2, and a failure
182
200
  // is an attempt superseded by an escalation or ended in an upstream error.
201
+ // A user verdict counts as feedbackWeight extra attempts of that outcome.
202
+ const good = fb?.good ?? 0;
203
+ const bad = fb?.bad ?? 0;
204
+ const w = feedbackWeight > 0 ? feedbackWeight : 0;
205
+ const attempts = row.attempts + w * (good + bad);
206
+ const failures = row.failures + w * bad;
183
207
  return {
184
208
  slug,
185
209
  attempts: row.attempts,
186
210
  escalations: row.escalations,
187
211
  errors: row.errors,
188
- successRate: (row.attempts - row.failures + 1) / (row.attempts + 2),
212
+ feedbackGood: good,
213
+ feedbackBad: bad,
214
+ successRate: (attempts - failures + 1) / (attempts + 2),
189
215
  meanCostError: row.mean_cost_error ?? 0,
190
216
  };
191
217
  }
@@ -236,7 +262,42 @@ function toEntry(row: LedgerRow): LedgerEntry {
236
262
  };
237
263
  }
238
264
 
265
+ export interface CacheReliabilityRow {
266
+ slug: string;
267
+ samples: number;
268
+ hit: number;
269
+ }
270
+
271
+ /**
272
+ * Observed cache hit rates when a warm cache was expected, per served slug.
273
+ * `sinceMs` bounds the rows scanned (0 ⇒ the newest `limitRows`). Rows whose
274
+ * cache count the router estimated (`usage.cachedEstimated`) are excluded.
275
+ */
276
+ export function queryCacheReliability(db: Database, opts: { warmTtlMs: number; sinceMs?: number; limitRows?: number }): CacheReliabilityRow[] {
277
+ const sinceMs = opts.sinceMs ?? 0;
278
+ const limitRows = opts.limitRows ?? CACHE_RELIABILITY_ROWS;
279
+ return db
280
+ .query(
281
+ `WITH recent AS (
282
+ SELECT conversation_key AS ck, created_at_ms AS t, COALESCE(served_slug, slug) AS s,
283
+ json_extract(usage, '$.promptTokens') AS p, json_extract(usage, '$.cachedTokens') AS c,
284
+ COALESCE(json_extract(usage, '$.cachedEstimated'), 0) AS est
285
+ FROM ledger WHERE wasted = 0 AND error IS NULL AND created_at_ms >= $since
286
+ ORDER BY created_at_ms DESC LIMIT $limit),
287
+ seq AS (
288
+ SELECT s, p, c, est, t,
289
+ LAG(s) OVER w AS prev_s, LAG(p) OVER w AS prev_p, LAG(t) OVER w AS prev_t
290
+ FROM recent WINDOW w AS (PARTITION BY ck ORDER BY t))
291
+ SELECT s AS slug, COUNT(*) AS samples, AVG(MIN(1.0, c * 1.0 / MIN(prev_p, p))) AS hit
292
+ FROM seq
293
+ WHERE prev_s = s AND p > 1000 AND prev_p > 1000 AND t - prev_t <= $ttl AND est = 0
294
+ GROUP BY s`,
295
+ )
296
+ .all({ $since: sinceMs, $limit: limitRows, $ttl: opts.warmTtlMs }) as CacheReliabilityRow[];
297
+ }
298
+
239
299
  export function createLedger(db: Database, cfg: RouterConfig): Ledger {
300
+ let cacheMemo: { atMs: number; map: Map<string, ModelCacheReliability> } | null = null;
240
301
  // Prepared once: record() runs on every turn.
241
302
  const insertStmt = db.query(
242
303
  `INSERT INTO ledger (
@@ -268,6 +329,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
268
329
  const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
269
330
  const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
270
331
  const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
332
+ const feedbackStmt = db.query(`SELECT ${FEEDBACK_SELECT} FROM feedback f WHERE f.slug = ? AND f.created_at_ms > ?`);
333
+ const feedbackHarnessStmt = db.query(
334
+ `SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND f.created_at_ms > ?`,
335
+ );
336
+ const allFeedbackStmt = db.query(`SELECT f.slug, ${FEEDBACK_SELECT} FROM feedback f WHERE f.created_at_ms > ? GROUP BY f.slug`);
337
+ const feedbackFor = (slug: string, harnessId: string | undefined, cutoff: number): FeedbackRow | null => {
338
+ if (cfg.filters.feedbackWeight <= 0) return null;
339
+ return harnessId !== undefined && harnessId !== ""
340
+ ? (feedbackHarnessStmt.get(slug, harnessId, cutoff) as FeedbackRow | null)
341
+ : (feedbackStmt.get(slug, cutoff) as FeedbackRow | null);
342
+ };
271
343
  const latencyStmt = db.query(
272
344
  `SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
273
345
  );
@@ -276,6 +348,10 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
276
348
  );
277
349
  const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
278
350
  const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
351
+ const providerSpendStmt = db.query(
352
+ "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND COALESCE(served_slug, slug) LIKE ?",
353
+ );
354
+ const sessionStmt = db.query("SELECT * FROM ledger WHERE omp_session_id = ? AND wasted = 0 ORDER BY created_at_ms DESC LIMIT ?");
279
355
  // What an escalated retry actually bills, per prompt token, over a window.
280
356
  // attempt > 0 rows are the re-dispatches that followed a rejected attempt;
281
357
  // errored ones carry no usage and are excluded.
@@ -399,13 +475,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
399
475
  ? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
400
476
  : (trustStmt.get(slug, cutoff) as TrustRow | null);
401
477
  if (row === null || row.attempts === 0) return null;
402
- return toTrust(slug, row);
478
+ return toTrust(slug, row, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight);
403
479
  },
404
480
 
405
481
  allTrust(): ModelTrust[] {
406
482
  const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
407
483
  const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
408
- return rows.map((row) => toTrust(row.slug, row));
484
+ const fb = new Map<string, FeedbackRow>();
485
+ if (cfg.filters.feedbackWeight > 0) {
486
+ for (const r of allFeedbackStmt.all(cutoff) as (FeedbackRow & { slug: string })[]) fb.set(r.slug, r);
487
+ }
488
+ return rows.map((row) => toTrust(row.slug, row, fb.get(row.slug) ?? null, cfg.filters.feedbackWeight));
409
489
  },
410
490
 
411
491
  latency(slug: string, harnessId?: string): ModelLatency | null {
@@ -428,13 +508,25 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
428
508
  ? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
429
509
  : (latencyStmt.get(slug) as LatencyRow | null);
430
510
  out.set(slug, {
431
- trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow),
511
+ trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight),
432
512
  latency: latencyRow === null ? null : toLatency(slug, latencyRow),
433
513
  });
434
514
  }
435
515
  return out;
436
516
  },
437
517
 
518
+ cacheReliability(slug: string): ModelCacheReliability | null {
519
+ const now = Date.now();
520
+ if (cacheMemo === null || now - cacheMemo.atMs > CACHE_RELIABILITY_MEMO_MS) {
521
+ const map = new Map<string, ModelCacheReliability>();
522
+ for (const r of queryCacheReliability(db, { warmTtlMs: cfg.hysteresis.cacheWarmTtlMs })) {
523
+ map.set(r.slug, { slug: r.slug, samples: r.samples, hitRate: Math.min(1, Math.max(0, r.hit)) });
524
+ }
525
+ cacheMemo = { atMs: now, map };
526
+ }
527
+ return cacheMemo.map.get(slug) ?? null;
528
+ },
529
+
438
530
  escalationCost(windowDays: number): EscalationCost | null {
439
531
  const now = Date.now();
440
532
  if (escalationMemo !== null && escalationMemo.windowDays === windowDays && now - escalationMemo.atMs < ESCALATION_COST_MEMO_MS) {
@@ -459,5 +551,18 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
459
551
  const rows = recentStmt.all(limit) as LedgerRow[];
460
552
  return rows.map(toEntry);
461
553
  },
554
+ providerSpendSince(slugPrefix: string, sinceMs: number): number {
555
+ const row = providerSpendStmt.get(sinceMs, `${slugPrefix}%`) as { total: number } | null;
556
+ return row?.total ?? 0;
557
+ },
558
+ latestForSession(ompSessionId: string): LedgerEntry | null {
559
+ if (ompSessionId === "") return null;
560
+ const row = sessionStmt.get(ompSessionId, 1) as LedgerRow | null;
561
+ return row === null ? null : toEntry(row);
562
+ },
563
+ entriesForSession(ompSessionId: string, limit: number): LedgerEntry[] {
564
+ if (ompSessionId === "") return [];
565
+ return (sessionStmt.all(ompSessionId, Math.max(1, limit)) as LedgerRow[]).map(toEntry);
566
+ },
462
567
  };
463
568
  }