auto-model-router 0.3.4 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +29 -4
- package/omp-extension/report-logic.ts +93 -0
- package/omp-extension/router-configure.ts +128 -2
- package/omp-extension/router-embed.ts +9 -6
- package/package.json +1 -1
- package/src/catalog/ollama-catalog.ts +30 -2
- package/src/cli/config-wizard.ts +11 -0
- package/src/cli/report.ts +7 -2
- package/src/config/defaults.ts +17 -0
- package/src/config/schema.ts +8 -0
- package/src/config/types.ts +65 -0
- package/src/cost/feedback.ts +81 -0
- package/src/cost/ledger.ts +110 -5
- package/src/cost/report.ts +118 -2
- package/src/cost/types.ts +32 -1
- package/src/router/candidates.ts +13 -4
- package/src/router/classify.ts +13 -0
- package/src/router/features.ts +59 -1
- package/src/router/index.ts +23 -5
- package/src/router/learned.ts +202 -0
- package/src/router/select.ts +64 -8
- package/src/router/types.ts +39 -1
- package/src/server/http.ts +82 -5
- package/src/server/overrides.ts +83 -0
- package/src/server/providers.ts +10 -2
- package/src/server/turn.ts +16 -1
- package/src/upstream/ollama-usage.ts +79 -2
- package/src/util/sqlite.ts +30 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +2 -0
- package/test/classify.test.ts +13 -0
- package/test/config-wizard.test.ts +8 -6
- package/test/controls.test.ts +238 -0
- package/test/escalate.test.ts +1 -0
- package/test/failover.test.ts +6 -4
- package/test/features.test.ts +63 -0
- package/test/http-resilience.test.ts +1 -1
- package/test/learned.test.ts +61 -0
- package/test/ollama.test.ts +74 -2
- package/test/report-hub.test.ts +6 -2
- package/test/report-logic.test.ts +3 -0
- package/test/report.test.ts +57 -0
- package/test/select.test.ts +126 -1
- package/test/trust-attribution.test.ts +95 -0
- package/test/turn.test.ts +36 -4
- package/tools/replay.ts +267 -156
- package/tools/train-classifier.ts +111 -0
package/src/config/defaults.ts
CHANGED
|
@@ -15,6 +15,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
15
15
|
// is deterministic, so peers reuse it), so this covers N sessions plus
|
|
16
16
|
// their subagents. Was effectively 8 per session when each bound its own.
|
|
17
17
|
maxConcurrentTurns: 24,
|
|
18
|
+
// omp subagents (no UI) route under this profile: delegated work, capped at moderate.
|
|
19
|
+
subagentProfile: "auto-sub",
|
|
18
20
|
},
|
|
19
21
|
openrouter: {
|
|
20
22
|
baseUrl: "https://openrouter.ai/api/v1",
|
|
@@ -99,6 +101,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
99
101
|
includeFree: false,
|
|
100
102
|
requireToolSupport: true,
|
|
101
103
|
minTrust: 0.7,
|
|
104
|
+
// Verdicts are recorded and reported first; weigh them once there are some.
|
|
105
|
+
feedbackWeight: 0,
|
|
102
106
|
minTrustSamples: 12,
|
|
103
107
|
// Shared trust by default: more samples, demotion guard stays effective
|
|
104
108
|
// even with a tiny guardrail-narrowed catalog.
|
|
@@ -118,6 +122,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
118
122
|
// slow models (e.g. deepseek-v4-flash ~20 tok/s) fall under it.
|
|
119
123
|
latencyReferenceTokensPerSec: 30,
|
|
120
124
|
latencyMinSamples: 20,
|
|
125
|
+
// Discount a warm model's stay price by its observed hit rate once this
|
|
126
|
+
// many warm-expected samples exist. See FilterConfig.cacheReliabilityMinSamples.
|
|
127
|
+
cacheReliabilityMinSamples: 10,
|
|
121
128
|
// Off: pricing a model's measured escalation rate at what an escalated
|
|
122
129
|
// retry actually costs changes rankings, so it is opt-in after a replay
|
|
123
130
|
// run prices it. See FilterConfig.escalationCostWeight.
|
|
@@ -127,6 +134,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
127
134
|
ambiguityThreshold: 0.6,
|
|
128
135
|
// Cheapest competent slug in the catalog; adjudication prompts are tiny.
|
|
129
136
|
model: "qwen/qwen3.7-flash",
|
|
137
|
+
// Off until a model is trained; see tools/train-classifier.ts.
|
|
138
|
+
learnedModelPath: "",
|
|
130
139
|
maxCostFraction: 0.02,
|
|
131
140
|
maxCostUsd: 0.002,
|
|
132
141
|
timeoutMs: 4_000,
|
|
@@ -137,6 +146,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
137
146
|
// A mechanical retry (failed tool call + tool-result continuation) keeps
|
|
138
147
|
// only a fifth of the +0.26; a user-visible failure keeps the full weight.
|
|
139
148
|
mechanicalRetryFactor: 0.2,
|
|
149
|
+
// Recorded, not acted on, until replay prices it. See ClassifierConfig.readOnlyToolWeight.
|
|
150
|
+
readOnlyToolWeight: 0,
|
|
140
151
|
// Shipped reasoning values, unchanged. See ClassifierConfig.reasoningWeights:
|
|
141
152
|
// a harness that pins the level for a whole session turns these into a
|
|
142
153
|
// constant tier offset, in which case `medium` belongs near 0.
|
|
@@ -281,6 +292,10 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
281
292
|
elideSupersededReads: true,
|
|
282
293
|
collapseDuplicateResults: true,
|
|
283
294
|
},
|
|
295
|
+
report: {
|
|
296
|
+
// The frontier pair most omp users would otherwise run on.
|
|
297
|
+
baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
|
|
298
|
+
},
|
|
284
299
|
budget: {
|
|
285
300
|
// No caps by default; at a configured ceiling, downgrade rather than fail.
|
|
286
301
|
onExceeded: "downgrade",
|
|
@@ -289,6 +304,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
289
304
|
{ id: "auto", name: "Auto (auto-model-router)", minTier: "trivial", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
|
|
290
305
|
{ id: "auto-cheap", name: "Auto Cheap (auto-model-router)", minTier: "trivial", maxTier: "simple", contextWindow: 400_000, maxTokens: 32_000 },
|
|
291
306
|
{ id: "auto-max", name: "Auto Max (auto-model-router)", minTier: "moderate", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000 },
|
|
307
|
+
// Subagent envelope (server.subagentProfile): never the top tier for delegated work.
|
|
308
|
+
{ id: "auto-sub", name: "Auto Subagent (auto-model-router)", minTier: "trivial", maxTier: "moderate", contextWindow: 400_000, maxTokens: 32_000 },
|
|
292
309
|
],
|
|
293
310
|
ledger: {
|
|
294
311
|
// Resolved by loadConfig: empty ⇒ `$AUTO_MODEL_ROUTER_HOME/router.db`.
|
package/src/config/schema.ts
CHANGED
|
@@ -20,6 +20,7 @@ const server = z.strictObject({
|
|
|
20
20
|
port: z.number().int().min(0).max(65_535).optional(),
|
|
21
21
|
apiKey: z.string().optional(),
|
|
22
22
|
harnessId: z.string().optional(),
|
|
23
|
+
subagentProfile: z.string().optional(),
|
|
23
24
|
maxConcurrentTurns: z.number().int().positive().max(1_000).optional(),
|
|
24
25
|
});
|
|
25
26
|
|
|
@@ -88,6 +89,7 @@ const filters = z.strictObject({
|
|
|
88
89
|
includeFree: z.boolean().optional(),
|
|
89
90
|
requireToolSupport: z.boolean().optional(),
|
|
90
91
|
minTrust: z.number().min(0).max(1).optional(),
|
|
92
|
+
feedbackWeight: z.number().nonnegative().optional(),
|
|
91
93
|
minTrustSamples: z.number().int().nonnegative().optional(),
|
|
92
94
|
trustScopedByHarness: z.boolean().optional(),
|
|
93
95
|
trustWindowDays: z.number().nonnegative().optional(),
|
|
@@ -96,13 +98,16 @@ const filters = z.strictObject({
|
|
|
96
98
|
latencyReferenceMs: z.number().positive().optional(),
|
|
97
99
|
latencyReferenceTokensPerSec: z.number().positive().optional(),
|
|
98
100
|
latencyMinSamples: z.number().int().nonnegative().optional(),
|
|
101
|
+
cacheReliabilityMinSamples: z.number().int().nonnegative().optional(),
|
|
99
102
|
maxExpectedWaitMs: z.number().positive().optional(),
|
|
103
|
+
latencyWeightContinuation: z.number().nonnegative().optional(),
|
|
100
104
|
escalationCostWeight: z.number().min(0).max(1).optional(),
|
|
101
105
|
});
|
|
102
106
|
|
|
103
107
|
const classifier = z.strictObject({
|
|
104
108
|
ambiguityThreshold: z.number().min(0).max(1).optional(),
|
|
105
109
|
model: z.string().min(1).optional(),
|
|
110
|
+
learnedModelPath: z.string().optional(),
|
|
106
111
|
maxCostFraction: z.number().min(0).max(1).optional(),
|
|
107
112
|
maxCostUsd: z.number().nonnegative().optional(),
|
|
108
113
|
timeoutMs: z.number().positive().optional(),
|
|
@@ -111,6 +116,7 @@ const classifier = z.strictObject({
|
|
|
111
116
|
chatAxis: qualityAxis.optional(),
|
|
112
117
|
agenticLoopDepth: z.number().int().nonnegative().optional(),
|
|
113
118
|
mechanicalRetryFactor: z.number().min(0).max(1).optional(),
|
|
119
|
+
readOnlyToolWeight: z.number().nonnegative().optional(),
|
|
114
120
|
reasoningWeights: z
|
|
115
121
|
.strictObject({
|
|
116
122
|
medium: z.number().nonnegative().optional(),
|
|
@@ -205,6 +211,7 @@ const budget = z.strictObject({
|
|
|
205
211
|
perTurnUsd: z.number().nonnegative().optional(),
|
|
206
212
|
perConversationUsd: z.number().nonnegative().optional(),
|
|
207
213
|
perDayUsd: z.number().nonnegative().optional(),
|
|
214
|
+
perMonthUsd: z.number().nonnegative().optional(),
|
|
208
215
|
onExceeded: z.enum(["downgrade", "reject"]).optional(),
|
|
209
216
|
});
|
|
210
217
|
|
|
@@ -265,6 +272,7 @@ export const configInputSchema = z.strictObject({
|
|
|
265
272
|
compaction: compaction.optional(),
|
|
266
273
|
budget: budget.optional(),
|
|
267
274
|
profiles: z.array(profile).optional(),
|
|
275
|
+
report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
|
|
268
276
|
ledger: ledger.optional(),
|
|
269
277
|
adaptiveTierFloors: z.boolean().optional(),
|
|
270
278
|
adaptivePriceCeilings: z.boolean().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -38,6 +38,14 @@ export interface ServerConfig {
|
|
|
38
38
|
* ⇒ no header (single-harness default).
|
|
39
39
|
*/
|
|
40
40
|
harnessId?: string;
|
|
41
|
+
/**
|
|
42
|
+
* Profile that requests from omp subagents (`X-Omp-Subagent: 1`, set by
|
|
43
|
+
* the embed extension for sessions without a UI) are routed under when they
|
|
44
|
+
* ask for the default profile. Subagents do delegated, bounded work — file
|
|
45
|
+
* reads, searches, summaries — that rarely needs the top tier. Empty
|
|
46
|
+
* disables the remap; a name with no matching profile is ignored.
|
|
47
|
+
*/
|
|
48
|
+
subagentProfile: string;
|
|
41
49
|
/**
|
|
42
50
|
* Concurrent in-flight turns this router process will accept; excess gets a
|
|
43
51
|
* 429 rather than being queued, so a local flood cannot pile up unbounded
|
|
@@ -215,6 +223,15 @@ export interface FilterConfig {
|
|
|
215
223
|
requireToolSupport: boolean;
|
|
216
224
|
/** Drop models whose ledger success rate is below this, once `minTrustSamples` is met. */
|
|
217
225
|
minTrust: number;
|
|
226
|
+
/**
|
|
227
|
+
* How much a user verdict (/router good|bad) weighs in a model's trust
|
|
228
|
+
* rate: each bad verdict counts as this many failures and each good one as
|
|
229
|
+
* this many successes, beside escalations and errors. 0 (default) records
|
|
230
|
+
* verdicts without acting on them. A person judging an answer wrong is a
|
|
231
|
+
* stronger signal than a probe rejection, so values of 2-5 are sensible
|
|
232
|
+
* once a week of verdicts is in the report.
|
|
233
|
+
*/
|
|
234
|
+
feedbackWeight: number;
|
|
218
235
|
/** Attempts required before `minTrust` is enforced against a model. */
|
|
219
236
|
minTrustSamples: number;
|
|
220
237
|
/**
|
|
@@ -264,6 +281,14 @@ export interface FilterConfig {
|
|
|
264
281
|
latencyReferenceTokensPerSec: number;
|
|
265
282
|
/** Streamed samples required before latency is scored against a model. */
|
|
266
283
|
latencyMinSamples: number;
|
|
284
|
+
/**
|
|
285
|
+
* Warm-expected samples a model needs before its observed cache hit rate
|
|
286
|
+
* (ledger `cacheReliability`) discounts the "stay warm" price in the
|
|
287
|
+
* stay/switch comparison. Below it, and when 0, a cache is assumed fully
|
|
288
|
+
* reliable. Measured 2026-09-06: same-model short-gap turns still ran cold
|
|
289
|
+
* 5-6% on glm/gemini, 11% on ling and 50% on nex.
|
|
290
|
+
*/
|
|
291
|
+
cacheReliabilityMinSamples: number;
|
|
267
292
|
/**
|
|
268
293
|
* Absolute expected-wait ceiling (ms). A hard drop, mirroring the price
|
|
269
294
|
* ceiling: any model whose expected total wait (TTFT + streaming the expected
|
|
@@ -277,6 +302,13 @@ export interface FilterConfig {
|
|
|
277
302
|
* Undefined ⇒ off (the default).
|
|
278
303
|
*/
|
|
279
304
|
maxExpectedWaitMs?: number;
|
|
305
|
+
/**
|
|
306
|
+
* Latency weight for tool-result continuations (the agent loop's own
|
|
307
|
+
* follow-ups, where no person is waiting on first token). Unset ⇒
|
|
308
|
+
* `latencyWeight` applies to every turn. Lower it to spend speed only on
|
|
309
|
+
* user-facing turns.
|
|
310
|
+
*/
|
|
311
|
+
latencyWeightContinuation?: number;
|
|
280
312
|
/**
|
|
281
313
|
* How much of a model's measured escalation risk to price into its effective
|
|
282
314
|
* cost, 0-1. 0 (the default) disables the term.
|
|
@@ -305,6 +337,13 @@ export interface ClassifierConfig {
|
|
|
305
337
|
ambiguityThreshold: number;
|
|
306
338
|
/** Slug used for adjudication. Must be cheap and fast. */
|
|
307
339
|
model: string;
|
|
340
|
+
/**
|
|
341
|
+
* Path of a model written by `tools/train-classifier.ts`. When set, every
|
|
342
|
+
* heuristic classification also carries the learned P(escalate) in its
|
|
343
|
+
* reasons (`learned: p(escalate)=…`) and `Classification.learnedRisk`.
|
|
344
|
+
* Advisory: it never moves a tier. Empty ⇒ off.
|
|
345
|
+
*/
|
|
346
|
+
learnedModelPath: string;
|
|
308
347
|
/** Skip adjudication when it would exceed this fraction of the forecast turn cost. */
|
|
309
348
|
maxCostFraction: number;
|
|
310
349
|
/** Absolute per-call ceiling, USD. */
|
|
@@ -325,6 +364,13 @@ export interface ClassifierConfig {
|
|
|
325
364
|
* loops buy the hard tier. 1 preserves the shipped behaviour.
|
|
326
365
|
*/
|
|
327
366
|
mechanicalRetryFactor: number;
|
|
367
|
+
/**
|
|
368
|
+
* Score subtracted when the newest assistant turn issued only read-only
|
|
369
|
+
* tools (read, grep, glob, ls, lsp…) and this is the tool-result
|
|
370
|
+
* continuation: the model is looking, not deciding. 0 (default) records
|
|
371
|
+
* the feature without acting on it — enable after a replay prices it.
|
|
372
|
+
*/
|
|
373
|
+
readOnlyToolWeight: number;
|
|
328
374
|
/**
|
|
329
375
|
* Score added when the CLIENT asks for a reasoning effort, per level. The
|
|
330
376
|
* premise is that asking for reasoning states expected difficulty directly.
|
|
@@ -508,6 +554,17 @@ export interface CacheConfig {
|
|
|
508
554
|
milestoneTokens: number;
|
|
509
555
|
}
|
|
510
556
|
|
|
557
|
+
/** Usage-report options. */
|
|
558
|
+
export interface ReportConfig {
|
|
559
|
+
/**
|
|
560
|
+
* Models to price the window's traffic on as if every turn had used that
|
|
561
|
+
* one model, at its list price with the window's own cache hit rate: the
|
|
562
|
+
* "what the router saved" counterfactual. Slugs missing from the catalog
|
|
563
|
+
* are skipped.
|
|
564
|
+
*/
|
|
565
|
+
baselines: string[];
|
|
566
|
+
}
|
|
567
|
+
|
|
511
568
|
export interface BudgetConfig {
|
|
512
569
|
/** Reject or downgrade when a turn's cold forecast exceeds this, USD. */
|
|
513
570
|
perTurnUsd?: number;
|
|
@@ -515,6 +572,13 @@ export interface BudgetConfig {
|
|
|
515
572
|
perConversationUsd?: number;
|
|
516
573
|
/** Rolling 24h ceiling, USD. */
|
|
517
574
|
perDayUsd?: number;
|
|
575
|
+
/**
|
|
576
|
+
* Calendar-month (UTC) target, USD. Paced: the per-day ceiling becomes
|
|
577
|
+
* min(perDayUsd, remaining ÷ days left in the month), so a month that runs
|
|
578
|
+
* ahead of pace tightens automatically instead of failing on its last day.
|
|
579
|
+
* Scoped per harness like perDayUsd.
|
|
580
|
+
*/
|
|
581
|
+
perMonthUsd?: number;
|
|
518
582
|
/** At the ceiling: drop to the cheapest viable model, or fail the request outright. */
|
|
519
583
|
onExceeded: "downgrade" | "reject";
|
|
520
584
|
}
|
|
@@ -676,6 +740,7 @@ export interface RouterConfig {
|
|
|
676
740
|
context: ContextConfig;
|
|
677
741
|
compaction: CompactionConfig;
|
|
678
742
|
budget: BudgetConfig;
|
|
743
|
+
report: ReportConfig;
|
|
679
744
|
profiles: ProfileConfig[];
|
|
680
745
|
ledger: LedgerConfig;
|
|
681
746
|
/**
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* User verdicts on routed turns, from omp (`/router feedback good|bad`).
|
|
3
|
+
*
|
|
4
|
+
* The router otherwise learns only from escalation signals. A person saying
|
|
5
|
+
* a cheap model's answer was wrong — or that it was fine — is the label the
|
|
6
|
+
* de-escalation question needs. Each verdict is tied to the ledger row it
|
|
7
|
+
* judged, so it aggregates by served model, tier and task.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Database } from "bun:sqlite";
|
|
11
|
+
|
|
12
|
+
export type Verdict = "good" | "bad";
|
|
13
|
+
|
|
14
|
+
export interface FeedbackRecord {
|
|
15
|
+
ledgerId: string;
|
|
16
|
+
ompSessionId: string;
|
|
17
|
+
slug: string;
|
|
18
|
+
tier: string;
|
|
19
|
+
verdict: Verdict;
|
|
20
|
+
note: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface FeedbackCounts {
|
|
24
|
+
good: number;
|
|
25
|
+
bad: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface FeedbackStore {
|
|
29
|
+
record(rec: FeedbackRecord, nowMs?: number): string;
|
|
30
|
+
/** Verdict counts per served slug since `sinceMs`. */
|
|
31
|
+
countsBySlug(sinceMs: number, harnessId?: string): Map<string, FeedbackCounts>;
|
|
32
|
+
/** Verdicts for one ledger row (a user may re-judge). */
|
|
33
|
+
forLedgerId(ledgerId: string): Array<{ verdict: Verdict; note: string; createdAtMs: number }>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function createFeedbackStore(db: Database): FeedbackStore {
|
|
37
|
+
const insert = db.query(
|
|
38
|
+
`INSERT INTO feedback (id, ledger_id, omp_session_id, slug, tier, verdict, note, created_at_ms)
|
|
39
|
+
VALUES ($id, $ledgerId, $ompSessionId, $slug, $tier, $verdict, $note, $createdAtMs)`,
|
|
40
|
+
);
|
|
41
|
+
const bySlug = db.query(
|
|
42
|
+
`SELECT f.slug, f.verdict, COUNT(*) AS n FROM feedback f
|
|
43
|
+
LEFT JOIN ledger l ON l.id = f.ledger_id
|
|
44
|
+
WHERE f.created_at_ms >= $since AND ($harness = '' OR l.harness_id = $harness)
|
|
45
|
+
GROUP BY f.slug, f.verdict`,
|
|
46
|
+
);
|
|
47
|
+
const forRow = db.query("SELECT verdict, note, created_at_ms FROM feedback WHERE ledger_id = ? ORDER BY created_at_ms DESC");
|
|
48
|
+
return {
|
|
49
|
+
record(rec, nowMs = Date.now()) {
|
|
50
|
+
const id = crypto.randomUUID();
|
|
51
|
+
insert.run({
|
|
52
|
+
$id: id,
|
|
53
|
+
$ledgerId: rec.ledgerId,
|
|
54
|
+
$ompSessionId: rec.ompSessionId,
|
|
55
|
+
$slug: rec.slug,
|
|
56
|
+
$tier: rec.tier,
|
|
57
|
+
$verdict: rec.verdict,
|
|
58
|
+
$note: rec.note.slice(0, 500),
|
|
59
|
+
$createdAtMs: nowMs,
|
|
60
|
+
});
|
|
61
|
+
return id;
|
|
62
|
+
},
|
|
63
|
+
countsBySlug(sinceMs, harnessId = "") {
|
|
64
|
+
const out = new Map<string, FeedbackCounts>();
|
|
65
|
+
for (const r of bySlug.all({ $since: sinceMs, $harness: harnessId }) as { slug: string; verdict: string; n: number }[]) {
|
|
66
|
+
const c = out.get(r.slug) ?? { good: 0, bad: 0 };
|
|
67
|
+
if (r.verdict === "good") c.good += r.n;
|
|
68
|
+
else c.bad += r.n;
|
|
69
|
+
out.set(r.slug, c);
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
},
|
|
73
|
+
forLedgerId(ledgerId) {
|
|
74
|
+
return (forRow.all(ledgerId) as { verdict: Verdict; note: string; created_at_ms: number }[]).map((r) => ({
|
|
75
|
+
verdict: r.verdict,
|
|
76
|
+
note: r.note,
|
|
77
|
+
createdAtMs: r.created_at_ms,
|
|
78
|
+
}));
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
package/src/cost/ledger.ts
CHANGED
|
@@ -24,6 +24,7 @@ import type {
|
|
|
24
24
|
Ledger,
|
|
25
25
|
LedgerEntry,
|
|
26
26
|
LedgerSignals,
|
|
27
|
+
ModelCacheReliability,
|
|
27
28
|
ModelLatency,
|
|
28
29
|
ModelTrust,
|
|
29
30
|
UsageCounts,
|
|
@@ -38,6 +39,14 @@ const MAX_SANE_BYTES_PER_TOKEN = 8;
|
|
|
38
39
|
const MIN_ESCALATION_SAMPLES = 10;
|
|
39
40
|
/** The escalation-cost aggregate scans a window of rows; memoised for this long. */
|
|
40
41
|
const ESCALATION_COST_MEMO_MS = 60_000;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Cache reliability is one window-function pass over the newest rows, memoised
|
|
45
|
+
* for a minute: the previous kept turn of each conversation is found with LAG,
|
|
46
|
+
* and a row counts when that turn was on the same model within the warm TTL.
|
|
47
|
+
*/
|
|
48
|
+
const CACHE_RELIABILITY_ROWS = 6_000;
|
|
49
|
+
const CACHE_RELIABILITY_MEMO_MS = 60_000;
|
|
41
50
|
const DAY_MS = 86_400_000;
|
|
42
51
|
|
|
43
52
|
// Row shapes below are fixed by our own schema in util/sqlite.ts.
|
|
@@ -85,6 +94,15 @@ interface TrustRow {
|
|
|
85
94
|
mean_cost_error: number | null;
|
|
86
95
|
}
|
|
87
96
|
|
|
97
|
+
interface FeedbackRow {
|
|
98
|
+
good: number | null;
|
|
99
|
+
bad: number | null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Verdict counts per served slug since a cutoff (optionally one harness). */
|
|
103
|
+
const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
|
|
104
|
+
COALESCE(SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END), 0) AS bad`;
|
|
105
|
+
|
|
88
106
|
interface LatencyRow {
|
|
89
107
|
samples: number;
|
|
90
108
|
ttft_ms: number | null;
|
|
@@ -177,15 +195,23 @@ function errorKindOf(error: string | null): string | null {
|
|
|
177
195
|
return error.slice(0, sep);
|
|
178
196
|
}
|
|
179
197
|
|
|
180
|
-
function toTrust(slug: string, row: TrustRow): ModelTrust {
|
|
198
|
+
function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
|
|
181
199
|
// Laplace smoothing: an untried model scores a neutral 1/2, and a failure
|
|
182
200
|
// is an attempt superseded by an escalation or ended in an upstream error.
|
|
201
|
+
// A user verdict counts as feedbackWeight extra attempts of that outcome.
|
|
202
|
+
const good = fb?.good ?? 0;
|
|
203
|
+
const bad = fb?.bad ?? 0;
|
|
204
|
+
const w = feedbackWeight > 0 ? feedbackWeight : 0;
|
|
205
|
+
const attempts = row.attempts + w * (good + bad);
|
|
206
|
+
const failures = row.failures + w * bad;
|
|
183
207
|
return {
|
|
184
208
|
slug,
|
|
185
209
|
attempts: row.attempts,
|
|
186
210
|
escalations: row.escalations,
|
|
187
211
|
errors: row.errors,
|
|
188
|
-
|
|
212
|
+
feedbackGood: good,
|
|
213
|
+
feedbackBad: bad,
|
|
214
|
+
successRate: (attempts - failures + 1) / (attempts + 2),
|
|
189
215
|
meanCostError: row.mean_cost_error ?? 0,
|
|
190
216
|
};
|
|
191
217
|
}
|
|
@@ -236,7 +262,42 @@ function toEntry(row: LedgerRow): LedgerEntry {
|
|
|
236
262
|
};
|
|
237
263
|
}
|
|
238
264
|
|
|
265
|
+
export interface CacheReliabilityRow {
|
|
266
|
+
slug: string;
|
|
267
|
+
samples: number;
|
|
268
|
+
hit: number;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Observed cache hit rates when a warm cache was expected, per served slug.
|
|
273
|
+
* `sinceMs` bounds the rows scanned (0 ⇒ the newest `limitRows`). Rows whose
|
|
274
|
+
* cache count the router estimated (`usage.cachedEstimated`) are excluded.
|
|
275
|
+
*/
|
|
276
|
+
export function queryCacheReliability(db: Database, opts: { warmTtlMs: number; sinceMs?: number; limitRows?: number }): CacheReliabilityRow[] {
|
|
277
|
+
const sinceMs = opts.sinceMs ?? 0;
|
|
278
|
+
const limitRows = opts.limitRows ?? CACHE_RELIABILITY_ROWS;
|
|
279
|
+
return db
|
|
280
|
+
.query(
|
|
281
|
+
`WITH recent AS (
|
|
282
|
+
SELECT conversation_key AS ck, created_at_ms AS t, COALESCE(served_slug, slug) AS s,
|
|
283
|
+
json_extract(usage, '$.promptTokens') AS p, json_extract(usage, '$.cachedTokens') AS c,
|
|
284
|
+
COALESCE(json_extract(usage, '$.cachedEstimated'), 0) AS est
|
|
285
|
+
FROM ledger WHERE wasted = 0 AND error IS NULL AND created_at_ms >= $since
|
|
286
|
+
ORDER BY created_at_ms DESC LIMIT $limit),
|
|
287
|
+
seq AS (
|
|
288
|
+
SELECT s, p, c, est, t,
|
|
289
|
+
LAG(s) OVER w AS prev_s, LAG(p) OVER w AS prev_p, LAG(t) OVER w AS prev_t
|
|
290
|
+
FROM recent WINDOW w AS (PARTITION BY ck ORDER BY t))
|
|
291
|
+
SELECT s AS slug, COUNT(*) AS samples, AVG(MIN(1.0, c * 1.0 / MIN(prev_p, p))) AS hit
|
|
292
|
+
FROM seq
|
|
293
|
+
WHERE prev_s = s AND p > 1000 AND prev_p > 1000 AND t - prev_t <= $ttl AND est = 0
|
|
294
|
+
GROUP BY s`,
|
|
295
|
+
)
|
|
296
|
+
.all({ $since: sinceMs, $limit: limitRows, $ttl: opts.warmTtlMs }) as CacheReliabilityRow[];
|
|
297
|
+
}
|
|
298
|
+
|
|
239
299
|
export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
300
|
+
let cacheMemo: { atMs: number; map: Map<string, ModelCacheReliability> } | null = null;
|
|
240
301
|
// Prepared once: record() runs on every turn.
|
|
241
302
|
const insertStmt = db.query(
|
|
242
303
|
`INSERT INTO ledger (
|
|
@@ -268,6 +329,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
268
329
|
const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
|
|
269
330
|
const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
|
|
270
331
|
const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
|
|
332
|
+
const feedbackStmt = db.query(`SELECT ${FEEDBACK_SELECT} FROM feedback f WHERE f.slug = ? AND f.created_at_ms > ?`);
|
|
333
|
+
const feedbackHarnessStmt = db.query(
|
|
334
|
+
`SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND f.created_at_ms > ?`,
|
|
335
|
+
);
|
|
336
|
+
const allFeedbackStmt = db.query(`SELECT f.slug, ${FEEDBACK_SELECT} FROM feedback f WHERE f.created_at_ms > ? GROUP BY f.slug`);
|
|
337
|
+
const feedbackFor = (slug: string, harnessId: string | undefined, cutoff: number): FeedbackRow | null => {
|
|
338
|
+
if (cfg.filters.feedbackWeight <= 0) return null;
|
|
339
|
+
return harnessId !== undefined && harnessId !== ""
|
|
340
|
+
? (feedbackHarnessStmt.get(slug, harnessId, cutoff) as FeedbackRow | null)
|
|
341
|
+
: (feedbackStmt.get(slug, cutoff) as FeedbackRow | null);
|
|
342
|
+
};
|
|
271
343
|
const latencyStmt = db.query(
|
|
272
344
|
`SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
|
|
273
345
|
);
|
|
@@ -276,6 +348,10 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
276
348
|
);
|
|
277
349
|
const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
|
|
278
350
|
const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
|
|
351
|
+
const providerSpendStmt = db.query(
|
|
352
|
+
"SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND COALESCE(served_slug, slug) LIKE ?",
|
|
353
|
+
);
|
|
354
|
+
const sessionStmt = db.query("SELECT * FROM ledger WHERE omp_session_id = ? AND wasted = 0 ORDER BY created_at_ms DESC LIMIT ?");
|
|
279
355
|
// What an escalated retry actually bills, per prompt token, over a window.
|
|
280
356
|
// attempt > 0 rows are the re-dispatches that followed a rejected attempt;
|
|
281
357
|
// errored ones carry no usage and are excluded.
|
|
@@ -399,13 +475,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
399
475
|
? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
|
|
400
476
|
: (trustStmt.get(slug, cutoff) as TrustRow | null);
|
|
401
477
|
if (row === null || row.attempts === 0) return null;
|
|
402
|
-
return toTrust(slug, row);
|
|
478
|
+
return toTrust(slug, row, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight);
|
|
403
479
|
},
|
|
404
480
|
|
|
405
481
|
allTrust(): ModelTrust[] {
|
|
406
482
|
const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
|
|
407
483
|
const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
|
|
408
|
-
|
|
484
|
+
const fb = new Map<string, FeedbackRow>();
|
|
485
|
+
if (cfg.filters.feedbackWeight > 0) {
|
|
486
|
+
for (const r of allFeedbackStmt.all(cutoff) as (FeedbackRow & { slug: string })[]) fb.set(r.slug, r);
|
|
487
|
+
}
|
|
488
|
+
return rows.map((row) => toTrust(row.slug, row, fb.get(row.slug) ?? null, cfg.filters.feedbackWeight));
|
|
409
489
|
},
|
|
410
490
|
|
|
411
491
|
latency(slug: string, harnessId?: string): ModelLatency | null {
|
|
@@ -428,13 +508,25 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
428
508
|
? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
|
|
429
509
|
: (latencyStmt.get(slug) as LatencyRow | null);
|
|
430
510
|
out.set(slug, {
|
|
431
|
-
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow),
|
|
511
|
+
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight),
|
|
432
512
|
latency: latencyRow === null ? null : toLatency(slug, latencyRow),
|
|
433
513
|
});
|
|
434
514
|
}
|
|
435
515
|
return out;
|
|
436
516
|
},
|
|
437
517
|
|
|
518
|
+
cacheReliability(slug: string): ModelCacheReliability | null {
|
|
519
|
+
const now = Date.now();
|
|
520
|
+
if (cacheMemo === null || now - cacheMemo.atMs > CACHE_RELIABILITY_MEMO_MS) {
|
|
521
|
+
const map = new Map<string, ModelCacheReliability>();
|
|
522
|
+
for (const r of queryCacheReliability(db, { warmTtlMs: cfg.hysteresis.cacheWarmTtlMs })) {
|
|
523
|
+
map.set(r.slug, { slug: r.slug, samples: r.samples, hitRate: Math.min(1, Math.max(0, r.hit)) });
|
|
524
|
+
}
|
|
525
|
+
cacheMemo = { atMs: now, map };
|
|
526
|
+
}
|
|
527
|
+
return cacheMemo.map.get(slug) ?? null;
|
|
528
|
+
},
|
|
529
|
+
|
|
438
530
|
escalationCost(windowDays: number): EscalationCost | null {
|
|
439
531
|
const now = Date.now();
|
|
440
532
|
if (escalationMemo !== null && escalationMemo.windowDays === windowDays && now - escalationMemo.atMs < ESCALATION_COST_MEMO_MS) {
|
|
@@ -459,5 +551,18 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
459
551
|
const rows = recentStmt.all(limit) as LedgerRow[];
|
|
460
552
|
return rows.map(toEntry);
|
|
461
553
|
},
|
|
554
|
+
providerSpendSince(slugPrefix: string, sinceMs: number): number {
|
|
555
|
+
const row = providerSpendStmt.get(sinceMs, `${slugPrefix}%`) as { total: number } | null;
|
|
556
|
+
return row?.total ?? 0;
|
|
557
|
+
},
|
|
558
|
+
latestForSession(ompSessionId: string): LedgerEntry | null {
|
|
559
|
+
if (ompSessionId === "") return null;
|
|
560
|
+
const row = sessionStmt.get(ompSessionId, 1) as LedgerRow | null;
|
|
561
|
+
return row === null ? null : toEntry(row);
|
|
562
|
+
},
|
|
563
|
+
entriesForSession(ompSessionId: string, limit: number): LedgerEntry[] {
|
|
564
|
+
if (ompSessionId === "") return [];
|
|
565
|
+
return (sessionStmt.all(ompSessionId, Math.max(1, limit)) as LedgerRow[]).map(toEntry);
|
|
566
|
+
},
|
|
462
567
|
};
|
|
463
568
|
}
|