auto-model-router 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +24 -2
- package/omp-extension/report-logic.ts +93 -0
- package/omp-extension/router-configure.ts +128 -2
- package/package.json +1 -1
- package/src/catalog/ollama-catalog.ts +30 -2
- package/src/cli/config-wizard.ts +8 -0
- package/src/cli/report.ts +7 -2
- package/src/config/defaults.ts +9 -0
- package/src/config/schema.ts +5 -0
- package/src/config/types.ts +41 -0
- package/src/cost/feedback.ts +81 -0
- package/src/cost/ledger.ts +73 -0
- package/src/cost/report.ts +106 -2
- package/src/cost/types.ts +28 -0
- package/src/router/candidates.ts +13 -4
- package/src/router/classify.ts +10 -0
- package/src/router/features.ts +20 -1
- package/src/router/index.ts +12 -1
- package/src/router/learned.ts +202 -0
- package/src/router/select.ts +64 -8
- package/src/router/types.ts +31 -1
- package/src/server/http.ts +82 -5
- package/src/server/overrides.ts +83 -0
- package/src/server/providers.ts +10 -2
- package/src/server/turn.ts +16 -1
- package/src/upstream/ollama-usage.ts +79 -2
- package/src/util/sqlite.ts +30 -0
- package/test/config-wizard.test.ts +2 -1
- package/test/controls.test.ts +223 -0
- package/test/failover.test.ts +3 -2
- package/test/features.test.ts +31 -0
- package/test/learned.test.ts +61 -0
- package/test/ollama.test.ts +74 -2
- package/test/report-hub.test.ts +4 -2
- package/test/report-logic.test.ts +3 -0
- package/test/report.test.ts +43 -0
- package/test/select.test.ts +126 -1
- package/test/trust-attribution.test.ts +42 -0
- package/test/turn.test.ts +33 -2
- package/tools/replay.ts +266 -156
- package/tools/train-classifier.ts +111 -0
package/test/ollama.test.ts
CHANGED
|
@@ -12,17 +12,30 @@ import {
|
|
|
12
12
|
parseOllamaListing,
|
|
13
13
|
parseOllamaShow,
|
|
14
14
|
type OllamaListing,
|
|
15
|
+
loadOllamaCatalogCache,
|
|
15
16
|
} from "../src/catalog/ollama-catalog.ts";
|
|
16
17
|
import { bareCloudName, ollamaRateFor } from "../src/catalog/ollama-prices.ts";
|
|
17
18
|
import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
|
|
18
19
|
import type { CatalogModel, CatalogSnapshot, CatalogSource } from "../src/catalog/types.ts";
|
|
19
20
|
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
21
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
20
22
|
import type { OllamaConfig, RouterConfig } from "../src/config/types.ts";
|
|
21
23
|
import { buildCandidates } from "../src/router/candidates.ts";
|
|
22
24
|
import { extractFeatures } from "../src/router/features.ts";
|
|
23
25
|
import { createMultiUpstream } from "../src/upstream/multi.ts";
|
|
24
26
|
import { classifyOllamaStatus, createOllamaClient, toOllamaBody } from "../src/upstream/ollama.ts";
|
|
25
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
CALIBRATION_MAX_FACTOR,
|
|
29
|
+
CALIBRATION_MIN_FACTOR,
|
|
30
|
+
calibrationFrom,
|
|
31
|
+
createOllamaUsageSource,
|
|
32
|
+
effectiveOllamaBias,
|
|
33
|
+
NO_USAGE,
|
|
34
|
+
ollamaMeter,
|
|
35
|
+
parseOllamaPlan,
|
|
36
|
+
parseOllamaUsage,
|
|
37
|
+
usageFraction,
|
|
38
|
+
} from "../src/upstream/ollama-usage.ts";
|
|
26
39
|
import type { Dispatch, DispatchOptions, UpstreamClient } from "../src/upstream/types.ts";
|
|
27
40
|
import { createLogger } from "../src/util/log.ts";
|
|
28
41
|
import { parseChatRequest } from "../src/wire/openai/request.ts";
|
|
@@ -195,6 +208,20 @@ describe("createOllamaCatalog", () => {
|
|
|
195
208
|
expect(src.peek()).toHaveLength(1);
|
|
196
209
|
});
|
|
197
210
|
|
|
211
|
+
test("a built set is persisted, and a fresh source hydrates it before its first listing", async () => {
|
|
212
|
+
const db = openDb(":memory:");
|
|
213
|
+
const fetchImpl = async (url: string): Promise<Response> => Response.json({ models: url.endsWith("/api/tags") ? DAEMON_TAGS : [] });
|
|
214
|
+
const src = createOllamaCatalog(OLLAMA, log, fetchImpl, db);
|
|
215
|
+
const built = await src.get(OR_MODELS);
|
|
216
|
+
expect(built.length).toBeGreaterThan(0);
|
|
217
|
+
expect(loadOllamaCatalogCache(db).models.map((m) => m.slug)).toEqual(built.map((m) => m.slug));
|
|
218
|
+
// A new process: peek() answers from disk with no network at all.
|
|
219
|
+
const dead = async (): Promise<Response> => { throw new Error("offline"); };
|
|
220
|
+
const again = createOllamaCatalog(OLLAMA, log, dead, db);
|
|
221
|
+
expect(again.peek().map((m) => m.slug)).toEqual(built.map((m) => m.slug));
|
|
222
|
+
db.close();
|
|
223
|
+
});
|
|
224
|
+
|
|
198
225
|
test("a failed listing keeps the previous set", async () => {
|
|
199
226
|
let fail = false;
|
|
200
227
|
const fetchImpl = async (url: string): Promise<Response> => {
|
|
@@ -488,7 +515,7 @@ describe("ollama plan usage (credit-aware bias)", () => {
|
|
|
488
515
|
const source = { get: async () => ollamaModels, peek: () => ollamaModels, invalidate: () => {} };
|
|
489
516
|
const breaker = { available: () => true, cooldownUntilMs: () => null, lastTrip: () => null };
|
|
490
517
|
let used = 0.2;
|
|
491
|
-
const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }) };
|
|
518
|
+
const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), calibration: () => null };
|
|
492
519
|
const catalog = createCompositeCatalog(openrouter, source, breaker, { costBias: 0.1, biasUntilUsage: 0.9, usage });
|
|
493
520
|
|
|
494
521
|
const a = await catalog.get();
|
|
@@ -538,3 +565,48 @@ describe("ollamaMeter", () => {
|
|
|
538
565
|
expect(parseOllamaPlan("nope")).toBeNull();
|
|
539
566
|
});
|
|
540
567
|
});
|
|
568
|
+
|
|
569
|
+
describe("ollama calibration", () => {
|
|
570
|
+
const s = (h: number, meterUsd: number, ledgerUsd: number) => ({ atMs: h * 3_600_000, meterUsd, ledgerUsd });
|
|
571
|
+
|
|
572
|
+
test("compares the newest reading with the oldest since the last meter reset", () => {
|
|
573
|
+
// Ledger estimated $4 over the span; the meter moved $5 ⇒ estimates run 20% low.
|
|
574
|
+
const c = calibrationFrom([s(0, 10, 20), s(12, 12.5, 22), s(24, 15, 24)])!;
|
|
575
|
+
expect(c.factor).toBeCloseTo(1.25, 6);
|
|
576
|
+
expect(c.spanHours).toBe(24);
|
|
577
|
+
expect(c.samples).toBe(3);
|
|
578
|
+
// A billing-cycle reset (meter drops) ends the span.
|
|
579
|
+
const reset = calibrationFrom([s(0, 50, 40), s(12, 0.5, 41), s(24, 1.5, 42)])!;
|
|
580
|
+
expect(reset.samples).toBe(2);
|
|
581
|
+
expect(reset.factor).toBeCloseTo(1, 6);
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
test("needs enough metered spend to mean anything, and clamps to 0.5–2×", () => {
|
|
585
|
+
expect(calibrationFrom([s(0, 1, 1), s(1, 1.03, 1.2)])).toBeNull(); // meter moved less than its resolution
|
|
586
|
+
expect(calibrationFrom([s(0, 1, 1), s(1, 2, 1.3)])).toBeNull(); // ledger moved less than $0.50
|
|
587
|
+
expect(calibrationFrom([s(0, 0, 0), s(1, 10, 1)])!.factor).toBe(CALIBRATION_MAX_FACTOR);
|
|
588
|
+
expect(calibrationFrom([s(0, 0, 0), s(1, 0.1, 10)])!.factor).toBe(CALIBRATION_MIN_FACTOR);
|
|
589
|
+
expect(calibrationFrom([s(0, 1, 1)])).toBeNull();
|
|
590
|
+
});
|
|
591
|
+
|
|
592
|
+
test("the source records a sample per poll and exposes the calibration", async () => {
|
|
593
|
+
const db = openDb(":memory:");
|
|
594
|
+
let ledgerUsd = 1;
|
|
595
|
+
let frac = 0.1;
|
|
596
|
+
const fetchImpl = async (url: string): Promise<Response> => {
|
|
597
|
+
if (url.endsWith("/api/me")) return Response.json({ Plan: "pro" });
|
|
598
|
+
return Response.json({ limits: { monthly: { usage: frac, models: [] } } });
|
|
599
|
+
};
|
|
600
|
+
const src = createOllamaUsageSource({ apiKey: "k", pollMs: 5, timeoutMs: 1000, log, fetchImpl, calibration: { db, ledgerUsd: () => ledgerUsd, planCreditsOverrideUsd: 0 } });
|
|
601
|
+
await src.get(); // meter $6 (10% of $60), ledger $1
|
|
602
|
+
expect(src.calibration()).toBeNull(); // one sample
|
|
603
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
604
|
+
ledgerUsd = 3; // ledger +$2
|
|
605
|
+
frac = 0.15; // meter +$3
|
|
606
|
+
await src.get();
|
|
607
|
+
const c = src.calibration()!;
|
|
608
|
+
expect(c.factor).toBeCloseTo(1.5, 6);
|
|
609
|
+
expect((db.query("SELECT COUNT(*) n FROM ollama_meter_samples").get() as { n: number }).n).toBe(2);
|
|
610
|
+
db.close();
|
|
611
|
+
});
|
|
612
|
+
});
|
package/test/report-hub.test.ts
CHANGED
|
@@ -79,14 +79,16 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
|
|
|
79
79
|
},
|
|
80
80
|
providers: [row("openrouter", 2), row("ollama", 1)],
|
|
81
81
|
models: [
|
|
82
|
-
{ ...row("z-ai/glm", 2), provider: "openrouter", tiers: { simple: 6, moderate: 4 } },
|
|
83
|
-
{ ...row("ollama/kimi", 1), provider: "ollama", tiers: { hard: 10 } },
|
|
82
|
+
{ ...row("z-ai/glm", 2), provider: "openrouter", tiers: { simple: 6, moderate: 4 }, feedback: { good: 0, bad: 0 } },
|
|
83
|
+
{ ...row("ollama/kimi", 1), provider: "ollama", tiers: { hard: 10 }, feedback: { good: 0, bad: 0 } },
|
|
84
84
|
],
|
|
85
85
|
tiers: [row("simple", 2), row("hard", 1)],
|
|
86
86
|
days: [
|
|
87
87
|
{ day: "2026-09-05", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
|
|
88
88
|
{ day: "2026-09-06", dispatches: 10, spendUsd: 1.5, cacheHitRate: 0.8 },
|
|
89
89
|
],
|
|
90
|
+
anatomy: null,
|
|
91
|
+
baselines: [],
|
|
90
92
|
...over,
|
|
91
93
|
};
|
|
92
94
|
}
|
|
@@ -66,6 +66,8 @@ describe("renderStatus", () => {
|
|
|
66
66
|
lastTrip: { kind: "quota", atMs: now - 120_000, message: "402" },
|
|
67
67
|
usage: { monthlyUsedFraction: 0.42, activityCostUsd: 3.1, fetchedAtMs: now },
|
|
68
68
|
meter: { usedUsd: 25.2, creditsUsd: 60, plan: "pro" },
|
|
69
|
+
calibration: { factor: 1.25, meterDeltaUsd: 5, ledgerDeltaUsd: 4, spanHours: 48 },
|
|
70
|
+
runway: { dailyBurnUsd: 1.5, creditsLeftUsd: 34.8, days: 23.2 },
|
|
69
71
|
costBias: { configured: 0.1, effective: 0.1, biasUntilUsage: 0.9 },
|
|
70
72
|
},
|
|
71
73
|
catalog: { models: 240, ageMs: 5 * 60_000, keyScoped: true, shrink: { fromModels: 300, toModels: 120, atMs: now } },
|
|
@@ -77,6 +79,7 @@ describe("renderStatus", () => {
|
|
|
77
79
|
expect(text).toContain("SHRANK 300 -> 120");
|
|
78
80
|
expect(text).toContain("COOLING DOWN");
|
|
79
81
|
expect(text).toContain("pro plan usage 42.0% ($25.20 of $60)");
|
|
82
|
+
expect(text).toContain("ollama billing: ledger estimate ×1.25 to match the meter (48h span) · burn $1.50/day · ~23 days of credits left");
|
|
80
83
|
expect(text).toContain("cost bias ×0.1 (until 90%)");
|
|
81
84
|
expect(text).toContain("last trip quota 2m ago");
|
|
82
85
|
expect(text).toContain("scope omp-router");
|
package/test/report.test.ts
CHANGED
|
@@ -156,6 +156,47 @@ describe("buildUsageReport", () => {
|
|
|
156
156
|
db.close();
|
|
157
157
|
});
|
|
158
158
|
|
|
159
|
+
test("prompt anatomy averages the recorded byte shares", () => {
|
|
160
|
+
const { db, ledger } = seeded();
|
|
161
|
+
const feat = (tool: number, older: number, stale: number) => ({ toolSchemaBytes: 1000, anatomy: { messages: 30, systemBytes: 1000, userBytes: 500, assistantBytes: 500, toolBytes: tool, olderHalfBytes: older, staleToolBytes: stale } });
|
|
162
|
+
ledger.record(entry({ features: feat(8000, 5000, 4000) }));
|
|
163
|
+
ledger.record(entry({ features: feat(6000, 4000, 2000) }));
|
|
164
|
+
ledger.record(entry({ features: null })); // pre-anatomy row: ignored
|
|
165
|
+
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
166
|
+
const a = r.anatomy!;
|
|
167
|
+
expect(a.rows).toBe(2);
|
|
168
|
+
// mean bytes: system 1000, user 500, assistant 500, tool 7000 ⇒ total 9000
|
|
169
|
+
expect(a.tool).toBeCloseTo(7000 / 9000, 6);
|
|
170
|
+
expect(a.system).toBeCloseTo(1000 / 9000, 6);
|
|
171
|
+
expect(a.schemas).toBeCloseTo(1000 / 9000, 6);
|
|
172
|
+
expect(a.olderHalf).toBeCloseTo(4500 / 9000, 6);
|
|
173
|
+
expect(a.staleTool).toBeCloseTo(3000 / 9000, 6);
|
|
174
|
+
expect(renderUsageReport(r)).toContain("prompt anatomy (mean of 2): tool results 78%");
|
|
175
|
+
db.close();
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("baselines price the window on one model with its own cache hit rate", () => {
|
|
179
|
+
const { db, ledger } = seeded();
|
|
180
|
+
ledger.record(entry({ reportedUsd: 0.5, usage: { promptTokens: 100_000, cachedTokens: 80_000, cacheWriteTokens: 0, completionTokens: 1_000, reasoningTokens: 0, images: 0 } }));
|
|
181
|
+
const r = buildUsageReport(db, {
|
|
182
|
+
windowDays: 7,
|
|
183
|
+
nowMs: NOW,
|
|
184
|
+
baselines: [
|
|
185
|
+
{ slug: "big/model", prompt: 15 / 1e6, completion: 75 / 1e6, cacheRead: 1.5 / 1e6 },
|
|
186
|
+
{ slug: "nocache/model", prompt: 3 / 1e6, completion: 15 / 1e6 },
|
|
187
|
+
],
|
|
188
|
+
});
|
|
189
|
+
// 20k fresh × $15/M + 80k cached × $1.5/M + 1k completion × $75/M = 0.30 + 0.12 + 0.075
|
|
190
|
+
expect(r.baselines[0]!.usd).toBeCloseTo(0.495, 6);
|
|
191
|
+
expect(r.baselines[0]!.savedShare).toBeCloseTo(1 - 0.5 / 0.495, 6);
|
|
192
|
+
// No cache rate published: every prompt token at list price.
|
|
193
|
+
expect(r.baselines[1]!.usd).toBeCloseTo(0.3 + 0.015, 6);
|
|
194
|
+
const text = renderUsageReport(r);
|
|
195
|
+
expect(text).toContain("same traffic on one model: big/model $0.4950 (router cost extra 1%)");
|
|
196
|
+
expect(text).toContain("nocache/model $0.3150 (router cost extra 59%)");
|
|
197
|
+
db.close();
|
|
198
|
+
});
|
|
199
|
+
|
|
159
200
|
test("empty ledger yields zeroed totals and null speeds", () => {
|
|
160
201
|
const { db } = seeded();
|
|
161
202
|
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
@@ -175,6 +216,8 @@ describe("buildUsageReport", () => {
|
|
|
175
216
|
});
|
|
176
217
|
expect(r.providers).toEqual([]);
|
|
177
218
|
expect(r.models).toEqual([]);
|
|
219
|
+
expect(r.anatomy).toBeNull();
|
|
220
|
+
expect(r.baselines).toEqual([]);
|
|
178
221
|
db.close();
|
|
179
222
|
});
|
|
180
223
|
|
package/test/select.test.ts
CHANGED
|
@@ -8,7 +8,8 @@ import type { ProfileConfig, RouterConfig } from "../src/config/types.ts";
|
|
|
8
8
|
import type { Ledger } from "../src/cost/types.ts";
|
|
9
9
|
import { extractFeatures } from "../src/router/features.ts";
|
|
10
10
|
import { scoreHeuristic } from "../src/router/classify.ts";
|
|
11
|
-
import {
|
|
11
|
+
import { latencyWeightFor } from "../src/router/candidates.ts";
|
|
12
|
+
import { BudgetExceededError, monthPace, monthStartMs, select } from "../src/router/select.ts";
|
|
12
13
|
import type { ConversationState, Tier } from "../src/router/types.ts";
|
|
13
14
|
import { parseChatRequest } from "../src/wire/openai/request.ts";
|
|
14
15
|
import type { NormRequest } from "../src/wire/types.ts";
|
|
@@ -1050,3 +1051,127 @@ describe("hysteresis.confirmUpgradesBelowConfidence", () => {
|
|
|
1050
1051
|
expect(d.upgradeDeferred).toBeNull();
|
|
1051
1052
|
});
|
|
1052
1053
|
});
|
|
1054
|
+
|
|
1055
|
+
describe("cache reliability in the stay/switch comparison", () => {
|
|
1056
|
+
const warmSlug = "x-ai/grok-4.6";
|
|
1057
|
+
function ledgerWithReliability(rate: number | null, samples = 50): Ledger {
|
|
1058
|
+
return {
|
|
1059
|
+
record: () => {},
|
|
1060
|
+
conversationSpend: () => 0,
|
|
1061
|
+
spendSince: () => 0,
|
|
1062
|
+
blendedRate: () => null,
|
|
1063
|
+
trust: () => null,
|
|
1064
|
+
allTrust: () => [],
|
|
1065
|
+
latency: () => null,
|
|
1066
|
+
tokenRatio: () => null,
|
|
1067
|
+
recentEntries: () => [],
|
|
1068
|
+
cacheReliability: (slug) => (rate === null || slug !== warmSlug ? null : { slug, samples, hitRate: rate }),
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
const stayCostOf = (ledger: Ledger, cfg: RouterConfig = BASE): number => {
|
|
1072
|
+
const d = run({
|
|
1073
|
+
tier: "hard",
|
|
1074
|
+
promptTokens: 80_000,
|
|
1075
|
+
cfg,
|
|
1076
|
+
ledger,
|
|
1077
|
+
st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }),
|
|
1078
|
+
});
|
|
1079
|
+
const m = /cache: (?:keeping warm|switch) .*?stay \$([0-9.]+)/.exec(d.reasons.join("\n"));
|
|
1080
|
+
if (m === null) throw new Error(`no stay/switch reason in ${d.reasons.join(" | ")}`);
|
|
1081
|
+
return Number(m[1]);
|
|
1082
|
+
};
|
|
1083
|
+
|
|
1084
|
+
test("an unreliable cache prices staying at the fresh rate, a reliable one at the cached rate", () => {
|
|
1085
|
+
const reliable = stayCostOf(ledgerWithReliability(1));
|
|
1086
|
+
const flaky = stayCostOf(ledgerWithReliability(0));
|
|
1087
|
+
const unknown = stayCostOf(ledgerWithReliability(null));
|
|
1088
|
+
expect(flaky).toBeGreaterThan(reliable);
|
|
1089
|
+
expect(unknown).toBeCloseTo(reliable, 6);
|
|
1090
|
+
});
|
|
1091
|
+
|
|
1092
|
+
test("too few samples, or the feature off, assume a reliable cache", () => {
|
|
1093
|
+
const reliable = stayCostOf(ledgerWithReliability(1));
|
|
1094
|
+
expect(stayCostOf(ledgerWithReliability(0, 3))).toBeCloseTo(reliable, 6);
|
|
1095
|
+
const off: RouterConfig = { ...BASE, filters: { ...BASE.filters, cacheReliabilityMinSamples: 0 } };
|
|
1096
|
+
expect(stayCostOf(ledgerWithReliability(0), off)).toBeCloseTo(reliable, 6);
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
test("the reason names the measured hit rate", () => {
|
|
1100
|
+
const d = run({
|
|
1101
|
+
tier: "hard",
|
|
1102
|
+
promptTokens: 80_000,
|
|
1103
|
+
ledger: ledgerWithReliability(0.5, 40),
|
|
1104
|
+
st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }),
|
|
1105
|
+
});
|
|
1106
|
+
expect(d.reasons.some((r) => r.includes("warm hit 50% over 40"))).toBe(true);
|
|
1107
|
+
});
|
|
1108
|
+
});
|
|
1109
|
+
|
|
1110
|
+
describe("session pin (forceSlug)", () => {
|
|
1111
|
+
test("a pinned catalog model wins over ranking and the warm model; an unknown pin is ignored with a reason", () => {
|
|
1112
|
+
const warmSlug = run({ tier: "moderate" }).slug;
|
|
1113
|
+
const pinSlug = run({ tier: "hard" }).slug; // a real, differently-ranked model
|
|
1114
|
+
const req = request("tidy the retry helper");
|
|
1115
|
+
const features = extractFeatures(req, 50_000);
|
|
1116
|
+
const base = {
|
|
1117
|
+
req,
|
|
1118
|
+
features,
|
|
1119
|
+
classification: scoreHeuristic(features, BASE),
|
|
1120
|
+
profile: PROFILE,
|
|
1121
|
+
state: state({ currentSlug: warmSlug, currentTier: "moderate", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 50_000 }),
|
|
1122
|
+
snapshot: SNAPSHOT,
|
|
1123
|
+
ledger: null,
|
|
1124
|
+
cfg: { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } },
|
|
1125
|
+
nowMs: Date.now(),
|
|
1126
|
+
};
|
|
1127
|
+
const pinned = select({ ...base, forceSlug: pinSlug });
|
|
1128
|
+
expect(pinned.slug).toBe(pinSlug);
|
|
1129
|
+
expect(pinned.sticky).toBe(false);
|
|
1130
|
+
expect(pinned.reasons.some((r) => r.includes(`pinned to ${pinSlug} by session override`))).toBe(true);
|
|
1131
|
+
const unknown = select({ ...base, forceSlug: "nope/model" });
|
|
1132
|
+
expect(unknown.slug).toBe(warmSlug); // the huge switch margin keeps the warm model
|
|
1133
|
+
expect(unknown.reasons.some((r) => r.includes("pin nope/model ignored"))).toBe(true);
|
|
1134
|
+
});
|
|
1135
|
+
});
|
|
1136
|
+
|
|
1137
|
+
describe("budget.perMonthUsd pacing", () => {
|
|
1138
|
+
test("monthPace spreads what is left over the days left, today included", () => {
|
|
1139
|
+
const sep7 = Date.UTC(2026, 8, 7, 12);
|
|
1140
|
+
expect(monthStartMs(sep7)).toBe(Date.UTC(2026, 8, 1));
|
|
1141
|
+
const p = monthPace(sep7, 60, 30);
|
|
1142
|
+
expect(p.daysLeft).toBe(24); // 7th..30th
|
|
1143
|
+
expect(p.dailyCapUsd).toBeCloseTo(30 / 24, 6);
|
|
1144
|
+
expect(monthPace(sep7, 60, 70).dailyCapUsd).toBe(0);
|
|
1145
|
+
expect(monthPace(Date.UTC(2026, 8, 30, 12), 60, 0).daysLeft).toBe(1);
|
|
1146
|
+
});
|
|
1147
|
+
|
|
1148
|
+
test("a month running ahead of pace tightens the daily cap and says so", () => {
|
|
1149
|
+
const ledger: Ledger = {
|
|
1150
|
+
record: () => {},
|
|
1151
|
+
conversationSpend: () => 0,
|
|
1152
|
+
spendSince: (sinceMs) => (sinceMs <= monthStartMs(Date.now()) + 1 ? 59.99 : 0), // month-to-date $59.99, last 24h $0
|
|
1153
|
+
blendedRate: () => null,
|
|
1154
|
+
trust: () => null,
|
|
1155
|
+
allTrust: () => [],
|
|
1156
|
+
latency: () => null,
|
|
1157
|
+
tokenRatio: () => null,
|
|
1158
|
+
recentEntries: () => [],
|
|
1159
|
+
};
|
|
1160
|
+
const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perMonthUsd: 60, onExceeded: "reject" } };
|
|
1161
|
+
expect(() => run({ tier: "hard", promptTokens: 50_000, cfg, ledger })).toThrow(/month pacing: \$59\.99 of \$60 spent/);
|
|
1162
|
+
// Under pace: the cap is generous and nothing breaches.
|
|
1163
|
+
const easy: Ledger = { ...ledger, spendSince: () => 1 };
|
|
1164
|
+
expect(run({ tier: "hard", promptTokens: 50_000, cfg, ledger: easy }).budgetDowngraded).toBe(false);
|
|
1165
|
+
});
|
|
1166
|
+
});
|
|
1167
|
+
|
|
1168
|
+
describe("filters.latencyWeightContinuation", () => {
|
|
1169
|
+
test("applies only to tool-result continuations, and only when set", () => {
|
|
1170
|
+
const f = { ...BASE.filters, latencyWeight: 0.75 };
|
|
1171
|
+
expect(latencyWeightFor(f, false)).toBe(0.75);
|
|
1172
|
+
expect(latencyWeightFor(f, true)).toBe(0.75);
|
|
1173
|
+
const g = { ...f, latencyWeightContinuation: 0.1 };
|
|
1174
|
+
expect(latencyWeightFor(g, false)).toBe(0.75);
|
|
1175
|
+
expect(latencyWeightFor(g, true)).toBe(0.1);
|
|
1176
|
+
});
|
|
1177
|
+
});
|
|
@@ -368,3 +368,45 @@ describe("v6 classifier instrumentation", () => {
|
|
|
368
368
|
}
|
|
369
369
|
});
|
|
370
370
|
});
|
|
371
|
+
|
|
372
|
+
describe("cache reliability signal", () => {
|
|
373
|
+
// Observed hit rate when a warm cache was expected: the previous kept turn
|
|
374
|
+
// of the conversation was on the same model within the warm TTL.
|
|
375
|
+
function seed(rows: Array<Partial<LedgerEntry>>) {
|
|
376
|
+
const db = openDb(":memory:");
|
|
377
|
+
const ledger = createLedger(db, cfg);
|
|
378
|
+
for (const r of rows) ledger.record(entry(r));
|
|
379
|
+
return { db, ledger };
|
|
380
|
+
}
|
|
381
|
+
const t0 = Date.UTC(2026, 8, 7, 12);
|
|
382
|
+
const usage = (prompt: number, cached: number, estimated = false) => ({ ...EMPTY_USAGE, promptTokens: prompt, cachedTokens: cached, ...(estimated ? { cachedEstimated: true } : {}) });
|
|
383
|
+
|
|
384
|
+
test("a model that hits when warm scores 1; one that misses scores 0; the first turn never counts", () => {
|
|
385
|
+
const { db, ledger } = seed([
|
|
386
|
+
{ conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0, usage: usage(50_000, 0) }, // first turn: no expectation
|
|
387
|
+
{ conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 50_000) },
|
|
388
|
+
{ conversationKey: "a", slug: "good/m", servedSlug: "good/m", createdAtMs: t0 + 120_000, usage: usage(70_000, 60_000) },
|
|
389
|
+
{ conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0, usage: usage(50_000, 0) },
|
|
390
|
+
{ conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 0) },
|
|
391
|
+
{ conversationKey: "b", slug: "flaky/m", servedSlug: "flaky/m", createdAtMs: t0 + 120_000, usage: usage(70_000, 30_000) },
|
|
392
|
+
]);
|
|
393
|
+
expect(ledger.cacheReliability?.("good/m")).toEqual({ slug: "good/m", samples: 2, hitRate: 1 });
|
|
394
|
+
const flaky = ledger.cacheReliability?.("flaky/m");
|
|
395
|
+
expect(flaky?.samples).toBe(2);
|
|
396
|
+
expect(flaky?.hitRate).toBeCloseTo(0.25, 6); // (0 + 30k/60k) / 2
|
|
397
|
+
expect(ledger.cacheReliability?.("never/m")).toBeNull();
|
|
398
|
+
db.close();
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
test("a switch, an idle gap past the TTL, or a router-estimated count is not a warm-expected sample", () => {
|
|
402
|
+
const { db, ledger } = seed([
|
|
403
|
+
{ conversationKey: "a", slug: "x/m", servedSlug: "x/m", createdAtMs: t0, usage: usage(50_000, 0) },
|
|
404
|
+
{ conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000, usage: usage(60_000, 0) }, // switch
|
|
405
|
+
{ conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000 + cfg.hysteresis.cacheWarmTtlMs + 1, usage: usage(70_000, 0) }, // gap
|
|
406
|
+
{ conversationKey: "a", slug: "y/m", servedSlug: "y/m", createdAtMs: t0 + 60_000 + cfg.hysteresis.cacheWarmTtlMs + 2, usage: usage(80_000, 70_000, true) }, // estimated
|
|
407
|
+
]);
|
|
408
|
+
expect(ledger.cacheReliability?.("x/m")).toBeNull();
|
|
409
|
+
expect(ledger.cacheReliability?.("y/m")).toBeNull();
|
|
410
|
+
db.close();
|
|
411
|
+
});
|
|
412
|
+
});
|
package/test/turn.test.ts
CHANGED
|
@@ -46,10 +46,10 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
46
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
47
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
48
48
|
},
|
|
49
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
50
50
|
classifier: {
|
|
51
51
|
ambiguityThreshold: 0,
|
|
52
|
-
model: "test/adjudicator",
|
|
52
|
+
model: "test/adjudicator", learnedModelPath: "",
|
|
53
53
|
maxCostFraction: 0.1,
|
|
54
54
|
maxCostUsd: 0.01,
|
|
55
55
|
timeoutMs: 5000,
|
|
@@ -76,6 +76,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
76
76
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
77
77
|
compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
|
|
78
78
|
budget: { onExceeded: "downgrade" },
|
|
79
|
+
report: { baselines: [] },
|
|
79
80
|
profiles: [],
|
|
80
81
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
81
82
|
adaptiveTierFloors: true,
|
|
@@ -896,4 +897,34 @@ describe("latency measurement covers the work the router actually does", () => {
|
|
|
896
897
|
expect(map.get("conv-test")!.cacheWarmSlug).toBe(ollamaModel.slug);
|
|
897
898
|
});
|
|
898
899
|
|
|
900
|
+
test("an Ollama estimate is scaled by the ledger-vs-meter calibration", async () => {
|
|
901
|
+
const ollamaModel: CatalogModel = {
|
|
902
|
+
slug: "ollama/glm-5.3-flash",
|
|
903
|
+
provider: "ollama",
|
|
904
|
+
canonicalSlug: "ollama/glm-5.3-flash",
|
|
905
|
+
name: "glm",
|
|
906
|
+
contextLength: 1_000_000,
|
|
907
|
+
supportsTools: true,
|
|
908
|
+
supportsReasoning: true,
|
|
909
|
+
reasoningMandatory: false,
|
|
910
|
+
supportsToolChoice: false,
|
|
911
|
+
inputModalities: ["text"],
|
|
912
|
+
price: { prompt: 0.15 / 1e6, cacheRead: 0.03 / 1e6, completion: 0.5 / 1e6 },
|
|
913
|
+
priceTiers: [],
|
|
914
|
+
quality: {},
|
|
915
|
+
tokenizer: "Other",
|
|
916
|
+
isFree: false,
|
|
917
|
+
createdAtMs: 0,
|
|
918
|
+
author: "ollama",
|
|
919
|
+
};
|
|
920
|
+
const priced = { ...catalog, find: (slug: string) => (slug === ollamaModel.slug ? ollamaModel : undefined) };
|
|
921
|
+
const { router } = mkRouter([mkDecision("simple", ollamaModel.slug)]);
|
|
922
|
+
const { upstream } = mkUpstream([{ kind: "chunks", chunks: [startChunk(ollamaModel.slug), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 100_000, completionTokens: 0 }, null)] }]);
|
|
923
|
+
const { ledger, entries } = mkLedger();
|
|
924
|
+
const { store } = mkConversations();
|
|
925
|
+
await runTurn(mkReq(), mkSink().sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog: priced, context: createDisabledBridge(), ollamaCostScale: () => 1.25 }, new AbortController().signal);
|
|
926
|
+
// 100k × $0.15/M = $0.015, scaled ×1.25.
|
|
927
|
+
expect(entries[0]!.reportedUsd).toBeCloseTo(0.01875, 6);
|
|
928
|
+
});
|
|
929
|
+
|
|
899
930
|
});
|