auto-model-router 0.2.32 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +208 -29
- package/docs/review-2026-09-05.md +267 -0
- package/omp-extension/configure-logic.ts +71 -15
- package/omp-extension/pi-coding-agent.d.ts +79 -2
- package/omp-extension/report-hub.ts +376 -0
- package/omp-extension/report-logic.ts +115 -0
- package/omp-extension/router-configure.ts +203 -51
- package/omp-extension/router-url.ts +52 -0
- package/omp-extension/toast-logic.ts +14 -2
- package/package.json +1 -1
- package/src/catalog/composite.ts +97 -0
- package/src/catalog/ollama-catalog.ts +309 -0
- package/src/catalog/ollama-prices.ts +85 -0
- package/src/catalog/openrouter-catalog.ts +39 -1
- package/src/catalog/types.ts +31 -1
- package/src/cli/args.ts +1 -0
- package/src/cli/config-wizard.ts +189 -28
- package/src/cli/explain.ts +2 -4
- package/src/cli/models.ts +2 -4
- package/src/cli/report.ts +37 -0
- package/src/config/defaults.ts +43 -2
- package/src/config/load.ts +25 -1
- package/src/config/omp-credentials.ts +31 -7
- package/src/config/schema.ts +27 -0
- package/src/config/types.ts +114 -2
- package/src/cost/ledger.ts +73 -4
- package/src/cost/report.ts +340 -0
- package/src/cost/types.ts +33 -1
- package/src/index.ts +5 -8
- package/src/router/candidates.ts +52 -4
- package/src/router/classify.ts +33 -6
- package/src/router/features.ts +13 -1
- package/src/router/select.ts +55 -8
- package/src/router/state.ts +6 -2
- package/src/router/tier-plan.ts +49 -11
- package/src/router/types.ts +10 -0
- package/src/server/http.ts +47 -6
- package/src/server/providers.ts +54 -0
- package/src/server/turn.ts +122 -34
- package/src/tokens/estimate.ts +16 -0
- package/src/upstream/multi.ts +26 -0
- package/src/upstream/ollama-usage.ts +157 -0
- package/src/upstream/ollama.ts +275 -0
- package/src/upstream/openrouter.ts +19 -1
- package/src/upstream/types.ts +2 -0
- package/src/util/sqlite.ts +25 -1
- package/test/catalog.test.ts +44 -0
- package/test/classify.test.ts +41 -5
- package/test/compaction.test.ts +1 -0
- package/test/config-wizard.test.ts +77 -1
- package/test/configure-logic.test.ts +129 -33
- package/test/embed-lifecycle.test.ts +1 -0
- package/test/failover.test.ts +148 -3
- package/test/features.test.ts +35 -0
- package/test/http-resilience.test.ts +24 -0
- package/test/ollama.test.ts +506 -0
- package/test/omp-credentials.test.ts +43 -1
- package/test/report-hub.test.ts +341 -0
- package/test/report-logic.test.ts +92 -0
- package/test/report.test.ts +217 -0
- package/test/select.test.ts +151 -1
- package/test/tier-plan.test.ts +159 -1
- package/test/toast-logic.test.ts +11 -2
- package/test/tokens.test.ts +71 -1
- package/test/trust-attribution.test.ts +2 -2
- package/test/turn.test.ts +124 -7
package/test/select.test.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
|
|
3
3
|
import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
|
|
4
4
|
import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
|
|
5
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
5
6
|
import { loadConfig } from "../src/config/load.ts";
|
|
6
7
|
import type { ProfileConfig, RouterConfig } from "../src/config/types.ts";
|
|
7
8
|
import type { Ledger } from "../src/cost/types.ts";
|
|
@@ -628,6 +629,7 @@ describe("context compaction", () => {
|
|
|
628
629
|
enabled: true,
|
|
629
630
|
budgetTokens: 1_000,
|
|
630
631
|
floorRatio: 1,
|
|
632
|
+
replanGrowthRatio: 1,
|
|
631
633
|
fitToWindow: false,
|
|
632
634
|
protectRecentTurns: 1,
|
|
633
635
|
maxToolResultBytes: 100,
|
|
@@ -801,7 +803,7 @@ describe("hysteresis.breakHoldOnMechanical", () => {
|
|
|
801
803
|
}
|
|
802
804
|
|
|
803
805
|
test("off by default, so a hold still pins the tier", () => {
|
|
804
|
-
expect(
|
|
806
|
+
expect(DEFAULT_CONFIG.hysteresis.breakHoldOnMechanical).toBe(false); // SHIPPED default, not the live config.yml (machine-dependent)
|
|
805
807
|
const { d, features } = decide(continuation(), false);
|
|
806
808
|
expect(features.isToolResultContinuation).toBe(true);
|
|
807
809
|
expect(d.tier).toBe("hard");
|
|
@@ -846,3 +848,151 @@ describe("hysteresis.breakHoldOnMechanical", () => {
|
|
|
846
848
|
expect(d.tier).toBe("moderate");
|
|
847
849
|
});
|
|
848
850
|
});
|
|
851
|
+
|
|
852
|
+
describe("hysteresis.switchHorizonTurns (review 2026-09-05 §4)", () => {
|
|
853
|
+
// Two moderate-eligible models built from raw catalog records: a kimi-shaped
|
|
854
|
+
// warm model (cheap to READ from cache, dear cold) and a gemini-shaped ranked
|
|
855
|
+
// winner (dear cold, cheap warm). With a one-turn horizon the cold write on
|
|
856
|
+
// the winner never pays for itself, so the dear model stays warm forever;
|
|
857
|
+
// over a run of turns the switch is obviously right.
|
|
858
|
+
function rawModel(id: string, coding: number, prompt: number, cacheRead: number, cacheWrite: number | null): Record<string, unknown> {
|
|
859
|
+
const pricing: Record<string, string> = {
|
|
860
|
+
prompt: String(prompt / 1e6),
|
|
861
|
+
completion: String((prompt * 4) / 1e6),
|
|
862
|
+
input_cache_read: String(cacheRead / 1e6),
|
|
863
|
+
};
|
|
864
|
+
if (cacheWrite !== null) pricing.input_cache_write = String(cacheWrite / 1e6);
|
|
865
|
+
return {
|
|
866
|
+
id,
|
|
867
|
+
canonical_slug: id,
|
|
868
|
+
name: id,
|
|
869
|
+
context_length: 1_000_000,
|
|
870
|
+
pricing,
|
|
871
|
+
supported_parameters: ["tools"],
|
|
872
|
+
architecture: { input_modalities: ["text"], tokenizer: "GPT" },
|
|
873
|
+
benchmarks: { artificial_analysis: { coding_index: coding, intelligence_index: coding, agentic_index: coding } },
|
|
874
|
+
created: 1_700_000_000,
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
const warmDear = normalizeCatalogModel(rawModel("test/warm-dear", 76.2, 2.55, 0.256, null))!;
|
|
878
|
+
const winner = normalizeCatalogModel(rawModel("test/winner", 76.0, 0.75, 0.075, 0.04))!;
|
|
879
|
+
const snap: CatalogSnapshot = { models: [warmDear, winner], fetchedAtMs: Date.now() };
|
|
880
|
+
const promptTokens = 100_000;
|
|
881
|
+
|
|
882
|
+
function decide(horizon: number) {
|
|
883
|
+
// BASE is the live config.yml, which may have exploration on; a
|
|
884
|
+
// deterministic exploration draw would route this turn down to `simple`,
|
|
885
|
+
// where the dear model is over the price ceiling and never compared.
|
|
886
|
+
const cfg: RouterConfig = {
|
|
887
|
+
...BASE,
|
|
888
|
+
exploration: { ...BASE.exploration, enabled: false },
|
|
889
|
+
hysteresis: { ...BASE.hysteresis, switchMargin: 1.3, switchHorizonTurns: horizon },
|
|
890
|
+
};
|
|
891
|
+
const req = request("keep going");
|
|
892
|
+
const features = extractFeatures(req, promptTokens);
|
|
893
|
+
return select({
|
|
894
|
+
req,
|
|
895
|
+
features,
|
|
896
|
+
classification: { ...scoreHeuristic(features, cfg), tier: "moderate" },
|
|
897
|
+
profile: PROFILE,
|
|
898
|
+
state: state({ currentSlug: "test/warm-dear", currentTier: "moderate", cacheWarmSlug: "test/warm-dear", cacheWarmAtMs: Date.now(), lastPromptTokens: promptTokens }),
|
|
899
|
+
snapshot: snap,
|
|
900
|
+
ledger: null,
|
|
901
|
+
cfg,
|
|
902
|
+
nowMs: Date.now(),
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
test("the ranked winner is the cheaper cold model", () => {
|
|
907
|
+
const d = decide(1);
|
|
908
|
+
expect(d.considered[0]!.model.slug).toBe("test/winner");
|
|
909
|
+
});
|
|
910
|
+
|
|
911
|
+
test("a one-turn horizon keeps the dear model warm (the shipped behaviour)", () => {
|
|
912
|
+
const d = decide(1);
|
|
913
|
+
expect(d.slug).toBe("test/warm-dear");
|
|
914
|
+
expect(d.sticky).toBe(true);
|
|
915
|
+
expect(d.reasons.some((r) => r.startsWith("cache: keeping warm test/warm-dear"))).toBe(true);
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
test("amortised over a run of turns, the switch is taken", () => {
|
|
919
|
+
const d = decide(8);
|
|
920
|
+
expect(d.slug).toBe("test/winner");
|
|
921
|
+
expect(d.sticky).toBe(false);
|
|
922
|
+
expect(d.reasons.some((r) => r.startsWith("cache: switch test/warm-dear → test/winner") && r.includes("over 8 turns"))).toBe(true);
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => {
|
|
927
|
+
const TOOL_RESULT = "y".repeat(3000);
|
|
928
|
+
function turn(n: number): NormRequest {
|
|
929
|
+
// n completed read/result/"continue" rounds; every result but the newest
|
|
930
|
+
// sits outside the protected window and is eligible for truncation.
|
|
931
|
+
const messages: unknown[] = [
|
|
932
|
+
{ role: "system", content: "You are a coding agent." },
|
|
933
|
+
{ role: "user", content: "read the files" },
|
|
934
|
+
];
|
|
935
|
+
for (let i = 0; i < n; i++) {
|
|
936
|
+
messages.push({ role: "assistant", content: null, tool_calls: [{ id: `c${i}`, type: "function", function: { name: "read", arguments: `{"path":"f${i}.ts"}` } }] });
|
|
937
|
+
messages.push({ role: "tool", tool_call_id: `c${i}`, content: TOOL_RESULT });
|
|
938
|
+
messages.push({ role: "user", content: "continue" });
|
|
939
|
+
}
|
|
940
|
+
return parseChatRequest({ model: "auto", tools: TOOLS, messages }, new Headers());
|
|
941
|
+
}
|
|
942
|
+
function cfgWith(ratio: number): RouterConfig {
|
|
943
|
+
return {
|
|
944
|
+
...BASE,
|
|
945
|
+
compaction: {
|
|
946
|
+
enabled: true,
|
|
947
|
+
budgetTokens: 100, // unreachable: every turn is over budget, as observed live
|
|
948
|
+
floorRatio: 1,
|
|
949
|
+
replanGrowthRatio: ratio,
|
|
950
|
+
fitToWindow: false,
|
|
951
|
+
protectRecentTurns: 2, // the newest result and its "continue" stay protected; older rounds are eligible
|
|
952
|
+
maxToolResultBytes: 100,
|
|
953
|
+
keepHeadBytes: 20,
|
|
954
|
+
keepTailBytes: 20,
|
|
955
|
+
elideSupersededReads: false,
|
|
956
|
+
collapseDuplicateResults: false,
|
|
957
|
+
},
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
function decide(req: NormRequest, cfg: RouterConfig, st: ConversationState, promptTokens: number) {
|
|
961
|
+
const features = extractFeatures(req, promptTokens);
|
|
962
|
+
return select({ req, features, classification: scoreHeuristic(features, cfg), profile: PROFILE, state: st, snapshot: SNAPSHOT, ledger: null, cfg, nowMs: Date.now() });
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
test("the plan records the compacted size it was made at", () => {
|
|
966
|
+
const first = decide(turn(2), cfgWith(1), state(), 4_000);
|
|
967
|
+
expect(first.compactionPlan.length).toBe(1);
|
|
968
|
+
expect(first.compactionPlanTokens).toBe(4_000 - first.promptTokensSaved);
|
|
969
|
+
expect(first.compactionSavedBytes).toBeGreaterThan(0);
|
|
970
|
+
});
|
|
971
|
+
|
|
972
|
+
test("at 1 (shipped) a newly eligible result is compacted on the very next turn", () => {
|
|
973
|
+
const first = decide(turn(2), cfgWith(1), state(), 4_000);
|
|
974
|
+
const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens });
|
|
975
|
+
// One more round: the prompt grew ~25%, one more result aged out.
|
|
976
|
+
const second = decide(turn(3), cfgWith(1), carried, 5_000);
|
|
977
|
+
expect(second.compactionPlan.length).toBe(2);
|
|
978
|
+
expect(second.reasons.some((r) => r.includes("(1 carried, 1 new)"))).toBe(true);
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
test("above 1, an existing plan holds until the compacted prompt has grown by the ratio", () => {
|
|
982
|
+
const first = decide(turn(2), cfgWith(2), state(), 4_000);
|
|
983
|
+
const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens });
|
|
984
|
+
// The raw prompt grew 25% and the COMPACTED prompt ~60% (the carried
|
|
985
|
+
// edit saves a smaller share of a bigger prompt) — still under 2x, so
|
|
986
|
+
// the carried plan is re-applied verbatim and nothing new is added.
|
|
987
|
+
const held = decide(turn(3), cfgWith(2), carried, 5_000);
|
|
988
|
+
expect(held.compactionPlan).toEqual(first.compactionPlan);
|
|
989
|
+
expect(held.compactionPlanTokens).toBe(first.compactionPlanTokens); // growth still accrues against the original size
|
|
990
|
+
expect(held.reasons.some((r) => r.includes("(1 carried, 0 new)") && r.includes("[re-plan rationed]"))).toBe(true);
|
|
991
|
+
// Past 2x the plan is extended and the new size is recorded.
|
|
992
|
+
const grown = decide(turn(3), cfgWith(2), carried, 8_000);
|
|
993
|
+
expect(grown.compactionPlan.length).toBe(2);
|
|
994
|
+
expect(grown.compactionPlanTokens).toBe(8_000 - grown.promptTokensSaved);
|
|
995
|
+
expect(grown.reasons.some((r) => r.includes("[re-plan rationed]"))).toBe(false);
|
|
996
|
+
});
|
|
997
|
+
});
|
|
998
|
+
|
package/test/tier-plan.test.ts
CHANGED
|
@@ -2,10 +2,11 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
|
|
3
3
|
import { joinBenchmarks, normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
|
|
4
4
|
import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
|
|
5
|
+
import type { Ledger } from "../src/cost/types.ts";
|
|
5
6
|
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
6
7
|
import { buildCandidates } from "../src/router/candidates.ts";
|
|
7
8
|
import { extractFeatures } from "../src/router/features.ts";
|
|
8
|
-
import { computeTierPlan, effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
|
|
9
|
+
import { computeTierPlan, countAdmitted, effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
|
|
9
10
|
import { TIER_ORDER } from "../src/router/types.ts";
|
|
10
11
|
import { parseChatRequest } from "../src/wire/openai/request.ts";
|
|
11
12
|
|
|
@@ -449,3 +450,160 @@ describe("quality normalization and capability floor (benchmark findings 4/6)",
|
|
|
449
450
|
expect(shipped.candidates.every((c) => !c.reasons.some((r) => r.includes("capability floor")))).toBe(true);
|
|
450
451
|
});
|
|
451
452
|
});
|
|
453
|
+
|
|
454
|
+
describe("thinness-gated relaxation (review 2026-09-05 §1)", () => {
|
|
455
|
+
// A WIDE catalog whose weak tail drags every quantile band far below the
|
|
456
|
+
// configured floors — the shape the key-admitted 347-model catalog has.
|
|
457
|
+
// Configured coding floors: simple 40, moderate 60, hard 72.
|
|
458
|
+
const wide = computeTierPlan(
|
|
459
|
+
models([
|
|
460
|
+
["w/1", 5, 0.02],
|
|
461
|
+
["w/2", 10, 0.02],
|
|
462
|
+
["w/3", 15, 0.03],
|
|
463
|
+
["w/4", 20, 0.03],
|
|
464
|
+
["w/5", 25, 0.05],
|
|
465
|
+
["w/6", 30, 0.05],
|
|
466
|
+
["w/7", 35, 0.1],
|
|
467
|
+
["w/8", 45, 0.1],
|
|
468
|
+
["w/9", 50, 0.2],
|
|
469
|
+
["w/10", 62, 0.5],
|
|
470
|
+
["w/11", 70, 0.7],
|
|
471
|
+
["w/12", 74, 1.0],
|
|
472
|
+
["w/13", 76, 2.0],
|
|
473
|
+
["w/14", 78, 5.0],
|
|
474
|
+
]),
|
|
475
|
+
BASE,
|
|
476
|
+
);
|
|
477
|
+
|
|
478
|
+
test("the bands sit below the configured floors on a wide catalog", () => {
|
|
479
|
+
// The premise the gate exists for: unconditional min() would relax here.
|
|
480
|
+
expect(wide.floors.coding.moderate).toBeLessThan(60);
|
|
481
|
+
expect(wide.floors.coding.hard).toBeLessThan(72);
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
test("a configured floor that three or more models meet stands as written", () => {
|
|
485
|
+
expect(effectiveQualityFloor(60, "moderate", "coding", wide)).toBe(60); // 62,70,74,76,78 meet it
|
|
486
|
+
expect(effectiveQualityFloor(72, "hard", "coding", wide)).toBe(72); // 74,76,78 meet it
|
|
487
|
+
expect(effectiveQualityFloor(40, "simple", "coding", wide)).toBe(40);
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
test("a floor fewer than three models meet is relaxed to the band", () => {
|
|
491
|
+
// Only 76 and 78 clear 75: thin, so the hard band applies.
|
|
492
|
+
expect(effectiveQualityFloor(75, "hard", "coding", wide)).toBe(Math.min(75, wide.floors.coding.hard));
|
|
493
|
+
// Nothing clears 90: relaxed as before.
|
|
494
|
+
expect(effectiveQualityFloor(90, "hard", "coding", wide)).toBe(wide.floors.coding.hard);
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
test("countAdmitted counts scores at or above the floor", () => {
|
|
498
|
+
expect(countAdmitted([10, 20, 30, 40], 25)).toBe(2);
|
|
499
|
+
expect(countAdmitted([10, 20, 30, 40], 40)).toBe(1);
|
|
500
|
+
expect(countAdmitted([10, 20, 30, 40], 41)).toBe(0);
|
|
501
|
+
expect(countAdmitted([10, 20, 30, 40], 0)).toBe(4);
|
|
502
|
+
expect(countAdmitted([], 0)).toBe(0);
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
test("in candidate selection the wide catalog keeps weak models out of moderate", () => {
|
|
506
|
+
const req = parseChatRequest(
|
|
507
|
+
{
|
|
508
|
+
model: "auto",
|
|
509
|
+
tools: [{ type: "function", function: { name: "read", description: "Read", parameters: { type: "object", properties: {} } } }],
|
|
510
|
+
messages: [{ role: "user", content: "refactor the auth module" }],
|
|
511
|
+
},
|
|
512
|
+
new Headers(),
|
|
513
|
+
);
|
|
514
|
+
const features = extractFeatures(req, 100);
|
|
515
|
+
const snap = snapshot(
|
|
516
|
+
models([
|
|
517
|
+
["w/8", 45, 0.1],
|
|
518
|
+
["w/9", 50, 0.2],
|
|
519
|
+
["w/10", 62, 0.5],
|
|
520
|
+
["w/11", 70, 0.7],
|
|
521
|
+
["w/12", 74, 1.0],
|
|
522
|
+
]),
|
|
523
|
+
);
|
|
524
|
+
const { candidates, rejected } = buildCandidates({
|
|
525
|
+
req,
|
|
526
|
+
features,
|
|
527
|
+
tier: "moderate",
|
|
528
|
+
task: "coding",
|
|
529
|
+
snapshot: snap,
|
|
530
|
+
ledger: null,
|
|
531
|
+
cfg: { ...BASE, adaptiveTierFloors: true },
|
|
532
|
+
expectedCompletionTokens: 512,
|
|
533
|
+
warmSlug: null,
|
|
534
|
+
});
|
|
535
|
+
// 62, 70 and 74 meet the configured 60, so 45 and 50 are excluded even
|
|
536
|
+
// though the adaptive band would have admitted them.
|
|
537
|
+
expect(candidates.map((c) => c.model.slug).sort()).toEqual(["w/10", "w/11", "w/12"]);
|
|
538
|
+
expect(rejected.filter((r) => r.reason === "below_quality_floor").map((r) => r.slug).sort()).toEqual(["w/8", "w/9"]);
|
|
539
|
+
});
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
describe("escalation-cost term (review 2026-09-05 §2)", () => {
|
|
543
|
+
const req = parseChatRequest(
|
|
544
|
+
{
|
|
545
|
+
model: "auto",
|
|
546
|
+
tools: [{ type: "function", function: { name: "read", description: "Read", parameters: { type: "object", properties: {} } } }],
|
|
547
|
+
messages: [{ role: "user", content: "rename the helper" }],
|
|
548
|
+
},
|
|
549
|
+
new Headers(),
|
|
550
|
+
);
|
|
551
|
+
const features = extractFeatures(req, 100_000);
|
|
552
|
+
// Identical price, quality AND success rate (so the trust divisor is
|
|
553
|
+
// neutral); only the escalation count tells them apart.
|
|
554
|
+
const snap = snapshot(
|
|
555
|
+
models([
|
|
556
|
+
["cheap/flaky", 50, 0.02],
|
|
557
|
+
["cheap/solid", 50, 0.02],
|
|
558
|
+
]),
|
|
559
|
+
);
|
|
560
|
+
const trustOf = (slug: string) =>
|
|
561
|
+
slug === "cheap/flaky"
|
|
562
|
+
? { slug, attempts: 100, escalations: 4, errors: 0, successRate: 0.96, meanCostError: 0 }
|
|
563
|
+
: { slug, attempts: 100, escalations: 0, errors: 4, successRate: 0.96, meanCostError: 0 };
|
|
564
|
+
const ledger: Ledger = {
|
|
565
|
+
record: () => {},
|
|
566
|
+
conversationSpend: () => 0,
|
|
567
|
+
spendSince: () => 0,
|
|
568
|
+
blendedRate: () => null,
|
|
569
|
+
trust: (slug) => trustOf(slug),
|
|
570
|
+
allTrust: () => [],
|
|
571
|
+
latency: () => null,
|
|
572
|
+
tokenRatio: () => null,
|
|
573
|
+
recentEntries: () => [],
|
|
574
|
+
};
|
|
575
|
+
function build(weight: number, usdPerPromptToken?: number) {
|
|
576
|
+
return buildCandidates({
|
|
577
|
+
req,
|
|
578
|
+
features,
|
|
579
|
+
tier: "trivial",
|
|
580
|
+
task: "coding",
|
|
581
|
+
snapshot: snap,
|
|
582
|
+
ledger,
|
|
583
|
+
cfg: { ...BASE, filters: { ...BASE.filters, escalationCostWeight: weight } },
|
|
584
|
+
expectedCompletionTokens: 512,
|
|
585
|
+
warmSlug: null,
|
|
586
|
+
...(usdPerPromptToken === undefined ? {} : { escalationUsdPerPromptToken: usdPerPromptToken }),
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
test("with the term off, nothing separates them and the tie falls lexically to the flaky model", () => {
|
|
591
|
+
const { candidates } = build(0, 1e-6);
|
|
592
|
+
// Same success rate, same trust divisor: escalations are invisible.
|
|
593
|
+
expect(candidates[0]!.model.slug).toBe("cheap/flaky");
|
|
594
|
+
expect(candidates[0]!.reasons.some((r) => r.startsWith("escalation risk"))).toBe(false);
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
test("priced at what an escalated retry actually bills, the flaky model loses", () => {
|
|
598
|
+
const { candidates } = build(1, 1e-6); // $1/Mtok of escalated-retry cost
|
|
599
|
+
expect(candidates[0]!.model.slug).toBe("cheap/solid");
|
|
600
|
+
const flaky = candidates.find((c) => c.model.slug === "cheap/flaky")!;
|
|
601
|
+
expect(flaky.reasons.some((r) => r.startsWith("escalation risk"))).toBe(true);
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
test("inert until the ledger can measure the retry cost", () => {
|
|
605
|
+
const { candidates } = build(1);
|
|
606
|
+
expect(candidates[0]!.model.slug).toBe("cheap/flaky");
|
|
607
|
+
});
|
|
608
|
+
});
|
|
609
|
+
|
package/test/toast-logic.test.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { parse as parseYaml } from "yaml";
|
|
|
5
5
|
import {
|
|
6
6
|
DEFAULT_ROUTER_URL,
|
|
7
7
|
newestId,
|
|
8
|
+
providerOf,
|
|
8
9
|
resolveRouterUrl,
|
|
9
10
|
selectToasts,
|
|
10
11
|
toToastText,
|
|
@@ -186,7 +187,15 @@ describe("toToastText", () => {
|
|
|
186
187
|
expect(toToastText(dec({ reportedUsd: null }))).not.toContain("$");
|
|
187
188
|
});
|
|
188
189
|
|
|
189
|
-
test("renders model [tier]", () => {
|
|
190
|
-
expect(toToastText(dec({ slug: "q/w", tier: "hard", reportedUsd: null }))).toBe("q/w [hard]");
|
|
190
|
+
test("renders provider · model [tier]", () => {
|
|
191
|
+
expect(toToastText(dec({ slug: "q/w", tier: "hard", reportedUsd: null }))).toBe("openrouter · q/w [hard]");
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
test("an Ollama slug is labelled with its provider and shown without the prefix", () => {
|
|
195
|
+
expect(toToastText(dec({ slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", tier: "moderate", reportedUsd: 0.0007 }))).toBe(
|
|
196
|
+
"ollama · glm-5.3-flash [moderate] · $0.00070",
|
|
197
|
+
);
|
|
198
|
+
expect(providerOf("ollama/gpt-oss:120b")).toEqual({ provider: "ollama", model: "gpt-oss:120b" });
|
|
199
|
+
expect(providerOf("z-ai/glm-5.3-flash")).toEqual({ provider: "openrouter", model: "z-ai/glm-5.3-flash" });
|
|
191
200
|
});
|
|
192
201
|
});
|
package/test/tokens.test.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
|
|
|
3
3
|
import { loadConfig } from "../src/config/load.ts";
|
|
4
4
|
import { createLedger } from "../src/cost/ledger.ts";
|
|
5
5
|
import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
|
|
6
|
-
import { DEFAULT_BYTES_PER_TOKEN, estimatePromptTokens, estimateTokens } from "../src/tokens/estimate.ts";
|
|
6
|
+
import { adjustPendingEstimate, DEFAULT_BYTES_PER_TOKEN, estimatePromptTokens, estimateTokens } from "../src/tokens/estimate.ts";
|
|
7
7
|
import { openDb } from "../src/util/sqlite.ts";
|
|
8
8
|
import { parseChatRequest } from "../src/wire/openai/request.ts";
|
|
9
9
|
|
|
@@ -167,3 +167,73 @@ describe("estimatePromptTokens", () => {
|
|
|
167
167
|
expect(estimatePromptTokens(withImage, "gpt", null)).toBeGreaterThan(estimatePromptTokens(text, "gpt", null) + 500);
|
|
168
168
|
});
|
|
169
169
|
});
|
|
170
|
+
|
|
171
|
+
describe("calibration hygiene (review 2026-09-05 §8)", () => {
|
|
172
|
+
function requestOf(text: string) {
|
|
173
|
+
return parseChatRequest({ model: "auto", messages: [{ role: "user", content: text }] }, new Headers());
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
test("a provider reporting impossible token counts never calibrates its family", () => {
|
|
177
|
+
const db = openDb(":memory:");
|
|
178
|
+
try {
|
|
179
|
+
const ledger = createLedger(db, cfg);
|
|
180
|
+
const req = requestOf("x".repeat(10_000));
|
|
181
|
+
for (let i = 0; i < 30; i++) {
|
|
182
|
+
estimatePromptTokens(req, "qwen3", ledger);
|
|
183
|
+
// 0.4 bytes/token: ~8x what the bytes imply (seen live from one provider).
|
|
184
|
+
ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 0.4) } }));
|
|
185
|
+
}
|
|
186
|
+
expect(ledger.tokenRatio("qwen3")).toBeNull();
|
|
187
|
+
for (let i = 0; i < 30; i++) {
|
|
188
|
+
estimatePromptTokens(req, "qwen3", ledger);
|
|
189
|
+
ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 3.2) } }));
|
|
190
|
+
}
|
|
191
|
+
expect(ledger.tokenRatio("qwen3")).toBeCloseTo(3.2, 1);
|
|
192
|
+
} finally {
|
|
193
|
+
db.close();
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test("adjustPendingEstimate calibrates against the dispatched bytes, not the raw request", () => {
|
|
198
|
+
const db = openDb(":memory:");
|
|
199
|
+
try {
|
|
200
|
+
const ledger = createLedger(db, cfg);
|
|
201
|
+
const req = requestOf("y".repeat(10_000));
|
|
202
|
+
for (let i = 0; i < 30; i++) {
|
|
203
|
+
estimatePromptTokens(req, "grok", ledger);
|
|
204
|
+
// Compaction halved the prompt before dispatch; the upstream billed the half.
|
|
205
|
+
adjustPendingEstimate(req.conversationKey, req.promptBytes / 2);
|
|
206
|
+
ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 2 / 3.5) } }));
|
|
207
|
+
}
|
|
208
|
+
// Paired with the raw bytes this would have learned 7.0; the dispatched bytes give the true 3.5.
|
|
209
|
+
expect(ledger.tokenRatio("grok")).toBeCloseTo(3.5, 1);
|
|
210
|
+
} finally {
|
|
211
|
+
db.close();
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
describe("ledger.escalationCost", () => {
|
|
217
|
+
test("measures what escalated retries bill per prompt token, once enough exist", () => {
|
|
218
|
+
const db = openDb(":memory:");
|
|
219
|
+
try {
|
|
220
|
+
const ledger = createLedger(db, cfg);
|
|
221
|
+
for (let i = 0; i < 9; i++) {
|
|
222
|
+
ledger.record(entry({ attempt: 1, reportedUsd: 0.02, usage: { ...EMPTY_USAGE, promptTokens: 1_000 } }));
|
|
223
|
+
}
|
|
224
|
+
expect(ledger.escalationCost?.(7)).toBeNull(); // 9 < the sample floor
|
|
225
|
+
ledger.record(entry({ attempt: 1, reportedUsd: 0.02, usage: { ...EMPTY_USAGE, promptTokens: 1_000 } }));
|
|
226
|
+
// Errored retries carry no usage and are excluded.
|
|
227
|
+
ledger.record(entry({ attempt: 1, reportedUsd: null, error: "upstream_error: boom", usage: EMPTY_USAGE }));
|
|
228
|
+
// Memoised: a fresh ledger reads through.
|
|
229
|
+
const fresh = createLedger(db, cfg);
|
|
230
|
+
const cost = fresh.escalationCost?.(7);
|
|
231
|
+
expect(cost).not.toBeNull();
|
|
232
|
+
expect(cost!.samples).toBe(10);
|
|
233
|
+
expect(cost!.usdPerPromptToken).toBeCloseTo(0.02 / 1_000, 8);
|
|
234
|
+
} finally {
|
|
235
|
+
db.close();
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
|
|
@@ -244,11 +244,11 @@ describe("v4 migration", () => {
|
|
|
244
244
|
}
|
|
245
245
|
});
|
|
246
246
|
|
|
247
|
-
test("schema is at user_version
|
|
247
|
+
test("schema is at user_version 16", () => {
|
|
248
248
|
const db = openDb(":memory:");
|
|
249
249
|
try {
|
|
250
250
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
251
|
-
expect(row.user_version).toBe(
|
|
251
|
+
expect(row.user_version).toBe(16);
|
|
252
252
|
} finally {
|
|
253
253
|
db.close();
|
|
254
254
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
import { createDisabledBridge } from "../src/context/bridge.ts";
|
|
3
3
|
import type { ContextBridge, TurnRecord } from "../src/context/types.ts";
|
|
4
|
-
import type { CatalogSource } from "../src/catalog/types.ts";
|
|
4
|
+
import type { CatalogModel, CatalogSource } from "../src/catalog/types.ts";
|
|
5
5
|
import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
|
|
6
6
|
import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
|
|
7
7
|
import type {
|
|
@@ -31,6 +31,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
31
31
|
return {
|
|
32
32
|
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
|
|
33
33
|
openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
|
|
34
|
+
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0 },
|
|
34
35
|
benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
|
|
35
36
|
tiers: {
|
|
36
37
|
trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
|
|
@@ -45,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
45
46
|
data: { axis: "intelligence", minQuality: 0 },
|
|
46
47
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
47
48
|
},
|
|
48
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
49
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20, escalationCostWeight: 0 },
|
|
49
50
|
classifier: {
|
|
50
51
|
ambiguityThreshold: 0,
|
|
51
52
|
model: "test/adjudicator",
|
|
@@ -56,6 +57,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
56
57
|
toolAxis: "coding",
|
|
57
58
|
chatAxis: "intelligence",
|
|
58
59
|
agenticLoopDepth: 3,
|
|
60
|
+
mechanicalRetryFactor: 0.2,
|
|
59
61
|
reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
|
|
60
62
|
},
|
|
61
63
|
escalation: {
|
|
@@ -68,11 +70,11 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
70
|
escalateOnLengthStop: false,
|
|
69
71
|
...escalation,
|
|
70
72
|
},
|
|
71
|
-
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false },
|
|
73
|
+
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
|
|
72
74
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
73
75
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
74
76
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
75
|
-
compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
77
|
+
compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
|
|
76
78
|
budget: { onExceeded: "downgrade" },
|
|
77
79
|
profiles: [],
|
|
78
80
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
@@ -147,6 +149,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
147
149
|
cacheBreakpointMessageIndices: [],
|
|
148
150
|
compactionPlan: [],
|
|
149
151
|
promptTokensSaved: 0,
|
|
152
|
+
compactionSavedBytes: 0,
|
|
153
|
+
compactionPlanTokens: 0,
|
|
150
154
|
reasoning: undefined,
|
|
151
155
|
maxTokens: undefined,
|
|
152
156
|
stripAssistantReasoning: false,
|
|
@@ -403,10 +407,13 @@ describe("runTurn", () => {
|
|
|
403
407
|
expect(entries[1]!.attempt).toBe(1);
|
|
404
408
|
expect(entries[1]!.slug).toBe("better/model");
|
|
405
409
|
|
|
406
|
-
//
|
|
407
|
-
|
|
410
|
+
// A refusal indicts the provider, so a same-tier sibling is probed first;
|
|
411
|
+
// this fake router only has the simple-tier decision left, which is the
|
|
412
|
+
// wrong tier, so the turn then escalates one tier up for real.
|
|
413
|
+
expect(calls).toHaveLength(3);
|
|
408
414
|
expect(calls[0]).toEqual({ attempt: 0 });
|
|
409
|
-
expect(calls[1]).toEqual({ attempt: 1
|
|
415
|
+
expect(calls[1]).toEqual({ attempt: 1 });
|
|
416
|
+
expect(calls[2]).toEqual({ attempt: 1, escalateFrom: "trivial" });
|
|
410
417
|
|
|
411
418
|
// The held refusal text never reached the client.
|
|
412
419
|
expect(textOut(chunks)).toBe("Here is the answer.");
|
|
@@ -729,4 +736,114 @@ describe("latency measurement covers the work the router actually does", () => {
|
|
|
729
736
|
expect(entries[0]?.ttftMs).not.toBeNull();
|
|
730
737
|
expect(entries[0]?.ttftMs ?? -1).toBeGreaterThanOrEqual(0);
|
|
731
738
|
});
|
|
739
|
+
|
|
740
|
+
test("a client hang-up AFTER the finish event is a completed turn, not an error", async () => {
|
|
741
|
+
// Measured live: 1,842 of 2,068 "request aborted" rows carried a finish
|
|
742
|
+
// reason and full usage — the generation had finished and the client
|
|
743
|
+
// closed before the trailing [DONE] was read. Recording that as an error
|
|
744
|
+
// skipped the state save on 12% of turns.
|
|
745
|
+
const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple" })]);
|
|
746
|
+
const { upstream } = mkUpstream([
|
|
747
|
+
{
|
|
748
|
+
kind: "die",
|
|
749
|
+
chunks: [
|
|
750
|
+
startChunk("cheap/model"),
|
|
751
|
+
textChunk("all done here"),
|
|
752
|
+
finishChunk("stop"),
|
|
753
|
+
usageChunk({ promptTokens: 120, cachedTokens: 100, completionTokens: 4 }, 0.0004),
|
|
754
|
+
],
|
|
755
|
+
error: new UpstreamError("aborted", 0, "request aborted", false),
|
|
756
|
+
},
|
|
757
|
+
]);
|
|
758
|
+
const { ledger, entries } = mkLedger();
|
|
759
|
+
const { store, map } = mkConversations();
|
|
760
|
+
const { sink, errors, finishes } = mkSink();
|
|
761
|
+
|
|
762
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
763
|
+
|
|
764
|
+
expect(errors).toHaveLength(0);
|
|
765
|
+
expect(entries).toHaveLength(1);
|
|
766
|
+
expect(entries[0]!.error).toBeNull();
|
|
767
|
+
expect(entries[0]!.wasted).toBe(false);
|
|
768
|
+
expect(entries[0]!.finishReason).toBe("stop");
|
|
769
|
+
expect(entries[0]!.reportedUsd).toBe(0.0004);
|
|
770
|
+
// The turn settled: hysteresis, cache warmth and the turn counter advance.
|
|
771
|
+
const state = map.get("conv-test")!;
|
|
772
|
+
expect(state.turn).toBe(1);
|
|
773
|
+
expect(state.currentSlug).toBe("cheap/model");
|
|
774
|
+
expect(state.cacheWarmSlug).toBe("cheap/model");
|
|
775
|
+
expect(state.lastPromptTokens).toBe(120);
|
|
776
|
+
expect(finishes).toHaveLength(1);
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
test("a client hang-up BEFORE the finish event is still recorded as aborted", async () => {
|
|
780
|
+
const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple", maxTokens: 1 })]);
|
|
781
|
+
const { upstream } = mkUpstream([
|
|
782
|
+
{
|
|
783
|
+
kind: "die",
|
|
784
|
+
chunks: [startChunk("cheap/model"), textChunk("partial answer that committed")],
|
|
785
|
+
error: new UpstreamError("aborted", 0, "request aborted", false),
|
|
786
|
+
},
|
|
787
|
+
]);
|
|
788
|
+
const { ledger, entries } = mkLedger();
|
|
789
|
+
const { store, map } = mkConversations();
|
|
790
|
+
const { sink, finishes } = mkSink();
|
|
791
|
+
|
|
792
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
793
|
+
|
|
794
|
+
expect(entries).toHaveLength(1);
|
|
795
|
+
expect(entries[0]!.error).toBe("request aborted");
|
|
796
|
+
expect(entries[0]!.finishReason).toBeNull();
|
|
797
|
+
expect(map.get("conv-test")!.turn).toBe(0);
|
|
798
|
+
expect(finishes).toHaveLength(0);
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
test("a provider that reports usage but no cost gets its actual tokens priced at the catalog rate", async () => {
|
|
803
|
+
// Ollama returns usage without `cost`. Leaving the forecast (assumed 1,024
|
|
804
|
+
// completion tokens) as the recorded figure overstated a 26-token turn ~3x.
|
|
805
|
+
const { router } = mkRouter([mkDecision("trivial", "ollama/glm-5.3-flash")]);
|
|
806
|
+
const { upstream } = mkUpstream([
|
|
807
|
+
{
|
|
808
|
+
kind: "chunks",
|
|
809
|
+
chunks: [startChunk("ollama/glm-5.3-flash"), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 1_000_000, completionTokens: 1_000_000 }, null)],
|
|
810
|
+
},
|
|
811
|
+
]);
|
|
812
|
+
const { ledger, entries } = mkLedger();
|
|
813
|
+
const { store } = mkConversations();
|
|
814
|
+
const { sink, finishes } = mkSink();
|
|
815
|
+
const priced = {
|
|
816
|
+
...catalog,
|
|
817
|
+
find: (slug: string) =>
|
|
818
|
+
slug === "ollama/glm-5.3-flash"
|
|
819
|
+
? ({
|
|
820
|
+
slug,
|
|
821
|
+
provider: "ollama",
|
|
822
|
+
canonicalSlug: slug,
|
|
823
|
+
name: slug,
|
|
824
|
+
contextLength: 1_000_000,
|
|
825
|
+
supportsTools: true,
|
|
826
|
+
supportsReasoning: true,
|
|
827
|
+
reasoningMandatory: false,
|
|
828
|
+
supportsToolChoice: false,
|
|
829
|
+
inputModalities: ["text"],
|
|
830
|
+
price: { prompt: 0.15 / 1e6, completion: 0.5 / 1e6 },
|
|
831
|
+
priceTiers: [],
|
|
832
|
+
quality: {},
|
|
833
|
+
tokenizer: "Other",
|
|
834
|
+
isFree: false,
|
|
835
|
+
createdAtMs: 0,
|
|
836
|
+
author: "ollama",
|
|
837
|
+
} satisfies CatalogModel)
|
|
838
|
+
: undefined,
|
|
839
|
+
};
|
|
840
|
+
|
|
841
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog: priced, context: createDisabledBridge() }, new AbortController().signal);
|
|
842
|
+
|
|
843
|
+
// 1M prompt tokens at $0.15/M + 1M completion tokens at $0.50/M.
|
|
844
|
+
expect(entries[0]!.reportedUsd).toBeCloseTo(0.65, 6);
|
|
845
|
+
expect(entries[0]!.priceModel?.slug).toBe("ollama/glm-5.3-flash");
|
|
846
|
+
expect(finishes[0]!.reportedUsd).toBeCloseTo(0.65, 6);
|
|
847
|
+
});
|
|
848
|
+
|
|
732
849
|
});
|