auto-model-router 0.2.32 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +208 -29
  3. package/docs/review-2026-09-05.md +267 -0
  4. package/omp-extension/configure-logic.ts +71 -15
  5. package/omp-extension/pi-coding-agent.d.ts +79 -2
  6. package/omp-extension/report-hub.ts +376 -0
  7. package/omp-extension/report-logic.ts +115 -0
  8. package/omp-extension/router-configure.ts +203 -51
  9. package/omp-extension/router-url.ts +52 -0
  10. package/omp-extension/toast-logic.ts +14 -2
  11. package/package.json +1 -1
  12. package/src/catalog/composite.ts +97 -0
  13. package/src/catalog/ollama-catalog.ts +309 -0
  14. package/src/catalog/ollama-prices.ts +85 -0
  15. package/src/catalog/openrouter-catalog.ts +39 -1
  16. package/src/catalog/types.ts +31 -1
  17. package/src/cli/args.ts +1 -0
  18. package/src/cli/config-wizard.ts +189 -28
  19. package/src/cli/explain.ts +2 -4
  20. package/src/cli/models.ts +2 -4
  21. package/src/cli/report.ts +37 -0
  22. package/src/config/defaults.ts +43 -2
  23. package/src/config/load.ts +25 -1
  24. package/src/config/omp-credentials.ts +31 -7
  25. package/src/config/schema.ts +27 -0
  26. package/src/config/types.ts +114 -2
  27. package/src/cost/ledger.ts +73 -4
  28. package/src/cost/report.ts +340 -0
  29. package/src/cost/types.ts +33 -1
  30. package/src/index.ts +5 -8
  31. package/src/router/candidates.ts +52 -4
  32. package/src/router/classify.ts +33 -6
  33. package/src/router/features.ts +13 -1
  34. package/src/router/select.ts +55 -8
  35. package/src/router/state.ts +6 -2
  36. package/src/router/tier-plan.ts +49 -11
  37. package/src/router/types.ts +10 -0
  38. package/src/server/http.ts +47 -6
  39. package/src/server/providers.ts +54 -0
  40. package/src/server/turn.ts +122 -34
  41. package/src/tokens/estimate.ts +16 -0
  42. package/src/upstream/multi.ts +26 -0
  43. package/src/upstream/ollama-usage.ts +157 -0
  44. package/src/upstream/ollama.ts +275 -0
  45. package/src/upstream/openrouter.ts +19 -1
  46. package/src/upstream/types.ts +2 -0
  47. package/src/util/sqlite.ts +25 -1
  48. package/test/catalog.test.ts +44 -0
  49. package/test/classify.test.ts +41 -5
  50. package/test/compaction.test.ts +1 -0
  51. package/test/config-wizard.test.ts +77 -1
  52. package/test/configure-logic.test.ts +129 -33
  53. package/test/embed-lifecycle.test.ts +1 -0
  54. package/test/failover.test.ts +148 -3
  55. package/test/features.test.ts +35 -0
  56. package/test/http-resilience.test.ts +24 -0
  57. package/test/ollama.test.ts +506 -0
  58. package/test/omp-credentials.test.ts +43 -1
  59. package/test/report-hub.test.ts +341 -0
  60. package/test/report-logic.test.ts +92 -0
  61. package/test/report.test.ts +217 -0
  62. package/test/select.test.ts +151 -1
  63. package/test/tier-plan.test.ts +159 -1
  64. package/test/toast-logic.test.ts +11 -2
  65. package/test/tokens.test.ts +71 -1
  66. package/test/trust-attribution.test.ts +2 -2
  67. package/test/turn.test.ts +124 -7
@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
2
2
 
3
3
  import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
4
4
  import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
5
+ import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
5
6
  import { loadConfig } from "../src/config/load.ts";
6
7
  import type { ProfileConfig, RouterConfig } from "../src/config/types.ts";
7
8
  import type { Ledger } from "../src/cost/types.ts";
@@ -628,6 +629,7 @@ describe("context compaction", () => {
628
629
  enabled: true,
629
630
  budgetTokens: 1_000,
630
631
  floorRatio: 1,
632
+ replanGrowthRatio: 1,
631
633
  fitToWindow: false,
632
634
  protectRecentTurns: 1,
633
635
  maxToolResultBytes: 100,
@@ -801,7 +803,7 @@ describe("hysteresis.breakHoldOnMechanical", () => {
801
803
  }
802
804
 
803
805
  test("off by default, so a hold still pins the tier", () => {
804
- expect(BASE.hysteresis.breakHoldOnMechanical).toBe(false);
806
+ expect(DEFAULT_CONFIG.hysteresis.breakHoldOnMechanical).toBe(false); // SHIPPED default, not the live config.yml (machine-dependent)
805
807
  const { d, features } = decide(continuation(), false);
806
808
  expect(features.isToolResultContinuation).toBe(true);
807
809
  expect(d.tier).toBe("hard");
@@ -846,3 +848,151 @@ describe("hysteresis.breakHoldOnMechanical", () => {
846
848
  expect(d.tier).toBe("moderate");
847
849
  });
848
850
  });
851
+
852
+ describe("hysteresis.switchHorizonTurns (review 2026-09-05 §4)", () => {
853
+ // Two moderate-eligible models built from raw catalog records: a kimi-shaped
854
+ // warm model (cheap to READ from cache, dear cold) and a gemini-shaped ranked
855
+ // winner (dear cold, cheap warm). With a one-turn horizon the cold write on
856
+ // the winner never pays for itself, so the dear model stays warm forever;
857
+ // over a run of turns the switch is obviously right.
858
+ function rawModel(id: string, coding: number, prompt: number, cacheRead: number, cacheWrite: number | null): Record<string, unknown> {
859
+ const pricing: Record<string, string> = {
860
+ prompt: String(prompt / 1e6),
861
+ completion: String((prompt * 4) / 1e6),
862
+ input_cache_read: String(cacheRead / 1e6),
863
+ };
864
+ if (cacheWrite !== null) pricing.input_cache_write = String(cacheWrite / 1e6);
865
+ return {
866
+ id,
867
+ canonical_slug: id,
868
+ name: id,
869
+ context_length: 1_000_000,
870
+ pricing,
871
+ supported_parameters: ["tools"],
872
+ architecture: { input_modalities: ["text"], tokenizer: "GPT" },
873
+ benchmarks: { artificial_analysis: { coding_index: coding, intelligence_index: coding, agentic_index: coding } },
874
+ created: 1_700_000_000,
875
+ };
876
+ }
877
+ const warmDear = normalizeCatalogModel(rawModel("test/warm-dear", 76.2, 2.55, 0.256, null))!;
878
+ const winner = normalizeCatalogModel(rawModel("test/winner", 76.0, 0.75, 0.075, 0.04))!;
879
+ const snap: CatalogSnapshot = { models: [warmDear, winner], fetchedAtMs: Date.now() };
880
+ const promptTokens = 100_000;
881
+
882
+ function decide(horizon: number) {
883
+ // BASE is the live config.yml, which may have exploration on; a
884
+ // deterministic exploration draw would route this turn down to `simple`,
885
+ // where the dear model is over the price ceiling and never compared.
886
+ const cfg: RouterConfig = {
887
+ ...BASE,
888
+ exploration: { ...BASE.exploration, enabled: false },
889
+ hysteresis: { ...BASE.hysteresis, switchMargin: 1.3, switchHorizonTurns: horizon },
890
+ };
891
+ const req = request("keep going");
892
+ const features = extractFeatures(req, promptTokens);
893
+ return select({
894
+ req,
895
+ features,
896
+ classification: { ...scoreHeuristic(features, cfg), tier: "moderate" },
897
+ profile: PROFILE,
898
+ state: state({ currentSlug: "test/warm-dear", currentTier: "moderate", cacheWarmSlug: "test/warm-dear", cacheWarmAtMs: Date.now(), lastPromptTokens: promptTokens }),
899
+ snapshot: snap,
900
+ ledger: null,
901
+ cfg,
902
+ nowMs: Date.now(),
903
+ });
904
+ }
905
+
906
+ test("the ranked winner is the cheaper cold model", () => {
907
+ const d = decide(1);
908
+ expect(d.considered[0]!.model.slug).toBe("test/winner");
909
+ });
910
+
911
+ test("a one-turn horizon keeps the dear model warm (the shipped behaviour)", () => {
912
+ const d = decide(1);
913
+ expect(d.slug).toBe("test/warm-dear");
914
+ expect(d.sticky).toBe(true);
915
+ expect(d.reasons.some((r) => r.startsWith("cache: keeping warm test/warm-dear"))).toBe(true);
916
+ });
917
+
918
+ test("amortised over a run of turns, the switch is taken", () => {
919
+ const d = decide(8);
920
+ expect(d.slug).toBe("test/winner");
921
+ expect(d.sticky).toBe(false);
922
+ expect(d.reasons.some((r) => r.startsWith("cache: switch test/warm-dear → test/winner") && r.includes("over 8 turns"))).toBe(true);
923
+ });
924
+ });
925
+
926
+ describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => {
927
+ const TOOL_RESULT = "y".repeat(3000);
928
+ function turn(n: number): NormRequest {
929
+ // n completed read/result/"continue" rounds; every result but the newest
930
+ // sits outside the protected window and is eligible for truncation.
931
+ const messages: unknown[] = [
932
+ { role: "system", content: "You are a coding agent." },
933
+ { role: "user", content: "read the files" },
934
+ ];
935
+ for (let i = 0; i < n; i++) {
936
+ messages.push({ role: "assistant", content: null, tool_calls: [{ id: `c${i}`, type: "function", function: { name: "read", arguments: `{"path":"f${i}.ts"}` } }] });
937
+ messages.push({ role: "tool", tool_call_id: `c${i}`, content: TOOL_RESULT });
938
+ messages.push({ role: "user", content: "continue" });
939
+ }
940
+ return parseChatRequest({ model: "auto", tools: TOOLS, messages }, new Headers());
941
+ }
942
+ function cfgWith(ratio: number): RouterConfig {
943
+ return {
944
+ ...BASE,
945
+ compaction: {
946
+ enabled: true,
947
+ budgetTokens: 100, // unreachable: every turn is over budget, as observed live
948
+ floorRatio: 1,
949
+ replanGrowthRatio: ratio,
950
+ fitToWindow: false,
951
+ protectRecentTurns: 2, // the newest result and its "continue" stay protected; older rounds are eligible
952
+ maxToolResultBytes: 100,
953
+ keepHeadBytes: 20,
954
+ keepTailBytes: 20,
955
+ elideSupersededReads: false,
956
+ collapseDuplicateResults: false,
957
+ },
958
+ };
959
+ }
960
+ function decide(req: NormRequest, cfg: RouterConfig, st: ConversationState, promptTokens: number) {
961
+ const features = extractFeatures(req, promptTokens);
962
+ return select({ req, features, classification: scoreHeuristic(features, cfg), profile: PROFILE, state: st, snapshot: SNAPSHOT, ledger: null, cfg, nowMs: Date.now() });
963
+ }
964
+
965
+ test("the plan records the compacted size it was made at", () => {
966
+ const first = decide(turn(2), cfgWith(1), state(), 4_000);
967
+ expect(first.compactionPlan.length).toBe(1);
968
+ expect(first.compactionPlanTokens).toBe(4_000 - first.promptTokensSaved);
969
+ expect(first.compactionSavedBytes).toBeGreaterThan(0);
970
+ });
971
+
972
+ test("at 1 (shipped) a newly eligible result is compacted on the very next turn", () => {
973
+ const first = decide(turn(2), cfgWith(1), state(), 4_000);
974
+ const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens });
975
+ // One more round: the prompt grew ~25%, one more result aged out.
976
+ const second = decide(turn(3), cfgWith(1), carried, 5_000);
977
+ expect(second.compactionPlan.length).toBe(2);
978
+ expect(second.reasons.some((r) => r.includes("(1 carried, 1 new)"))).toBe(true);
979
+ });
980
+
981
+ test("above 1, an existing plan holds until the compacted prompt has grown by the ratio", () => {
982
+ const first = decide(turn(2), cfgWith(2), state(), 4_000);
983
+ const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens });
984
+ // The raw prompt grew 25% and the COMPACTED prompt ~60% (the carried
985
+ // edit saves a smaller share of a bigger prompt) — still under 2x, so
986
+ // the carried plan is re-applied verbatim and nothing new is added.
987
+ const held = decide(turn(3), cfgWith(2), carried, 5_000);
988
+ expect(held.compactionPlan).toEqual(first.compactionPlan);
989
+ expect(held.compactionPlanTokens).toBe(first.compactionPlanTokens); // growth still accrues against the original size
990
+ expect(held.reasons.some((r) => r.includes("(1 carried, 0 new)") && r.includes("[re-plan rationed]"))).toBe(true);
991
+ // Past 2x the plan is extended and the new size is recorded.
992
+ const grown = decide(turn(3), cfgWith(2), carried, 8_000);
993
+ expect(grown.compactionPlan.length).toBe(2);
994
+ expect(grown.compactionPlanTokens).toBe(8_000 - grown.promptTokensSaved);
995
+ expect(grown.reasons.some((r) => r.includes("[re-plan rationed]"))).toBe(false);
996
+ });
997
+ });
998
+
@@ -2,10 +2,11 @@ import { describe, expect, test } from "bun:test";
2
2
 
3
3
  import { joinBenchmarks, normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
4
4
  import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
5
+ import type { Ledger } from "../src/cost/types.ts";
5
6
  import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
6
7
  import { buildCandidates } from "../src/router/candidates.ts";
7
8
  import { extractFeatures } from "../src/router/features.ts";
8
- import { computeTierPlan, effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
9
+ import { computeTierPlan, countAdmitted, effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
9
10
  import { TIER_ORDER } from "../src/router/types.ts";
10
11
  import { parseChatRequest } from "../src/wire/openai/request.ts";
11
12
 
@@ -449,3 +450,160 @@ describe("quality normalization and capability floor (benchmark findings 4/6)",
449
450
  expect(shipped.candidates.every((c) => !c.reasons.some((r) => r.includes("capability floor")))).toBe(true);
450
451
  });
451
452
  });
453
+
454
+ describe("thinness-gated relaxation (review 2026-09-05 §1)", () => {
455
+ // A WIDE catalog whose weak tail drags every quantile band far below the
456
+ // configured floors — the shape the key-admitted 347-model catalog has.
457
+ // Configured coding floors: simple 40, moderate 60, hard 72.
458
+ const wide = computeTierPlan(
459
+ models([
460
+ ["w/1", 5, 0.02],
461
+ ["w/2", 10, 0.02],
462
+ ["w/3", 15, 0.03],
463
+ ["w/4", 20, 0.03],
464
+ ["w/5", 25, 0.05],
465
+ ["w/6", 30, 0.05],
466
+ ["w/7", 35, 0.1],
467
+ ["w/8", 45, 0.1],
468
+ ["w/9", 50, 0.2],
469
+ ["w/10", 62, 0.5],
470
+ ["w/11", 70, 0.7],
471
+ ["w/12", 74, 1.0],
472
+ ["w/13", 76, 2.0],
473
+ ["w/14", 78, 5.0],
474
+ ]),
475
+ BASE,
476
+ );
477
+
478
+ test("the bands sit below the configured floors on a wide catalog", () => {
479
+ // The premise the gate exists for: unconditional min() would relax here.
480
+ expect(wide.floors.coding.moderate).toBeLessThan(60);
481
+ expect(wide.floors.coding.hard).toBeLessThan(72);
482
+ });
483
+
484
+ test("a configured floor that three or more models meet stands as written", () => {
485
+ expect(effectiveQualityFloor(60, "moderate", "coding", wide)).toBe(60); // 62,70,74,76,78 meet it
486
+ expect(effectiveQualityFloor(72, "hard", "coding", wide)).toBe(72); // 74,76,78 meet it
487
+ expect(effectiveQualityFloor(40, "simple", "coding", wide)).toBe(40);
488
+ });
489
+
490
+ test("a floor fewer than three models meet is relaxed to the band", () => {
491
+ // Only 76 and 78 clear 75: thin, so the hard band applies.
492
+ expect(effectiveQualityFloor(75, "hard", "coding", wide)).toBe(Math.min(75, wide.floors.coding.hard));
493
+ // Nothing clears 90: relaxed as before.
494
+ expect(effectiveQualityFloor(90, "hard", "coding", wide)).toBe(wide.floors.coding.hard);
495
+ });
496
+
497
+ test("countAdmitted counts scores at or above the floor", () => {
498
+ expect(countAdmitted([10, 20, 30, 40], 25)).toBe(2);
499
+ expect(countAdmitted([10, 20, 30, 40], 40)).toBe(1);
500
+ expect(countAdmitted([10, 20, 30, 40], 41)).toBe(0);
501
+ expect(countAdmitted([10, 20, 30, 40], 0)).toBe(4);
502
+ expect(countAdmitted([], 0)).toBe(0);
503
+ });
504
+
505
+ test("in candidate selection the wide catalog keeps weak models out of moderate", () => {
506
+ const req = parseChatRequest(
507
+ {
508
+ model: "auto",
509
+ tools: [{ type: "function", function: { name: "read", description: "Read", parameters: { type: "object", properties: {} } } }],
510
+ messages: [{ role: "user", content: "refactor the auth module" }],
511
+ },
512
+ new Headers(),
513
+ );
514
+ const features = extractFeatures(req, 100);
515
+ const snap = snapshot(
516
+ models([
517
+ ["w/8", 45, 0.1],
518
+ ["w/9", 50, 0.2],
519
+ ["w/10", 62, 0.5],
520
+ ["w/11", 70, 0.7],
521
+ ["w/12", 74, 1.0],
522
+ ]),
523
+ );
524
+ const { candidates, rejected } = buildCandidates({
525
+ req,
526
+ features,
527
+ tier: "moderate",
528
+ task: "coding",
529
+ snapshot: snap,
530
+ ledger: null,
531
+ cfg: { ...BASE, adaptiveTierFloors: true },
532
+ expectedCompletionTokens: 512,
533
+ warmSlug: null,
534
+ });
535
+ // 62, 70 and 74 meet the configured 60, so 45 and 50 are excluded even
536
+ // though the adaptive band would have admitted them.
537
+ expect(candidates.map((c) => c.model.slug).sort()).toEqual(["w/10", "w/11", "w/12"]);
538
+ expect(rejected.filter((r) => r.reason === "below_quality_floor").map((r) => r.slug).sort()).toEqual(["w/8", "w/9"]);
539
+ });
540
+ });
541
+
542
+ describe("escalation-cost term (review 2026-09-05 §2)", () => {
543
+ const req = parseChatRequest(
544
+ {
545
+ model: "auto",
546
+ tools: [{ type: "function", function: { name: "read", description: "Read", parameters: { type: "object", properties: {} } } }],
547
+ messages: [{ role: "user", content: "rename the helper" }],
548
+ },
549
+ new Headers(),
550
+ );
551
+ const features = extractFeatures(req, 100_000);
552
+ // Identical price, quality AND success rate (so the trust divisor is
553
+ // neutral); only the escalation count tells them apart.
554
+ const snap = snapshot(
555
+ models([
556
+ ["cheap/flaky", 50, 0.02],
557
+ ["cheap/solid", 50, 0.02],
558
+ ]),
559
+ );
560
+ const trustOf = (slug: string) =>
561
+ slug === "cheap/flaky"
562
+ ? { slug, attempts: 100, escalations: 4, errors: 0, successRate: 0.96, meanCostError: 0 }
563
+ : { slug, attempts: 100, escalations: 0, errors: 4, successRate: 0.96, meanCostError: 0 };
564
+ const ledger: Ledger = {
565
+ record: () => {},
566
+ conversationSpend: () => 0,
567
+ spendSince: () => 0,
568
+ blendedRate: () => null,
569
+ trust: (slug) => trustOf(slug),
570
+ allTrust: () => [],
571
+ latency: () => null,
572
+ tokenRatio: () => null,
573
+ recentEntries: () => [],
574
+ };
575
+ function build(weight: number, usdPerPromptToken?: number) {
576
+ return buildCandidates({
577
+ req,
578
+ features,
579
+ tier: "trivial",
580
+ task: "coding",
581
+ snapshot: snap,
582
+ ledger,
583
+ cfg: { ...BASE, filters: { ...BASE.filters, escalationCostWeight: weight } },
584
+ expectedCompletionTokens: 512,
585
+ warmSlug: null,
586
+ ...(usdPerPromptToken === undefined ? {} : { escalationUsdPerPromptToken: usdPerPromptToken }),
587
+ });
588
+ }
589
+
590
+ test("with the term off, nothing separates them and the tie falls lexically to the flaky model", () => {
591
+ const { candidates } = build(0, 1e-6);
592
+ // Same success rate, same trust divisor: escalations are invisible.
593
+ expect(candidates[0]!.model.slug).toBe("cheap/flaky");
594
+ expect(candidates[0]!.reasons.some((r) => r.startsWith("escalation risk"))).toBe(false);
595
+ });
596
+
597
+ test("priced at what an escalated retry actually bills, the flaky model loses", () => {
598
+ const { candidates } = build(1, 1e-6); // $1/Mtok of escalated-retry cost
599
+ expect(candidates[0]!.model.slug).toBe("cheap/solid");
600
+ const flaky = candidates.find((c) => c.model.slug === "cheap/flaky")!;
601
+ expect(flaky.reasons.some((r) => r.startsWith("escalation risk"))).toBe(true);
602
+ });
603
+
604
+ test("inert until the ledger can measure the retry cost", () => {
605
+ const { candidates } = build(1);
606
+ expect(candidates[0]!.model.slug).toBe("cheap/flaky");
607
+ });
608
+ });
609
+
@@ -5,6 +5,7 @@ import { parse as parseYaml } from "yaml";
5
5
  import {
6
6
  DEFAULT_ROUTER_URL,
7
7
  newestId,
8
+ providerOf,
8
9
  resolveRouterUrl,
9
10
  selectToasts,
10
11
  toToastText,
@@ -186,7 +187,15 @@ describe("toToastText", () => {
186
187
  expect(toToastText(dec({ reportedUsd: null }))).not.toContain("$");
187
188
  });
188
189
 
189
- test("renders model [tier]", () => {
190
- expect(toToastText(dec({ slug: "q/w", tier: "hard", reportedUsd: null }))).toBe("q/w [hard]");
190
+ test("renders provider · model [tier]", () => {
191
+ expect(toToastText(dec({ slug: "q/w", tier: "hard", reportedUsd: null }))).toBe("openrouter · q/w [hard]");
192
+ });
193
+
194
+ test("an Ollama slug is labelled with its provider and shown without the prefix", () => {
195
+ expect(toToastText(dec({ slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", tier: "moderate", reportedUsd: 0.0007 }))).toBe(
196
+ "ollama · glm-5.3-flash [moderate] · $0.00070",
197
+ );
198
+ expect(providerOf("ollama/gpt-oss:120b")).toEqual({ provider: "ollama", model: "gpt-oss:120b" });
199
+ expect(providerOf("z-ai/glm-5.3-flash")).toEqual({ provider: "openrouter", model: "z-ai/glm-5.3-flash" });
191
200
  });
192
201
  });
@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
3
3
  import { loadConfig } from "../src/config/load.ts";
4
4
  import { createLedger } from "../src/cost/ledger.ts";
5
5
  import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
6
- import { DEFAULT_BYTES_PER_TOKEN, estimatePromptTokens, estimateTokens } from "../src/tokens/estimate.ts";
6
+ import { adjustPendingEstimate, DEFAULT_BYTES_PER_TOKEN, estimatePromptTokens, estimateTokens } from "../src/tokens/estimate.ts";
7
7
  import { openDb } from "../src/util/sqlite.ts";
8
8
  import { parseChatRequest } from "../src/wire/openai/request.ts";
9
9
 
@@ -167,3 +167,73 @@ describe("estimatePromptTokens", () => {
167
167
  expect(estimatePromptTokens(withImage, "gpt", null)).toBeGreaterThan(estimatePromptTokens(text, "gpt", null) + 500);
168
168
  });
169
169
  });
170
+
171
+ describe("calibration hygiene (review 2026-09-05 §8)", () => {
172
+ function requestOf(text: string) {
173
+ return parseChatRequest({ model: "auto", messages: [{ role: "user", content: text }] }, new Headers());
174
+ }
175
+
176
+ test("a provider reporting impossible token counts never calibrates its family", () => {
177
+ const db = openDb(":memory:");
178
+ try {
179
+ const ledger = createLedger(db, cfg);
180
+ const req = requestOf("x".repeat(10_000));
181
+ for (let i = 0; i < 30; i++) {
182
+ estimatePromptTokens(req, "qwen3", ledger);
183
+ // 0.4 bytes/token: ~8x what the bytes imply (seen live from one provider).
184
+ ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 0.4) } }));
185
+ }
186
+ expect(ledger.tokenRatio("qwen3")).toBeNull();
187
+ for (let i = 0; i < 30; i++) {
188
+ estimatePromptTokens(req, "qwen3", ledger);
189
+ ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 3.2) } }));
190
+ }
191
+ expect(ledger.tokenRatio("qwen3")).toBeCloseTo(3.2, 1);
192
+ } finally {
193
+ db.close();
194
+ }
195
+ });
196
+
197
+ test("adjustPendingEstimate calibrates against the dispatched bytes, not the raw request", () => {
198
+ const db = openDb(":memory:");
199
+ try {
200
+ const ledger = createLedger(db, cfg);
201
+ const req = requestOf("y".repeat(10_000));
202
+ for (let i = 0; i < 30; i++) {
203
+ estimatePromptTokens(req, "grok", ledger);
204
+ // Compaction halved the prompt before dispatch; the upstream billed the half.
205
+ adjustPendingEstimate(req.conversationKey, req.promptBytes / 2);
206
+ ledger.record(entry({ conversationKey: req.conversationKey, usage: { ...EMPTY_USAGE, promptTokens: Math.round(req.promptBytes / 2 / 3.5) } }));
207
+ }
208
+ // Paired with the raw bytes this would have learned 7.0; the dispatched bytes give the true 3.5.
209
+ expect(ledger.tokenRatio("grok")).toBeCloseTo(3.5, 1);
210
+ } finally {
211
+ db.close();
212
+ }
213
+ });
214
+ });
215
+
216
+ describe("ledger.escalationCost", () => {
217
+ test("measures what escalated retries bill per prompt token, once enough exist", () => {
218
+ const db = openDb(":memory:");
219
+ try {
220
+ const ledger = createLedger(db, cfg);
221
+ for (let i = 0; i < 9; i++) {
222
+ ledger.record(entry({ attempt: 1, reportedUsd: 0.02, usage: { ...EMPTY_USAGE, promptTokens: 1_000 } }));
223
+ }
224
+ expect(ledger.escalationCost?.(7)).toBeNull(); // 9 < the sample floor
225
+ ledger.record(entry({ attempt: 1, reportedUsd: 0.02, usage: { ...EMPTY_USAGE, promptTokens: 1_000 } }));
226
+ // Errored retries carry no usage and are excluded.
227
+ ledger.record(entry({ attempt: 1, reportedUsd: null, error: "upstream_error: boom", usage: EMPTY_USAGE }));
228
+ // Memoised: a fresh ledger reads through.
229
+ const fresh = createLedger(db, cfg);
230
+ const cost = fresh.escalationCost?.(7);
231
+ expect(cost).not.toBeNull();
232
+ expect(cost!.samples).toBe(10);
233
+ expect(cost!.usdPerPromptToken).toBeCloseTo(0.02 / 1_000, 8);
234
+ } finally {
235
+ db.close();
236
+ }
237
+ });
238
+ });
239
+
@@ -244,11 +244,11 @@ describe("v4 migration", () => {
244
244
  }
245
245
  });
246
246
 
247
- test("schema is at user_version 14", () => {
247
+ test("schema is at user_version 16", () => {
248
248
  const db = openDb(":memory:");
249
249
  try {
250
250
  const row = db.query("PRAGMA user_version").get() as { user_version: number };
251
- expect(row.user_version).toBe(14);
251
+ expect(row.user_version).toBe(16);
252
252
  } finally {
253
253
  db.close();
254
254
  }
package/test/turn.test.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { describe, expect, test } from "bun:test";
2
2
  import { createDisabledBridge } from "../src/context/bridge.ts";
3
3
  import type { ContextBridge, TurnRecord } from "../src/context/types.ts";
4
- import type { CatalogSource } from "../src/catalog/types.ts";
4
+ import type { CatalogModel, CatalogSource } from "../src/catalog/types.ts";
5
5
  import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
6
6
  import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
7
7
  import type {
@@ -31,6 +31,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
31
31
  return {
32
32
  server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
33
33
  openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
34
+ ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0 },
34
35
  benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
35
36
  tiers: {
36
37
  trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
@@ -45,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
45
46
  data: { axis: "intelligence", minQuality: 0 },
46
47
  chat: { axis: "intelligence", minQuality: 0 },
47
48
  },
48
- filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
49
+ filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20, escalationCostWeight: 0 },
49
50
  classifier: {
50
51
  ambiguityThreshold: 0,
51
52
  model: "test/adjudicator",
@@ -56,6 +57,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
56
57
  toolAxis: "coding",
57
58
  chatAxis: "intelligence",
58
59
  agenticLoopDepth: 3,
60
+ mechanicalRetryFactor: 0.2,
59
61
  reasoningWeights: { medium: 0.14, high: 0.24, xhigh: 0.3, max: 0.34 },
60
62
  },
61
63
  escalation: {
@@ -68,11 +70,11 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
68
70
  escalateOnLengthStop: false,
69
71
  ...escalation,
70
72
  },
71
- hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false },
73
+ hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
72
74
  exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
73
75
  cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
74
76
  context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
75
- compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
77
+ compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
76
78
  budget: { onExceeded: "downgrade" },
77
79
  profiles: [],
78
80
  ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
@@ -147,6 +149,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
147
149
  cacheBreakpointMessageIndices: [],
148
150
  compactionPlan: [],
149
151
  promptTokensSaved: 0,
152
+ compactionSavedBytes: 0,
153
+ compactionPlanTokens: 0,
150
154
  reasoning: undefined,
151
155
  maxTokens: undefined,
152
156
  stripAssistantReasoning: false,
@@ -403,10 +407,13 @@ describe("runTurn", () => {
403
407
  expect(entries[1]!.attempt).toBe(1);
404
408
  expect(entries[1]!.slug).toBe("better/model");
405
409
 
406
- // The escalated attempt re-routed one tier up.
407
- expect(calls).toHaveLength(2);
410
+ // A refusal indicts the provider, so a same-tier sibling is probed first;
411
+ // this fake router only has the simple-tier decision left, which is the
412
+ // wrong tier, so the turn then escalates one tier up for real.
413
+ expect(calls).toHaveLength(3);
408
414
  expect(calls[0]).toEqual({ attempt: 0 });
409
- expect(calls[1]).toEqual({ attempt: 1, escalateFrom: "trivial" });
415
+ expect(calls[1]).toEqual({ attempt: 1 });
416
+ expect(calls[2]).toEqual({ attempt: 1, escalateFrom: "trivial" });
410
417
 
411
418
  // The held refusal text never reached the client.
412
419
  expect(textOut(chunks)).toBe("Here is the answer.");
@@ -729,4 +736,114 @@ describe("latency measurement covers the work the router actually does", () => {
729
736
  expect(entries[0]?.ttftMs).not.toBeNull();
730
737
  expect(entries[0]?.ttftMs ?? -1).toBeGreaterThanOrEqual(0);
731
738
  });
739
+
740
+ test("a client hang-up AFTER the finish event is a completed turn, not an error", async () => {
741
+ // Measured live: 1,842 of 2,068 "request aborted" rows carried a finish
742
+ // reason and full usage — the generation had finished and the client
743
+ // closed before the trailing [DONE] was read. Recording that as an error
744
+ // skipped the state save on 12% of turns.
745
+ const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple" })]);
746
+ const { upstream } = mkUpstream([
747
+ {
748
+ kind: "die",
749
+ chunks: [
750
+ startChunk("cheap/model"),
751
+ textChunk("all done here"),
752
+ finishChunk("stop"),
753
+ usageChunk({ promptTokens: 120, cachedTokens: 100, completionTokens: 4 }, 0.0004),
754
+ ],
755
+ error: new UpstreamError("aborted", 0, "request aborted", false),
756
+ },
757
+ ]);
758
+ const { ledger, entries } = mkLedger();
759
+ const { store, map } = mkConversations();
760
+ const { sink, errors, finishes } = mkSink();
761
+
762
+ await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
763
+
764
+ expect(errors).toHaveLength(0);
765
+ expect(entries).toHaveLength(1);
766
+ expect(entries[0]!.error).toBeNull();
767
+ expect(entries[0]!.wasted).toBe(false);
768
+ expect(entries[0]!.finishReason).toBe("stop");
769
+ expect(entries[0]!.reportedUsd).toBe(0.0004);
770
+ // The turn settled: hysteresis, cache warmth and the turn counter advance.
771
+ const state = map.get("conv-test")!;
772
+ expect(state.turn).toBe(1);
773
+ expect(state.currentSlug).toBe("cheap/model");
774
+ expect(state.cacheWarmSlug).toBe("cheap/model");
775
+ expect(state.lastPromptTokens).toBe(120);
776
+ expect(finishes).toHaveLength(1);
777
+ });
778
+
779
+ test("a client hang-up BEFORE the finish event is still recorded as aborted", async () => {
780
+ const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple", maxTokens: 1 })]);
781
+ const { upstream } = mkUpstream([
782
+ {
783
+ kind: "die",
784
+ chunks: [startChunk("cheap/model"), textChunk("partial answer that committed")],
785
+ error: new UpstreamError("aborted", 0, "request aborted", false),
786
+ },
787
+ ]);
788
+ const { ledger, entries } = mkLedger();
789
+ const { store, map } = mkConversations();
790
+ const { sink, finishes } = mkSink();
791
+
792
+ await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
793
+
794
+ expect(entries).toHaveLength(1);
795
+ expect(entries[0]!.error).toBe("request aborted");
796
+ expect(entries[0]!.finishReason).toBeNull();
797
+ expect(map.get("conv-test")!.turn).toBe(0);
798
+ expect(finishes).toHaveLength(0);
799
+ });
800
+
801
+
802
+ test("a provider that reports usage but no cost gets its actual tokens priced at the catalog rate", async () => {
803
+ // Ollama returns usage without `cost`. Leaving the forecast (assumed 1,024
804
+ // completion tokens) as the recorded figure overstated a 26-token turn ~3x.
805
+ const { router } = mkRouter([mkDecision("trivial", "ollama/glm-5.3-flash")]);
806
+ const { upstream } = mkUpstream([
807
+ {
808
+ kind: "chunks",
809
+ chunks: [startChunk("ollama/glm-5.3-flash"), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: 1_000_000, completionTokens: 1_000_000 }, null)],
810
+ },
811
+ ]);
812
+ const { ledger, entries } = mkLedger();
813
+ const { store } = mkConversations();
814
+ const { sink, finishes } = mkSink();
815
+ const priced = {
816
+ ...catalog,
817
+ find: (slug: string) =>
818
+ slug === "ollama/glm-5.3-flash"
819
+ ? ({
820
+ slug,
821
+ provider: "ollama",
822
+ canonicalSlug: slug,
823
+ name: slug,
824
+ contextLength: 1_000_000,
825
+ supportsTools: true,
826
+ supportsReasoning: true,
827
+ reasoningMandatory: false,
828
+ supportsToolChoice: false,
829
+ inputModalities: ["text"],
830
+ price: { prompt: 0.15 / 1e6, completion: 0.5 / 1e6 },
831
+ priceTiers: [],
832
+ quality: {},
833
+ tokenizer: "Other",
834
+ isFree: false,
835
+ createdAtMs: 0,
836
+ author: "ollama",
837
+ } satisfies CatalogModel)
838
+ : undefined,
839
+ };
840
+
841
+ await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog: priced, context: createDisabledBridge() }, new AbortController().signal);
842
+
843
+ // 1M prompt tokens at $0.15/M + 1M completion tokens at $0.50/M.
844
+ expect(entries[0]!.reportedUsd).toBeCloseTo(0.65, 6);
845
+ expect(entries[0]!.priceModel?.slug).toBe("ollama/glm-5.3-flash");
846
+ expect(finishes[0]!.reportedUsd).toBeCloseTo(0.65, 6);
847
+ });
848
+
732
849
  });