auto-model-router 0.2.29 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/pages.yml +50 -0
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +22 -0
- package/package.json +3 -1
- package/site/assets/style.css +108 -0
- package/site/data/benchmarks.json +139 -0
- package/src/config/defaults.ts +6 -0
- package/src/config/schema.ts +1 -0
- package/src/config/types.ts +19 -0
- package/src/router/select.ts +14 -1
- package/test/failover.test.ts +1 -1
- package/test/select.test.ts +78 -0
- package/test/turn.test.ts +1 -1
- package/tools/build-site.ts +384 -0
- package/tools/export-benchmarks.ts +82 -0
- package/tools/replay.ts +83 -16
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: pages
|
|
2
|
+
|
|
3
|
+
# Build the static site (tools/build-site.ts) and publish it to GitHub Pages.
|
|
4
|
+
# Runs on pushes to main that touch the site, and on demand. The build is
|
|
5
|
+
# dependency-free; benchmark numbers come from the committed
|
|
6
|
+
# site/data/benchmarks.json, regenerated locally with `bun run site:data`
|
|
7
|
+
# (CI has no ledger, so it never overwrites that data).
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches: [main]
|
|
11
|
+
paths:
|
|
12
|
+
- 'site/**'
|
|
13
|
+
- 'tools/build-site.ts'
|
|
14
|
+
- '.github/workflows/pages.yml'
|
|
15
|
+
workflow_dispatch:
|
|
16
|
+
|
|
17
|
+
permissions:
|
|
18
|
+
contents: read
|
|
19
|
+
pages: write
|
|
20
|
+
id-token: write
|
|
21
|
+
|
|
22
|
+
# One live deploy at a time; don't cancel an in-flight publish.
|
|
23
|
+
concurrency:
|
|
24
|
+
group: pages
|
|
25
|
+
cancel-in-progress: false
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
build:
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/checkout@v7
|
|
32
|
+
- uses: oven-sh/setup-bun@v2
|
|
33
|
+
with:
|
|
34
|
+
bun-version: latest
|
|
35
|
+
- name: Build site
|
|
36
|
+
run: bun run tools/build-site.ts
|
|
37
|
+
- uses: actions/configure-pages@v6
|
|
38
|
+
- uses: actions/upload-pages-artifact@v5
|
|
39
|
+
with:
|
|
40
|
+
path: site/dist
|
|
41
|
+
|
|
42
|
+
deploy:
|
|
43
|
+
needs: build
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
environment:
|
|
46
|
+
name: github-pages
|
|
47
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
48
|
+
steps:
|
|
49
|
+
- id: deployment
|
|
50
|
+
uses: actions/deploy-pages@v5
|
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.2.
|
|
10
|
+
"version": "0.2.31",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.2.
|
|
17
|
+
"version": "0.2.31",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# auto-model-router
|
|
2
2
|
|
|
3
|
+
**[Website & benchmarks →](https://drewappling.github.io/auto-model-router/)**
|
|
4
|
+
|
|
3
5
|
A local model router for [Oh My Pi](https://github.com/oh-my-pi). It presents
|
|
4
6
|
itself as one keyless OpenAI-compatible provider, then picks a concrete
|
|
5
7
|
OpenRouter model **per turn** based on measured price and estimated task
|
|
@@ -389,6 +391,26 @@ replaces it.
|
|
|
389
391
|
The embedded router reports the key source via its in-process `GET /health`
|
|
390
392
|
(`config` | `env` | `omp-auth-store` | `none`) — never the key itself.
|
|
391
393
|
|
|
394
|
+
### Available models & guardrails
|
|
395
|
+
|
|
396
|
+
The router never ships a hand-curated model list. With a key configured it
|
|
397
|
+
fetches the **key-scoped catalog** (`GET /models/user`) — the exact set of
|
|
398
|
+
models that key is *entitled to* under your account's active
|
|
399
|
+
[OpenRouter guardrails](https://openrouter.ai/docs/guides/features/guardrails),
|
|
400
|
+
provider preferences, and data policies — and routes only within it. Keyless, it
|
|
401
|
+
falls back to the public `/models` for pricing and capability discovery, but
|
|
402
|
+
dispatch still needs a key.
|
|
403
|
+
|
|
404
|
+
Your OpenRouter guardrails — model and provider allowlists, budget limits,
|
|
405
|
+
Zero-Data-Retention and privacy rules — are therefore the router's outer
|
|
406
|
+
boundary: a model your key cannot reach is never a routing candidate. The
|
|
407
|
+
catalog is refetched in the background every `catalogRefreshMs` (default 5 min),
|
|
408
|
+
so tightening or relaxing a guardrail is picked up without a restart. If a
|
|
409
|
+
guardrail narrows the eligible set below a tier's quality floor,
|
|
410
|
+
`adaptiveTierFloors` (on by default) relaxes that tier to the best available
|
|
411
|
+
models rather than leaving it empty — see [Adaptive tier floors](#adaptive-tier-floors)
|
|
412
|
+
and [Tier rescue](#tier-rescue) below.
|
|
413
|
+
|
|
392
414
|
---
|
|
393
415
|
|
|
394
416
|
## How it runs
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "auto-model-router",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.31",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
6
6
|
"type": "module",
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
"typecheck:src": "tsc --noEmit",
|
|
13
13
|
"test": "bun test",
|
|
14
14
|
"smoke": "bun run tools/smoke.ts",
|
|
15
|
+
"site": "bun run tools/build-site.ts",
|
|
16
|
+
"site:data": "bun run tools/export-benchmarks.ts",
|
|
15
17
|
"version": "bun run tools/sync-marketplace-version.ts",
|
|
16
18
|
"release": "npm version $1 && git push --follow-tags"
|
|
17
19
|
},
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/* auto-model-router site. Hand-authored, dependency-free. */
|
|
2
|
+
:root {
|
|
3
|
+
--bg: #0d1117;
|
|
4
|
+
--bg-alt: #161b22;
|
|
5
|
+
--border: #30363d;
|
|
6
|
+
--fg: #e6edf3;
|
|
7
|
+
--fg-dim: #9198a1;
|
|
8
|
+
--accent: #4ea1ff;
|
|
9
|
+
--accent-dim: #1f6feb;
|
|
10
|
+
--good: #3fb950;
|
|
11
|
+
--code-bg: #161b22;
|
|
12
|
+
--max: 900px;
|
|
13
|
+
}
|
|
14
|
+
* { box-sizing: border-box; }
|
|
15
|
+
html { scroll-behavior: smooth; }
|
|
16
|
+
body {
|
|
17
|
+
margin: 0;
|
|
18
|
+
background: var(--bg);
|
|
19
|
+
color: var(--fg);
|
|
20
|
+
font: 16px/1.65 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
21
|
+
-webkit-font-smoothing: antialiased;
|
|
22
|
+
}
|
|
23
|
+
a { color: var(--accent); text-decoration: none; }
|
|
24
|
+
a:hover { text-decoration: underline; }
|
|
25
|
+
code, pre, kbd {
|
|
26
|
+
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
|
|
27
|
+
font-size: 0.9em;
|
|
28
|
+
}
|
|
29
|
+
code { background: var(--code-bg); padding: 0.15em 0.4em; border-radius: 4px; border: 1px solid var(--border); }
|
|
30
|
+
pre {
|
|
31
|
+
background: var(--code-bg);
|
|
32
|
+
border: 1px solid var(--border);
|
|
33
|
+
border-radius: 8px;
|
|
34
|
+
padding: 1rem 1.15rem;
|
|
35
|
+
overflow-x: auto;
|
|
36
|
+
line-height: 1.5;
|
|
37
|
+
}
|
|
38
|
+
pre code { background: none; border: none; padding: 0; }
|
|
39
|
+
|
|
40
|
+
header.nav {
|
|
41
|
+
position: sticky; top: 0; z-index: 10;
|
|
42
|
+
background: rgba(13,17,23,0.85);
|
|
43
|
+
backdrop-filter: blur(8px);
|
|
44
|
+
border-bottom: 1px solid var(--border);
|
|
45
|
+
}
|
|
46
|
+
header.nav .inner {
|
|
47
|
+
max-width: var(--max); margin: 0 auto; padding: 0.75rem 1.25rem;
|
|
48
|
+
display: flex; align-items: center; gap: 1.5rem;
|
|
49
|
+
}
|
|
50
|
+
header.nav .brand { font-weight: 700; color: var(--fg); letter-spacing: -0.02em; }
|
|
51
|
+
header.nav nav { display: flex; gap: 1.15rem; flex-wrap: wrap; }
|
|
52
|
+
header.nav nav a { color: var(--fg-dim); font-size: 0.94rem; }
|
|
53
|
+
header.nav nav a.active, header.nav nav a:hover { color: var(--fg); text-decoration: none; }
|
|
54
|
+
header.nav .spacer { flex: 1; }
|
|
55
|
+
|
|
56
|
+
main { max-width: var(--max); margin: 0 auto; padding: 2.5rem 1.25rem 4rem; }
|
|
57
|
+
|
|
58
|
+
.hero { text-align: center; padding: 3rem 0 2rem; }
|
|
59
|
+
.hero h1 { font-size: 2.6rem; margin: 0 0 0.5rem; letter-spacing: -0.03em; }
|
|
60
|
+
.hero p.tagline { font-size: 1.25rem; color: var(--fg-dim); margin: 0 auto 1.75rem; max-width: 40rem; }
|
|
61
|
+
.hero .cta { display: inline-flex; gap: 0.75rem; flex-wrap: wrap; justify-content: center; }
|
|
62
|
+
.btn {
|
|
63
|
+
display: inline-block; padding: 0.6rem 1.25rem; border-radius: 8px;
|
|
64
|
+
font-weight: 600; border: 1px solid var(--border);
|
|
65
|
+
}
|
|
66
|
+
.btn.primary { background: var(--accent-dim); border-color: var(--accent-dim); color: #fff; }
|
|
67
|
+
.btn.primary:hover { background: var(--accent); text-decoration: none; }
|
|
68
|
+
.btn.ghost { color: var(--fg); }
|
|
69
|
+
.btn.ghost:hover { border-color: var(--fg-dim); text-decoration: none; }
|
|
70
|
+
|
|
71
|
+
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 1rem; margin: 2.5rem 0; }
|
|
72
|
+
.stat { background: var(--bg-alt); border: 1px solid var(--border); border-radius: 10px; padding: 1.25rem; text-align: center; }
|
|
73
|
+
.stat .n { font-size: 2rem; font-weight: 700; color: var(--good); letter-spacing: -0.02em; }
|
|
74
|
+
.stat .l { color: var(--fg-dim); font-size: 0.9rem; margin-top: 0.25rem; }
|
|
75
|
+
|
|
76
|
+
h2 { font-size: 1.6rem; margin: 2.75rem 0 1rem; letter-spacing: -0.02em; padding-top: 0.5rem; }
|
|
77
|
+
h3 { font-size: 1.2rem; margin: 2rem 0 0.75rem; }
|
|
78
|
+
h2:first-child { margin-top: 0; }
|
|
79
|
+
p, ul, ol { margin: 0 0 1rem; }
|
|
80
|
+
ul, ol { padding-left: 1.4rem; }
|
|
81
|
+
li { margin: 0.3rem 0; }
|
|
82
|
+
|
|
83
|
+
table { width: 100%; border-collapse: collapse; margin: 1rem 0 1.5rem; font-size: 0.95rem; }
|
|
84
|
+
th, td { text-align: left; padding: 0.6rem 0.8rem; border-bottom: 1px solid var(--border); }
|
|
85
|
+
th { color: var(--fg-dim); font-weight: 600; }
|
|
86
|
+
td.win { color: var(--good); font-weight: 600; }
|
|
87
|
+
tbody tr:hover { background: var(--bg-alt); }
|
|
88
|
+
table caption { text-align: left; color: var(--fg-dim); font-size: 0.9rem; margin-bottom: 0.6rem; caption-side: bottom; }
|
|
89
|
+
|
|
90
|
+
.card { background: var(--bg-alt); border: 1px solid var(--border); border-radius: 10px; padding: 1.25rem 1.4rem; margin: 1rem 0; }
|
|
91
|
+
.card h3 { margin-top: 0; }
|
|
92
|
+
.note { color: var(--fg-dim); font-size: 0.95rem; }
|
|
93
|
+
.pill { display: inline-block; background: var(--bg-alt); border: 1px solid var(--border); border-radius: 999px; padding: 0.15rem 0.7rem; font-size: 0.8rem; color: var(--fg-dim); }
|
|
94
|
+
|
|
95
|
+
dl.knobs { margin: 0; }
|
|
96
|
+
dl.knobs dt { font-family: ui-monospace, monospace; color: var(--accent); margin-top: 1.1rem; font-size: 0.95rem; }
|
|
97
|
+
dl.knobs dd { margin: 0.25rem 0 0; color: var(--fg); }
|
|
98
|
+
dl.knobs dd .default { color: var(--fg-dim); font-size: 0.88rem; }
|
|
99
|
+
|
|
100
|
+
footer { border-top: 1px solid var(--border); color: var(--fg-dim); font-size: 0.9rem; }
|
|
101
|
+
footer .inner { max-width: var(--max); margin: 0 auto; padding: 2rem 1.25rem; display: flex; gap: 1rem; flex-wrap: wrap; }
|
|
102
|
+
footer .spacer { flex: 1; }
|
|
103
|
+
|
|
104
|
+
@media (max-width: 600px) {
|
|
105
|
+
.hero h1 { font-size: 2rem; }
|
|
106
|
+
header.nav .inner { gap: 1rem; }
|
|
107
|
+
main { padding-top: 1.5rem; }
|
|
108
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
{
|
|
2
|
+
"generatedAt": "2026-08-29",
|
|
3
|
+
"baseline": "claude-opus-5",
|
|
4
|
+
"headline": {
|
|
5
|
+
"coreCostMultiple": "26.5×",
|
|
6
|
+
"coreCostMultipleLabel": "cheaper at identical correctness",
|
|
7
|
+
"realWorldMultiple": "≈15×",
|
|
8
|
+
"realWorldSavedPct": "~93%"
|
|
9
|
+
},
|
|
10
|
+
"suites": {
|
|
11
|
+
"core": {
|
|
12
|
+
"title": "Core suite — 10 coding tasks × 3 trials",
|
|
13
|
+
"note": "Both engines solved everything, so this measures cost at equal correctness. 26.5× cheaper — in fewer turns, fewer tool calls, and 19 minutes less wall clock. The one regression is time to first token: a routed turn pays for classification and dispatch before anything streams.",
|
|
14
|
+
"columns": [
|
|
15
|
+
"",
|
|
16
|
+
"auto-model-router",
|
|
17
|
+
"Claude Opus 5"
|
|
18
|
+
],
|
|
19
|
+
"rows": [
|
|
20
|
+
[
|
|
21
|
+
"Tasks solved",
|
|
22
|
+
"30 / 30",
|
|
23
|
+
"30 / 30"
|
|
24
|
+
],
|
|
25
|
+
[
|
|
26
|
+
"Total cost",
|
|
27
|
+
"$0.63",
|
|
28
|
+
"$16.61"
|
|
29
|
+
],
|
|
30
|
+
[
|
|
31
|
+
"Cost per solved task",
|
|
32
|
+
"$0.0209",
|
|
33
|
+
"$0.5538"
|
|
34
|
+
],
|
|
35
|
+
[
|
|
36
|
+
"Turns to finish",
|
|
37
|
+
"278",
|
|
38
|
+
"303"
|
|
39
|
+
],
|
|
40
|
+
[
|
|
41
|
+
"Tool calls",
|
|
42
|
+
"265",
|
|
43
|
+
"337"
|
|
44
|
+
],
|
|
45
|
+
[
|
|
46
|
+
"Wall clock",
|
|
47
|
+
"2 057 s",
|
|
48
|
+
"3 185 s"
|
|
49
|
+
],
|
|
50
|
+
[
|
|
51
|
+
"Median time to first token",
|
|
52
|
+
"5 776 ms",
|
|
53
|
+
"1 490 ms"
|
|
54
|
+
]
|
|
55
|
+
],
|
|
56
|
+
"winnerCol": 1
|
|
57
|
+
},
|
|
58
|
+
"ladder": {
|
|
59
|
+
"title": "Difficulty ladder — 7 rungs, run twice",
|
|
60
|
+
"note": "A second suite of deliberately escalating difficulty, ending in npm semver range semantics and a minimal diff with a specified tie-break. At the top of the ladder the engines separate.",
|
|
61
|
+
"columns": [
|
|
62
|
+
"",
|
|
63
|
+
"auto-model-router",
|
|
64
|
+
"Claude Opus 5"
|
|
65
|
+
],
|
|
66
|
+
"rows": [
|
|
67
|
+
[
|
|
68
|
+
"Run 1",
|
|
69
|
+
"5 / 7 · $0.30",
|
|
70
|
+
"5 / 7 · $6.25"
|
|
71
|
+
],
|
|
72
|
+
[
|
|
73
|
+
"Run 2",
|
|
74
|
+
"5 / 7 · $0.46",
|
|
75
|
+
"6 / 7 · $6.60"
|
|
76
|
+
]
|
|
77
|
+
],
|
|
78
|
+
"winnerCol": 1
|
|
79
|
+
},
|
|
80
|
+
"routed": {
|
|
81
|
+
"title": "What it routed to",
|
|
82
|
+
"note": "Across 464 routed turns in all five runs. Tier escalation converts to a costlier model roughly one-for-one; the escalation target is chosen live from trust and latency history, so it differs between runs on the same catalog.",
|
|
83
|
+
"columns": [
|
|
84
|
+
"Model",
|
|
85
|
+
"Turns",
|
|
86
|
+
"Input price",
|
|
87
|
+
"Role"
|
|
88
|
+
],
|
|
89
|
+
"rows": [
|
|
90
|
+
[
|
|
91
|
+
"z-ai/glm-5.3-flash",
|
|
92
|
+
"389 (84%)",
|
|
93
|
+
"$0.07 / MTok",
|
|
94
|
+
"default"
|
|
95
|
+
],
|
|
96
|
+
[
|
|
97
|
+
"google/gemini-3.7-flash",
|
|
98
|
+
"56 (12%)",
|
|
99
|
+
"$0.75 / MTok",
|
|
100
|
+
"escalation target"
|
|
101
|
+
],
|
|
102
|
+
[
|
|
103
|
+
"x-ai/grok-4.6",
|
|
104
|
+
"18 (4%)",
|
|
105
|
+
"$2.00 / MTok",
|
|
106
|
+
"escalation target"
|
|
107
|
+
]
|
|
108
|
+
]
|
|
109
|
+
},
|
|
110
|
+
"realWorld": {
|
|
111
|
+
"title": "Real-world — a week on the live ledger",
|
|
112
|
+
"note": "6 918 billed turns across 299 conversations, 7 days, 410:1 input-to-output, 68% cache hit — the identical token stream repriced against a single Opus 5 model with its own cache namespace. ≈15× cheaper, ~93% saved: a four-figure monthly bill becomes a three-figure one.",
|
|
113
|
+
"columns": [
|
|
114
|
+
"",
|
|
115
|
+
"auto-model-router",
|
|
116
|
+
"Claude Opus 5 (single-model)"
|
|
117
|
+
],
|
|
118
|
+
"rows": [
|
|
119
|
+
[
|
|
120
|
+
"Spend over the week",
|
|
121
|
+
"$61.69",
|
|
122
|
+
"$921.20"
|
|
123
|
+
],
|
|
124
|
+
[
|
|
125
|
+
"Per turn",
|
|
126
|
+
"$0.0089",
|
|
127
|
+
"$0.133"
|
|
128
|
+
],
|
|
129
|
+
[
|
|
130
|
+
"Extrapolated / month",
|
|
131
|
+
"$263",
|
|
132
|
+
"$3 932"
|
|
133
|
+
]
|
|
134
|
+
],
|
|
135
|
+
"winnerCol": 1
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
"ledgerSnapshot": null
|
|
139
|
+
}
|
package/src/config/defaults.ts
CHANGED
|
@@ -20,6 +20,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
20
20
|
baseUrl: "https://openrouter.ai/api/v1",
|
|
21
21
|
// May stay empty: catalog and `config` work keyless; only dispatch fails.
|
|
22
22
|
apiKey: "",
|
|
23
|
+
// App attribution for OpenRouter's Activity/Apps ranking. `title` is the
|
|
24
|
+
// display name; `referer` is the identity OpenRouter groups requests by.
|
|
25
|
+
referer: "https://github.com/drewappling/auto-model-router",
|
|
23
26
|
title: "auto-model-router",
|
|
24
27
|
// Agent turns are long; a frontier model with tools can stream for minutes.
|
|
25
28
|
timeoutMs: 600_000,
|
|
@@ -135,6 +138,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
135
138
|
// OpenRouter sticky sessions expire in 5-10 minutes.
|
|
136
139
|
cacheWarmTtlMs: 300_000,
|
|
137
140
|
maxDowngradePerTurn: 1,
|
|
141
|
+
// Off: breaking a hold means a model switch, which costs a cache write.
|
|
142
|
+
// Enable where the held tier is expensive; see HysteresisConfig.
|
|
143
|
+
breakHoldOnMechanical: false,
|
|
138
144
|
},
|
|
139
145
|
exploration: {
|
|
140
146
|
// Opt-in. Exploration knowingly routes some turns below the tier that
|
package/src/config/schema.ts
CHANGED
|
@@ -111,6 +111,7 @@ const hysteresis = z.strictObject({
|
|
|
111
111
|
switchMargin: z.number().positive().optional(),
|
|
112
112
|
cacheWarmTtlMs: z.number().nonnegative().optional(),
|
|
113
113
|
maxDowngradePerTurn: z.number().int().nonnegative().optional(),
|
|
114
|
+
breakHoldOnMechanical: z.boolean().optional(),
|
|
114
115
|
});
|
|
115
116
|
|
|
116
117
|
const exploration = z.strictObject({
|
package/src/config/types.ts
CHANGED
|
@@ -287,6 +287,25 @@ export interface HysteresisConfig {
|
|
|
287
287
|
cacheWarmTtlMs: number;
|
|
288
288
|
/** Downgrade at most this many tiers per turn, so quality never falls off a cliff. */
|
|
289
289
|
maxDowngradePerTurn: number;
|
|
290
|
+
/**
|
|
291
|
+
* Let a mechanical tool-result continuation escape a hold that sits above its
|
|
292
|
+
* own classification.
|
|
293
|
+
*
|
|
294
|
+
* A hold bets that the next turn resembles the one that armed it, and it is
|
|
295
|
+
* usually right — flapping cold-starts prompt caches. But a continuation the
|
|
296
|
+
* classifier has already docked for being a mechanical next step, and whose
|
|
297
|
+
* score lands below the held tier, is evidence against that bet. Measured on
|
|
298
|
+
* 24h of live traffic: 37 of 44 sticky `hard` dispatches were exactly that,
|
|
299
|
+
* one scoring 0.154 (trivial) yet served by claude-opus-5 — $2.66 billed
|
|
300
|
+
* against $0.05 for the identical tokens on the moderate pick.
|
|
301
|
+
*
|
|
302
|
+
* Off by default: breaking a hold means a model switch, and switching costs a
|
|
303
|
+
* cache write. Worth it when the held tier is expensive, not obviously worth
|
|
304
|
+
* it when the tiers are close, so it is opt-in per deployment.
|
|
305
|
+
* `maxDowngradePerTurn` still applies, so quality steps down rather than
|
|
306
|
+
* falling off a cliff.
|
|
307
|
+
*/
|
|
308
|
+
breakHoldOnMechanical: boolean;
|
|
290
309
|
}
|
|
291
310
|
|
|
292
311
|
/**
|
package/src/router/select.ts
CHANGED
|
@@ -117,10 +117,23 @@ export function select(args: SelectArgs): Decision {
|
|
|
117
117
|
|
|
118
118
|
// 2. Hysteresis: while the sticky window is open, never route below the
|
|
119
119
|
// held tier — per-turn flapping would repeatedly cold-start prompt caches.
|
|
120
|
+
//
|
|
121
|
+
// Exception, when `breakHoldOnMechanical` is on: a hold is a bet that the
|
|
122
|
+
// NEXT turn resembles the one that armed it. A tool-result continuation
|
|
123
|
+
// whose own score lands below the held tier is direct evidence against
|
|
124
|
+
// that bet — the classifier already docks it for being a mechanical next
|
|
125
|
+
// step — so paying the held tier for it buys nothing. Measured on 24h of
|
|
126
|
+
// live traffic: 37 of 44 sticky `hard` dispatches were exactly this shape,
|
|
127
|
+
// one scoring 0.154 (trivial) yet served by claude-opus-5; $2.66 billed
|
|
128
|
+
// against $0.05 for the identical tokens on the moderate pick.
|
|
120
129
|
let cls = classification;
|
|
130
|
+
const mechanicalOverride =
|
|
131
|
+
cfg.hysteresis.breakHoldOnMechanical && features.isToolResultContinuation && tierIdx(effective) < tierIdx(clampTier(state.currentTier ?? effective));
|
|
121
132
|
if (state.stickyUntilTurn > state.turn && state.currentTier !== null && tierIdx(state.currentTier) >= tierIdx(effective)) {
|
|
122
133
|
const held = clampTier(state.currentTier);
|
|
123
|
-
if (
|
|
134
|
+
if (mechanicalOverride) {
|
|
135
|
+
reasons.push(`hysteresis hold ${held} broken: mechanical tool-result continuation classified ${effective}`);
|
|
136
|
+
} else if (held !== effective) {
|
|
124
137
|
reasons.push(`hysteresis: holding ${held} until turn ${state.stickyUntilTurn} (classified ${effective})`);
|
|
125
138
|
cls = {
|
|
126
139
|
...classification,
|
package/test/failover.test.ts
CHANGED
|
@@ -67,7 +67,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
67
67
|
escalateOnLengthStop: false,
|
|
68
68
|
...escalation,
|
|
69
69
|
},
|
|
70
|
-
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
70
|
+
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false },
|
|
71
71
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
72
72
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
73
73
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
package/test/select.test.ts
CHANGED
|
@@ -743,3 +743,81 @@ describe("context compaction", () => {
|
|
|
743
743
|
expect(tight.promptTokensSaved).toBeGreaterThan(loose.promptTokensSaved);
|
|
744
744
|
});
|
|
745
745
|
});
|
|
746
|
+
|
|
747
|
+
describe("hysteresis.breakHoldOnMechanical", () => {
|
|
748
|
+
// A hold bets the next turn resembles the one that armed it. A tool-result
|
|
749
|
+
// continuation the classifier has already docked, scoring below the held
|
|
750
|
+
// tier, is evidence against that bet. Measured on 24h of live traffic: 37 of
|
|
751
|
+
// 44 sticky `hard` dispatches were exactly that shape — one scoring 0.154
|
|
752
|
+
// (trivial) yet served by claude-opus-5 — $2.66 billed against $0.05 for the
|
|
753
|
+
// same tokens on the moderate pick.
|
|
754
|
+
function continuation(): NormRequest {
|
|
755
|
+
return parseChatRequest(
|
|
756
|
+
{
|
|
757
|
+
model: "auto",
|
|
758
|
+
tools: TOOLS,
|
|
759
|
+
messages: [
|
|
760
|
+
{ role: "system", content: "You are a coding agent." },
|
|
761
|
+
{ role: "user", content: "read the file" },
|
|
762
|
+
{ role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"a.ts"}' } }] },
|
|
763
|
+
{ role: "tool", tool_call_id: "c1", content: "export const x = 1;" },
|
|
764
|
+
],
|
|
765
|
+
},
|
|
766
|
+
new Headers(),
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
const held = state({ currentTier: "hard", currentSlug: "x-ai/grok-4.6", stickyUntilTurn: 9, turn: 1 });
|
|
771
|
+
|
|
772
|
+
function decide(req: NormRequest, breakHold: boolean) {
|
|
773
|
+
const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, breakHoldOnMechanical: breakHold } };
|
|
774
|
+
const features = extractFeatures(req, 4_000);
|
|
775
|
+
return { d: select({ req, features, classification: scoreHeuristic(features, cfg), profile: PROFILE, state: held, snapshot: SNAPSHOT, ledger: null, cfg, nowMs: Date.now() }), features };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
test("off by default, so a hold still pins the tier", () => {
|
|
779
|
+
expect(BASE.hysteresis.breakHoldOnMechanical).toBe(false);
|
|
780
|
+
const { d, features } = decide(continuation(), false);
|
|
781
|
+
expect(features.isToolResultContinuation).toBe(true);
|
|
782
|
+
expect(d.tier).toBe("hard");
|
|
783
|
+
expect(d.classification.source).toBe("sticky");
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
test("on, a mechanical continuation escapes the hold", () => {
|
|
787
|
+
const { d } = decide(continuation(), true);
|
|
788
|
+
expect(d.tier).not.toBe("hard");
|
|
789
|
+
expect(d.classification.source).not.toBe("sticky");
|
|
790
|
+
expect(d.reasons.some((r) => /hold hard broken/.test(r))).toBe(true);
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
test("a NON-mechanical turn still gets the hold, so flap protection survives", () => {
|
|
794
|
+
// This is the case hysteresis exists for: fresh user work mid-conversation
|
|
795
|
+
// must not bounce the model and cold-start its cache.
|
|
796
|
+
const { d, features } = decide(request("now refactor the retry helper"), true);
|
|
797
|
+
expect(features.isToolResultContinuation).toBe(false);
|
|
798
|
+
expect(d.tier).toBe("hard");
|
|
799
|
+
expect(d.classification.source).toBe("sticky");
|
|
800
|
+
});
|
|
801
|
+
|
|
802
|
+
test("the downgrade clamp still applies, so quality steps rather than falls", () => {
|
|
803
|
+
const cfg: RouterConfig = {
|
|
804
|
+
...BASE,
|
|
805
|
+
hysteresis: { ...BASE.hysteresis, breakHoldOnMechanical: true, maxDowngradePerTurn: 1 },
|
|
806
|
+
};
|
|
807
|
+
const req = continuation();
|
|
808
|
+
const features = extractFeatures(req, 4_000);
|
|
809
|
+
// Force the fresh classification far below the hold to exercise the clamp.
|
|
810
|
+
const d = select({
|
|
811
|
+
req,
|
|
812
|
+
features,
|
|
813
|
+
classification: { ...scoreHeuristic(features, cfg), tier: "trivial" },
|
|
814
|
+
profile: PROFILE,
|
|
815
|
+
state: held,
|
|
816
|
+
snapshot: SNAPSHOT,
|
|
817
|
+
ledger: null,
|
|
818
|
+
cfg,
|
|
819
|
+
nowMs: Date.now(),
|
|
820
|
+
});
|
|
821
|
+
expect(d.tier).toBe("moderate");
|
|
822
|
+
});
|
|
823
|
+
});
|
package/test/turn.test.ts
CHANGED
|
@@ -68,7 +68,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
68
|
escalateOnLengthStop: false,
|
|
69
69
|
...escalation,
|
|
70
70
|
},
|
|
71
|
-
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
71
|
+
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false },
|
|
72
72
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
73
73
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
74
74
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Static site generator for the auto-model-router GitHub Pages site.
|
|
4
|
+
*
|
|
5
|
+
* Dependency-free by design: content is authored as HTML in this file and the
|
|
6
|
+
* only dynamic input is `site/data/benchmarks.json`, which the head-to-head
|
|
7
|
+
* suite tables render from and which `tools/export-benchmarks.ts` regenerates
|
|
8
|
+
* from a live ledger at release time. Emits `site/dist/`, ready for
|
|
9
|
+
* `actions/upload-pages-artifact`.
|
|
10
|
+
*
|
|
11
|
+
* Run by hand: `bun tools/build-site.ts` (then open site/dist/index.html).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { cpSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { join, resolve } from "node:path";
|
|
16
|
+
|
|
17
|
+
const ROOT = resolve(import.meta.dir, "..");
|
|
18
|
+
const SITE = join(ROOT, "site");
|
|
19
|
+
const DIST = join(SITE, "dist");
|
|
20
|
+
const REPO = "https://github.com/drewappling/auto-model-router";
|
|
21
|
+
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// Benchmark data
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
interface SuiteTable {
|
|
27
|
+
title: string;
|
|
28
|
+
note: string;
|
|
29
|
+
columns: string[];
|
|
30
|
+
rows: string[][];
|
|
31
|
+
winnerCol?: number;
|
|
32
|
+
}
|
|
33
|
+
interface LedgerSnapshot {
|
|
34
|
+
generatedAt: string;
|
|
35
|
+
windowDays: number | null;
|
|
36
|
+
requests: number;
|
|
37
|
+
spendAllTimeUsd: number;
|
|
38
|
+
spend7dUsd: number;
|
|
39
|
+
perTurnUsd: number;
|
|
40
|
+
escalationRatePct: number;
|
|
41
|
+
perModel: { slug: string; requests: number; sharePct: number }[];
|
|
42
|
+
}
|
|
43
|
+
interface Benchmarks {
|
|
44
|
+
generatedAt: string;
|
|
45
|
+
baseline: string;
|
|
46
|
+
headline: {
|
|
47
|
+
coreCostMultiple: string;
|
|
48
|
+
coreCostMultipleLabel: string;
|
|
49
|
+
realWorldMultiple: string;
|
|
50
|
+
realWorldSavedPct: string;
|
|
51
|
+
};
|
|
52
|
+
suites: { core: SuiteTable; ladder: SuiteTable; routed: SuiteTable; realWorld: SuiteTable };
|
|
53
|
+
ledgerSnapshot: LedgerSnapshot | null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const bench = JSON.parse(readFileSync(join(SITE, "data", "benchmarks.json"), "utf8")) as Benchmarks;
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// HTML helpers
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
function esc(s: string): string {
|
|
63
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function suiteTable(t: SuiteTable): string {
|
|
67
|
+
const head = t.columns.map((c) => `<th>${esc(c)}</th>`).join("");
|
|
68
|
+
const body = t.rows
|
|
69
|
+
.map((row) => {
|
|
70
|
+
const cells = row
|
|
71
|
+
.map((cell, i) => {
|
|
72
|
+
const win = t.winnerCol !== undefined && i === t.winnerCol && i > 0;
|
|
73
|
+
return `<td${win ? ' class="win"' : ""}>${esc(cell)}</td>`;
|
|
74
|
+
})
|
|
75
|
+
.join("");
|
|
76
|
+
return `<tr>${cells}</tr>`;
|
|
77
|
+
})
|
|
78
|
+
.join("\n");
|
|
79
|
+
return `<h3>${esc(t.title)}</h3>
|
|
80
|
+
<table>
|
|
81
|
+
<thead><tr>${head}</tr></thead>
|
|
82
|
+
<tbody>
|
|
83
|
+
${body}
|
|
84
|
+
</tbody>
|
|
85
|
+
</table>
|
|
86
|
+
<p class="note">${esc(t.note)}</p>`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function ledgerPanel(s: LedgerSnapshot | null): string {
|
|
90
|
+
if (s === null) {
|
|
91
|
+
return `<p class="note">No live ledger snapshot is bundled with this build. Maintainers regenerate one with <code>bun tools/export-benchmarks.ts</code> against a real install before a release.</p>`;
|
|
92
|
+
}
|
|
93
|
+
const window = s.windowDays === null ? "all time" : `last ${s.windowDays} days`;
|
|
94
|
+
const rows = s.perModel
|
|
95
|
+
.map((m) => `<tr><td><code>${esc(m.slug)}</code></td><td>${m.requests}</td><td>${m.sharePct.toFixed(1)}%</td></tr>`)
|
|
96
|
+
.join("\n");
|
|
97
|
+
return `<p class="note">Generated ${esc(s.generatedAt)} from a real install's ledger (${window}).</p>
|
|
98
|
+
<div class="stats">
|
|
99
|
+
<div class="stat"><div class="n">${s.requests.toLocaleString()}</div><div class="l">billed turns</div></div>
|
|
100
|
+
<div class="stat"><div class="n">$${s.perTurnUsd.toFixed(4)}</div><div class="l">per turn</div></div>
|
|
101
|
+
<div class="stat"><div class="n">$${s.spend7dUsd.toFixed(2)}</div><div class="l">spend, 7 days</div></div>
|
|
102
|
+
<div class="stat"><div class="n">${s.escalationRatePct.toFixed(1)}%</div><div class="l">escalation rate</div></div>
|
|
103
|
+
</div>
|
|
104
|
+
<table>
|
|
105
|
+
<thead><tr><th>Model</th><th>Requests</th><th>Spend share</th></tr></thead>
|
|
106
|
+
<tbody>
|
|
107
|
+
${rows}
|
|
108
|
+
</tbody>
|
|
109
|
+
</table>`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
// Layout
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
interface Page {
|
|
117
|
+
slug: string; // "" for index
|
|
118
|
+
title: string;
|
|
119
|
+
nav: string;
|
|
120
|
+
body: string;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const NAV: { href: string; label: string; key: string }[] = [
|
|
124
|
+
{ href: "index.html", label: "Overview", key: "home" },
|
|
125
|
+
{ href: "install.html", label: "Install", key: "install" },
|
|
126
|
+
{ href: "config.html", label: "Configuration", key: "config" },
|
|
127
|
+
{ href: "benchmarks.html", label: "Benchmarks", key: "benchmarks" },
|
|
128
|
+
];
|
|
129
|
+
|
|
130
|
+
function layout(p: Page): string {
|
|
131
|
+
const nav = NAV.map(
|
|
132
|
+
(n) => `<a href="${n.href}"${n.key === p.nav ? ' class="active"' : ""}>${esc(n.label)}</a>`,
|
|
133
|
+
).join("\n ");
|
|
134
|
+
return `<!doctype html>
|
|
135
|
+
<html lang="en">
|
|
136
|
+
<head>
|
|
137
|
+
<meta charset="utf-8">
|
|
138
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
139
|
+
<title>${esc(p.title)}</title>
|
|
140
|
+
<meta name="description" content="A local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Well over an order of magnitude cheaper at equal correctness.">
|
|
141
|
+
<link rel="stylesheet" href="assets/style.css">
|
|
142
|
+
</head>
|
|
143
|
+
<body>
|
|
144
|
+
<header class="nav">
|
|
145
|
+
<div class="inner">
|
|
146
|
+
<a class="brand" href="index.html">auto-model-router</a>
|
|
147
|
+
<div class="spacer"></div>
|
|
148
|
+
<nav>
|
|
149
|
+
${nav}
|
|
150
|
+
<a href="${REPO}">GitHub</a>
|
|
151
|
+
</nav>
|
|
152
|
+
</div>
|
|
153
|
+
</header>
|
|
154
|
+
<main>
|
|
155
|
+
${p.body}
|
|
156
|
+
</main>
|
|
157
|
+
<footer>
|
|
158
|
+
<div class="inner">
|
|
159
|
+
<span>auto-model-router \u2014 MIT licensed</span>
|
|
160
|
+
<div class="spacer"></div>
|
|
161
|
+
<a href="${REPO}">GitHub</a>
|
|
162
|
+
<a href="https://www.npmjs.com/package/auto-model-router">npm</a>
|
|
163
|
+
<a href="${REPO}/issues">Issues</a>
|
|
164
|
+
</div>
|
|
165
|
+
</footer>
|
|
166
|
+
</body>
|
|
167
|
+
</html>
|
|
168
|
+
`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ---------------------------------------------------------------------------
|
|
172
|
+
// Pages
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
const indexBody = `<section class="hero">
|
|
176
|
+
<h1>The right model for every turn</h1>
|
|
177
|
+
<p class="tagline">A local, keyless model router for <a href="https://github.com/oh-my-pi">Oh My Pi</a>. One OpenAI-compatible provider that picks a concrete OpenRouter model <strong>per turn</strong> from measured price and estimated task complexity \u2014 including mid-conversation.</p>
|
|
178
|
+
<div class="cta">
|
|
179
|
+
<a class="btn primary" href="install.html">Get started</a>
|
|
180
|
+
<a class="btn ghost" href="benchmarks.html">See the benchmarks</a>
|
|
181
|
+
</div>
|
|
182
|
+
</section>
|
|
183
|
+
|
|
184
|
+
<div class="stats">
|
|
185
|
+
<div class="stat"><div class="n">${esc(bench.headline.coreCostMultiple)}</div><div class="l">${esc(bench.headline.coreCostMultipleLabel)}</div></div>
|
|
186
|
+
<div class="stat"><div class="n">${esc(bench.headline.realWorldMultiple)}</div><div class="l">cheaper on a real week of traffic</div></div>
|
|
187
|
+
<div class="stat"><div class="n">${esc(bench.headline.realWorldSavedPct)}</div><div class="l">of spend saved</div></div>
|
|
188
|
+
</div>
|
|
189
|
+
|
|
190
|
+
<h2>Why this exists when OpenRouter already ships routers</h2>
|
|
191
|
+
<p>OpenRouter has <code>openrouter/auto</code> and <code>openrouter/pareto-code</code>. Both are opaque, server-side, and \u2014 per Pareto's own docs \u2014 <em>"you can't directly cap cost or latency per request."</em> This router does the things a prompt classifier structurally cannot:</p>
|
|
192
|
+
<div class="card">
|
|
193
|
+
<h3>Agent-loop awareness</h3>
|
|
194
|
+
<p class="note">OpenRouter sees a prompt. We see omp's tool array, tool-result depth, and whether the previous tool call failed. Most agent turns are mechanical post-tool-result continuations \u2014 the largest cost lever in agent traffic, and invisible upstream.</p>
|
|
195
|
+
</div>
|
|
196
|
+
<div class="card">
|
|
197
|
+
<h3>Budget enforcement</h3>
|
|
198
|
+
<p class="note">Per-turn, per-conversation, and rolling-24h caps, checked against a <strong>cold-cache forecast</strong> before dispatch, with forced downgrade at the ceiling.</p>
|
|
199
|
+
</div>
|
|
200
|
+
<div class="card">
|
|
201
|
+
<h3>Mid-stream escalation</h3>
|
|
202
|
+
<p class="note">Hold the first N tokens; on a malformed tool call, refusal, empty completion, or repeated tool call, abort and re-dispatch upward. omp never observes the failure.</p>
|
|
203
|
+
</div>
|
|
204
|
+
<div class="card">
|
|
205
|
+
<h3>Cache-aware hysteresis</h3>
|
|
206
|
+
<p class="note">Switching models forfeits the warm prompt cache. The decision is arithmetic, not vibes: expected saving must beat the forfeited cache-read discount by a configured margin.</p>
|
|
207
|
+
</div>
|
|
208
|
+
<div class="card">
|
|
209
|
+
<h3>Closed-loop trust</h3>
|
|
210
|
+
<p class="note">Per-model escalation and error rates from <em>your</em> traffic demote cheap-but-flaky models automatically.</p>
|
|
211
|
+
</div>
|
|
212
|
+
<div class="card">
|
|
213
|
+
<h3>Explainability</h3>
|
|
214
|
+
<p class="note">Every decision \u2014 candidates, rejections, forecasts, reasons \u2014 is persisted and replayable via <code>auto-model-router explain</code>.</p>
|
|
215
|
+
</div>
|
|
216
|
+
|
|
217
|
+
<h2>How it runs</h2>
|
|
218
|
+
<p>auto-model-router runs <strong>embedded inside the omp process</strong> as an extension \u2014 no separate server, no orphaned process. It binds a free OS-assigned port and lives and dies with the omp session. For non-omp harnesses (Hermes, Claude, any OpenAI-compatible client), run it standalone with <code>auto-model-router serve --port <n></code>.</p>
|
|
219
|
+
<p><a href="install.html">Install it →</a></p>`;
|
|
220
|
+
|
|
221
|
+
const installBody = `<h2>Installing</h2>
|
|
222
|
+
<p>No separate Bun install is needed for the embedded path. The standalone <code>serve</code> binary bundles Bun.</p>
|
|
223
|
+
|
|
224
|
+
<h3>Via npm (recommended)</h3>
|
|
225
|
+
<pre><code>npm install -g auto-model-router</code></pre>
|
|
226
|
+
<p>Then add the shipped extensions to omp's <code>~/.omp/agent/config.yml</code> (<code>$PI_CODING_AGENT_DIR/config.yml</code> when that env var relocates the agent dir):</p>
|
|
227
|
+
<pre><code># ~/.omp/agent/config.yml
|
|
228
|
+
extensions:
|
|
229
|
+
- auto-model-router/omp-extension/router-embed.ts
|
|
230
|
+
- auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
|
|
231
|
+
- auto-model-router/omp-extension/router-configure.ts # optional: /router command</code></pre>
|
|
232
|
+
|
|
233
|
+
<h3>From the repo (cross-platform installer)</h3>
|
|
234
|
+
<pre><code>bun tools/install.ts</code></pre>
|
|
235
|
+
<p>It wires the extensions into omp's <code>~/.omp/agent/config.yml</code>, backing up the previous file first. It is idempotent. Use <code>--no-toast --no-configure</code> for only the required embed extension.</p>
|
|
236
|
+
|
|
237
|
+
<h3>From the marketplace</h3>
|
|
238
|
+
<p>This repo doubles as its own marketplace. Add it as a source, then install:</p>
|
|
239
|
+
<pre><code>omp plugin marketplace add drewappling/auto-model-router
|
|
240
|
+
omp plugin install auto-model-router@auto-model-router</code></pre>
|
|
241
|
+
<p>Or in the TUI: <code>/marketplace add drewappling/auto-model-router</code> then <code>/marketplace install auto-model-router@auto-model-router</code>.</p>
|
|
242
|
+
|
|
243
|
+
<h3>As a Pi package</h3>
|
|
244
|
+
<pre><code>pi install npm:auto-model-router
|
|
245
|
+
# or from git:
|
|
246
|
+
pi install git:github.com/drewappling/auto-model-router</code></pre>
|
|
247
|
+
|
|
248
|
+
<h2>Setup \u2014 the OpenRouter key</h2>
|
|
249
|
+
<p>There is exactly one OpenRouter key on the machine, owned by omp. Once you have run <code>/login openrouter</code> inside omp, auto-model-router borrows that key with no config and no second copy to rotate or leak. Alternatively set <code>OPENROUTER_API_KEY</code> in the environment, or <code>openrouter.apiKey</code> in <code>config.yml</code>.</p>
|
|
250
|
+
<p class="note">The catalog and the <code>config</code> command work keyless; only dispatch needs a key.</p>
|
|
251
|
+
|
|
252
|
+
<h2>Available models & guardrails</h2>
|
|
253
|
+
<p>The router never ships a hand-curated model list. When an OpenRouter key is configured it fetches the key-scoped catalog (<code>GET /models/user</code>) \u2014 the exact set of models that key is <strong>entitled to</strong> under your account's active <a href="https://openrouter.ai/docs/guides/features/guardrails">guardrails</a>, provider preferences, and data policies \u2014 and routes only within it. Keyless, it falls back to the public catalog for pricing and capability discovery, but dispatch still needs a key.</p>
|
|
254
|
+
<p>This means your OpenRouter <a href="https://openrouter.ai/docs/guides/features/guardrails">guardrails</a> \u2014 model and provider allowlists, budget limits, Zero-Data-Retention and privacy rules \u2014 are the router's outer boundary: a model your key cannot reach is never a routing candidate. The catalog is refetched in the background every few minutes, so tightening or relaxing a guardrail is picked up without a restart.</p>
|
|
255
|
+
<div class="card">
|
|
256
|
+
<p class="note"><strong>Narrow guardrails still route.</strong> If a guardrail shrinks the eligible set so far that a complexity tier's quality floor admits nothing, <code>adaptiveTierFloors</code> (on by default) relaxes that tier's economic envelope to the best available models rather than leaving it empty \u2014 so the router keeps working on a tightly restricted key instead of stalling on the cheapest tier.</p>
|
|
257
|
+
</div>
|
|
258
|
+
|
|
259
|
+
<h2>Activating it</h2>
|
|
260
|
+
<p>After installing, <strong>restart the omp session</strong> (extensions load at session start), then run <code>/model</code> and pick <code>auto-model-router/auto</code>.</p>
|
|
261
|
+
<div class="card">
|
|
262
|
+
<p class="note"><strong>Note on updates.</strong> The embedded router is long-lived per omp session and reads its config and code at boot. Config changes to hot-reloadable knobs apply live; changes to the listening socket, the OpenRouter client, or the agentdox bridge require a session restart.</p>
|
|
263
|
+
</div>
|
|
264
|
+
|
|
265
|
+
<h2>Standalone (Hermes / any OpenAI-compatible client)</h2>
|
|
266
|
+
<pre><code>auto-model-router serve --port 8788</code></pre>
|
|
267
|
+
<p>Register it as a plain OpenAI-compatible provider pointing at <code>http://127.0.0.1:8788/v1</code>. No API key is enforced unless you set <code>server.apiKey</code>.</p>
|
|
268
|
+
|
|
269
|
+
<h2>Configuring behaviour</h2>
|
|
270
|
+
<p>Every routing lever lives in <code>$AUTO_MODEL_ROUTER_HOME/config.yml</code>. See the <a href="config.html">configuration reference</a> for the knobs and their shipped defaults.</p>`;
|
|
271
|
+
|
|
272
|
+
const configBody = `<h2>Configuration reference</h2>
|
|
273
|
+
<p>auto-model-router is configured through <code>$AUTO_MODEL_ROUTER_HOME/config.yml</code> (defaults to <code>~/.auto-model-router/config.yml</code>). The file is a deep-partial overlay on the built-in defaults: set only the keys you want to change. Most per-turn knobs <strong>hot-reload</strong> \u2014 edits apply on the next turn with no restart. The listening socket (<code>server.*</code>), the OpenRouter client (<code>openrouter.*</code>), and the agentdox bridge (<code>context.*</code>) are captured at boot and need a session restart.</p>
|
|
274
|
+
<p class="pill">All values below are the shipped defaults.</p>
|
|
275
|
+
|
|
276
|
+
<h3>openrouter \u2014 upstream & attribution</h3>
|
|
277
|
+
<dl class="knobs">
|
|
278
|
+
<dt>openrouter.apiKey</dt><dd>OpenRouter key. The router routes only within the models this key is entitled to under your <a href="https://openrouter.ai/docs/guides/features/guardrails">OpenRouter guardrails</a> (fetched via <code>/models/user</code>). <span class="default">Default: empty \u2014 borrowed from omp's credential store, or <code>OPENROUTER_API_KEY</code>.</span></dd>
|
|
279
|
+
<dt>openrouter.title / openrouter.referer</dt><dd>App attribution for OpenRouter's Activity/Apps ranking. <code>title</code> is the display name; <code>referer</code> is the identity requests are grouped by. <span class="default">Default: <code>auto-model-router</code> and the project URL.</span></dd>
|
|
280
|
+
<dt>openrouter.timeoutMs</dt><dd>Per-request timeout. Agent turns are long. <span class="default">Default: 600000 (10 min).</span></dd>
|
|
281
|
+
</dl>
|
|
282
|
+
|
|
283
|
+
<h3>tiers \u2014 the complexity ladder</h3>
|
|
284
|
+
<p>Each complexity tier sets a quality floor and a price ceiling. A model priced above a tier's ceiling is excluded before ranking; within the tier, <code>score = (quality/100) ^ qualityExponent / effectiveUsd</code> picks the winner. <code>hard</code> has no ceiling \u2014 quality is the point of the top tier.</p>
|
|
285
|
+
<dl class="knobs">
|
|
286
|
+
<dt>tiers.trivial</dt><dd>minQuality 0, maxInputPerMtok $0.30, qualityExponent 0 <span class="default">(cheapest above the floor).</span></dd>
|
|
287
|
+
<dt>tiers.simple</dt><dd>minQuality 40, maxInputPerMtok $1.50, qualityExponent 0.</dd>
|
|
288
|
+
<dt>tiers.moderate</dt><dd>minQuality 60, maxInputPerMtok $4.00, qualityExponent 1.</dd>
|
|
289
|
+
<dt>tiers.hard</dt><dd>minQuality 72, no price ceiling, qualityExponent 3.</dd>
|
|
290
|
+
<dt>tiers.<tier>.capabilityFloorUsd</dt><dd>Optional. Pick the highest-quality candidate whose cold-cache cost fits this cap, ignoring quality-per-dollar. Buys quality with money deliberately. <span class="default">Default: unset.</span></dd>
|
|
291
|
+
<dt>tiers.<tier>.pin</dt><dd>Force a specific slug set for the tier. <span class="default">Default: none.</span></dd>
|
|
292
|
+
</dl>
|
|
293
|
+
|
|
294
|
+
<h3>filters \u2014 the eligible catalog</h3>
|
|
295
|
+
<dl class="knobs">
|
|
296
|
+
<dt>filters.includeFree</dt><dd>Include $0 models. <span class="default">Default: false \u2014 free models are rate-limited enough that retries cost more than they save.</span></dd>
|
|
297
|
+
<dt>filters.requireToolSupport</dt><dd><span class="default">Default: true.</span></dd>
|
|
298
|
+
<dt>filters.minTrust / minTrustSamples</dt><dd>Demote models whose measured reliability falls below the floor once enough samples exist. <span class="default">Default: 0.7 over 12 samples.</span></dd>
|
|
299
|
+
<dt>filters.contextHeadroom</dt><dd>Require a context window this multiple of the estimated prompt. <span class="default">Default: 1.25.</span></dd>
|
|
300
|
+
<dt>filters.latencyWeight</dt><dd>Inflate a model's effective cost by expected wait (TTFT + completion time). <span class="default">Default: 0 (off) \u2014 opt in after establishing a baseline.</span></dd>
|
|
301
|
+
</dl>
|
|
302
|
+
|
|
303
|
+
<h3>escalation \u2014 mid-stream recovery</h3>
|
|
304
|
+
<dl class="knobs">
|
|
305
|
+
<dt>escalation.enabled</dt><dd><span class="default">Default: true.</span></dd>
|
|
306
|
+
<dt>escalation.probeTokens</dt><dd>Hold this many tokens before committing, to catch a bad start. <span class="default">Default: 48.</span></dd>
|
|
307
|
+
<dt>escalation.maxAttempts</dt><dd>Original try plus retries. Each retry beyond the first can abandon generated tokens. <span class="default">Default: 3.</span></dd>
|
|
308
|
+
<dt>escalation.triggers</dt><dd>malformed_tool_args, refusal, empty_completion, repeat_tool_call, missing_expected_tool_call.</dd>
|
|
309
|
+
<dt>escalation.probeTiers</dt><dd>trivial, simple, moderate \u2014 never <code>hard</code>, which has nowhere to escalate to.</dd>
|
|
310
|
+
</dl>
|
|
311
|
+
|
|
312
|
+
<h3>hysteresis \u2014 cache-aware stickiness</h3>
|
|
313
|
+
<dl class="knobs">
|
|
314
|
+
<dt>hysteresis.holdTurns / holdTurnsAfterEscalation</dt><dd>Hold the current tier for N turns to protect the warm cache. <span class="default">Default: 2, and 4 after an escalation.</span></dd>
|
|
315
|
+
<dt>hysteresis.switchMargin</dt><dd>Expected saving must beat the forfeited cache discount by this factor to switch. <span class="default">Default: 1.3.</span></dd>
|
|
316
|
+
<dt>hysteresis.maxDowngradePerTurn</dt><dd>Step tiers down at most this fast. <span class="default">Default: 1.</span></dd>
|
|
317
|
+
<dt>hysteresis.breakHoldOnMechanical</dt><dd>Let a mechanical tool-result continuation break a hold that sits above the fresh classification. <span class="default">Default: false.</span></dd>
|
|
318
|
+
</dl>
|
|
319
|
+
|
|
320
|
+
<h3>budget \u2014 spend caps</h3>
|
|
321
|
+
<dl class="knobs">
|
|
322
|
+
<dt>budget.perTurnUsd / perConversationUsd / rolling24hUsd</dt><dd>Optional ceilings, checked against the cold-cache forecast before dispatch. <span class="default">Default: no caps.</span></dd>
|
|
323
|
+
<dt>budget.onExceeded</dt><dd><code>downgrade</code> or <code>fail</code> at the ceiling. <span class="default">Default: downgrade.</span></dd>
|
|
324
|
+
</dl>
|
|
325
|
+
|
|
326
|
+
<h3>context \u2014 agentdox bridge (restart to change)</h3>
|
|
327
|
+
<dl class="knobs">
|
|
328
|
+
<dt>context.enabled</dt><dd>Inject one shared project-context block per conversation. <span class="default">Default: false \u2014 needs a URL and token.</span></dd>
|
|
329
|
+
<dt>context.baseUrl / token / defaultScope</dt><dd>agentdox endpoint, bearer, and fallback project scope.</dd>
|
|
330
|
+
<dt>context.memoryLimit / docsLimit / sessionLimit / briefChars</dt><dd>Bound what the server selects, so the block is ranked rather than byte-truncated. <span class="default">Default: 8 / 2 / 6 / 12000, inside a 24000-char cap.</span></dd>
|
|
331
|
+
</dl>
|
|
332
|
+
|
|
333
|
+
<h3>compaction \u2014 prompt shrinking</h3>
|
|
334
|
+
<dl class="knobs">
|
|
335
|
+
<dt>compaction.enabled</dt><dd>Shrink stale, low-value context before dispatch. <span class="default">Default: false \u2014 elision is lossy, never implicit.</span></dd>
|
|
336
|
+
<dt>compaction.budgetTokens</dt><dd>Fire above this prompt size. <span class="default">Default: 40000.</span></dd>
|
|
337
|
+
<dt>compaction.floorRatio</dt><dd>Compact to this fraction of the budget. Below 1 overshoots and holds the plan (cache-friendly); 1 re-tightens every turn. <span class="default">Default: 1; 0.75 recommended once you have watched your ledger.</span></dd>
|
|
338
|
+
</dl>
|
|
339
|
+
|
|
340
|
+
<h2>Inspecting decisions</h2>
|
|
341
|
+
<pre><code>auto-model-router stats # spend and per-model distribution
|
|
342
|
+
auto-model-router explain # candidates, rejections, forecasts for the last turn
|
|
343
|
+
auto-model-router models # the eligible catalog per tier</code></pre>
|
|
344
|
+
<p class="note">The full type surface and every field's doc-comment live in <a href="${REPO}/blob/main/src/config/types.ts"><code>src/config/types.ts</code></a>.</p>`;
|
|
345
|
+
|
|
346
|
+
const benchmarksBody = `<h2>Benchmarks</h2>
|
|
347
|
+
<p>Measured against Claude Opus 5 on Anthropic first-party. Each task is a real omp session working in a pristine git workspace from a written spec; hidden tests are copied in only <em>after</em> the agent exits, so they cannot be read or edited. Every task is verified to fail an untouched workspace and to pass a reference solution. Both arms are metered from omp's own event stream under an identical tool surface. The router arm routes freely \u2014 nothing pinned.</p>
|
|
348
|
+
<p class="pill">Data generated ${esc(bench.generatedAt)} \u00b7 baseline <code>${esc(bench.baseline)}</code></p>
|
|
349
|
+
|
|
350
|
+
${suiteTable(bench.suites.core)}
|
|
351
|
+
${suiteTable(bench.suites.ladder)}
|
|
352
|
+
${suiteTable(bench.suites.routed)}
|
|
353
|
+
${suiteTable(bench.suites.realWorld)}
|
|
354
|
+
|
|
355
|
+
<h2>Live ledger snapshot</h2>
|
|
356
|
+
${ledgerPanel(bench.ledgerSnapshot)}
|
|
357
|
+
|
|
358
|
+
<h2>Scope & honesty</h2>
|
|
359
|
+
<p class="note">These are small, self-contained tasks of one to three files. On the core suite both engines solved everything, so it measures cost at equal correctness rather than capability; the ladder is where capability separates. The cost multiple varied between 14\u00d7 and 32\u00d7 across runs depending on which task the baseline stalled on \u2014 treat "well over an order of magnitude" as the claim, not a specific figure. Full harness, tasks, and raw per-turn data are in <a href="${REPO}/blob/main/docs/routing-benchmark-findings.md"><code>docs/routing-benchmark-findings.md</code></a>.</p>`;
|
|
360
|
+
|
|
361
|
+
const PAGES: Page[] = [
|
|
362
|
+
{ slug: "index", title: "auto-model-router \u2014 the right model for every turn", nav: "home", body: indexBody },
|
|
363
|
+
{ slug: "install", title: "Install \u2014 auto-model-router", nav: "install", body: installBody },
|
|
364
|
+
{ slug: "config", title: "Configuration \u2014 auto-model-router", nav: "config", body: configBody },
|
|
365
|
+
{ slug: "benchmarks", title: "Benchmarks \u2014 auto-model-router", nav: "benchmarks", body: benchmarksBody },
|
|
366
|
+
];
|
|
367
|
+
|
|
368
|
+
// ---------------------------------------------------------------------------
|
|
369
|
+
// Emit
|
|
370
|
+
// ---------------------------------------------------------------------------
|
|
371
|
+
|
|
372
|
+
function main(): void {
|
|
373
|
+
rmSync(DIST, { recursive: true, force: true });
|
|
374
|
+
mkdirSync(DIST, { recursive: true });
|
|
375
|
+
for (const p of PAGES) {
|
|
376
|
+
writeFileSync(join(DIST, `${p.slug}.html`), layout(p), "utf8");
|
|
377
|
+
}
|
|
378
|
+
cpSync(join(SITE, "assets"), join(DIST, "assets"), { recursive: true });
|
|
379
|
+
// .nojekyll: the artifact is already built HTML; skip GitHub's Jekyll pass.
|
|
380
|
+
writeFileSync(join(DIST, ".nojekyll"), "", "utf8");
|
|
381
|
+
console.log(`built ${PAGES.length} pages \u2192 ${DIST}`);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
main();
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Regenerates the `ledgerSnapshot` section of `site/data/benchmarks.json` from a
|
|
4
|
+
* real install's ledger, so the published site can show live routing economics
|
|
5
|
+
* instead of only the authored head-to-head suites.
|
|
6
|
+
*
|
|
7
|
+
* It reuses `computeStats` \u2014 the exact aggregation behind the `stats` command
|
|
8
|
+
* and the `/stats` endpoint \u2014 so the site can never drift from what the tool
|
|
9
|
+
* reports. The authored `suites` block is preserved untouched; only
|
|
10
|
+
* `ledgerSnapshot` is rewritten.
|
|
11
|
+
*
|
|
12
|
+
* Intended to run at release time, on the machine that holds the live ledger,
|
|
13
|
+
* and to commit the refreshed JSON alongside the version bump. When no ledger
|
|
14
|
+
* exists the snapshot is set to null and the site renders a "no snapshot"
|
|
15
|
+
* placeholder rather than fabricating numbers.
|
|
16
|
+
*
|
|
17
|
+
* bun tools/export-benchmarks.ts # all-time snapshot
|
|
18
|
+
* bun tools/export-benchmarks.ts --days 7 # last 7 days
|
|
19
|
+
* bun tools/export-benchmarks.ts --db path.db # a specific ledger
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { join, resolve } from "node:path";
|
|
24
|
+
import { loadConfig } from "../src/config/load.ts";
|
|
25
|
+
import { createLedger } from "../src/cost/ledger.ts";
|
|
26
|
+
import { computeStats } from "../src/server/http.ts";
|
|
27
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
28
|
+
|
|
29
|
+
const ROOT = resolve(import.meta.dir, "..");
|
|
30
|
+
const DATA_PATH = join(ROOT, "site", "data", "benchmarks.json");
|
|
31
|
+
|
|
32
|
+
function parseDays(argv: string[]): number | undefined {
|
|
33
|
+
const i = argv.indexOf("--days");
|
|
34
|
+
if (i === -1) return undefined;
|
|
35
|
+
const n = Number(argv[i + 1]);
|
|
36
|
+
if (!Number.isFinite(n) || n <= 0) throw new Error(`--days needs a positive number, got ${argv[i + 1]}`);
|
|
37
|
+
return n;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function parseDb(argv: string[]): string | undefined {
|
|
41
|
+
const i = argv.indexOf("--db");
|
|
42
|
+
return i === -1 ? undefined : argv[i + 1];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const argv = process.argv.slice(2);
|
|
46
|
+
const days = parseDays(argv);
|
|
47
|
+
const cfg = loadConfig({});
|
|
48
|
+
const dbPath = parseDb(argv) ?? cfg.ledger.path;
|
|
49
|
+
|
|
50
|
+
const data = JSON.parse(readFileSync(DATA_PATH, "utf8")) as Record<string, unknown>;
|
|
51
|
+
|
|
52
|
+
if (!existsSync(dbPath)) {
|
|
53
|
+
console.error(`no ledger at ${dbPath}; setting ledgerSnapshot to null`);
|
|
54
|
+
data.ledgerSnapshot = null;
|
|
55
|
+
writeFileSync(DATA_PATH, `${JSON.stringify(data, null, 2)}\n`, "utf8");
|
|
56
|
+
process.exit(0);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const db = openDb(dbPath);
|
|
60
|
+
try {
|
|
61
|
+
const stats = computeStats(createLedger(db, cfg), days === undefined ? {} : { windowDays: days });
|
|
62
|
+
const perTurnUsd = stats.requests > 0 ? stats.windowSpendUsd / stats.requests : 0;
|
|
63
|
+
data.ledgerSnapshot = {
|
|
64
|
+
generatedAt: new Date(stats.generatedAtMs).toISOString().slice(0, 10),
|
|
65
|
+
windowDays: stats.windowDays,
|
|
66
|
+
requests: stats.requests,
|
|
67
|
+
spendAllTimeUsd: Number(stats.spendAllTimeUsd.toFixed(2)),
|
|
68
|
+
spend7dUsd: Number(stats.spend7dUsd.toFixed(2)),
|
|
69
|
+
perTurnUsd: Number(perTurnUsd.toFixed(4)),
|
|
70
|
+
escalationRatePct: Number((stats.escalationRate * 100).toFixed(1)),
|
|
71
|
+
// Top models by spend share; the long tail adds noise, not signal.
|
|
72
|
+
perModel: stats.perModel.slice(0, 8).map((m) => ({
|
|
73
|
+
slug: m.slug,
|
|
74
|
+
requests: m.requests,
|
|
75
|
+
sharePct: Number((m.share * 100).toFixed(1)),
|
|
76
|
+
})),
|
|
77
|
+
};
|
|
78
|
+
writeFileSync(DATA_PATH, `${JSON.stringify(data, null, 2)}\n`, "utf8");
|
|
79
|
+
console.log(`ledgerSnapshot \u2190 ${stats.requests} turns from ${dbPath} (${stats.windowDays === null ? "all time" : `${stats.windowDays}d`})`);
|
|
80
|
+
} finally {
|
|
81
|
+
db.close();
|
|
82
|
+
}
|
package/tools/replay.ts
CHANGED
|
@@ -35,8 +35,12 @@
|
|
|
35
35
|
* - `messages` are not recorded, so compaction cannot be re-planned. Replay
|
|
36
36
|
* forces `compaction.enabled=false` and feeds the POST-compaction prompt
|
|
37
37
|
* size (`usage.promptTokens`), i.e. the prompt selection actually saw.
|
|
38
|
-
* -
|
|
39
|
-
*
|
|
38
|
+
* - Hysteresis holds ARE modelled: the window is re-armed after each replayed
|
|
39
|
+
* decision exactly as `turn.ts` does, and evolved PER VARIANT so a change
|
|
40
|
+
* that stops arming an expensive tier also drops the holds that followed it.
|
|
41
|
+
* What remains absent is escalation-lengthened holds, since replay does not
|
|
42
|
+
* retry, and `hold_arm` exploration draws are reproduced from the
|
|
43
|
+
* conversation key rather than read back from the row.
|
|
40
44
|
* - `requestedReasoning` IS recorded and is now used. It was previously forced
|
|
41
45
|
* to undefined here on the belief the ledger omitted it, which under-scored
|
|
42
46
|
* ~42% of dispatches and reproduced 27 hard decisions against 120 served.
|
|
@@ -63,6 +67,7 @@ import { computeCost } from "../src/cost/forecast.ts";
|
|
|
63
67
|
import { createLedger } from "../src/cost/ledger.ts";
|
|
64
68
|
import type { UsageCounts } from "../src/cost/types.ts";
|
|
65
69
|
import { scoreHeuristic } from "../src/router/classify.ts";
|
|
70
|
+
import { resolveHoldTurns } from "../src/router/explore.ts";
|
|
66
71
|
import { select } from "../src/router/select.ts";
|
|
67
72
|
import type { ConversationState, Decision, Features, Tier } from "../src/router/types.ts";
|
|
68
73
|
import type { UpstreamClient } from "../src/upstream/types.ts";
|
|
@@ -140,6 +145,7 @@ interface Row {
|
|
|
140
145
|
reported_usd: number | null;
|
|
141
146
|
predicted_usd: number;
|
|
142
147
|
created_at_ms: number;
|
|
148
|
+
error_kind: string | null;
|
|
143
149
|
}
|
|
144
150
|
|
|
145
151
|
/**
|
|
@@ -201,17 +207,24 @@ function requestOf(row: Row, f: Features): NormRequest {
|
|
|
201
207
|
* prompt size. Deriving state from the RECORDED outcome rather than the
|
|
202
208
|
* replayed one also stops replay error compounding down a conversation.
|
|
203
209
|
*
|
|
204
|
-
*
|
|
205
|
-
* the
|
|
210
|
+
* `stickyUntilTurn` and `currentTier` are the exception: they are SIMULATED per
|
|
211
|
+
* variant, by re-arming the hold exactly as `turn.ts` does after each replayed
|
|
212
|
+
* decision. Without that, replay never held a tier and every hysteresis change
|
|
213
|
+
* priced as zero.
|
|
206
214
|
*/
|
|
207
|
-
function stateOf(row: Row, prior: PriorTurn | undefined): ConversationState {
|
|
215
|
+
function stateOf(row: Row, prior: PriorTurn | undefined, hold: HoldState | undefined): ConversationState {
|
|
208
216
|
return {
|
|
209
217
|
key: row.conversation_key,
|
|
210
218
|
sessionId: `omp-${row.conversation_key}`,
|
|
211
|
-
turn
|
|
219
|
+
// `turn.ts` computes turnNumber = state.turn + 1 and records THAT, so the
|
|
220
|
+
// state `select` sees carries the PREVIOUS turn number. Passing row.turn
|
|
221
|
+
// would expire every hold a turn early.
|
|
222
|
+
turn: row.turn - 1,
|
|
212
223
|
currentSlug: prior?.slug ?? null,
|
|
213
|
-
|
|
214
|
-
|
|
224
|
+
// Tier and hold window come from THIS VARIANT's own history (see
|
|
225
|
+
// HoldState); everything else comes from the recorded outcome.
|
|
226
|
+
currentTier: hold?.tier ?? ((prior?.tier as Tier | undefined) ?? null),
|
|
227
|
+
stickyUntilTurn: hold?.stickyUntilTurn ?? 0,
|
|
215
228
|
escalations: 0,
|
|
216
229
|
spentUsd: prior?.spentUsd ?? 0,
|
|
217
230
|
lastPromptTokens: prior?.promptTokens ?? 0,
|
|
@@ -235,6 +248,24 @@ interface PriorTurn {
|
|
|
235
248
|
atMs: number;
|
|
236
249
|
}
|
|
237
250
|
|
|
251
|
+
/**
|
|
252
|
+
* Hysteresis state, evolved PER VARIANT.
|
|
253
|
+
*
|
|
254
|
+
* A hold is a consequence of the decisions a variant made, so A and B must each
|
|
255
|
+
* carry their own: if both read the recorded holds, a change that stops arming
|
|
256
|
+
* `hard` would still be charged for the holds that followed it in production,
|
|
257
|
+
* and the change would price as smaller than it is.
|
|
258
|
+
*
|
|
259
|
+
* This is the one place replay departs from "inputs come from the recorded
|
|
260
|
+
* outcome". The cost is that hold state compounds a variant's own replay error
|
|
261
|
+
* down a conversation; the benefit is that hold policy becomes measurable at
|
|
262
|
+
* all, which it was not.
|
|
263
|
+
*/
|
|
264
|
+
interface HoldState {
|
|
265
|
+
tier: Tier | null;
|
|
266
|
+
stickyUntilTurn: number;
|
|
267
|
+
}
|
|
268
|
+
|
|
238
269
|
/**
|
|
239
270
|
* Re-prices a decision against the tokens the turn ACTUALLY used, via the real
|
|
240
271
|
* `computeCost` so price tiers, the cache split and reasoning/request fees are
|
|
@@ -281,7 +312,7 @@ const predicate = args.where === "" ? "" : ` AND (${args.where})`;
|
|
|
281
312
|
const rows = (
|
|
282
313
|
db
|
|
283
314
|
.query(
|
|
284
|
-
`SELECT id, conversation_key, turn, requested_model, harness_id, served_slug, tier, features, usage, reported_usd, predicted_usd, created_at_ms
|
|
315
|
+
`SELECT id, conversation_key, turn, requested_model, harness_id, served_slug, tier, features, usage, reported_usd, predicted_usd, created_at_ms, error_kind
|
|
285
316
|
FROM ledger
|
|
286
317
|
WHERE features IS NOT NULL AND wasted = 0${predicate}
|
|
287
318
|
ORDER BY created_at_ms DESC LIMIT ?`,
|
|
@@ -307,23 +338,49 @@ interface Outcome {
|
|
|
307
338
|
tier: Tier;
|
|
308
339
|
slug: string;
|
|
309
340
|
usd: number;
|
|
341
|
+
/** Whether the hysteresis hold bound this dispatch, for reporting. */
|
|
342
|
+
held: boolean;
|
|
343
|
+
/** Hold state to carry into this variant's next dispatch. */
|
|
344
|
+
hold: HoldState;
|
|
310
345
|
}
|
|
311
346
|
|
|
312
|
-
function run(cfg: RouterConfig, row: Row, usage: UsageCounts, prior: PriorTurn | undefined): Outcome {
|
|
347
|
+
function run(cfg: RouterConfig, row: Row, usage: UsageCounts, prior: PriorTurn | undefined, hold: HoldState | undefined): Outcome {
|
|
313
348
|
const f = featuresOf(row, usage.promptTokens);
|
|
314
349
|
const req = requestOf(row, f);
|
|
350
|
+
const state = stateOf(row, prior, hold);
|
|
315
351
|
const decision: Decision = select({
|
|
316
352
|
req,
|
|
317
353
|
features: f,
|
|
318
354
|
classification: scoreHeuristic(f, cfg),
|
|
319
355
|
profile: profileOf(cfg, row.requested_model),
|
|
320
|
-
state
|
|
356
|
+
state,
|
|
321
357
|
snapshot: catalogSnapshot,
|
|
322
358
|
ledger,
|
|
323
359
|
cfg,
|
|
324
360
|
nowMs: Date.now(),
|
|
325
361
|
});
|
|
326
|
-
|
|
362
|
+
|
|
363
|
+
// Re-arm exactly as turn.ts does: only when the served tier CHANGED, because
|
|
364
|
+
// re-arming every turn extends the window forever and the router then never
|
|
365
|
+
// downgrades. `escalated` is false — replay does not model escalation
|
|
366
|
+
// retries, so escalation-lengthened holds are still absent.
|
|
367
|
+
// Only a dispatch that reaches the COMMIT path re-arms, as in turn.ts: an
|
|
368
|
+
// aborted one never gets there, and 27% of rows abort (omp closing the
|
|
369
|
+
// stream once it has the tool calls). Re-arming on those inflated the hold
|
|
370
|
+
// count roughly 4x against what production recorded.
|
|
371
|
+
const committed = row.error_kind === null;
|
|
372
|
+
const tierChanged = committed && (hold?.tier ?? null) !== decision.tier;
|
|
373
|
+
const next: HoldState = tierChanged
|
|
374
|
+
? { tier: decision.tier, stickyUntilTurn: row.turn + resolveHoldTurns(cfg, row.conversation_key, false).turns }
|
|
375
|
+
: { tier: committed ? decision.tier : (hold?.tier ?? null), stickyUntilTurn: hold?.stickyUntilTurn ?? 0 };
|
|
376
|
+
|
|
377
|
+
return {
|
|
378
|
+
tier: decision.tier,
|
|
379
|
+
slug: decision.slug,
|
|
380
|
+
usd: repriceUsd(bySlug.get(decision.slug), usage),
|
|
381
|
+
held: decision.classification.source === "sticky",
|
|
382
|
+
hold: next,
|
|
383
|
+
};
|
|
327
384
|
}
|
|
328
385
|
|
|
329
386
|
const tallyA = new Map<string, number>();
|
|
@@ -344,13 +401,23 @@ const bump = (m: Map<string, number>, k: string) => m.set(k, (m.get(k) ?? 0) + 1
|
|
|
344
401
|
// Carries the RECORDED outcome of each conversation's previous dispatch forward,
|
|
345
402
|
// so cache warmth and the prior slug are real rather than assumed absent.
|
|
346
403
|
const priorByConv = new Map<string, PriorTurn>();
|
|
404
|
+
// Hold state is per VARIANT, since a hold follows from that variant's own
|
|
405
|
+
// decisions. See HoldState.
|
|
406
|
+
const holdA = new Map<string, HoldState>();
|
|
407
|
+
const holdB = new Map<string, HoldState>();
|
|
408
|
+
let heldA = 0;
|
|
409
|
+
let heldB = 0;
|
|
347
410
|
|
|
348
411
|
for (const row of rows) {
|
|
349
412
|
const u = JSON.parse(row.usage) as UsageCounts;
|
|
350
413
|
if (!(u.promptTokens > 0)) continue;
|
|
351
414
|
const prior = priorByConv.get(row.conversation_key);
|
|
352
|
-
const a = run(cfgA, row, u, prior);
|
|
353
|
-
const b = run(cfgB, row, u, prior);
|
|
415
|
+
const a = run(cfgA, row, u, prior, holdA.get(row.conversation_key));
|
|
416
|
+
const b = run(cfgB, row, u, prior, holdB.get(row.conversation_key));
|
|
417
|
+
holdA.set(row.conversation_key, a.hold);
|
|
418
|
+
holdB.set(row.conversation_key, b.hold);
|
|
419
|
+
if (a.held) heldA++;
|
|
420
|
+
if (b.held) heldB++;
|
|
354
421
|
priorByConv.set(row.conversation_key, {
|
|
355
422
|
slug: row.served_slug,
|
|
356
423
|
tier: row.tier,
|
|
@@ -386,8 +453,8 @@ console.log(`variant B overrides: ${args.setB.length ? args.setB.join(" ") : "(n
|
|
|
386
453
|
console.log(`\nFIDELITY vs what actually ran:`);
|
|
387
454
|
console.log(` same model ${fidelitySlug}/${comparable} (${pct(fidelitySlug, comparable)}%) same tier ${fidelityTier}/${comparable} (${pct(fidelityTier, comparable)}%)`);
|
|
388
455
|
console.log(" Divergence is expected where code has changed since those rows were served");
|
|
389
|
-
console.log(" (replay runs CURRENT code); the rest is
|
|
390
|
-
console.log(
|
|
456
|
+
console.log(" (replay runs CURRENT code); the rest is what replay cannot model.");
|
|
457
|
+
console.log(` hysteresis holds bound ${heldA} dispatches in A, ${heldB} in B (simulated per variant).`);
|
|
391
458
|
|
|
392
459
|
function table(label: string, rec: Map<string, number>, A: Map<string, number>, B: Map<string, number>) {
|
|
393
460
|
const keys = [...new Set([...rec.keys(), ...A.keys(), ...B.keys()])].sort((x, y) => (B.get(y) ?? 0) - (B.get(x) ?? 0));
|