omp-llm-role 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -6
- package/docs/dev/architecture.md +42 -19
- package/docs/dev/ci.md +9 -0
- package/docs/dev/current-state.md +126 -20
- package/docs/dev/explorer.md +126 -23
- package/docs/dev/releasing.md +66 -3
- package/docs/dev/scoring.md +76 -8
- package/docs/dev/testing.md +105 -14
- package/docs/dev/writes-and-state.md +161 -26
- package/docs/llm-role-rankings.md +115 -115
- package/package.json +19 -1
- package/src/agent-create.ts +4 -2
- package/src/availability.ts +40 -7
- package/src/engine.ts +33 -2
- package/src/explorer/boot.ts +43 -1
- package/src/explorer/explain.ts +101 -11
- package/src/explorer/levers.ts +100 -0
- package/src/explorer/pool.ts +46 -0
- package/src/explorer/server.ts +92 -21
- package/src/extension.ts +132 -2
- package/src/settings.ts +50 -0
- package/src/updater.ts +132 -33
- package/src/usage-ledger.ts +127 -0
- package/web/app.js +139 -1
- package/web/style.css +3 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
llm-stats.com best-fit ranking per omp model role — 408 models, 2026-10-
|
|
1
|
+
llm-stats.com best-fit ranking per omp model role — 408 models, 2026-10-19
|
|
2
2
|
Value ranking per role: each metric is cardinal-normalized with fixed anchors
|
|
3
3
|
(no ranks): llm-stats index_* affine (v+20)/80 (interval scale, observed −16..+60),
|
|
4
4
|
benchmarks chance-anchored pass rates, throughput log-anchored 10..300 tok/s.
|
|
@@ -21,179 +21,179 @@ agon=Design Arena `agents/agon_webapps` Elo from the same endpoint: context
|
|
|
21
21
|
only, unweighted (22/87 of the designer pool, below the ~35–40% coverage
|
|
22
22
|
bar at which a metric earns weight).
|
|
23
23
|
★ = Pareto-frontier: no eligible model is both cheaper and better (q).
|
|
24
|
-
Throughput + price: OpenRouter per-provider routes under the default price-based routing (weight 1/price² over stable standard-tier providers; p50 tok/s of routed traffic; $/M 3:1 in:out), throughput
|
|
24
|
+
Throughput + price: OpenRouter per-provider routes under the default price-based routing (weight 1/price² over stable standard-tier providers; p50 tok/s of routed traffic; $/M 3:1 in:out), throughput 149/408, priced 148; models without OpenRouter throughput or a billed route are not ranked.
|
|
25
25
|
|
|
26
26
|
## @default — Main workhorse: strong general coding-agent quality, sane cost
|
|
27
27
|
eligible: 140 — λ 0.00556 $/quality-point (thinking ×1.857)
|
|
28
28
|
|
|
29
29
|
| # | value | q | model | org | $/M | tok/s | ctx | gen 54.7% | code 18.9% | ag 12.6% | tool 10.5% | tput 3.2% |
|
|
30
30
|
| -: | ----: | ----: | --------------------- | ----------- | ----: | ----: | ---: | --------: | ---------: | -------: | ---------: | --------: |
|
|
31
|
-
| 1 | 0.
|
|
32
|
-
| 2 | 0.
|
|
33
|
-
| 3 | 0.
|
|
34
|
-
| 4 | 0.
|
|
35
|
-
| 5 | 0.776 | 0.
|
|
36
|
-
| 6 | 0.
|
|
37
|
-
| 7 | 0.
|
|
38
|
-
| 8 | 0.
|
|
39
|
-
| 9 | 0.
|
|
40
|
-
| 10 | 0.
|
|
31
|
+
| 1 | 0.807 | 0.809 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.483 | 0.146 | 0.093 | 0.069 | 0.019 |
|
|
32
|
+
| 2 | 0.799 | 0.841 | ★ Gemini 4 Argon | Google | $7.65 | 38 | — | 0.516 | 0.149 | 0.096 | 0.068 | 0.012 |
|
|
33
|
+
| 3 | 0.794 | 0.806 | GLM-5.3 | Zhipu AI | $2.09 | 73 | 1.0M | 0.488 | 0.142 | 0.090 | 0.067 | 0.018 |
|
|
34
|
+
| 4 | 0.789 | 0.810 | ★ Muse Spark 1.3 | Meta | $3.71 | 70 | 1.0M | 0.495 | 0.139 | 0.091 | 0.067 | 0.018 |
|
|
35
|
+
| 5 | 0.776 | 0.793 | DeepSeek-V4-Pro-0813 | DeepSeek | $3.09 | 78 | 1.0M | 0.484 | 0.139 | 0.088 | 0.063 | 0.019 |
|
|
36
|
+
| 6 | 0.776 | 0.781 | MiMo-V2.6-Pro | Xiaomi | $1.01 | 29 | 1.0M | 0.476 | 0.141 | 0.089 | 0.066 | 0.010 |
|
|
37
|
+
| 7 | 0.773 | 0.811 | ★ Kimi K3 | Moonshot AI | $6.81 | 52 | 1.0M | 0.494 | 0.144 | 0.091 | 0.067 | 0.015 |
|
|
38
|
+
| 8 | 0.766 | 0.809 | GPT-6.1 Sol | OpenAI | $7.89 | 40 | 1.1M | 0.506 | 0.141 | 0.092 | 0.057 | 0.013 |
|
|
39
|
+
| 9 | 0.764 | 0.777 | Hy4 preview | Tencent | $2.32 | 37 | — | 0.478 | 0.137 | 0.087 | 0.063 | 0.012 |
|
|
40
|
+
| 10 | 0.763 | 0.765 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.473 | 0.123 | 0.087 | 0.065 | 0.018 |
|
|
41
41
|
|
|
42
42
|
## @smol — Fast lightweight model: cheap and quick, still competent
|
|
43
43
|
eligible: 140 — λ 0.04091 $/quality-point
|
|
44
44
|
|
|
45
|
-
| # | value | q | model | org
|
|
46
|
-
| -: | ----: | ----: | ---------------------------- |
|
|
47
|
-
| 1 | 0.
|
|
48
|
-
| 2 | 0.
|
|
49
|
-
| 3 | 0.
|
|
50
|
-
| 4 | 0.
|
|
51
|
-
| 5 | 0.
|
|
52
|
-
| 6 | 0.
|
|
53
|
-
| 7 | 0.
|
|
54
|
-
| 8 | 0.
|
|
55
|
-
| 9 | 0.
|
|
56
|
-
| 10 | 0.
|
|
45
|
+
| # | value | q | model | org | $/M | tok/s | ctx | tput 40% | gen 28.6% | code 17.1% | tool 14.3% |
|
|
46
|
+
| -: | ----: | ----: | ---------------------------- | --------------------- | ----: | ----: | ---: | -------: | --------: | ---------: | ---------: |
|
|
47
|
+
| 1 | 0.706 | 0.714 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.20 | 75 | 1.0M | 0.237 | 0.252 | 0.132 | 0.094 |
|
|
48
|
+
| 2 | 0.687 | 0.713 | Inkling-Small | Thinking Machines Lab | $0.64 | 167 | 524k | 0.331 | 0.208 | 0.104 | 0.070 |
|
|
49
|
+
| 3 | 0.669 | 0.671 | ★ Ling 3.0 Flash | InclusionAI | $0.04 | 140 | 131k | 0.310 | 0.200 | 0.096 | 0.065 |
|
|
50
|
+
| 4 | 0.662 | 0.744 | ★ Muse Spark 1.1 | Meta | $2.00 | 127 | 1.0M | 0.298 | 0.245 | 0.113 | 0.087 |
|
|
51
|
+
| 5 | 0.661 | 0.669 | GLM-5.3-Flash | Zhipu AI | $0.18 | 66 | 1.0M | 0.223 | 0.247 | 0.112 | 0.088 |
|
|
52
|
+
| 6 | 0.661 | 0.707 | GLM-5.3 | Zhipu AI | $1.12 | 73 | 1.0M | 0.233 | 0.255 | 0.129 | 0.091 |
|
|
53
|
+
| 7 | 0.652 | 0.670 | DeepSeek-V4-Flash-Vision-Exp | DeepSeek | $0.43 | 83 | 1.0M | 0.249 | 0.231 | 0.115 | 0.075 |
|
|
54
|
+
| 8 | 0.652 | 0.733 | Muse Spark 1.2 | Meta | $2.00 | 224 | 1.0M | 0.366 | 0.204 | 0.102 | 0.062 |
|
|
55
|
+
| 9 | 0.647 | 0.667 | GPT-5.6 Luna | OpenAI | $0.47 | 80 | 1.1M | 0.244 | 0.229 | 0.117 | 0.076 |
|
|
56
|
+
| 10 | 0.643 | 0.707 | Gemini 3.8 Flash | Google | $1.57 | 95 | 1.0M | 0.264 | 0.247 | 0.119 | 0.076 |
|
|
57
57
|
|
|
58
58
|
## @slow — Most capable model for hard problems; cost and speed as tiebreakers
|
|
59
59
|
eligible: 140 — λ 0.00263 $/quality-point (thinking ×2.714)
|
|
60
60
|
|
|
61
61
|
| # | value | q | model | org | $/M | tok/s | ctx | gen 63.2% | code 18.9% | ag 13.7% | tput 4.2% |
|
|
62
62
|
| -: | ----: | ----: | --------------------- | ----------- | ----: | ----: | ---: | --------: | ---------: | -------: | --------: |
|
|
63
|
-
| 1 | 0.
|
|
64
|
-
| 2 | 0.
|
|
65
|
-
| 3 | 0.
|
|
66
|
-
| 4 | 0.
|
|
67
|
-
| 5 | 0.812 | 0.
|
|
68
|
-
| 6 | 0.
|
|
69
|
-
| 7 | 0.
|
|
70
|
-
| 8 | 0.796 | 0.
|
|
71
|
-
| 9 | 0.
|
|
72
|
-
| 10 | 0.
|
|
63
|
+
| 1 | 0.835 | 0.865 | ★ Gemini 4 Argon | Google | $11.2 | 38 | — | 0.595 | 0.149 | 0.104 | 0.016 |
|
|
64
|
+
| 2 | 0.827 | 0.829 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.54 | 75 | 1.0M | 0.558 | 0.146 | 0.100 | 0.025 |
|
|
65
|
+
| 3 | 0.819 | 0.827 | GLM-5.3 | Zhipu AI | $3.05 | 73 | 1.0M | 0.563 | 0.142 | 0.098 | 0.025 |
|
|
66
|
+
| 4 | 0.818 | 0.833 | ★ Muse Spark 1.3 | Meta | $5.43 | 70 | 1.0M | 0.572 | 0.139 | 0.099 | 0.024 |
|
|
67
|
+
| 5 | 0.812 | 0.842 | GPT-6.1 Sol | OpenAI | $11.5 | 40 | 1.1M | 0.584 | 0.141 | 0.100 | 0.017 |
|
|
68
|
+
| 6 | 0.806 | 0.833 | Kimi K3 | Moonshot AI | $9.95 | 52 | 1.0M | 0.570 | 0.144 | 0.098 | 0.020 |
|
|
69
|
+
| 7 | 0.806 | 0.818 | DeepSeek-V4-Pro-0813 | DeepSeek | $4.51 | 78 | 1.0M | 0.559 | 0.139 | 0.095 | 0.025 |
|
|
70
|
+
| 8 | 0.796 | 0.800 | MiMo-V2.6-Pro | Xiaomi | $1.47 | 29 | 1.0M | 0.549 | 0.141 | 0.097 | 0.013 |
|
|
71
|
+
| 9 | 0.792 | 0.852 | GPT-5.6 Sol | OpenAI | $22.9 | 54 | 1.1M | 0.580 | 0.150 | 0.101 | 0.021 |
|
|
72
|
+
| 10 | 0.790 | 0.798 | Hy4 preview | Tencent | $3.39 | 37 | — | 0.552 | 0.137 | 0.094 | 0.016 |
|
|
73
73
|
|
|
74
74
|
## @vision — Image understanding: vision index dominates
|
|
75
|
-
eligible:
|
|
75
|
+
eligible: 73 — λ 0.00682 $/quality-point (thinking ×1.857)
|
|
76
76
|
|
|
77
77
|
| # | value | q | model | org | $/M | tok/s | ctx | vis 54.2% | gen 20.8% | rea 15.6% | code 6.3% | tput 3.1% |
|
|
78
78
|
| -: | ----: | ----: | --------------------- | ------------------------- | ----: | ----: | ---: | --------: | --------: | --------: | --------: | --------: |
|
|
79
|
-
| 1 | 0.
|
|
79
|
+
| 1 | 0.733 | 0.786 | ★ Gemini 4 Argon | Google | $7.65 | 38 | — | 0.385 | 0.196 | 0.143 | 0.049 | 0.012 |
|
|
80
80
|
| 2 | 0.728 | 0.731 | ★ Qwen3.8 Flash | Alibaba Cloud / Qwen Team | $0.43 | 47 | 1.0M | 0.363 | 0.178 | 0.133 | 0.042 | 0.014 |
|
|
81
|
-
| 3 | 0.
|
|
82
|
-
| 4 | 0.
|
|
83
|
-
| 5 | 0.
|
|
84
|
-
| 6 | 0.
|
|
85
|
-
| 7 | 0.
|
|
86
|
-
| 8 | 0.
|
|
87
|
-
| 9 | 0.
|
|
88
|
-
| 10 | 0.
|
|
81
|
+
| 3 | 0.727 | 0.730 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.345 | 0.184 | 0.134 | 0.048 | 0.018 |
|
|
82
|
+
| 4 | 0.724 | 0.750 | ★ Muse Spark 1.1 | Meta | $3.71 | 127 | 1.0M | 0.371 | 0.179 | 0.136 | 0.041 | 0.023 |
|
|
83
|
+
| 5 | 0.722 | 0.768 | ★ Kimi K3 | Moonshot AI | $6.81 | 52 | 1.0M | 0.378 | 0.188 | 0.139 | 0.048 | 0.015 |
|
|
84
|
+
| 6 | 0.715 | 0.769 | GPT-6.1 Sol | OpenAI | $7.89 | 40 | 1.1M | 0.377 | 0.193 | 0.140 | 0.047 | 0.013 |
|
|
85
|
+
| 7 | 0.714 | 0.734 | ★ Gemini 3.8 Flash | Google | $2.91 | 95 | 1.0M | 0.361 | 0.180 | 0.129 | 0.043 | 0.021 |
|
|
86
|
+
| 8 | 0.713 | 0.715 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.344 | 0.180 | 0.133 | 0.041 | 0.017 |
|
|
87
|
+
| 9 | 0.702 | 0.707 | Qwen3.8-27B | Alibaba Cloud / Qwen Team | $0.72 | 55 | 262k | 0.358 | 0.168 | 0.126 | 0.039 | 0.016 |
|
|
88
|
+
| 10 | 0.697 | 0.717 | Gemini 3.7 Flash | Google | $2.87 | 86 | 1.0M | 0.353 | 0.176 | 0.127 | 0.041 | 0.020 |
|
|
89
89
|
|
|
90
90
|
## @plan — Planning: reasoning and long-context coherence, priced so the cheapest capable planner wins
|
|
91
91
|
eligible: 140 — λ 0.01667 $/quality-point (thinking ×1.857)
|
|
92
92
|
|
|
93
|
-
| # | value | q | model
|
|
94
|
-
| -: | ----: | ----: |
|
|
95
|
-
| 1 | 0.
|
|
96
|
-
| 2 | 0.
|
|
97
|
-
| 3 | 0.
|
|
98
|
-
| 4 | 0.
|
|
99
|
-
| 5 | 0.
|
|
100
|
-
| 6 | 0.
|
|
101
|
-
| 7 | 0.
|
|
102
|
-
| 8 | 0.
|
|
103
|
-
| 9 | 0.735 | 0.
|
|
104
|
-
| 10 | 0.
|
|
93
|
+
| # | value | q | model | org | $/M | tok/s | ctx | gen 80% | lc 15.6% | tput 4.4% |
|
|
94
|
+
| -: | ----: | ----: | --------------------- | ------------------------- | ----: | ----: | ---: | ------: | -------: | --------: |
|
|
95
|
+
| 1 | 0.789 | 0.851 | ★ Muse Spark 1.3 | Meta | $3.71 | 70 | 1.0M | 0.724 | 0.101 | 0.025 |
|
|
96
|
+
| 2 | 0.774 | 0.781 | ★ Qwen3.8 Flash | Alibaba Cloud / Qwen Team | $0.43 | 47 | 1.0M | 0.682 | 0.079 | 0.020 |
|
|
97
|
+
| 3 | 0.764 | 0.813 | ★ Gemini 3.8 Flash | Google | $2.91 | 95 | 1.0M | 0.692 | 0.091 | 0.029 |
|
|
98
|
+
| 4 | 0.757 | 0.763 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.707 | 0.030 | 0.026 |
|
|
99
|
+
| 5 | 0.755 | 0.802 | ★ Gemini 3.7 Flash | Google | $2.87 | 86 | 1.0M | 0.676 | 0.098 | 0.028 |
|
|
100
|
+
| 6 | 0.752 | 0.790 | ★ Hy4 preview | Tencent | $2.32 | 37 | — | 0.699 | 0.075 | 0.017 |
|
|
101
|
+
| 7 | 0.742 | 0.870 | ★ Gemini 4 Argon | Google | $7.65 | 38 | — | 0.754 | 0.098 | 0.017 |
|
|
102
|
+
| 8 | 0.740 | 0.746 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.691 | 0.030 | 0.025 |
|
|
103
|
+
| 9 | 0.735 | 0.749 | GPT-5.6 Luna | OpenAI | $0.88 | 80 | 1.1M | 0.642 | 0.080 | 0.027 |
|
|
104
|
+
| 10 | 0.735 | 0.769 | GLM-5.3 | Zhipu AI | $2.09 | 73 | 1.0M | 0.713 | 0.030 | 0.026 |
|
|
105
105
|
|
|
106
106
|
## @commit — Commit messages: cheap and fast with decent general quality
|
|
107
107
|
eligible: 140 — λ 0.04091 $/quality-point
|
|
108
108
|
|
|
109
|
-
| # | value | q | model | org
|
|
110
|
-
| -: | ----: | ----: | ---------------------------- |
|
|
111
|
-
| 1 | 0.
|
|
112
|
-
| 2 | 0.
|
|
113
|
-
| 3 | 0.
|
|
114
|
-
| 4 | 0.
|
|
115
|
-
| 5 | 0.
|
|
116
|
-
| 6 | 0.
|
|
117
|
-
| 7 | 0.
|
|
118
|
-
| 8 | 0.
|
|
119
|
-
| 9 | 0.
|
|
120
|
-
| 10 | 0.
|
|
109
|
+
| # | value | q | model | org | $/M | tok/s | ctx | tput 41.5% | gen 43.1% | code 15.4% |
|
|
110
|
+
| -: | ----: | ----: | ---------------------------- | --------------------- | ----: | ----: | ---: | ---------: | --------: | ---------: |
|
|
111
|
+
| 1 | 0.736 | 0.745 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.20 | 75 | 1.0M | 0.246 | 0.380 | 0.118 |
|
|
112
|
+
| 2 | 0.724 | 0.750 | ★ Inkling-Small | Thinking Machines Lab | $0.64 | 167 | 524k | 0.344 | 0.313 | 0.094 |
|
|
113
|
+
| 3 | 0.709 | 0.710 | ★ Ling 3.0 Flash | InclusionAI | $0.04 | 140 | 131k | 0.322 | 0.302 | 0.086 |
|
|
114
|
+
| 4 | 0.699 | 0.781 | ★ Muse Spark 1.1 | Meta | $2.00 | 127 | 1.0M | 0.310 | 0.370 | 0.101 |
|
|
115
|
+
| 5 | 0.697 | 0.779 | Muse Spark 1.2 | Meta | $2.00 | 224 | 1.0M | 0.380 | 0.308 | 0.091 |
|
|
116
|
+
| 6 | 0.696 | 0.703 | GLM-5.3-Flash | Zhipu AI | $0.18 | 66 | 1.0M | 0.231 | 0.372 | 0.100 |
|
|
117
|
+
| 7 | 0.695 | 0.741 | GLM-5.3 | Zhipu AI | $1.12 | 73 | 1.0M | 0.242 | 0.384 | 0.115 |
|
|
118
|
+
| 8 | 0.692 | 0.710 | DeepSeek-V4-Flash-Vision-Exp | DeepSeek | $0.43 | 83 | 1.0M | 0.259 | 0.348 | 0.103 |
|
|
119
|
+
| 9 | 0.690 | 0.754 | ★ Gemini 3.8 Flash | Google | $1.57 | 95 | 1.0M | 0.275 | 0.373 | 0.106 |
|
|
120
|
+
| 10 | 0.685 | 0.704 | GPT-5.6 Luna | OpenAI | $0.47 | 80 | 1.1M | 0.254 | 0.346 | 0.105 |
|
|
121
121
|
|
|
122
122
|
## @tiny — Background tasks (titles, memory): cheapest and fastest wins
|
|
123
123
|
eligible: 140 — λ 0.03333 $/quality-point
|
|
124
124
|
|
|
125
|
-
| # | value | q | model
|
|
126
|
-
| -: | ----: | ----: |
|
|
127
|
-
| 1 | 0.
|
|
128
|
-
| 2 | 0.
|
|
129
|
-
| 3 | 0.
|
|
130
|
-
| 4 | 0.
|
|
131
|
-
| 5 | 0.
|
|
132
|
-
| 6 | 0.
|
|
133
|
-
| 7 | 0.
|
|
134
|
-
| 8 | 0.
|
|
135
|
-
| 9 | 0.
|
|
136
|
-
| 10 | 0.
|
|
125
|
+
| # | value | q | model | org | $/M | tok/s | ctx | tput 58.3% | gen 41.7% |
|
|
126
|
+
| -: | ----: | ----: | ----------------------------- | ------------------------- | ----: | ----: | ---: | ---------: | --------: |
|
|
127
|
+
| 1 | 0.769 | 0.782 | ★ Mercury 2 | Inception | $0.38 | 255 | 128k | 0.555 | 0.227 |
|
|
128
|
+
| 2 | 0.765 | 0.831 | ★ Muse Spark 1.2 | Meta | $2.00 | 224 | 1.0M | 0.533 | 0.298 |
|
|
129
|
+
| 3 | 0.764 | 0.786 | ★ Inkling-Small | Thinking Machines Lab | $0.64 | 167 | 524k | 0.483 | 0.303 |
|
|
130
|
+
| 4 | 0.743 | 0.745 | ★ Ling 3.0 Flash | InclusionAI | $0.04 | 140 | 131k | 0.453 | 0.292 |
|
|
131
|
+
| 5 | 0.738 | 0.756 | Gemini 3.1 Flash-Lite | Google | $0.56 | 176 | 1.0M | 0.492 | 0.265 |
|
|
132
|
+
| 6 | 0.726 | 0.793 | Muse Spark 1.1 | Meta | $2.00 | 127 | 1.0M | 0.435 | 0.358 |
|
|
133
|
+
| 7 | 0.717 | 0.768 | Gemini 3.6 Flash | Google | $1.54 | 132 | 1.0M | 0.443 | 0.325 |
|
|
134
|
+
| 8 | 0.707 | 0.713 | DeepSeek-V4.1-Flash | DeepSeek | $0.20 | 75 | 1.0M | 0.345 | 0.368 |
|
|
135
|
+
| 9 | 0.705 | 0.707 | Laguna XS 2.1 | Poolside | $0.07 | 161 | 262k | 0.477 | 0.231 |
|
|
136
|
+
| 10 | 0.702 | 0.743 | Qwen3-235B-A22B-Thinking-2507 | Alibaba Cloud / Qwen Team | $1.21 | 177 | — | 0.493 | 0.250 |
|
|
137
137
|
|
|
138
138
|
## @task — Subagents: agentic + tool calling, moderate cost sensitivity
|
|
139
139
|
eligible: 140 — λ 0.00747 $/quality-point (thinking ×1.857)
|
|
140
140
|
|
|
141
141
|
| # | value | q | model | org | $/M | tok/s | ctx | ag 29.9% | tool 20.7% | code 18.4% | gen 25.3% | tput 5.7% |
|
|
142
142
|
| -: | ----: | ----: | --------------------- | ------------------------- | ----: | ----: | ---: | -------: | ---------: | ---------: | --------: | --------: |
|
|
143
|
-
| 1 | 0.
|
|
144
|
-
| 2 | 0.
|
|
145
|
-
| 3 | 0.
|
|
146
|
-
| 4 | 0.
|
|
147
|
-
| 5 | 0.
|
|
148
|
-
| 6 | 0.
|
|
149
|
-
| 7 | 0.
|
|
150
|
-
| 8 | 0.
|
|
151
|
-
| 9 | 0.
|
|
152
|
-
| 10 | 0.
|
|
143
|
+
| 1 | 0.750 | 0.753 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.219 | 0.136 | 0.141 | 0.223 | 0.034 |
|
|
144
|
+
| 2 | 0.726 | 0.742 | GLM-5.3 | Zhipu AI | $2.09 | 73 | 1.0M | 0.213 | 0.131 | 0.138 | 0.225 | 0.033 |
|
|
145
|
+
| 3 | 0.716 | 0.743 | Muse Spark 1.3 | Meta | $3.71 | 70 | 1.0M | 0.215 | 0.132 | 0.134 | 0.229 | 0.033 |
|
|
146
|
+
| 4 | 0.710 | 0.767 | ★ Gemini 4 Argon | Google | $7.65 | 38 | — | 0.228 | 0.134 | 0.145 | 0.238 | 0.022 |
|
|
147
|
+
| 5 | 0.707 | 0.715 | MiMo-V2.6-Pro | Xiaomi | $1.01 | 29 | 1.0M | 0.211 | 0.129 | 0.137 | 0.220 | 0.018 |
|
|
148
|
+
| 6 | 0.702 | 0.725 | DeepSeek-V4-Pro-0813 | DeepSeek | $3.09 | 78 | 1.0M | 0.208 | 0.124 | 0.135 | 0.224 | 0.035 |
|
|
149
|
+
| 7 | 0.700 | 0.703 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.206 | 0.127 | 0.120 | 0.218 | 0.032 |
|
|
150
|
+
| 8 | 0.692 | 0.743 | Kimi K3 | Moonshot AI | $6.81 | 52 | 1.0M | 0.215 | 0.132 | 0.140 | 0.228 | 0.028 |
|
|
151
|
+
| 9 | 0.688 | 0.691 | Qwen3.8 Flash | Alibaba Cloud / Qwen Team | $0.43 | 47 | 1.0M | 0.198 | 0.127 | 0.124 | 0.216 | 0.026 |
|
|
152
|
+
| 10 | 0.687 | 0.705 | Hy4 preview | Tencent | $2.32 | 37 | — | 0.205 | 0.124 | 0.133 | 0.221 | 0.022 |
|
|
153
153
|
|
|
154
154
|
## @advisor — Advisor/watchdog: deep reasoning over long context
|
|
155
155
|
eligible: 140 — λ 0.01250 $/quality-point (thinking ×1.857)
|
|
156
156
|
|
|
157
157
|
| # | value | q | model | org | $/M | tok/s | ctx | gen 74.3% | lc 12.6% | tput 13.1% |
|
|
158
158
|
| -: | ----: | ----: | --------------------- | ------------------------- | ----: | ----: | ---: | --------: | -------: | ---------: |
|
|
159
|
-
| 1 | 0.
|
|
160
|
-
| 2 | 0.
|
|
161
|
-
| 3 | 0.
|
|
162
|
-
| 4 | 0.
|
|
163
|
-
| 5 | 0.
|
|
164
|
-
| 6 | 0.
|
|
165
|
-
| 7 | 0.
|
|
166
|
-
| 8 | 0.
|
|
167
|
-
| 9 | 0.
|
|
168
|
-
| 10 | 0.
|
|
159
|
+
| 1 | 0.783 | 0.830 | ★ Muse Spark 1.3 | Meta | $3.71 | 70 | 1.0M | 0.673 | 0.082 | 0.075 |
|
|
160
|
+
| 2 | 0.767 | 0.803 | ★ Gemini 3.8 Flash | Google | $2.91 | 95 | 1.0M | 0.643 | 0.073 | 0.087 |
|
|
161
|
+
| 3 | 0.755 | 0.790 | ★ Gemini 3.7 Flash | Google | $2.87 | 86 | 1.0M | 0.628 | 0.079 | 0.083 |
|
|
162
|
+
| 4 | 0.754 | 0.759 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.657 | 0.024 | 0.078 |
|
|
163
|
+
| 5 | 0.752 | 0.757 | Qwen3.8 Flash | Alibaba Cloud / Qwen Team | $0.43 | 47 | 1.0M | 0.634 | 0.063 | 0.060 |
|
|
164
|
+
| 6 | 0.737 | 0.764 | ★ GLM-5.3 | Zhipu AI | $2.09 | 73 | 1.0M | 0.663 | 0.024 | 0.076 |
|
|
165
|
+
| 7 | 0.735 | 0.831 | ★ Gemini 4 Argon | Google | $7.65 | 38 | — | 0.700 | 0.079 | 0.051 |
|
|
166
|
+
| 8 | 0.735 | 0.739 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.642 | 0.024 | 0.073 |
|
|
167
|
+
| 9 | 0.731 | 0.760 | Hy4 preview | Tencent | $2.32 | 37 | — | 0.649 | 0.060 | 0.050 |
|
|
168
|
+
| 10 | 0.730 | 0.741 | GPT-5.6 Luna | OpenAI | $0.88 | 80 | 1.1M | 0.597 | 0.065 | 0.080 |
|
|
169
169
|
|
|
170
170
|
## @designer — Design work: visual/UX judgement on image-capable models
|
|
171
|
-
eligible:
|
|
171
|
+
eligible: 87 — λ 0.01024 $/quality-point (thinking ×1.857)
|
|
172
172
|
|
|
173
173
|
| # | value | q | model | org | $/M | tok/s | ctx | gen 31.3% | code 7.2% | vis 21.7% | tput 18.1% | web 21.7% | agon |
|
|
174
174
|
| -: | ----: | ----: | --------------------- | ----------- | ----: | ----: | ---: | --------: | --------: | --------: | ---------: | --------: | ---: |
|
|
175
|
-
| 1 | 0.
|
|
176
|
-
| 2 | 0.
|
|
177
|
-
| 3 | 0.
|
|
178
|
-
| 4 | 0.
|
|
179
|
-
| 5 | 0.
|
|
180
|
-
| 6 | 0.
|
|
181
|
-
| 7 | 0.
|
|
182
|
-
| 8 | 0.
|
|
183
|
-
| 9 | 0.
|
|
184
|
-
| 10 | 0.
|
|
185
|
-
focus: website — coverage
|
|
175
|
+
| 1 | 0.771 | 0.775 | ★ DeepSeek-V4.1-Flash | DeepSeek | $0.37 | 75 | 1.0M | 0.277 | 0.056 | 0.138 | 0.107 | 0.197 | — |
|
|
176
|
+
| 2 | 0.744 | 0.774 | Gemini 3.8 Flash | Google | $2.91 | 95 | 1.0M | 0.271 | 0.050 | 0.144 | 0.119 | 0.189 | 1236 |
|
|
177
|
+
| 3 | 0.741 | 0.779 | ★ Muse Spark 1.3 | Meta | $3.71 | 70 | 1.0M | 0.283 | 0.053 | 0.125 | 0.103 | 0.214 | 1296 |
|
|
178
|
+
| 4 | 0.730 | 0.760 | Gemini 3.7 Flash | Google | $2.87 | 86 | 1.0M | 0.265 | 0.047 | 0.141 | 0.114 | 0.192 | 1223 |
|
|
179
|
+
| 5 | 0.727 | 0.765 | Muse Spark 1.1 | Meta | $3.71 | 127 | 1.0M | 0.269 | 0.048 | 0.148 | 0.135 | 0.165 | — |
|
|
180
|
+
| 6 | 0.725 | 0.755 | Gemini 3.6 Flash | Google | $2.87 | 132 | 1.0M | 0.244 | 0.042 | 0.140 | 0.137 | 0.190 | 1194 |
|
|
181
|
+
| 7 | 0.721 | 0.724 | ★ GLM-5.3-Flash | Zhipu AI | $0.33 | 66 | 1.0M | 0.270 | 0.047 | 0.138 | 0.101 | 0.168 | — |
|
|
182
|
+
| 8 | 0.718 | 0.788 | ★ Kimi K3 | Moonshot AI | $6.81 | 52 | 1.0M | 0.283 | 0.055 | 0.152 | 0.088 | 0.211 | 1292 |
|
|
183
|
+
| 9 | 0.702 | 0.781 | Gemini 4 Argon | Google | $7.65 | 38 | — | 0.295 | 0.057 | 0.154 | 0.070 | 0.204 | — |
|
|
184
|
+
| 10 | 0.698 | 0.709 | MiMo-V2.6-Pro | Xiaomi | $1.01 | 29 | 1.0M | 0.272 | 0.054 | 0.127 | 0.056 | 0.199 | — |
|
|
185
|
+
focus: website — coverage 127/408 (31.1%) ok; dispersion 0.992 ok; composition ok; freshness 0.0mo ok; trust unknown; modality unknown; maintenance unknown; provenance unknown; cross-source unknown
|
|
186
186
|
|
|
187
187
|
# Suggested settings.modelRoles (best-fit #1 per role, resolved via the omp catalog).
|
|
188
188
|
# Selectors are openrouter/<id>; the omp-llm-role plugin writes this block daily.
|
|
189
189
|
modelRoles:
|
|
190
190
|
default: "openrouter/deepseek/deepseek-v4.1-flash"
|
|
191
191
|
smol: "openrouter/deepseek/deepseek-v4.1-flash"
|
|
192
|
-
slow: "openrouter/
|
|
193
|
-
vision: "openrouter/
|
|
192
|
+
slow: "openrouter/google/gemini-4-argon"
|
|
193
|
+
vision: "openrouter/google/gemini-4-argon"
|
|
194
194
|
plan: "openrouter/meta/muse-spark-1.3"
|
|
195
195
|
commit: "openrouter/deepseek/deepseek-v4.1-flash"
|
|
196
|
-
tiny: "openrouter/
|
|
196
|
+
tiny: "openrouter/inception/mercury-2"
|
|
197
197
|
task: "openrouter/deepseek/deepseek-v4.1-flash"
|
|
198
198
|
advisor: "openrouter/meta/muse-spark-1.3"
|
|
199
199
|
designer: "openrouter/deepseek/deepseek-v4.1-flash"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-llm-role",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "omp plugin: daily model-role updater driven by llm-stats/OpenRouter rankings",
|
|
6
6
|
"license": "MIT",
|
|
@@ -63,6 +63,14 @@
|
|
|
63
63
|
"min": 0,
|
|
64
64
|
"step": 1
|
|
65
65
|
},
|
|
66
|
+
"fallbackQualityFloor": {
|
|
67
|
+
"type": "number",
|
|
68
|
+
"description": "Quality floor for a cheaper-or-equal fallback: the fraction of the chosen model's quality it must retain (0 disables)",
|
|
69
|
+
"default": 0.9,
|
|
70
|
+
"min": 0,
|
|
71
|
+
"max": 1,
|
|
72
|
+
"step": 0.001
|
|
73
|
+
},
|
|
66
74
|
"activateDefaultOnEmptySession": {
|
|
67
75
|
"type": "boolean",
|
|
68
76
|
"description": "Reapply the new default selector to an empty session's active model",
|
|
@@ -78,6 +86,16 @@
|
|
|
78
86
|
"auto"
|
|
79
87
|
]
|
|
80
88
|
},
|
|
89
|
+
"tokenProfile.inputTokens": {
|
|
90
|
+
"type": "number",
|
|
91
|
+
"description": "Input tokens per task for the per-task price (used when the role has no recorded usage)",
|
|
92
|
+
"min": 0
|
|
93
|
+
},
|
|
94
|
+
"tokenProfile.outputTokens": {
|
|
95
|
+
"type": "number",
|
|
96
|
+
"description": "Output tokens per task for the per-task price (used when the role has no recorded usage)",
|
|
97
|
+
"min": 0
|
|
98
|
+
},
|
|
81
99
|
"features.endpointCeilings": {
|
|
82
100
|
"type": "boolean",
|
|
83
101
|
"description": "Drop routes that cannot call tools and routes whose output ceiling is below 16 384 tokens",
|
package/src/agent-create.ts
CHANGED
|
@@ -424,8 +424,10 @@ function coveredValues(models: readonly Model[], metric: string, fill: number):
|
|
|
424
424
|
return values;
|
|
425
425
|
}
|
|
426
426
|
|
|
427
|
-
/** Linear-interpolated quantile (R type 7) of a value list
|
|
428
|
-
|
|
427
|
+
/** Linear-interpolated quantile (R type 7) of a value list — the repo's one
|
|
428
|
+
* quantile convention: the focus-axis dispersion stats and the usage ledger's
|
|
429
|
+
* per-turn median (`quantile(values, 0.5)`) both read it. */
|
|
430
|
+
export function quantile(values: readonly number[], p: number): number {
|
|
429
431
|
const sorted = [...values].sort((a, b) => a - b);
|
|
430
432
|
const h = (sorted.length - 1) * p;
|
|
431
433
|
const lo = Math.floor(h);
|
package/src/availability.ts
CHANGED
|
@@ -89,8 +89,8 @@ export type ProbeVerdict = "ok" | "blocked" | "unknown";
|
|
|
89
89
|
const OPENROUTER_COMPLETIONS_URL = "https://openrouter.ai/api/v1/chat/completions";
|
|
90
90
|
|
|
91
91
|
/** POST a one-token completion to OpenRouter with the shared request skeleton
|
|
92
|
-
* (Bearer auth, JSON body, 20s timeout). Shared by `probeModel`
|
|
93
|
-
* `fetchAllowedProviders`. */
|
|
92
|
+
* (Bearer auth, JSON body, 20s timeout). Shared by `probeModel`, `probeRoute`
|
|
93
|
+
* and `fetchAllowedProviders`. */
|
|
94
94
|
function completionRequest(token: string, body: unknown, fetchImpl: typeof fetch): Promise<Response> {
|
|
95
95
|
return fetchImpl(OPENROUTER_COMPLETIONS_URL, {
|
|
96
96
|
method: "POST",
|
|
@@ -110,6 +110,20 @@ function errorMessage(body: unknown): string {
|
|
|
110
110
|
* whitelist (or a data-policy toggle) leaves a model without a runnable endpoint. */
|
|
111
111
|
const BLOCKED_RE = /no allowed providers/i;
|
|
112
112
|
|
|
113
|
+
/**
|
|
114
|
+
* The verdict a probe response carries, shared by `probeModel` and `probeRoute`
|
|
115
|
+
* so the model probe and the route probe cannot drift apart: a 2xx is `ok`, the
|
|
116
|
+
* narrow no-allowed-providers 404 is `blocked`, and everything else is
|
|
117
|
+
* `unknown` (usable) — any other status (a 429, a 5xx), a 404 with any other
|
|
118
|
+
* body, an unparseable body. Never throws.
|
|
119
|
+
*/
|
|
120
|
+
async function verdictOf(res: Response): Promise<ProbeVerdict> {
|
|
121
|
+
if (res.ok) return "ok";
|
|
122
|
+
if (res.status !== 404) return "unknown";
|
|
123
|
+
const body: unknown = await res.json().catch(() => null);
|
|
124
|
+
return BLOCKED_RE.test(errorMessage(body)) ? "blocked" : "unknown";
|
|
125
|
+
}
|
|
126
|
+
|
|
113
127
|
/**
|
|
114
128
|
* One-token completion probe: the only reliable signal for
|
|
115
129
|
* account-level provider restrictions — neither `/api/v1/key` nor the catalog
|
|
@@ -121,11 +135,30 @@ const BLOCKED_RE = /no allowed providers/i;
|
|
|
121
135
|
export async function probeModel(token: string, catalogId: string, fetchImpl: typeof fetch = fetch): Promise<ProbeVerdict> {
|
|
122
136
|
try {
|
|
123
137
|
const res = await completionRequest(token, { model: catalogId, messages: [{ role: "user", content: "ok" }], max_tokens: 1 }, fetchImpl);
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
return await verdictOf(res);
|
|
139
|
+
} catch {
|
|
140
|
+
return "unknown";
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* One-token completion bound to a single provider route (`provider.only`), the
|
|
146
|
+
* same request skeleton as `probeModel` plus the route pin. The endpoints API
|
|
147
|
+
* and OpenRouter's routing table disagree — an endpoint can list `status: 0`
|
|
148
|
+
* with 100% uptime and still 404 — so a real request is the only reliable
|
|
149
|
+
* signal for whether a route is routable. Only the narrow no-allowed-providers
|
|
150
|
+
* 404 blocks a route; a non-404 status, a 404 with any other body and a
|
|
151
|
+
* transport throw are all "unknown" and count as usable, exactly as
|
|
152
|
+
* `probeModel` does. Never throws.
|
|
153
|
+
*/
|
|
154
|
+
export async function probeRoute(token: string, catalogId: string, providerSlug: string, fetchImpl: typeof fetch = fetch): Promise<ProbeVerdict> {
|
|
155
|
+
try {
|
|
156
|
+
const res = await completionRequest(
|
|
157
|
+
token,
|
|
158
|
+
{ model: catalogId, messages: [{ role: "user", content: "ok" }], max_tokens: 1, provider: { only: [providerSlug] } },
|
|
159
|
+
fetchImpl,
|
|
160
|
+
);
|
|
161
|
+
return await verdictOf(res);
|
|
129
162
|
} catch {
|
|
130
163
|
return "unknown";
|
|
131
164
|
}
|
package/src/engine.ts
CHANGED
|
@@ -313,6 +313,19 @@ export type OpenRouterEndpointPages = Record<string, OpenRouterEndpointRecord[]>
|
|
|
313
313
|
|
|
314
314
|
export type OrEnrichment = { tput: number; latency: number | null; price: number | null; thinking: boolean; routes: OpenRouterEndpointRecord[] };
|
|
315
315
|
|
|
316
|
+
/** The route's input (prompt) $/M under an assumed cache-hit rate: the input term
|
|
317
|
+
* `routeEffectivePrice` blends into the billed price. A hit rate `h` blends the
|
|
318
|
+
* cache-read input in, `h·cacheRead + (1−h)·input`; with `h = 0` or no cache-read
|
|
319
|
+
* price this is the listed input price, and a route with no input price has none —
|
|
320
|
+
* missing data keeps the full input price, never a silent discount. The single
|
|
321
|
+
* accessor for the input term: the billed blend, the output-price inverse
|
|
322
|
+
* (`routeOutputPrice`) and the explorer's per-task price all read it. */
|
|
323
|
+
export function routeInputPrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
324
|
+
if (route.weightPrice == null) return null;
|
|
325
|
+
if (cacheHitRate <= 0 || route.cacheReadPrice == null) return route.weightPrice;
|
|
326
|
+
return cacheHitRate * route.cacheReadPrice + (1 - cacheHitRate) * route.weightPrice;
|
|
327
|
+
}
|
|
328
|
+
|
|
316
329
|
/** The effective billed $/M of one route under an assumed cache-hit rate. The route's
|
|
317
330
|
* billed 3:1 price is `price = (3·input + output)/4` on the listed input price; a hit
|
|
318
331
|
* rate `h` blends the cache-read input into the input term,
|
|
@@ -320,15 +333,33 @@ export type OrEnrichment = { tput: number; latency: number | null; price: number
|
|
|
320
333
|
* (the output term is unchanged). With `h = 0`, no cache-read price, or no input price
|
|
321
334
|
* this is the route's listed `price` — missing data keeps the full input price, never a
|
|
322
335
|
* silent discount. Shared by the blend (`blendRoutePool`), the pin path (`rankRole`) and
|
|
323
|
-
* `roleMetricValue`. */
|
|
336
|
+
* `roleMetricValue`; the input term itself is `routeInputPrice`. */
|
|
324
337
|
export function routeEffectivePrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
325
338
|
if (route.price == null) return null;
|
|
326
339
|
if (cacheHitRate <= 0 || route.cacheReadPrice == null || route.weightPrice == null) return route.price;
|
|
327
340
|
const input = route.weightPrice;
|
|
328
|
-
const effInput =
|
|
341
|
+
const effInput = routeInputPrice(route, cacheHitRate) as number;
|
|
329
342
|
return route.price + (3 * (effInput - input)) / 4;
|
|
330
343
|
}
|
|
331
344
|
|
|
345
|
+
/** The output (completion) $/M implied by a billed 3:1 blend and the input price
|
|
346
|
+
* behind it: the inverse of `billed = IO_PRICE_RATIO·(3·input + output)`, i.e.
|
|
347
|
+
* `output = billed/IO_PRICE_RATIO − (1−IO_PRICE_RATIO)/IO_PRICE_RATIO · input`.
|
|
348
|
+
* The 3:1 ratio lives here (and in `IO_PRICE_RATIO`), not in the callers; dividing
|
|
349
|
+
* by `IO_PRICE_RATIO` (2⁻²) is exact, so this is the plain `4·billed − 3·input`. */
|
|
350
|
+
export function blendOutputPrice(billed: number, input: number): number {
|
|
351
|
+
return billed / IO_PRICE_RATIO - ((1 - IO_PRICE_RATIO) / IO_PRICE_RATIO) * input;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** The route's output (completion) $/M: the inverse blend of its own billed and
|
|
355
|
+
* input prices (see `blendOutputPrice`). Null when either of them is missing. */
|
|
356
|
+
export function routeOutputPrice(route: OpenRouterEndpointRecord, cacheHitRate = 0): number | null {
|
|
357
|
+
const billed = routeEffectivePrice(route, cacheHitRate);
|
|
358
|
+
const input = routeInputPrice(route, cacheHitRate);
|
|
359
|
+
if (billed === null || input === null) return null;
|
|
360
|
+
return blendOutputPrice(billed, input);
|
|
361
|
+
}
|
|
362
|
+
|
|
332
363
|
/** Expected price/throughput/latency of one request over a route pool under the
|
|
333
364
|
* router's default price-based load balancing: P(route i) ∝ 1/input_price_i² — the
|
|
334
365
|
* router sorts on the input (prompt) price alone, so the weight basis is
|