model-compare-opencode-plugin 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +186 -2
- package/dist/tui.js +741 -0
- package/package.json +60 -4
- package/src/config.ts +197 -0
- package/src/core/catalog.ts +189 -0
- package/src/core/hash.ts +12 -0
- package/src/core/pins.ts +158 -0
- package/src/core/rank.ts +132 -0
- package/src/core/types.ts +50 -0
- package/src/index.ts +478 -0
- package/src/opencode/aliases.ts +58 -0
- package/src/opencode/commands.ts +251 -0
- package/src/opencode/guard.ts +44 -0
- package/src/opencode/refresh.ts +188 -0
- package/src/opencode/resolution.ts +69 -0
- package/src/opencode/storage.ts +97 -0
- package/src/opencode/usebest.ts +121 -0
- package/src/rpc.ts +78 -0
- package/src/sources/index.ts +115 -0
- package/src/sources/local.ts +175 -0
- package/src/sources/remote.ts +155 -0
- package/src/tui.tsx +772 -0
- package/vendor/UPSTREAM.json +6 -0
- package/vendor/model_compare.py +1950 -0
|
@@ -0,0 +1,1950 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""model-compare: pick the best value-for-money LLM on OpenRouter.
|
|
3
|
+
|
|
4
|
+
Run it before launching a coding agent to auto-select the model that gives
|
|
5
|
+
the most capability per dollar today. Data sources:
|
|
6
|
+
|
|
7
|
+
* OpenRouter public API (/api/v1/models) - catalog, pricing (input and
|
|
8
|
+
output), context window, release date and capabilities.
|
|
9
|
+
* Artificial Analysis intelligence index - model quality. Primary
|
|
10
|
+
source: OpenRouter's republished AA benchmarks (data.benchmarks in
|
|
11
|
+
the frontend models API, exact per-model keys). When a model is not
|
|
12
|
+
covered there: the AA API v2 (--aa-api-key or the AA_API_KEY
|
|
13
|
+
environment variable; free key at artificialanalysis.ai), matched by
|
|
14
|
+
exact slug/name only. Without a key, AA data comes only from
|
|
15
|
+
OpenRouter. Unmatched models rank on price/context/age.
|
|
16
|
+
|
|
17
|
+
Ranking: every criterion is normalized to [0, 1] and combined with
|
|
18
|
+
priority-dependent weights:
|
|
19
|
+
|
|
20
|
+
balanced: quality 0.40 / price 0.40 / context 0.10 / age 0.10
|
|
21
|
+
price: quality 0.20 / price 0.60 / context 0.10 / age 0.10
|
|
22
|
+
quality: quality 0.60 / price 0.20 / context 0.10 / age 0.10
|
|
23
|
+
|
|
24
|
+
price_score log-scaled blended price (USD per 1M tokens,
|
|
25
|
+
--input-share input / rest output) within the candidate
|
|
26
|
+
pool; free listings score 1.0.
|
|
27
|
+
quality_score AA intelligence index divided by --quality-ref (default
|
|
28
|
+
70), clamped to [0, 1]; unmatched models score 0.
|
|
29
|
+
context_score logarithmic bonus from --min-context up to 4x that
|
|
30
|
+
threshold (extra headroom helps, but with diminishing
|
|
31
|
+
returns).
|
|
32
|
+
age_score exponential decay with --recency-half-life (default 120
|
|
33
|
+
days): the fresher the listing, the better.
|
|
34
|
+
|
|
35
|
+
Examples:
|
|
36
|
+
model_compare.py top 5, balanced
|
|
37
|
+
model_compare.py --best print only "#1 opencode id" (openrouter/<model>)
|
|
38
|
+
model_compare.py --priority price --top 3 three cheapest-sensible picks
|
|
39
|
+
model_compare.py --discount only currently-discounted models
|
|
40
|
+
model_compare.py --catalog full catalog (ranked + filtered) as one JSON document
|
|
41
|
+
MODEL=$(model_compare.py --best) shell integration (opencode --model $MODEL)
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import argparse
|
|
47
|
+
import gzip
|
|
48
|
+
import json
|
|
49
|
+
import math
|
|
50
|
+
import os
|
|
51
|
+
import re
|
|
52
|
+
import sys
|
|
53
|
+
import time
|
|
54
|
+
import urllib.error
|
|
55
|
+
import urllib.parse
|
|
56
|
+
import urllib.request
|
|
57
|
+
from datetime import datetime, timezone
|
|
58
|
+
|
|
59
|
+
OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
|
|
60
|
+
OPENROUTER_DISCOUNTS_URL = (
|
|
61
|
+
"https://openrouter.ai/api/frontend/v1/models/find?output_modalities=text"
|
|
62
|
+
)
|
|
63
|
+
# Public per-endpoint list of zero-data-retention endpoints; DESIGN.md
|
|
64
|
+
# ("Zero data retention") describes how its entries are interpreted.
|
|
65
|
+
OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
|
|
66
|
+
# Supported V2 free-tier replacement for the retired /api/v2/data/llms/models
|
|
67
|
+
# (legacy endpoints 410 after 2026-11-04; see
|
|
68
|
+
# https://artificialanalysis.ai/data-api/migrate-v2-data).
|
|
69
|
+
AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
|
|
70
|
+
VERSION = "0.3.1"
|
|
71
|
+
USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
|
|
72
|
+
|
|
73
|
+
# opencode expects provider-qualified model ids: openrouter/<provider/model>.
|
|
74
|
+
# Single source of truth -- other provider namespaces can be supported later
|
|
75
|
+
# without touching consumers of --best / --json.
|
|
76
|
+
OPENCODE_PROVIDER = "openrouter"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def opencode_model_id(model_id: str) -> str:
|
|
80
|
+
return f"{OPENCODE_PROVIDER}/{model_id}"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
PRIORITY_WEIGHTS = {
|
|
84
|
+
"balanced": {"quality": 0.40, "price": 0.40, "context": 0.10, "age": 0.10},
|
|
85
|
+
"price": {"quality": 0.20, "price": 0.60, "context": 0.10, "age": 0.10},
|
|
86
|
+
"quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
CATALOG_SCHEMA_VERSION = 2
|
|
90
|
+
|
|
91
|
+
# The drop-reason keys build_candidates counts, verbatim -- keep in sync with
|
|
92
|
+
# its drop() call sites. pool.dropped lists all of them zero-filled so the
|
|
93
|
+
# contract is self-describing.
|
|
94
|
+
CATALOG_DROP_REASONS = (
|
|
95
|
+
"malformed id",
|
|
96
|
+
"context",
|
|
97
|
+
"pricing",
|
|
98
|
+
"schedule",
|
|
99
|
+
"free",
|
|
100
|
+
"batch",
|
|
101
|
+
"no discount",
|
|
102
|
+
"not ZDR",
|
|
103
|
+
"modality",
|
|
104
|
+
"tool calling",
|
|
105
|
+
"expired",
|
|
106
|
+
"age",
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
PAREN_RE = re.compile(r"\([^)]*\)")
|
|
110
|
+
PROVIDER_PREFIX_RE = re.compile(r"^[^:\s]{1,30}:\s*")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def warn(msg: str) -> None:
|
|
114
|
+
print(f"warning: {msg}", file=sys.stderr)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def norm_key(s: str) -> str:
|
|
118
|
+
return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# HTTP + cache
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class StripApiKeyRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
127
|
+
"""Drop the AA API key when a redirect leaves the original origin.
|
|
128
|
+
|
|
129
|
+
urllib copies custom request headers onto the redirected request even
|
|
130
|
+
across hosts. The key is kept only when scheme, host and port all match,
|
|
131
|
+
so a cross-host redirect or an https->http downgrade never carries it.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
135
|
+
new = super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
136
|
+
if new is not None:
|
|
137
|
+
old, dest = (
|
|
138
|
+
urllib.parse.urlsplit(req.full_url),
|
|
139
|
+
urllib.parse.urlsplit(new.full_url),
|
|
140
|
+
)
|
|
141
|
+
same_origin = (old.scheme, old.hostname, old.port) == (
|
|
142
|
+
dest.scheme,
|
|
143
|
+
dest.hostname,
|
|
144
|
+
dest.port,
|
|
145
|
+
)
|
|
146
|
+
if not same_origin:
|
|
147
|
+
# Request stores header names capitalize()d: "X-api-key".
|
|
148
|
+
new.remove_header("X-api-key")
|
|
149
|
+
return new
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def http_get(url: str, headers: dict | None = None, timeout: int = 30) -> bytes:
|
|
153
|
+
req = urllib.request.Request(
|
|
154
|
+
url,
|
|
155
|
+
headers={
|
|
156
|
+
"User-Agent": USER_AGENT,
|
|
157
|
+
"Accept-Encoding": "gzip",
|
|
158
|
+
**(headers or {}),
|
|
159
|
+
},
|
|
160
|
+
)
|
|
161
|
+
opener = urllib.request.build_opener(StripApiKeyRedirectHandler)
|
|
162
|
+
with opener.open(req, timeout=timeout) as resp:
|
|
163
|
+
data = resp.read()
|
|
164
|
+
if data[:2] == b"\x1f\x8b":
|
|
165
|
+
data = gzip.decompress(data)
|
|
166
|
+
return data
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def fetch_json(
|
|
170
|
+
url: str, headers: dict | None = None, timeout: int = 30, retries: int = 1
|
|
171
|
+
):
|
|
172
|
+
last_exc: Exception = RuntimeError("request failed")
|
|
173
|
+
for attempt in range(retries + 1):
|
|
174
|
+
try:
|
|
175
|
+
return json.loads(http_get(url, headers, timeout))
|
|
176
|
+
except urllib.error.HTTPError as exc:
|
|
177
|
+
# HTTPError is a URLError, so this clause must come first. A
|
|
178
|
+
# 4xx (bad key, forbidden, rate limited) will not change on an
|
|
179
|
+
# immediate retry; raise it at once. 5xx falls through to retry.
|
|
180
|
+
if 400 <= exc.code < 500:
|
|
181
|
+
raise
|
|
182
|
+
last_exc = exc
|
|
183
|
+
if attempt < retries:
|
|
184
|
+
time.sleep(1)
|
|
185
|
+
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
|
186
|
+
last_exc = exc
|
|
187
|
+
if attempt < retries:
|
|
188
|
+
time.sleep(1)
|
|
189
|
+
raise last_exc
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def cache_path(name: str):
|
|
193
|
+
base = os.environ.get("XDG_CACHE_HOME") or os.path.join(
|
|
194
|
+
os.path.expanduser("~"), ".cache"
|
|
195
|
+
)
|
|
196
|
+
return base and os.path.join(base, "model-compare", f"{name}.json")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def load_cache(name: str, ttl_seconds: float):
|
|
200
|
+
path = cache_path(name)
|
|
201
|
+
if not path:
|
|
202
|
+
return None
|
|
203
|
+
try:
|
|
204
|
+
with open(path) as fh:
|
|
205
|
+
blob = json.load(fh)
|
|
206
|
+
if time.time() - blob["fetched_at"] <= ttl_seconds:
|
|
207
|
+
return blob["payload"]
|
|
208
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
209
|
+
pass
|
|
210
|
+
return None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def save_cache(name: str, payload) -> None:
|
|
214
|
+
path = cache_path(name)
|
|
215
|
+
if not path:
|
|
216
|
+
return
|
|
217
|
+
try:
|
|
218
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
219
|
+
tmp_path = f"{path}.tmp"
|
|
220
|
+
with open(tmp_path, "w") as fh:
|
|
221
|
+
json.dump({"fetched_at": time.time(), "payload": payload}, fh)
|
|
222
|
+
os.replace(tmp_path, path)
|
|
223
|
+
except OSError as exc:
|
|
224
|
+
warn(f"could not write cache: {exc}")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# ---------------------------------------------------------------------------
|
|
228
|
+
# OpenRouter catalog
|
|
229
|
+
# ---------------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def fetch_openrouter_models(args):
|
|
233
|
+
if not args.no_cache:
|
|
234
|
+
cached = load_cache("openrouter-models", args.cache_ttl)
|
|
235
|
+
if cached:
|
|
236
|
+
return cached, True
|
|
237
|
+
payload = fetch_json(OPENROUTER_MODELS_URL, timeout=30)
|
|
238
|
+
models = payload.get("data") or []
|
|
239
|
+
if not models:
|
|
240
|
+
raise RuntimeError("OpenRouter API returned no models")
|
|
241
|
+
save_cache("openrouter-models", models)
|
|
242
|
+
return models, False
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
FRONTEND_DISCOUNTS_CACHE = "openrouter-frontend-discounts"
|
|
246
|
+
FRONTEND_AA_CACHE = "openrouter-frontend-aa"
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def fetch_openrouter_frontend(args):
|
|
250
|
+
"""Derive discounts, ZDR ids and OR-published AA benchmarks.
|
|
251
|
+
|
|
252
|
+
Two loads: the base frontend URL serves discounts and benchmarks in one
|
|
253
|
+
response; the per-endpoint ZDR list serves ZDR and is skipped entirely
|
|
254
|
+
under --no-zdr. Each payload caches independently under its own key and is
|
|
255
|
+
cached only when non-empty, so a degraded payload self-heals on the
|
|
256
|
+
next run. A base-URL outage never blocks the ZDR fetch, and vice
|
|
257
|
+
versa. Returns (discounts, zdr_ids, aa_by_id, cache_hits) where
|
|
258
|
+
cache_hits names the payloads served from cache ("discounts", "zdr",
|
|
259
|
+
"aa").
|
|
260
|
+
"""
|
|
261
|
+
discounts = {}
|
|
262
|
+
aa_by_id = {}
|
|
263
|
+
zdr_ids = set()
|
|
264
|
+
cache_hits = set()
|
|
265
|
+
|
|
266
|
+
aa_from_cache = False
|
|
267
|
+
if not args.no_cache:
|
|
268
|
+
cached = load_cache(FRONTEND_DISCOUNTS_CACHE, args.cache_ttl)
|
|
269
|
+
if cached:
|
|
270
|
+
discounts = cached
|
|
271
|
+
cache_hits.add("discounts")
|
|
272
|
+
cached = load_cache(FRONTEND_AA_CACHE, args.cache_ttl)
|
|
273
|
+
if cached:
|
|
274
|
+
aa_by_id = cached
|
|
275
|
+
aa_from_cache = True
|
|
276
|
+
cache_hits.add("aa")
|
|
277
|
+
|
|
278
|
+
if not discounts or not aa_from_cache:
|
|
279
|
+
try:
|
|
280
|
+
payload = fetch_json(OPENROUTER_DISCOUNTS_URL, timeout=30)
|
|
281
|
+
except Exception as exc:
|
|
282
|
+
if not discounts:
|
|
283
|
+
warn(f"could not fetch discount data: {exc}")
|
|
284
|
+
if not aa_from_cache:
|
|
285
|
+
warn(f"could not fetch AA benchmark data: {exc}")
|
|
286
|
+
else:
|
|
287
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
288
|
+
entries = data.get("models") if isinstance(data, dict) else None
|
|
289
|
+
fresh_discounts = {}
|
|
290
|
+
for entry in entries or []:
|
|
291
|
+
if not isinstance(entry, dict):
|
|
292
|
+
continue
|
|
293
|
+
slug = entry.get("slug") or ""
|
|
294
|
+
if not slug or slug.startswith("~"):
|
|
295
|
+
continue
|
|
296
|
+
endpoint = entry.get("endpoint") or {}
|
|
297
|
+
raw = (endpoint.get("pricing") or {}).get("discount")
|
|
298
|
+
if not isinstance(raw, (int, float)):
|
|
299
|
+
continue
|
|
300
|
+
variant = endpoint.get("variant") or ""
|
|
301
|
+
key = slug if variant in ("", "standard") else f"{slug}:{variant}"
|
|
302
|
+
fresh_discounts.setdefault(key, float(raw))
|
|
303
|
+
fresh_aa = build_aa_benchmarks(
|
|
304
|
+
data.get("benchmarks") if isinstance(data, dict) else None
|
|
305
|
+
)
|
|
306
|
+
if fresh_discounts:
|
|
307
|
+
discounts = fresh_discounts
|
|
308
|
+
save_cache(FRONTEND_DISCOUNTS_CACHE, discounts)
|
|
309
|
+
cache_hits.discard("discounts")
|
|
310
|
+
elif not discounts:
|
|
311
|
+
# An intact endpoint always yields hundreds of entries
|
|
312
|
+
# (discount: 0 is still an entry); an empty map means the
|
|
313
|
+
# response shape changed -- never cache that, so the next
|
|
314
|
+
# run recovers on its own.
|
|
315
|
+
warn("no discount entries found; treating discounts as unavailable")
|
|
316
|
+
if not aa_from_cache:
|
|
317
|
+
if fresh_aa:
|
|
318
|
+
aa_by_id = fresh_aa
|
|
319
|
+
save_cache(FRONTEND_AA_CACHE, aa_by_id)
|
|
320
|
+
else:
|
|
321
|
+
warn(
|
|
322
|
+
"no AA benchmark entries found; treating OpenRouter "
|
|
323
|
+
"benchmarks as unavailable"
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
if args.no_zdr:
|
|
327
|
+
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
328
|
+
if not args.no_cache:
|
|
329
|
+
# v3: the id set now comes from the per-endpoint ZDR list; v1/v2
|
|
330
|
+
# sets were derived from the models/find response and must never
|
|
331
|
+
# be read.
|
|
332
|
+
cached = load_cache("openrouter-zdr-v3", args.cache_ttl)
|
|
333
|
+
if cached:
|
|
334
|
+
return discounts, set(cached), aa_by_id, cache_hits | {"zdr"}
|
|
335
|
+
try:
|
|
336
|
+
payload = fetch_json(OPENROUTER_ZDR_URL, timeout=30)
|
|
337
|
+
except Exception as exc:
|
|
338
|
+
warn(f"could not fetch ZDR data: {exc}")
|
|
339
|
+
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
340
|
+
# One entry per ZDR endpoint, keyed by the exact model id (variants
|
|
341
|
+
# such as :free are listed individually). An id is ZDR iff it has at
|
|
342
|
+
# least one ZDR endpoint. The models/find ?zdr=true filter cannot be
|
|
343
|
+
# used: it is model-level, so it lists non-ZDR variants (NVIDIA's
|
|
344
|
+
# :free endpoints) too. No "~" router aliases occur in this list, so
|
|
345
|
+
# unlike the find parsing there is no "~" skip; any stray id is
|
|
346
|
+
# harmless because build_candidates only matches real model ids.
|
|
347
|
+
# An endpoint counts only when its status is exactly the integer 0.
|
|
348
|
+
# OpenRouter's OpenAPI schema (EndpointStatus) lists 0, -1, -2, -3, -5
|
|
349
|
+
# and -10 without describing them, so the meaning of the non-zero values
|
|
350
|
+
# is unverified; treat anything other than 0, including a missing
|
|
351
|
+
# status, as not serving (fail closed).
|
|
352
|
+
entries = payload.get("data") if isinstance(payload, dict) else None
|
|
353
|
+
ids = set()
|
|
354
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
355
|
+
if not isinstance(entry, dict):
|
|
356
|
+
continue
|
|
357
|
+
status = entry.get("status")
|
|
358
|
+
if type(status) is not int or status != 0:
|
|
359
|
+
continue
|
|
360
|
+
model_id = entry.get("model_id")
|
|
361
|
+
if isinstance(model_id, str) and model_id:
|
|
362
|
+
ids.add(model_id)
|
|
363
|
+
if not ids:
|
|
364
|
+
warn("no ZDR entries found; treating ZDR data as unavailable")
|
|
365
|
+
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
366
|
+
save_cache("openrouter-zdr-v3", sorted(ids))
|
|
367
|
+
return discounts, ids, aa_by_id, cache_hits
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def parse_price(value) -> float | None:
|
|
371
|
+
# bools are not prices; a huge JSON int (json.loads yields an exact
|
|
372
|
+
# Python int) overflows float() -- both fail soft to None.
|
|
373
|
+
if isinstance(value, bool):
|
|
374
|
+
return None
|
|
375
|
+
try:
|
|
376
|
+
return float(value)
|
|
377
|
+
except (TypeError, ValueError, OverflowError):
|
|
378
|
+
return None
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
# ---------------------------------------------------------------------------
|
|
382
|
+
# OpenRouter tiered pricing (pricing.overrides)
|
|
383
|
+
# ---------------------------------------------------------------------------
|
|
384
|
+
|
|
385
|
+
# Keys of a pricing.overrides entry (OpenRouter docs, verified 2026-10-09).
|
|
386
|
+
# An entry carrying any other key is skipped whole (docs forward-compat rule).
|
|
387
|
+
TIER_CONDITION_KEYS = (
|
|
388
|
+
"min_prompt_tokens",
|
|
389
|
+
"utc_start",
|
|
390
|
+
"utc_end",
|
|
391
|
+
"utc_days",
|
|
392
|
+
)
|
|
393
|
+
TIER_PRICE_KEYS = (
|
|
394
|
+
"prompt",
|
|
395
|
+
"completion",
|
|
396
|
+
"request",
|
|
397
|
+
"image",
|
|
398
|
+
"image_output",
|
|
399
|
+
"audio",
|
|
400
|
+
"audio_output",
|
|
401
|
+
"web_search",
|
|
402
|
+
"internal_reasoning",
|
|
403
|
+
"input_cache_read",
|
|
404
|
+
"input_cache_write",
|
|
405
|
+
"input_cache_write_1h",
|
|
406
|
+
"input_audio_cache",
|
|
407
|
+
)
|
|
408
|
+
WEEKDAYS = (
|
|
409
|
+
"monday",
|
|
410
|
+
"tuesday",
|
|
411
|
+
"wednesday",
|
|
412
|
+
"thursday",
|
|
413
|
+
"friday",
|
|
414
|
+
"saturday",
|
|
415
|
+
"sunday",
|
|
416
|
+
)
|
|
417
|
+
_WEEK_MINUTES = 7 * 24 * 60
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _integral(value) -> int | None:
|
|
421
|
+
"""Finite, non-bool, integral number -> int; else None (fail-soft)."""
|
|
422
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
423
|
+
return None
|
|
424
|
+
# Magnitude guard first: math.isfinite raises OverflowError on huge ints
|
|
425
|
+
# (json.loads of a 400-digit literal); no real token count or clock
|
|
426
|
+
# comes anywhere near 1e15.
|
|
427
|
+
if abs(value) > 10**15:
|
|
428
|
+
return None
|
|
429
|
+
if not math.isfinite(value) or value != int(value):
|
|
430
|
+
return None
|
|
431
|
+
return int(value)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _hhmm(value) -> int | None:
|
|
435
|
+
"""Integer HHMM clock in 0..2359 with minutes < 60; else None."""
|
|
436
|
+
clock = _integral(value)
|
|
437
|
+
if clock is None or not 0 <= clock <= 2359 or clock % 100 > 59:
|
|
438
|
+
return None
|
|
439
|
+
return clock
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _hhmm_minutes(clock: int) -> int:
|
|
443
|
+
return (clock // 100) * 60 + clock % 100
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _window_minutes(start: int, end: int) -> int:
|
|
447
|
+
"""Minutes a HHMM window covers per day (wrap-aware; whole day when equal)."""
|
|
448
|
+
span = _hhmm_minutes(end) - _hhmm_minutes(start)
|
|
449
|
+
if span == 0:
|
|
450
|
+
return 24 * 60
|
|
451
|
+
return span % (24 * 60)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _window_span(start: int, end: int):
|
|
455
|
+
"""The per-day minute values a window matches (wrap-aware)."""
|
|
456
|
+
start_min = _hhmm_minutes(start)
|
|
457
|
+
end_min = _hhmm_minutes(end)
|
|
458
|
+
if start_min == end_min:
|
|
459
|
+
return range(24 * 60)
|
|
460
|
+
if start_min < end_min:
|
|
461
|
+
return range(start_min, end_min)
|
|
462
|
+
return list(range(start_min, 24 * 60)) + list(range(0, end_min))
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _coverage(days_count: int, minutes: int) -> float:
|
|
466
|
+
"""Fraction of the 7-day week a window covers, 4 decimals."""
|
|
467
|
+
return round(days_count * minutes / _WEEK_MINUTES, 4)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _entry_prices(entry):
|
|
471
|
+
"""Parse an override entry's prompt/completion prices (spec rule 5).
|
|
472
|
+
|
|
473
|
+
Returns {"prompt": float|None, "completion": float|None} where None means
|
|
474
|
+
the key is absent (inheritance), or None when a present price is
|
|
475
|
+
malformed, negative or non-finite (the entry is skipped whole).
|
|
476
|
+
"""
|
|
477
|
+
values = {}
|
|
478
|
+
for key in ("prompt", "completion"):
|
|
479
|
+
if key not in entry:
|
|
480
|
+
values[key] = None
|
|
481
|
+
continue
|
|
482
|
+
value = parse_price(entry[key])
|
|
483
|
+
if value is None or not math.isfinite(value) or value < 0:
|
|
484
|
+
return None
|
|
485
|
+
values[key] = value
|
|
486
|
+
return values
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _schedule_windows(window_entries):
|
|
490
|
+
"""Normalize validated time-window entries (spec rules 6, 9, 10).
|
|
491
|
+
|
|
492
|
+
Returns the window list (API order, days-only windows normalized to 0/0,
|
|
493
|
+
utc_days deduplicated in week order, per-window coverage), or None when
|
|
494
|
+
the windows do not tile the 7-day week exactly (gaps or overlaps), or a
|
|
495
|
+
window is missing prompt/completion (rule 7: no top-level inheritance --
|
|
496
|
+
the top level is fetch-time dependent).
|
|
497
|
+
"""
|
|
498
|
+
windows = []
|
|
499
|
+
covered = set()
|
|
500
|
+
matched = 0
|
|
501
|
+
for entry in window_entries:
|
|
502
|
+
start, end = entry["start"], entry["end"]
|
|
503
|
+
days = entry["days"] # None, or a list of weekday indices
|
|
504
|
+
if start is None: # days-only window: the whole UTC day
|
|
505
|
+
start, end = 0, 0
|
|
506
|
+
price_in, price_out = entry["price_in"], entry["price_out"]
|
|
507
|
+
if price_in is None or price_out is None:
|
|
508
|
+
return None
|
|
509
|
+
day_indices = list(range(len(WEEKDAYS))) if days is None else sorted(set(days))
|
|
510
|
+
span = _window_span(start, end)
|
|
511
|
+
for day in day_indices:
|
|
512
|
+
for minute in span:
|
|
513
|
+
covered.add((day, minute))
|
|
514
|
+
matched += len(span)
|
|
515
|
+
windows.append(
|
|
516
|
+
{
|
|
517
|
+
"utc_days": None
|
|
518
|
+
if days is None
|
|
519
|
+
else [WEEKDAYS[i] for i in day_indices],
|
|
520
|
+
"utc_start": start,
|
|
521
|
+
"utc_end": end,
|
|
522
|
+
"coverage": _coverage(len(day_indices), len(span)),
|
|
523
|
+
"price_in": price_in,
|
|
524
|
+
"price_out": price_out,
|
|
525
|
+
}
|
|
526
|
+
)
|
|
527
|
+
if matched != _WEEK_MINUTES or len(covered) != _WEEK_MINUTES:
|
|
528
|
+
return None # gaps or overlaps: the schedule does not tile the week
|
|
529
|
+
return windows
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def effective_pricing(pricing, prompt_tokens, input_share):
|
|
533
|
+
"""Resolve pricing.overrides into one effective price (spec section 5.1).
|
|
534
|
+
|
|
535
|
+
Prices are USD per token. Returns a dict with the effective and base
|
|
536
|
+
prices (per token), the applied tier threshold, the cumulative tier list,
|
|
537
|
+
the normalized schedule (or None), whether the schedule is invalid, the
|
|
538
|
+
rounded maximum off-peak discount, and the peak window's blended price.
|
|
539
|
+
"""
|
|
540
|
+
source = pricing if isinstance(pricing, dict) else {}
|
|
541
|
+
top_in = parse_price(source.get("prompt"))
|
|
542
|
+
top_out = parse_price(source.get("completion"))
|
|
543
|
+
base_in, base_out = top_in, top_out
|
|
544
|
+
overrides = source.get("overrides")
|
|
545
|
+
|
|
546
|
+
tier_entries = [] # (threshold, price_in|None, price_out|None), API order
|
|
547
|
+
window_entries = [] # validated time-window pre-windows, API order
|
|
548
|
+
time_touched = False
|
|
549
|
+
schedule_broken = False
|
|
550
|
+
|
|
551
|
+
if isinstance(overrides, list):
|
|
552
|
+
for entry in overrides:
|
|
553
|
+
if not isinstance(entry, dict):
|
|
554
|
+
continue
|
|
555
|
+
time_keys = [
|
|
556
|
+
key for key in ("utc_start", "utc_end", "utc_days") if key in entry
|
|
557
|
+
]
|
|
558
|
+
if any(
|
|
559
|
+
key not in TIER_CONDITION_KEYS and key not in TIER_PRICE_KEYS
|
|
560
|
+
for key in entry
|
|
561
|
+
):
|
|
562
|
+
if time_keys:
|
|
563
|
+
schedule_broken = True # skipped time window: tiling gap
|
|
564
|
+
continue
|
|
565
|
+
has_threshold = "min_prompt_tokens" in entry
|
|
566
|
+
threshold = _integral(entry["min_prompt_tokens"]) if has_threshold else None
|
|
567
|
+
if has_threshold and (threshold is None or threshold < 0):
|
|
568
|
+
if time_keys:
|
|
569
|
+
schedule_broken = True
|
|
570
|
+
continue
|
|
571
|
+
if ("utc_start" in entry) != ("utc_end" in entry):
|
|
572
|
+
schedule_broken = True # exactly one of the pair
|
|
573
|
+
continue
|
|
574
|
+
start = end = None
|
|
575
|
+
if "utc_start" in entry:
|
|
576
|
+
start = _hhmm(entry["utc_start"])
|
|
577
|
+
end = _hhmm(entry["utc_end"])
|
|
578
|
+
if start is None or end is None:
|
|
579
|
+
schedule_broken = True
|
|
580
|
+
continue
|
|
581
|
+
days = None
|
|
582
|
+
if "utc_days" in entry:
|
|
583
|
+
names = entry["utc_days"]
|
|
584
|
+
if (
|
|
585
|
+
not isinstance(names, list)
|
|
586
|
+
or not names
|
|
587
|
+
or any(day not in WEEKDAYS for day in names)
|
|
588
|
+
):
|
|
589
|
+
schedule_broken = True
|
|
590
|
+
continue
|
|
591
|
+
days = sorted({WEEKDAYS.index(day) for day in names})
|
|
592
|
+
prices = _entry_prices(entry)
|
|
593
|
+
if prices is None:
|
|
594
|
+
if time_keys:
|
|
595
|
+
schedule_broken = True
|
|
596
|
+
continue
|
|
597
|
+
if time_keys:
|
|
598
|
+
time_touched = True
|
|
599
|
+
if has_threshold:
|
|
600
|
+
# combined token+time entry: invalid time window (rule 9)
|
|
601
|
+
schedule_broken = True
|
|
602
|
+
continue
|
|
603
|
+
window_entries.append(
|
|
604
|
+
{
|
|
605
|
+
"start": start,
|
|
606
|
+
"end": end,
|
|
607
|
+
"days": days,
|
|
608
|
+
"price_in": prices["prompt"],
|
|
609
|
+
"price_out": prices["completion"],
|
|
610
|
+
}
|
|
611
|
+
)
|
|
612
|
+
elif has_threshold:
|
|
613
|
+
tier_entries.append((threshold, prices["prompt"], prices["completion"]))
|
|
614
|
+
# else: conditionless entry (price keys only) -- ignored (plan
|
|
615
|
+
# Review Focus; none live, and honoring an unconditional price
|
|
616
|
+
# swap would be dangerous)
|
|
617
|
+
|
|
618
|
+
schedule = None
|
|
619
|
+
peak_blended = None
|
|
620
|
+
max_discount = None
|
|
621
|
+
schedule_error = False
|
|
622
|
+
|
|
623
|
+
if time_touched or schedule_broken:
|
|
624
|
+
if schedule_broken:
|
|
625
|
+
schedule_error = True
|
|
626
|
+
else:
|
|
627
|
+
schedule = _schedule_windows(window_entries)
|
|
628
|
+
if schedule is None:
|
|
629
|
+
schedule_error = True
|
|
630
|
+
schedule = None
|
|
631
|
+
if schedule is not None and not schedule_error:
|
|
632
|
+
for window in schedule:
|
|
633
|
+
window["blended"] = (
|
|
634
|
+
input_share * window["price_in"]
|
|
635
|
+
+ (1 - input_share) * window["price_out"]
|
|
636
|
+
)
|
|
637
|
+
peak = max(schedule, key=lambda window: window["blended"])
|
|
638
|
+
peak_blended = peak["blended"] # ties: first in API order
|
|
639
|
+
# Deterministic base (rule 7): freeze at the peak window's prices.
|
|
640
|
+
base_in, base_out = peak["price_in"], peak["price_out"]
|
|
641
|
+
if peak_blended == 0:
|
|
642
|
+
max_discount = 0.0
|
|
643
|
+
else:
|
|
644
|
+
max_discount = round(
|
|
645
|
+
max(1 - window["blended"] / peak_blended for window in schedule), 4
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
# Token tiers (rule 8) over the (possibly frozen) base: later entries win
|
|
649
|
+
# per key in array order; a threshold equal to prompt_tokens never applies.
|
|
650
|
+
applied = None
|
|
651
|
+
current_in, current_out = base_in, base_out
|
|
652
|
+
by_threshold = {}
|
|
653
|
+
for threshold, tier_in, tier_out in tier_entries:
|
|
654
|
+
if prompt_tokens > threshold:
|
|
655
|
+
if tier_in is not None:
|
|
656
|
+
current_in = tier_in
|
|
657
|
+
if tier_out is not None:
|
|
658
|
+
current_out = tier_out
|
|
659
|
+
applied = threshold
|
|
660
|
+
slot = by_threshold.setdefault(threshold, {"price_in": None, "price_out": None})
|
|
661
|
+
if tier_in is not None:
|
|
662
|
+
slot["price_in"] = tier_in
|
|
663
|
+
if tier_out is not None:
|
|
664
|
+
slot["price_out"] = tier_out
|
|
665
|
+
tiers = []
|
|
666
|
+
running_in, running_out = base_in, base_out
|
|
667
|
+
for threshold in sorted(by_threshold):
|
|
668
|
+
slot = by_threshold[threshold]
|
|
669
|
+
if slot["price_in"] is not None:
|
|
670
|
+
running_in = slot["price_in"]
|
|
671
|
+
if slot["price_out"] is not None:
|
|
672
|
+
running_out = slot["price_out"]
|
|
673
|
+
tiers.append(
|
|
674
|
+
{
|
|
675
|
+
"min_prompt_tokens": threshold,
|
|
676
|
+
"price_in": running_in,
|
|
677
|
+
"price_out": running_out,
|
|
678
|
+
}
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
return {
|
|
682
|
+
"price_in": current_in,
|
|
683
|
+
"price_out": current_out,
|
|
684
|
+
# top_*: the RAW OpenRouter top-level prices (spec rule 1) -- the
|
|
685
|
+
# validity gate checks these even for windowed models, whose base_*
|
|
686
|
+
# fields are the frozen peak window (rule 7).
|
|
687
|
+
"top_price_in": top_in,
|
|
688
|
+
"top_price_out": top_out,
|
|
689
|
+
"base_price_in": base_in,
|
|
690
|
+
"base_price_out": base_out,
|
|
691
|
+
"tier_prompt_tokens": applied,
|
|
692
|
+
"tiers": tiers,
|
|
693
|
+
"schedule": schedule,
|
|
694
|
+
"schedule_error": schedule_error,
|
|
695
|
+
"max_discount": max_discount,
|
|
696
|
+
"peak_blended": peak_blended,
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def fmt_tier_note(threshold):
|
|
701
|
+
""">100k-style marker for the applied token tier; None passes through."""
|
|
702
|
+
if threshold is None:
|
|
703
|
+
return None
|
|
704
|
+
value = int(threshold)
|
|
705
|
+
if value >= 1000:
|
|
706
|
+
return f">{value // 1000}k"
|
|
707
|
+
return f">{value}"
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def fmt_sched_note(max_discount):
|
|
711
|
+
"""-38%-style marker for a schedule's max off-peak discount."""
|
|
712
|
+
if max_discount is None:
|
|
713
|
+
return None
|
|
714
|
+
formatted = f"{max_discount:.0%}"
|
|
715
|
+
if formatted == "0%":
|
|
716
|
+
return "SCHED"
|
|
717
|
+
return f"-{formatted}"
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def fmt_sched_detail(schedule, peak_blended):
|
|
721
|
+
"""Human-readable off-peak schedule summary (spec section 5.6 grammar).
|
|
722
|
+
|
|
723
|
+
Off-peak windows are those whose blended price is strictly below the
|
|
724
|
+
peak's; schedule windows carry their parser-computed "blended" price.
|
|
725
|
+
"""
|
|
726
|
+
if not schedule or peak_blended is None:
|
|
727
|
+
return None
|
|
728
|
+
offpeak = [window for window in schedule if window["blended"] < peak_blended]
|
|
729
|
+
if not offpeak:
|
|
730
|
+
return None
|
|
731
|
+
|
|
732
|
+
def _clock(clock):
|
|
733
|
+
return f"{clock // 100:02d}:{clock % 100:02d}"
|
|
734
|
+
|
|
735
|
+
groups = {}
|
|
736
|
+
for window in offpeak:
|
|
737
|
+
days = window["utc_days"]
|
|
738
|
+
# All seven weekdays listed explicitly is the same day set as
|
|
739
|
+
# utc_days absent: same group, same "daily" label (spec 5.6).
|
|
740
|
+
if days is None or set(days) == set(WEEKDAYS):
|
|
741
|
+
key = (0, -1, ())
|
|
742
|
+
label = "daily"
|
|
743
|
+
else:
|
|
744
|
+
indices = [WEEKDAYS.index(day) for day in days]
|
|
745
|
+
key = (1, min(indices), tuple(days))
|
|
746
|
+
exact = {WEEKDAYS[i] for i in range(5)}
|
|
747
|
+
if set(days) == exact:
|
|
748
|
+
label = "weekdays"
|
|
749
|
+
elif set(days) == {"saturday", "sunday"}:
|
|
750
|
+
label = "weekends"
|
|
751
|
+
else:
|
|
752
|
+
label = ",".join(day[:3] for day in days) # week order
|
|
753
|
+
group = groups.setdefault(key, {"label": label, "windows": []})
|
|
754
|
+
group["windows"].append(window)
|
|
755
|
+
parts = []
|
|
756
|
+
for key in sorted(groups):
|
|
757
|
+
group = groups[key]
|
|
758
|
+
bits = []
|
|
759
|
+
for window in sorted(group["windows"], key=lambda w: w["utc_start"]):
|
|
760
|
+
if window["utc_start"] == window["utc_end"]:
|
|
761
|
+
bits.append("all day")
|
|
762
|
+
else:
|
|
763
|
+
bits.append(
|
|
764
|
+
f"{_clock(window['utc_start'])}-{_clock(window['utc_end'])}"
|
|
765
|
+
)
|
|
766
|
+
parts.append(f"{group['label']} {', '.join(bits)}")
|
|
767
|
+
return "; ".join(parts) + " UTC"
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def coerce_int(value, default: int = 0) -> int:
|
|
771
|
+
"""Best-effort int coercion for external fields that may be str/float/None."""
|
|
772
|
+
try:
|
|
773
|
+
return int(float(value))
|
|
774
|
+
except (TypeError, ValueError):
|
|
775
|
+
return default
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def parse_iso_datetime(value):
|
|
779
|
+
if not value:
|
|
780
|
+
return None
|
|
781
|
+
try:
|
|
782
|
+
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
783
|
+
except ValueError:
|
|
784
|
+
return None
|
|
785
|
+
if parsed.tzinfo is None:
|
|
786
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
787
|
+
return parsed
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
# ---------------------------------------------------------------------------
|
|
791
|
+
# Artificial Analysis intelligence index
|
|
792
|
+
# ---------------------------------------------------------------------------
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def aa_api_entries(api_key: str) -> list:
|
|
796
|
+
"""Intelligence entries from AA's V2 language models endpoint.
|
|
797
|
+
|
|
798
|
+
Documented envelope: {tier, intelligence_index_version, pagination,
|
|
799
|
+
data[]} with snake_case fields; the index lives at
|
|
800
|
+
data[].evaluations.artificial_analysis_intelligence_index. Items
|
|
801
|
+
without a usable slug/name or without a finite numeric index are
|
|
802
|
+
skipped (nulls mean "not measured"). Pagination follows the documented
|
|
803
|
+
has_more/total_pages contract with a 25-page hard cap; a failure past
|
|
804
|
+
page 1 is best-effort (warn, keep earlier pages), a page-1 failure
|
|
805
|
+
propagates.
|
|
806
|
+
"""
|
|
807
|
+
entries = {}
|
|
808
|
+
page = 1
|
|
809
|
+
while page <= 25: # hard cap: halt after processing page 25
|
|
810
|
+
try:
|
|
811
|
+
payload = fetch_json(
|
|
812
|
+
f"{AA_API_URL}?page={page}",
|
|
813
|
+
headers={"x-api-key": api_key},
|
|
814
|
+
timeout=30,
|
|
815
|
+
)
|
|
816
|
+
except Exception as exc:
|
|
817
|
+
if page == 1:
|
|
818
|
+
raise
|
|
819
|
+
warn(f"AA API page {page} failed ({exc}); using entries collected so far")
|
|
820
|
+
break
|
|
821
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
822
|
+
for item in data if isinstance(data, list) else []:
|
|
823
|
+
if not isinstance(item, dict):
|
|
824
|
+
continue
|
|
825
|
+
slug = item.get("slug")
|
|
826
|
+
name = item.get("name")
|
|
827
|
+
key = (
|
|
828
|
+
slug
|
|
829
|
+
if isinstance(slug, str) and slug
|
|
830
|
+
else (name if isinstance(name, str) and name else None)
|
|
831
|
+
)
|
|
832
|
+
if not key:
|
|
833
|
+
continue
|
|
834
|
+
evals = item.get("evaluations")
|
|
835
|
+
raw = (
|
|
836
|
+
evals.get("artificial_analysis_intelligence_index")
|
|
837
|
+
if isinstance(evals, dict)
|
|
838
|
+
else None
|
|
839
|
+
)
|
|
840
|
+
if not isinstance(raw, (int, float)) or isinstance(raw, bool):
|
|
841
|
+
continue
|
|
842
|
+
try:
|
|
843
|
+
index = float(raw) # huge ints raise OverflowError
|
|
844
|
+
except OverflowError:
|
|
845
|
+
continue
|
|
846
|
+
if not math.isfinite(index):
|
|
847
|
+
continue
|
|
848
|
+
entries.setdefault(
|
|
849
|
+
key,
|
|
850
|
+
{
|
|
851
|
+
"key": key,
|
|
852
|
+
"name": name if isinstance(name, str) and name else key,
|
|
853
|
+
"index": index,
|
|
854
|
+
},
|
|
855
|
+
)
|
|
856
|
+
pag = payload.get("pagination") if isinstance(payload, dict) else None
|
|
857
|
+
if not isinstance(pag, dict):
|
|
858
|
+
break # undocumented shape: single-page degrade
|
|
859
|
+
if pag.get("has_more") is not True:
|
|
860
|
+
break
|
|
861
|
+
total_pages = pag.get("total_pages")
|
|
862
|
+
if isinstance(total_pages, int) and page >= total_pages:
|
|
863
|
+
break
|
|
864
|
+
page += 1
|
|
865
|
+
return list(entries.values())
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def fetch_aa_entries(args):
|
|
869
|
+
"""AA intelligence entries from the keyed AA API v2, or nothing.
|
|
870
|
+
|
|
871
|
+
Returns (entries, source, cached). With a key, an API failure or an
|
|
872
|
+
empty result warns and yields ([], None, False). Without a key this is
|
|
873
|
+
silent and yields ([], None, False): OpenRouter's benchmarks are the
|
|
874
|
+
primary source and the catalog records the absence (sources.aa).
|
|
875
|
+
"""
|
|
876
|
+
if not args.no_cache:
|
|
877
|
+
# v2: entries cached under the old key predate the V2 endpoint
|
|
878
|
+
# migration and must never be read. Only AA API entries are read;
|
|
879
|
+
# build_catalog rejects any other source as unknown.
|
|
880
|
+
cached = load_cache("aa-intelligence-v2", args.cache_ttl)
|
|
881
|
+
if isinstance(cached, dict) and cached.get("source") == "AA API v2":
|
|
882
|
+
return cached.get("entries", []), cached.get("source"), True
|
|
883
|
+
api_key = args.aa_api_key or os.environ.get("AA_API_KEY")
|
|
884
|
+
if not api_key:
|
|
885
|
+
return [], None, False
|
|
886
|
+
try:
|
|
887
|
+
entries = aa_api_entries(api_key)
|
|
888
|
+
except Exception as exc:
|
|
889
|
+
warn(f"AA API request failed ({exc})")
|
|
890
|
+
return [], None, False
|
|
891
|
+
if not entries:
|
|
892
|
+
warn("AA API returned no intelligence scores")
|
|
893
|
+
return [], None, False
|
|
894
|
+
save_cache("aa-intelligence-v2", {"entries": entries, "source": "AA API v2"})
|
|
895
|
+
return entries, "AA API v2", False
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
# ---------------------------------------------------------------------------
|
|
899
|
+
# Matching AA entries to OpenRouter model ids
|
|
900
|
+
# ---------------------------------------------------------------------------
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def build_aa_lookup(entries):
|
|
904
|
+
exact = {}
|
|
905
|
+
fuzzy = []
|
|
906
|
+
for entry in entries:
|
|
907
|
+
key = entry.get("key") or ""
|
|
908
|
+
name = entry.get("name") or key
|
|
909
|
+
raw_index = entry.get("index")
|
|
910
|
+
if not isinstance(raw_index, (int, float)) or isinstance(raw_index, bool):
|
|
911
|
+
continue
|
|
912
|
+
index = float(raw_index)
|
|
913
|
+
# Non-finite indexes would serialize as NaN and poison quality.
|
|
914
|
+
if not math.isfinite(index):
|
|
915
|
+
continue
|
|
916
|
+
|
|
917
|
+
def put(tier, raw):
|
|
918
|
+
normalized = norm_key(raw)
|
|
919
|
+
if normalized:
|
|
920
|
+
exact.setdefault((tier, normalized), index)
|
|
921
|
+
|
|
922
|
+
if "/" in key:
|
|
923
|
+
put(0, key)
|
|
924
|
+
put(1, key.rsplit("/", 1)[-1])
|
|
925
|
+
else:
|
|
926
|
+
put(1, key)
|
|
927
|
+
put(2, name)
|
|
928
|
+
put(3, PAREN_RE.sub(" ", name))
|
|
929
|
+
tokens = set(norm_key(key).split()) | set(norm_key(name).split())
|
|
930
|
+
if tokens:
|
|
931
|
+
fuzzy.append((tokens, index))
|
|
932
|
+
return exact, fuzzy
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def match_quality(model, exact, fuzzy, allow_fuzzy=False):
|
|
936
|
+
model_id = model["id"]
|
|
937
|
+
name = model.get("name") or ""
|
|
938
|
+
display = PROVIDER_PREFIX_RE.sub("", name).strip()
|
|
939
|
+
# Assumes model_id contains a "/" (provider/base). build_candidates drops
|
|
940
|
+
# malformed ids upstream, so base is never empty here.
|
|
941
|
+
_, _, base = model_id.split(":", 1)[0].partition("/")
|
|
942
|
+
|
|
943
|
+
for tier, raw in (
|
|
944
|
+
(0, model_id),
|
|
945
|
+
(1, base),
|
|
946
|
+
(2, display),
|
|
947
|
+
(3, PAREN_RE.sub(" ", display)),
|
|
948
|
+
):
|
|
949
|
+
normalized = norm_key(raw)
|
|
950
|
+
if normalized and (tier, normalized) in exact:
|
|
951
|
+
return exact[(tier, normalized)]
|
|
952
|
+
|
|
953
|
+
if not allow_fuzzy:
|
|
954
|
+
return None
|
|
955
|
+
own = set(norm_key(base).split()) | set(norm_key(display).split())
|
|
956
|
+
if own:
|
|
957
|
+
best = 0.0
|
|
958
|
+
best_index = None
|
|
959
|
+
for tokens, index in fuzzy:
|
|
960
|
+
overlap = len(own & tokens) / len(own | tokens)
|
|
961
|
+
if overlap > best:
|
|
962
|
+
best = overlap
|
|
963
|
+
best_index = index
|
|
964
|
+
if best >= 0.5 and best_index is not None:
|
|
965
|
+
return best_index
|
|
966
|
+
return None
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def base_model_id(model_id: str) -> str:
|
|
970
|
+
"""Candidate id without its :variant suffix (OR benchmarks are per base model)."""
|
|
971
|
+
return model_id.split(":", 1)[0]
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
AA_BENCHMARK_FIELDS = ("intelligence_index", "coding_index", "agentic_index")
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def build_aa_benchmarks(benchmarks) -> dict:
|
|
978
|
+
"""Map bare OpenRouter id -> OR-published AA trio from data.benchmarks.
|
|
979
|
+
|
|
980
|
+
Keys are dated permaslugs (z-ai/glm-5.3-flash-20260826): the trailing
|
|
981
|
+
-YYYYMMDD is stripped and undated keys map to themselves; when two keys
|
|
982
|
+
strip to the same id the latest date wins. Values failing the
|
|
983
|
+
finite-number guard are treated as absent; entries with at least one
|
|
984
|
+
valid field are kept.
|
|
985
|
+
"""
|
|
986
|
+
aa_by_id = {}
|
|
987
|
+
seen_date = {}
|
|
988
|
+
|
|
989
|
+
def accept(bare, date, trio):
|
|
990
|
+
if bare in seen_date and seen_date[bare] >= date:
|
|
991
|
+
return
|
|
992
|
+
seen_date[bare] = date
|
|
993
|
+
aa_by_id[bare] = trio
|
|
994
|
+
|
|
995
|
+
for key, node in (benchmarks or {}).items():
|
|
996
|
+
aa = node.get("aa") if isinstance(node, dict) else None
|
|
997
|
+
if not isinstance(aa, dict):
|
|
998
|
+
continue
|
|
999
|
+
trio = {}
|
|
1000
|
+
for field in AA_BENCHMARK_FIELDS:
|
|
1001
|
+
value = aa.get(field)
|
|
1002
|
+
if (
|
|
1003
|
+
isinstance(value, (int, float))
|
|
1004
|
+
and not isinstance(value, bool)
|
|
1005
|
+
and math.isfinite(value)
|
|
1006
|
+
):
|
|
1007
|
+
trio[field] = float(value)
|
|
1008
|
+
if not trio:
|
|
1009
|
+
continue
|
|
1010
|
+
match = re.match(r"^(.*)-(\d{8})$", key)
|
|
1011
|
+
if match:
|
|
1012
|
+
accept(match.group(1), match.group(2), trio)
|
|
1013
|
+
else:
|
|
1014
|
+
accept(key, "", trio)
|
|
1015
|
+
return aa_by_id
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def resolve_quality(candidates, aa_by_id, exact, fuzzy, aa_source):
|
|
1019
|
+
"""Per-candidate quality: OR benchmarks first, AA exact fallback, else None.
|
|
1020
|
+
|
|
1021
|
+
OR's per-slug values cannot mispair; the AA fallback is exact-tier only
|
|
1022
|
+
(match_quality's fuzzy tier is the proven variant-conflation bug and
|
|
1023
|
+
stays off in production). Source values match the catalog contract:
|
|
1024
|
+
"openrouter", "api".
|
|
1025
|
+
"""
|
|
1026
|
+
fallback_source = {"AA API v2": "api"}.get(aa_source)
|
|
1027
|
+
quality_by_id = {}
|
|
1028
|
+
source_by_id = {}
|
|
1029
|
+
for cand in candidates:
|
|
1030
|
+
bench = aa_by_id.get(base_model_id(cand["id"]))
|
|
1031
|
+
intelligence = bench.get("intelligence_index") if bench else None
|
|
1032
|
+
if intelligence is not None:
|
|
1033
|
+
quality_by_id[cand["id"]] = intelligence
|
|
1034
|
+
source_by_id[cand["id"]] = "openrouter"
|
|
1035
|
+
continue
|
|
1036
|
+
index = match_quality({"id": cand["id"], "name": cand["name"]}, exact, fuzzy)
|
|
1037
|
+
if index is not None:
|
|
1038
|
+
quality_by_id[cand["id"]] = index
|
|
1039
|
+
source_by_id[cand["id"]] = fallback_source
|
|
1040
|
+
return quality_by_id, source_by_id
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def model_family(model_id: str) -> str | None:
|
|
1044
|
+
"""Best-effort model family from the base slug (documented heuristic).
|
|
1045
|
+
|
|
1046
|
+
The leading token delimited by dash, underscore or digit of the
|
|
1047
|
+
lowercased base slug: glm-5.3 -> glm, gpt-5.2-mini -> gpt,
|
|
1048
|
+
deepseek-chat-v4 -> deepseek. Oddballs yield oddballs (o4-mini -> "o");
|
|
1049
|
+
a bare letter-run followed by a trailing digit version (k2) yields None.
|
|
1050
|
+
"""
|
|
1051
|
+
base = model_id.split(":", 1)[0].partition("/")[2].lower()
|
|
1052
|
+
parts = re.split(r"[-_\d]", base, maxsplit=1)
|
|
1053
|
+
token = parts[0]
|
|
1054
|
+
if len(parts) > 1 and not parts[1] and base[len(token) : len(token) + 1].isdigit():
|
|
1055
|
+
return None
|
|
1056
|
+
return token or None
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def _per_1m(price_per_token: float) -> float:
|
|
1060
|
+
"""USD per token -> USD per 1M tokens, 6 decimals."""
|
|
1061
|
+
return round(price_per_token * 1_000_000.0, 6)
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
|
|
1065
|
+
"""Catalog-wide off-peak index built from the RAW model list (spec 5.4).
|
|
1066
|
+
|
|
1067
|
+
Deliberately unfiltered by the candidate pool: sub-floor-context and
|
|
1068
|
+
non-ZDR schedule models are listed (that is the point -- today's live
|
|
1069
|
+
schedule models mostly sit below the 1M context floor). Router aliases
|
|
1070
|
+
(~ids) and malformed ids are excluded; :batch variants are included as
|
|
1071
|
+
distinct deals. Sorted by id.
|
|
1072
|
+
"""
|
|
1073
|
+
entries = []
|
|
1074
|
+
for model in models:
|
|
1075
|
+
model_id = model.get("id") or ""
|
|
1076
|
+
if not model_id or model_id.startswith("~") or "/" not in model_id:
|
|
1077
|
+
continue
|
|
1078
|
+
eff = effective_pricing(
|
|
1079
|
+
model.get("pricing") or {}, args.min_context, args.input_share
|
|
1080
|
+
)
|
|
1081
|
+
top_in, top_out = eff["top_price_in"], eff["top_price_out"]
|
|
1082
|
+
if (
|
|
1083
|
+
top_in is None
|
|
1084
|
+
or top_out is None
|
|
1085
|
+
or top_in < 0
|
|
1086
|
+
or top_out < 0
|
|
1087
|
+
or not math.isfinite(top_in)
|
|
1088
|
+
or not math.isfinite(top_out)
|
|
1089
|
+
):
|
|
1090
|
+
continue # invalid top level: not a deal worth listing
|
|
1091
|
+
windows = eff["schedule"]
|
|
1092
|
+
if eff["schedule_error"] or not windows:
|
|
1093
|
+
continue
|
|
1094
|
+
peak = max(windows, key=lambda window: window["blended"])
|
|
1095
|
+
offpeak = min(windows, key=lambda window: window["blended"])
|
|
1096
|
+
quality = quality_by_id.get(model_id)
|
|
1097
|
+
if quality is None:
|
|
1098
|
+
benchmark = aa_by_id.get(base_model_id(model_id)) or {}
|
|
1099
|
+
quality = benchmark.get("intelligence_index")
|
|
1100
|
+
name = model.get("name") or model_id
|
|
1101
|
+
entries.append(
|
|
1102
|
+
{
|
|
1103
|
+
"id": model_id,
|
|
1104
|
+
"name": PROVIDER_PREFIX_RE.sub("", name).strip() or name,
|
|
1105
|
+
"context": coerce_int(model.get("context_length"), 0),
|
|
1106
|
+
"quality": quality,
|
|
1107
|
+
"score": balanced_by_id.get(model_id),
|
|
1108
|
+
"max_discount": eff["max_discount"],
|
|
1109
|
+
"sched_note": fmt_sched_note(eff["max_discount"]),
|
|
1110
|
+
"sched_detail": fmt_sched_detail(windows, eff["peak_blended"]),
|
|
1111
|
+
"peak": {
|
|
1112
|
+
"input_per_1m": _per_1m(peak["price_in"]),
|
|
1113
|
+
"output_per_1m": _per_1m(peak["price_out"]),
|
|
1114
|
+
"blended_per_1m": _per_1m(peak["blended"]),
|
|
1115
|
+
},
|
|
1116
|
+
"offpeak": {
|
|
1117
|
+
"input_per_1m": _per_1m(offpeak["price_in"]),
|
|
1118
|
+
"output_per_1m": _per_1m(offpeak["price_out"]),
|
|
1119
|
+
"blended_per_1m": _per_1m(offpeak["blended"]),
|
|
1120
|
+
},
|
|
1121
|
+
"schedule": [
|
|
1122
|
+
{
|
|
1123
|
+
"utc_days": window["utc_days"],
|
|
1124
|
+
"utc_start": window["utc_start"],
|
|
1125
|
+
"utc_end": window["utc_end"],
|
|
1126
|
+
"coverage": window["coverage"],
|
|
1127
|
+
"input_per_1m": _per_1m(window["price_in"]),
|
|
1128
|
+
"output_per_1m": _per_1m(window["price_out"]),
|
|
1129
|
+
"blended_per_1m": _per_1m(window["blended"]),
|
|
1130
|
+
}
|
|
1131
|
+
for window in windows
|
|
1132
|
+
],
|
|
1133
|
+
}
|
|
1134
|
+
)
|
|
1135
|
+
entries.sort(key=lambda entry: entry["id"])
|
|
1136
|
+
return entries
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def catalog_weights(candidates, quality_by_id):
|
|
1140
|
+
"""Effective per-priority weights for the catalog document.
|
|
1141
|
+
|
|
1142
|
+
Mirrors compute_scores' quality-blind rule: when no candidate has a
|
|
1143
|
+
quality score the quality weight is dropped and the rest renormalize,
|
|
1144
|
+
so scores.overall in the catalog is exactly reproducible downstream.
|
|
1145
|
+
"""
|
|
1146
|
+
quality_blind = not any(c["id"] in quality_by_id for c in candidates)
|
|
1147
|
+
weights = {}
|
|
1148
|
+
for priority, base in PRIORITY_WEIGHTS.items():
|
|
1149
|
+
effective = dict(base)
|
|
1150
|
+
if quality_blind and "quality" in effective:
|
|
1151
|
+
effective.pop("quality")
|
|
1152
|
+
total = sum(effective.values())
|
|
1153
|
+
effective = {name: value / total for name, value in effective.items()}
|
|
1154
|
+
weights[priority] = effective
|
|
1155
|
+
return weights
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
# ---------------------------------------------------------------------------
|
|
1159
|
+
# Filtering and scoring
|
|
1160
|
+
# ---------------------------------------------------------------------------
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
|
|
1164
|
+
now = time.time()
|
|
1165
|
+
require_tools = not args.no_require_tools
|
|
1166
|
+
dropped = {}
|
|
1167
|
+
candidates = []
|
|
1168
|
+
|
|
1169
|
+
def drop(reason, model_id, name):
|
|
1170
|
+
dropped[reason] = dropped.get(reason, 0) + 1
|
|
1171
|
+
if filtered_out is None:
|
|
1172
|
+
return
|
|
1173
|
+
# Empty/malformed ids would fail the site validator's filtered-id
|
|
1174
|
+
# rule, so they stay counted in dropped only.
|
|
1175
|
+
if not model_id or "/" not in model_id:
|
|
1176
|
+
return
|
|
1177
|
+
filtered_out.append(
|
|
1178
|
+
{"id": model_id, "name": name or model_id, "reasons": [reason]}
|
|
1179
|
+
)
|
|
1180
|
+
|
|
1181
|
+
for model in models:
|
|
1182
|
+
model_id = model.get("id") or ""
|
|
1183
|
+
if "/" not in model_id:
|
|
1184
|
+
drop("malformed id", model_id, model.get("name"))
|
|
1185
|
+
continue
|
|
1186
|
+
context = coerce_int(model.get("context_length"), 0)
|
|
1187
|
+
if context < args.min_context:
|
|
1188
|
+
drop("context", model_id, model.get("name"))
|
|
1189
|
+
continue
|
|
1190
|
+
pricing = model.get("pricing") or {}
|
|
1191
|
+
eff = effective_pricing(pricing, args.min_context, args.input_share)
|
|
1192
|
+
top_in, top_out = eff["top_price_in"], eff["top_price_out"]
|
|
1193
|
+
# The RAW top-level prices carry the validity drop (spec rules 1+7):
|
|
1194
|
+
# windowed models freeze their base at the peak window, so the gate
|
|
1195
|
+
# must check the top level itself. A bad override must never drop a
|
|
1196
|
+
# model, and effective prices inherit validity from the validated
|
|
1197
|
+
# base and override prices.
|
|
1198
|
+
if (
|
|
1199
|
+
top_in is None
|
|
1200
|
+
or top_out is None
|
|
1201
|
+
or top_in < 0
|
|
1202
|
+
or top_out < 0
|
|
1203
|
+
or not math.isfinite(top_in)
|
|
1204
|
+
or not math.isfinite(top_out)
|
|
1205
|
+
):
|
|
1206
|
+
drop("pricing", model_id, model.get("name"))
|
|
1207
|
+
continue
|
|
1208
|
+
if eff["schedule_error"]:
|
|
1209
|
+
# No valid peak window means no deterministic base (spec rule 9):
|
|
1210
|
+
# schedule-only AND mixed models fail closed here.
|
|
1211
|
+
drop("schedule", model_id, model.get("name"))
|
|
1212
|
+
continue
|
|
1213
|
+
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
1214
|
+
if args.exclude_free and price_in == 0 and price_out == 0:
|
|
1215
|
+
drop("free", model_id, model.get("name"))
|
|
1216
|
+
continue
|
|
1217
|
+
if not args.include_batch and model_id.endswith(":batch"):
|
|
1218
|
+
drop("batch", model_id, model.get("name"))
|
|
1219
|
+
continue
|
|
1220
|
+
discount = (discounts or {}).get(model_id)
|
|
1221
|
+
if args.discount and not has_discount(discount):
|
|
1222
|
+
drop("no discount", model_id, model.get("name"))
|
|
1223
|
+
continue
|
|
1224
|
+
if not args.no_zdr and model_id not in (zdr_ids or set()):
|
|
1225
|
+
drop("not ZDR", model_id, model.get("name"))
|
|
1226
|
+
continue
|
|
1227
|
+
modality = (model.get("architecture") or {}).get("modality") or ""
|
|
1228
|
+
output_modality = (
|
|
1229
|
+
modality.split("->")[-1].strip() if "->" in modality else "text"
|
|
1230
|
+
)
|
|
1231
|
+
if output_modality != "text":
|
|
1232
|
+
drop("modality", model_id, model.get("name"))
|
|
1233
|
+
continue
|
|
1234
|
+
params = model.get("supported_parameters") or []
|
|
1235
|
+
tool_calling = "tools" in params and "tool_choice" in params
|
|
1236
|
+
if require_tools and not tool_calling:
|
|
1237
|
+
drop("tool calling", model_id, model.get("name"))
|
|
1238
|
+
continue
|
|
1239
|
+
expiry = parse_iso_datetime(model.get("expiration_date"))
|
|
1240
|
+
expired = bool(expiry and expiry < datetime.now(timezone.utc))
|
|
1241
|
+
if expired:
|
|
1242
|
+
drop("expired", model_id, model.get("name"))
|
|
1243
|
+
continue
|
|
1244
|
+
created = model.get("created") or 0
|
|
1245
|
+
try:
|
|
1246
|
+
created = float(created)
|
|
1247
|
+
except (TypeError, ValueError):
|
|
1248
|
+
created = 0.0
|
|
1249
|
+
age_days = max(0.0, (now - created) / 86400.0) if created > 0 else None
|
|
1250
|
+
if args.max_age_days and age_days is not None and age_days > args.max_age_days:
|
|
1251
|
+
drop("age", model_id, model.get("name"))
|
|
1252
|
+
continue
|
|
1253
|
+
price_in_m = price_in * 1_000_000.0
|
|
1254
|
+
price_out_m = price_out * 1_000_000.0
|
|
1255
|
+
blended_m = (
|
|
1256
|
+
args.input_share * price_in_m + (1.0 - args.input_share) * price_out_m
|
|
1257
|
+
)
|
|
1258
|
+
candidates.append(
|
|
1259
|
+
{
|
|
1260
|
+
"id": model_id,
|
|
1261
|
+
"name": model.get("name") or model_id,
|
|
1262
|
+
"context": context,
|
|
1263
|
+
"price_in": price_in_m,
|
|
1264
|
+
"price_out": price_out_m,
|
|
1265
|
+
"blended": blended_m,
|
|
1266
|
+
"base_price_in": eff["base_price_in"] * 1_000_000.0,
|
|
1267
|
+
"base_price_out": eff["base_price_out"] * 1_000_000.0,
|
|
1268
|
+
"tier_prompt_tokens": eff["tier_prompt_tokens"],
|
|
1269
|
+
"tiers": eff["tiers"],
|
|
1270
|
+
"schedule": eff["schedule"],
|
|
1271
|
+
"max_discount": eff["max_discount"],
|
|
1272
|
+
"tier_note": fmt_tier_note(eff["tier_prompt_tokens"]),
|
|
1273
|
+
"sched_note": fmt_sched_note(eff["max_discount"]),
|
|
1274
|
+
"sched_detail": fmt_sched_detail(eff["schedule"], eff["peak_blended"]),
|
|
1275
|
+
"age_days": age_days,
|
|
1276
|
+
"discount": discount,
|
|
1277
|
+
"created": created,
|
|
1278
|
+
"tool_calling": tool_calling,
|
|
1279
|
+
"zdr": None if args.no_zdr else model_id in (zdr_ids or set()),
|
|
1280
|
+
"expired": expired,
|
|
1281
|
+
}
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
return candidates, dropped
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
def compute_scores(candidates, args, quality_by_id):
|
|
1288
|
+
weights = dict(PRIORITY_WEIGHTS[args.priority])
|
|
1289
|
+
# Deliberate asymmetry: the quality weight is dropped (and the remaining
|
|
1290
|
+
# weights renormalized) only when *no* candidate has a quality score, so a
|
|
1291
|
+
# quality-blind pool is ranked purely on price/context/age. When *some*
|
|
1292
|
+
# candidates match, the quality weight is kept and unmatched candidates take
|
|
1293
|
+
# quality_score = 0 -- they are penalized rather than silently reweighted.
|
|
1294
|
+
if not any(c["id"] in quality_by_id for c in candidates):
|
|
1295
|
+
weights.pop("quality")
|
|
1296
|
+
total = sum(weights.values())
|
|
1297
|
+
weights = {name: value / total for name, value in weights.items()}
|
|
1298
|
+
|
|
1299
|
+
prices = [c["blended"] for c in candidates]
|
|
1300
|
+
low = math.log10(min(prices) + 0.01)
|
|
1301
|
+
high = math.log10(max(prices) + 0.01)
|
|
1302
|
+
price_span = (high - low) or 1.0
|
|
1303
|
+
floor_ctx = max(args.min_context, 1000)
|
|
1304
|
+
cap_ctx = 4.0 * floor_ctx
|
|
1305
|
+
ctx_span = math.log(cap_ctx / floor_ctx) or 1.0
|
|
1306
|
+
|
|
1307
|
+
for cand in candidates:
|
|
1308
|
+
price = cand["blended"]
|
|
1309
|
+
if price <= 0:
|
|
1310
|
+
cand["price_score"] = 1.0
|
|
1311
|
+
else:
|
|
1312
|
+
cand["price_score"] = max(
|
|
1313
|
+
0.0, min(1.0, 1.0 - (math.log10(price + 0.01) - low) / price_span)
|
|
1314
|
+
)
|
|
1315
|
+
quality = quality_by_id.get(cand["id"])
|
|
1316
|
+
cand["quality"] = quality
|
|
1317
|
+
if quality is None:
|
|
1318
|
+
cand["quality_score"] = 0.0
|
|
1319
|
+
else:
|
|
1320
|
+
cand["quality_score"] = max(0.0, min(1.0, quality / args.quality_ref))
|
|
1321
|
+
context = cand["context"]
|
|
1322
|
+
cand["context_score"] = (
|
|
1323
|
+
max(0.0, min(1.0, math.log(context / floor_ctx) / ctx_span))
|
|
1324
|
+
if context > floor_ctx
|
|
1325
|
+
else 0.0
|
|
1326
|
+
)
|
|
1327
|
+
age_days = cand["age_days"]
|
|
1328
|
+
cand["age_score"] = (
|
|
1329
|
+
0.5 if age_days is None else 0.5 ** (age_days / args.recency_half_life)
|
|
1330
|
+
)
|
|
1331
|
+
cand["score"] = weighted_score(cand, weights)
|
|
1332
|
+
|
|
1333
|
+
candidates.sort(key=lambda c: ranking_key(c["score"], c))
|
|
1334
|
+
return weights
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def weighted_score(cand, w):
|
|
1338
|
+
"""The one unrounded weighted score, shared by compute_scores and
|
|
1339
|
+
catalog_rankings. Keep the operand order (quality, price, context, age):
|
|
1340
|
+
float addition is non-associative, and a reordered sum can differ by one
|
|
1341
|
+
ULP and flip a near-tie between the CLI and the catalog."""
|
|
1342
|
+
return (
|
|
1343
|
+
w.get("quality", 0.0) * cand["quality_score"]
|
|
1344
|
+
+ w.get("price", 0.0) * cand["price_score"]
|
|
1345
|
+
+ w.get("context", 0.0) * cand["context_score"]
|
|
1346
|
+
+ w.get("age", 0.0) * cand["age_score"]
|
|
1347
|
+
)
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
def ranking_key(score, cand):
|
|
1351
|
+
"""The one sort key: score desc, then raw quality desc, cheaper first, id."""
|
|
1352
|
+
return (-score, -(cand["quality"] or 0.0), cand["blended"], cand["id"])
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def catalog_rankings(candidates, weights):
|
|
1356
|
+
"""Per-priority full ordered id lists: the order a `--priority P` run
|
|
1357
|
+
produces. Pure: sorted() only, never writes onto the candidate dicts."""
|
|
1358
|
+
return {
|
|
1359
|
+
priority: [
|
|
1360
|
+
c["id"]
|
|
1361
|
+
for c in sorted(
|
|
1362
|
+
candidates,
|
|
1363
|
+
key=lambda c, w=weights[priority]: ranking_key(weighted_score(c, w), c),
|
|
1364
|
+
)
|
|
1365
|
+
]
|
|
1366
|
+
for priority in PRIORITY_WEIGHTS
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
# ---------------------------------------------------------------------------
|
|
1371
|
+
# Output
|
|
1372
|
+
# ---------------------------------------------------------------------------
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def fmt_price(value: float) -> str:
|
|
1376
|
+
if value == 0:
|
|
1377
|
+
return "0"
|
|
1378
|
+
if value >= 100:
|
|
1379
|
+
return f"{value:.0f}"
|
|
1380
|
+
if value >= 10:
|
|
1381
|
+
return f"{value:.1f}"
|
|
1382
|
+
if value >= 1:
|
|
1383
|
+
return f"{value:.2f}"
|
|
1384
|
+
return f"{value:.3f}"
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def fmt_context(value: int) -> str:
|
|
1388
|
+
if value >= 1_000_000:
|
|
1389
|
+
return f"{value / 1_000_000:.2f}M"
|
|
1390
|
+
if value >= 1_000:
|
|
1391
|
+
return f"{value / 1_000:.0f}K"
|
|
1392
|
+
return str(value)
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def fmt_age(age_days) -> str:
|
|
1396
|
+
if age_days is None:
|
|
1397
|
+
return "-"
|
|
1398
|
+
if age_days >= 365:
|
|
1399
|
+
return f"{age_days / 365:.1f}y"
|
|
1400
|
+
if age_days >= 60:
|
|
1401
|
+
return f"{age_days / 30.44:.0f}mo"
|
|
1402
|
+
return f"{age_days:.0f}d"
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
def has_discount(value) -> bool:
|
|
1406
|
+
# Only discounts that survive .0% rounding count; smaller slivers and
|
|
1407
|
+
# malformed (negative) values are treated as no discount everywhere:
|
|
1408
|
+
# the DISC column, --json output, and the --discount filter.
|
|
1409
|
+
return bool(value) and value > 0 and f"{value:.0%}" != "0%"
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
def fmt_discount(value) -> str:
|
|
1413
|
+
return f"{value:.0%}" if has_discount(value) else "--"
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
def print_table(top, total_candidates, weights, quality_note):
|
|
1417
|
+
headers = [
|
|
1418
|
+
"RANK",
|
|
1419
|
+
"MODEL",
|
|
1420
|
+
"QUAL",
|
|
1421
|
+
"$IN/M",
|
|
1422
|
+
"$OUT/M",
|
|
1423
|
+
"DISC",
|
|
1424
|
+
"TIER",
|
|
1425
|
+
"CTX",
|
|
1426
|
+
"AGE",
|
|
1427
|
+
"SCORE",
|
|
1428
|
+
]
|
|
1429
|
+
rows = []
|
|
1430
|
+
for rank, cand in enumerate(top, 1):
|
|
1431
|
+
rows.append(
|
|
1432
|
+
[
|
|
1433
|
+
str(rank),
|
|
1434
|
+
cand["id"],
|
|
1435
|
+
"-" if cand["quality"] is None else f"{cand['quality']:.1f}",
|
|
1436
|
+
fmt_price(cand["price_in"]),
|
|
1437
|
+
fmt_price(cand["price_out"]),
|
|
1438
|
+
fmt_discount(cand["discount"]),
|
|
1439
|
+
cand["tier_note"] or cand["sched_note"] or "--",
|
|
1440
|
+
fmt_context(cand["context"]),
|
|
1441
|
+
fmt_age(cand["age_days"]),
|
|
1442
|
+
f"{cand['score']:.3f}",
|
|
1443
|
+
]
|
|
1444
|
+
)
|
|
1445
|
+
widths = [len(h) for h in headers]
|
|
1446
|
+
for row in rows:
|
|
1447
|
+
for i, cell in enumerate(row):
|
|
1448
|
+
widths[i] = max(widths[i], len(cell))
|
|
1449
|
+
|
|
1450
|
+
def emit(row):
|
|
1451
|
+
if row[1] == "MODEL":
|
|
1452
|
+
line = " ".join(h.ljust(w) for h, w in zip(row, widths))
|
|
1453
|
+
print(line)
|
|
1454
|
+
print(" ".join("-" * w for w in widths))
|
|
1455
|
+
else:
|
|
1456
|
+
cells = [row[0].rjust(widths[0]), row[1].ljust(widths[1])]
|
|
1457
|
+
cells += [c.rjust(w) for c, w in zip(row[2:], widths[2:])]
|
|
1458
|
+
print(" ".join(cells))
|
|
1459
|
+
|
|
1460
|
+
emit(headers)
|
|
1461
|
+
for row in rows:
|
|
1462
|
+
emit(row)
|
|
1463
|
+
weight_bits = ", ".join(
|
|
1464
|
+
f"{k} {v:.0%}" for k, v in sorted(weights.items(), reverse=True)
|
|
1465
|
+
)
|
|
1466
|
+
print()
|
|
1467
|
+
print(
|
|
1468
|
+
f"pool: {total_candidates} candidates | weights: {weight_bits} | {quality_note}"
|
|
1469
|
+
)
|
|
1470
|
+
print(
|
|
1471
|
+
"quality: Artificial Analysis intelligence index (- = unknown); prices in USD per 1M tokens; "
|
|
1472
|
+
"DISC = active discount; AGE = time since listed on OpenRouter"
|
|
1473
|
+
)
|
|
1474
|
+
print(
|
|
1475
|
+
"Prices shown are what a long session pays: several models bill at a "
|
|
1476
|
+
"higher rate once the prompt outgrows their cheap short-context tier."
|
|
1477
|
+
)
|
|
1478
|
+
print(
|
|
1479
|
+
"TIER marks how prices vary: a token threshold is the prompt size "
|
|
1480
|
+
"above which the higher rate applies; a percentage is a scheduled "
|
|
1481
|
+
"off-peak discount (price shown: the standard rate); SCHED marks a "
|
|
1482
|
+
"schedule whose discount is under 1%. --json lists each model's "
|
|
1483
|
+
"schedule."
|
|
1484
|
+
)
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def print_json(top):
|
|
1488
|
+
payload = [
|
|
1489
|
+
{
|
|
1490
|
+
"model": cand["id"],
|
|
1491
|
+
"opencode_model": opencode_model_id(cand["id"]),
|
|
1492
|
+
"name": cand["name"],
|
|
1493
|
+
"score": round(cand["score"], 4),
|
|
1494
|
+
"quality_index": cand["quality"],
|
|
1495
|
+
"input_usd_per_m": round(cand["price_in"], 6),
|
|
1496
|
+
"output_usd_per_m": round(cand["price_out"], 6),
|
|
1497
|
+
"blended_usd_per_m": round(cand["blended"], 6),
|
|
1498
|
+
"discount": round(cand["discount"], 4)
|
|
1499
|
+
if has_discount(cand["discount"])
|
|
1500
|
+
else None,
|
|
1501
|
+
# Display string from the DISC column's formatter, so the site
|
|
1502
|
+
# shows exactly what the CLI prints instead of re-rounding.
|
|
1503
|
+
"discount_pct": fmt_discount(cand["discount"]),
|
|
1504
|
+
# Tier/schedule fields follow the same precedent: the strings the
|
|
1505
|
+
# site shows are these, never re-formatted downstream.
|
|
1506
|
+
"pricing_tier_prompt_tokens": cand["tier_prompt_tokens"],
|
|
1507
|
+
"base_input_usd_per_m": round(cand["base_price_in"], 6),
|
|
1508
|
+
"base_output_usd_per_m": round(cand["base_price_out"], 6),
|
|
1509
|
+
"tier_note": cand["tier_note"],
|
|
1510
|
+
"sched_note": cand["sched_note"],
|
|
1511
|
+
"sched_detail": cand["sched_detail"],
|
|
1512
|
+
"time_schedule": (
|
|
1513
|
+
cand["sched_note"] + " " + cand["sched_detail"]
|
|
1514
|
+
if cand["sched_detail"]
|
|
1515
|
+
else cand["sched_note"]
|
|
1516
|
+
),
|
|
1517
|
+
"context_tokens": cand["context"],
|
|
1518
|
+
"age_days": round(cand["age_days"], 1)
|
|
1519
|
+
if cand["age_days"] is not None
|
|
1520
|
+
else None,
|
|
1521
|
+
}
|
|
1522
|
+
for cand in top
|
|
1523
|
+
]
|
|
1524
|
+
print(json.dumps(payload, indent=2))
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
def build_catalog(
|
|
1528
|
+
args,
|
|
1529
|
+
models,
|
|
1530
|
+
candidates,
|
|
1531
|
+
dropped,
|
|
1532
|
+
filtered,
|
|
1533
|
+
discounts,
|
|
1534
|
+
quality_by_id,
|
|
1535
|
+
aa_source,
|
|
1536
|
+
aa_by_id,
|
|
1537
|
+
quality_source_by_id,
|
|
1538
|
+
):
|
|
1539
|
+
"""Assemble the full-catalog document (see README "Catalog output").
|
|
1540
|
+
|
|
1541
|
+
Pure: reads candidates post-compute_scores (which already carry the
|
|
1542
|
+
component scores) and never mutates them. Deterministic apart from
|
|
1543
|
+
generated_at: models sort by (-overall.balanced, id), filtered by id,
|
|
1544
|
+
and age uses date precision so two runs in the same UTC day match.
|
|
1545
|
+
"""
|
|
1546
|
+
now = datetime.now(timezone.utc)
|
|
1547
|
+
weights = catalog_weights(candidates, quality_by_id)
|
|
1548
|
+
aa_modes = {"AA API v2": "api"}
|
|
1549
|
+
if aa_source is not None and aa_source not in aa_modes:
|
|
1550
|
+
# Never claim mode "none" for a source we do not know: the document
|
|
1551
|
+
# would contradict itself (mode none with matched quality scores).
|
|
1552
|
+
raise ValueError(f"unknown AA source: {aa_source!r}")
|
|
1553
|
+
matched_openrouter = sum(
|
|
1554
|
+
1 for s in quality_source_by_id.values() if s == "openrouter"
|
|
1555
|
+
)
|
|
1556
|
+
# fallback is what the AA API yielded on its own; mode
|
|
1557
|
+
# hides it behind "openrouter" as soon as one OpenRouter match exists.
|
|
1558
|
+
aa_fallback = aa_modes.get(aa_source, "none")
|
|
1559
|
+
aa_mode = "openrouter" if matched_openrouter else aa_fallback
|
|
1560
|
+
|
|
1561
|
+
entries = []
|
|
1562
|
+
for cand in candidates:
|
|
1563
|
+
provider, _, _base = cand["id"].partition("/")
|
|
1564
|
+
created = cand["created"] or 0.0
|
|
1565
|
+
listed_date = (
|
|
1566
|
+
datetime.fromtimestamp(created, tz=timezone.utc).date()
|
|
1567
|
+
if created > 0
|
|
1568
|
+
else None
|
|
1569
|
+
)
|
|
1570
|
+
# Round the component scores first, then derive overall from the
|
|
1571
|
+
# rounded values so scores.overall is exactly reproducible from the
|
|
1572
|
+
# document's own numbers.
|
|
1573
|
+
scores = {
|
|
1574
|
+
"price": round(cand["price_score"], 4),
|
|
1575
|
+
"quality": round(cand["quality_score"], 4),
|
|
1576
|
+
"context": round(cand["context_score"], 4),
|
|
1577
|
+
"age": round(cand["age_score"], 4),
|
|
1578
|
+
}
|
|
1579
|
+
overall = {}
|
|
1580
|
+
for priority in PRIORITY_WEIGHTS:
|
|
1581
|
+
w = weights[priority]
|
|
1582
|
+
overall[priority] = round(
|
|
1583
|
+
w.get("quality", 0.0) * scores["quality"]
|
|
1584
|
+
+ w.get("price", 0.0) * scores["price"]
|
|
1585
|
+
+ w.get("context", 0.0) * scores["context"]
|
|
1586
|
+
+ w.get("age", 0.0) * scores["age"],
|
|
1587
|
+
4,
|
|
1588
|
+
)
|
|
1589
|
+
bench = aa_by_id.get(base_model_id(cand["id"])) or {}
|
|
1590
|
+
entries.append(
|
|
1591
|
+
{
|
|
1592
|
+
"id": cand["id"],
|
|
1593
|
+
"name": PROVIDER_PREFIX_RE.sub("", cand["name"]).strip()
|
|
1594
|
+
or cand["name"],
|
|
1595
|
+
"provider": provider,
|
|
1596
|
+
"family": model_family(cand["id"]),
|
|
1597
|
+
"pricing": {
|
|
1598
|
+
"input_per_1m": round(cand["price_in"], 6),
|
|
1599
|
+
"output_per_1m": round(cand["price_out"], 6),
|
|
1600
|
+
"blended_per_1m": round(cand["blended"], 6),
|
|
1601
|
+
"base": {
|
|
1602
|
+
"input_per_1m": round(cand["base_price_in"], 6),
|
|
1603
|
+
"output_per_1m": round(cand["base_price_out"], 6),
|
|
1604
|
+
"blended_per_1m": round(
|
|
1605
|
+
args.input_share * cand["base_price_in"]
|
|
1606
|
+
+ (1 - args.input_share) * cand["base_price_out"],
|
|
1607
|
+
6,
|
|
1608
|
+
),
|
|
1609
|
+
},
|
|
1610
|
+
"tiers": [
|
|
1611
|
+
{
|
|
1612
|
+
"min_prompt_tokens": tier["min_prompt_tokens"],
|
|
1613
|
+
"input_per_1m": _per_1m(tier["price_in"]),
|
|
1614
|
+
"output_per_1m": _per_1m(tier["price_out"]),
|
|
1615
|
+
"blended_per_1m": _per_1m(
|
|
1616
|
+
args.input_share * tier["price_in"]
|
|
1617
|
+
+ (1 - args.input_share) * tier["price_out"]
|
|
1618
|
+
),
|
|
1619
|
+
}
|
|
1620
|
+
for tier in cand["tiers"]
|
|
1621
|
+
],
|
|
1622
|
+
"schedule": (
|
|
1623
|
+
[
|
|
1624
|
+
{
|
|
1625
|
+
"utc_days": window["utc_days"],
|
|
1626
|
+
"utc_start": window["utc_start"],
|
|
1627
|
+
"utc_end": window["utc_end"],
|
|
1628
|
+
"coverage": window["coverage"],
|
|
1629
|
+
"input_per_1m": _per_1m(window["price_in"]),
|
|
1630
|
+
"output_per_1m": _per_1m(window["price_out"]),
|
|
1631
|
+
"blended_per_1m": _per_1m(window["blended"]),
|
|
1632
|
+
}
|
|
1633
|
+
for window in cand["schedule"]
|
|
1634
|
+
]
|
|
1635
|
+
if cand["schedule"]
|
|
1636
|
+
else None
|
|
1637
|
+
),
|
|
1638
|
+
},
|
|
1639
|
+
"context": cand["context"],
|
|
1640
|
+
"listed_at": listed_date.isoformat() if listed_date else None,
|
|
1641
|
+
"age_days": max(0, (now.date() - listed_date).days)
|
|
1642
|
+
if listed_date
|
|
1643
|
+
else None,
|
|
1644
|
+
"tool_calling": cand["tool_calling"],
|
|
1645
|
+
"zdr": cand["zdr"],
|
|
1646
|
+
"discount": round(cand["discount"], 4)
|
|
1647
|
+
if has_discount(cand["discount"])
|
|
1648
|
+
else None,
|
|
1649
|
+
"expired": cand["expired"],
|
|
1650
|
+
"aa": {
|
|
1651
|
+
"intelligence_index": bench.get("intelligence_index"),
|
|
1652
|
+
"coding_index": bench.get("coding_index"),
|
|
1653
|
+
"agentic_index": bench.get("agentic_index"),
|
|
1654
|
+
},
|
|
1655
|
+
"quality": cand["quality"],
|
|
1656
|
+
"quality_match": quality_source_by_id.get(cand["id"]),
|
|
1657
|
+
"scores": {**scores, "overall": overall},
|
|
1658
|
+
}
|
|
1659
|
+
)
|
|
1660
|
+
entries.sort(key=lambda e: (-e["scores"]["overall"]["balanced"], e["id"]))
|
|
1661
|
+
|
|
1662
|
+
balanced_by_id = {
|
|
1663
|
+
entry["id"]: entry["scores"]["overall"]["balanced"] for entry in entries
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
return {
|
|
1667
|
+
"schedules": build_schedules(
|
|
1668
|
+
models, args, quality_by_id, aa_by_id, balanced_by_id
|
|
1669
|
+
),
|
|
1670
|
+
"schema_version": CATALOG_SCHEMA_VERSION,
|
|
1671
|
+
"tool": "model-compare",
|
|
1672
|
+
"generated_at": now.isoformat(timespec="seconds"),
|
|
1673
|
+
"parameters": {
|
|
1674
|
+
"input_share": args.input_share,
|
|
1675
|
+
"quality_ref": args.quality_ref,
|
|
1676
|
+
"min_context": args.min_context,
|
|
1677
|
+
"recency_half_life": args.recency_half_life,
|
|
1678
|
+
"max_age_days": args.max_age_days,
|
|
1679
|
+
"zdr_required": not args.no_zdr,
|
|
1680
|
+
"require_tools": not args.no_require_tools,
|
|
1681
|
+
"exclude_free": args.exclude_free,
|
|
1682
|
+
"include_batch": args.include_batch,
|
|
1683
|
+
"weights": weights,
|
|
1684
|
+
},
|
|
1685
|
+
"sources": {
|
|
1686
|
+
"openrouter": "ok",
|
|
1687
|
+
"aa": {
|
|
1688
|
+
"mode": aa_mode,
|
|
1689
|
+
"fallback": aa_fallback,
|
|
1690
|
+
"matched": len(quality_by_id),
|
|
1691
|
+
"matched_openrouter": matched_openrouter,
|
|
1692
|
+
},
|
|
1693
|
+
"zdr": "skipped" if args.no_zdr else "ok",
|
|
1694
|
+
"discounts": "ok" if discounts else "unavailable",
|
|
1695
|
+
},
|
|
1696
|
+
"pool": {
|
|
1697
|
+
"listed": len(models),
|
|
1698
|
+
"candidates": len(candidates),
|
|
1699
|
+
"dropped": {
|
|
1700
|
+
reason: dropped.get(reason, 0) for reason in CATALOG_DROP_REASONS
|
|
1701
|
+
},
|
|
1702
|
+
},
|
|
1703
|
+
"models": entries,
|
|
1704
|
+
# The single ranking authority: every site artifact (data.json rows,
|
|
1705
|
+
# history tabs, the highlights diff, best.txt) projects from or is
|
|
1706
|
+
# checked against it. models keeps its own balanced sort.
|
|
1707
|
+
"rankings": catalog_rankings(candidates, weights),
|
|
1708
|
+
"filtered": sorted(filtered, key=lambda e: e["id"]),
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
def print_catalog(document):
|
|
1713
|
+
print(json.dumps(document, indent=2))
|
|
1714
|
+
|
|
1715
|
+
|
|
1716
|
+
# ---------------------------------------------------------------------------
|
|
1717
|
+
# CLI
|
|
1718
|
+
# ---------------------------------------------------------------------------
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
def parse_args(argv=None):
|
|
1722
|
+
parser = argparse.ArgumentParser(
|
|
1723
|
+
prog="model-compare",
|
|
1724
|
+
description="Pick the best value-for-money LLM on OpenRouter for coding-agent work.",
|
|
1725
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
1726
|
+
epilog=(__doc__ or "").split("Examples:", 1)[-1],
|
|
1727
|
+
)
|
|
1728
|
+
parser.add_argument(
|
|
1729
|
+
"--version",
|
|
1730
|
+
action="version",
|
|
1731
|
+
version=f"%(prog)s {VERSION}",
|
|
1732
|
+
)
|
|
1733
|
+
parser.add_argument(
|
|
1734
|
+
"--min-context",
|
|
1735
|
+
type=int,
|
|
1736
|
+
default=1_000_000,
|
|
1737
|
+
help="hard minimum context window in tokens (default: 1000000)",
|
|
1738
|
+
)
|
|
1739
|
+
parser.add_argument(
|
|
1740
|
+
"--priority",
|
|
1741
|
+
choices=sorted(PRIORITY_WEIGHTS),
|
|
1742
|
+
default="balanced",
|
|
1743
|
+
help="ranking emphasis (default: balanced)",
|
|
1744
|
+
)
|
|
1745
|
+
parser.add_argument(
|
|
1746
|
+
"--top", type=int, default=5, help="how many models to list (default: 5)"
|
|
1747
|
+
)
|
|
1748
|
+
parser.add_argument(
|
|
1749
|
+
"--best",
|
|
1750
|
+
action="store_true",
|
|
1751
|
+
help="print only the #1 model as an opencode id (openrouter/<model>), for scripting",
|
|
1752
|
+
)
|
|
1753
|
+
parser.add_argument(
|
|
1754
|
+
"--json", action="store_true", help="print the ranked candidates as JSON"
|
|
1755
|
+
)
|
|
1756
|
+
parser.add_argument(
|
|
1757
|
+
"--catalog",
|
|
1758
|
+
action="store_true",
|
|
1759
|
+
help="print the full model catalog (ranked candidates and filtered-out models with reasons) as one JSON document; ignores --top and --priority, cannot be combined with --best or --json",
|
|
1760
|
+
)
|
|
1761
|
+
parser.add_argument(
|
|
1762
|
+
"--input-share",
|
|
1763
|
+
type=float,
|
|
1764
|
+
default=0.75,
|
|
1765
|
+
help="input-token share used for the blended price, 0-1 (default: 0.75)",
|
|
1766
|
+
)
|
|
1767
|
+
parser.add_argument(
|
|
1768
|
+
"--recency-half-life",
|
|
1769
|
+
type=float,
|
|
1770
|
+
default=120.0,
|
|
1771
|
+
help="age decay half-life in days (default: 120)",
|
|
1772
|
+
)
|
|
1773
|
+
parser.add_argument(
|
|
1774
|
+
"--max-age-days",
|
|
1775
|
+
type=float,
|
|
1776
|
+
default=0.0,
|
|
1777
|
+
help="drop models listed more than this many days ago (0 = off)",
|
|
1778
|
+
)
|
|
1779
|
+
parser.add_argument(
|
|
1780
|
+
"--quality-ref",
|
|
1781
|
+
type=float,
|
|
1782
|
+
default=70.0,
|
|
1783
|
+
help="AA intelligence index that counts as full quality score (default: 70)",
|
|
1784
|
+
)
|
|
1785
|
+
parser.add_argument(
|
|
1786
|
+
"--aa-api-key",
|
|
1787
|
+
default=None,
|
|
1788
|
+
help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
|
|
1789
|
+
)
|
|
1790
|
+
parser.add_argument(
|
|
1791
|
+
"--no-require-tools",
|
|
1792
|
+
action="store_true",
|
|
1793
|
+
help="do not require tool-calling support (coding agents want it)",
|
|
1794
|
+
)
|
|
1795
|
+
parser.add_argument(
|
|
1796
|
+
"--exclude-free",
|
|
1797
|
+
action="store_true",
|
|
1798
|
+
help="drop zero-cost listings such as rate-limited :free variants",
|
|
1799
|
+
)
|
|
1800
|
+
parser.add_argument(
|
|
1801
|
+
"--include-batch",
|
|
1802
|
+
action="store_true",
|
|
1803
|
+
help="keep ':batch' variants (asynchronous completion; cheaper but unsuitable for interactive agents)",
|
|
1804
|
+
)
|
|
1805
|
+
parser.add_argument(
|
|
1806
|
+
"--discount",
|
|
1807
|
+
action="store_true",
|
|
1808
|
+
help="only list models with an active discount",
|
|
1809
|
+
)
|
|
1810
|
+
parser.add_argument(
|
|
1811
|
+
"--no-zdr",
|
|
1812
|
+
action="store_true",
|
|
1813
|
+
help="consider all models, not just zero-data-retention (ZDR) ones",
|
|
1814
|
+
)
|
|
1815
|
+
parser.add_argument(
|
|
1816
|
+
"--no-cache",
|
|
1817
|
+
action="store_true",
|
|
1818
|
+
help="bypass the local response cache and refetch",
|
|
1819
|
+
)
|
|
1820
|
+
parser.add_argument(
|
|
1821
|
+
"--cache-ttl",
|
|
1822
|
+
type=int,
|
|
1823
|
+
default=6 * 3600,
|
|
1824
|
+
help="cache lifetime in seconds (default: 21600 = 6h)",
|
|
1825
|
+
)
|
|
1826
|
+
args = parser.parse_args(argv)
|
|
1827
|
+
if not 0.0 <= args.input_share <= 1.0:
|
|
1828
|
+
parser.error("--input-share must be between 0 and 1")
|
|
1829
|
+
if args.top < 1:
|
|
1830
|
+
parser.error("--top must be at least 1")
|
|
1831
|
+
if args.recency_half_life <= 0 or args.quality_ref <= 0:
|
|
1832
|
+
parser.error("--recency-half-life and --quality-ref must be positive")
|
|
1833
|
+
if args.catalog and (args.best or args.json):
|
|
1834
|
+
parser.error("--catalog cannot be combined with --best or --json")
|
|
1835
|
+
return args
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def main(argv=None) -> int:
|
|
1839
|
+
args = parse_args(argv)
|
|
1840
|
+
try:
|
|
1841
|
+
models, or_cached = fetch_openrouter_models(args)
|
|
1842
|
+
except Exception as exc:
|
|
1843
|
+
warn(f"could not fetch the OpenRouter catalog: {exc}")
|
|
1844
|
+
return 1
|
|
1845
|
+
try:
|
|
1846
|
+
return run(args, models, or_cached)
|
|
1847
|
+
except Exception as exc:
|
|
1848
|
+
warn(f"could not rank models: {exc}")
|
|
1849
|
+
return 1
|
|
1850
|
+
|
|
1851
|
+
|
|
1852
|
+
def run(args, models, or_cached) -> int:
|
|
1853
|
+
discounts, zdr_ids, aa_by_id, frontend_cache_hits = fetch_openrouter_frontend(args)
|
|
1854
|
+
if not args.no_zdr and not zdr_ids:
|
|
1855
|
+
warn(
|
|
1856
|
+
"ZDR data unavailable; refusing to rank possibly non-ZDR models "
|
|
1857
|
+
"(--no-zdr to override)"
|
|
1858
|
+
)
|
|
1859
|
+
return 1
|
|
1860
|
+
aa_entries, aa_source, aa_cached = fetch_aa_entries(args)
|
|
1861
|
+
exact, fuzzy = build_aa_lookup(aa_entries)
|
|
1862
|
+
|
|
1863
|
+
filtered = []
|
|
1864
|
+
candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
|
|
1865
|
+
if not candidates:
|
|
1866
|
+
warn(
|
|
1867
|
+
f"no models satisfy the filters (min-context={args.min_context}, "
|
|
1868
|
+
f"tools={'off' if args.no_require_tools else 'required'}); relax them and retry"
|
|
1869
|
+
)
|
|
1870
|
+
return 2
|
|
1871
|
+
|
|
1872
|
+
quality_by_id, quality_source_by_id = resolve_quality(
|
|
1873
|
+
candidates, aa_by_id, exact, fuzzy, aa_source
|
|
1874
|
+
)
|
|
1875
|
+
weights = compute_scores(candidates, args, quality_by_id)
|
|
1876
|
+
|
|
1877
|
+
if args.catalog:
|
|
1878
|
+
print_catalog(
|
|
1879
|
+
build_catalog(
|
|
1880
|
+
args,
|
|
1881
|
+
models,
|
|
1882
|
+
candidates,
|
|
1883
|
+
dropped,
|
|
1884
|
+
filtered,
|
|
1885
|
+
discounts,
|
|
1886
|
+
quality_by_id,
|
|
1887
|
+
aa_source,
|
|
1888
|
+
aa_by_id,
|
|
1889
|
+
quality_source_by_id,
|
|
1890
|
+
)
|
|
1891
|
+
)
|
|
1892
|
+
return 0
|
|
1893
|
+
|
|
1894
|
+
limit = 1 if args.best else args.top
|
|
1895
|
+
top = candidates[:limit]
|
|
1896
|
+
|
|
1897
|
+
if args.best:
|
|
1898
|
+
print(opencode_model_id(top[0]["id"]))
|
|
1899
|
+
elif args.json:
|
|
1900
|
+
print_json(top)
|
|
1901
|
+
else:
|
|
1902
|
+
drop_note = ""
|
|
1903
|
+
if dropped:
|
|
1904
|
+
bits = ", ".join(
|
|
1905
|
+
f"{v} {k}" for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
|
|
1906
|
+
)
|
|
1907
|
+
drop_note = f" (dropped: {bits})"
|
|
1908
|
+
matched_or = sum(1 for s in quality_source_by_id.values() if s == "openrouter")
|
|
1909
|
+
matched_aa = len(quality_by_id) - matched_or
|
|
1910
|
+
unmatched = len(candidates) - len(quality_by_id)
|
|
1911
|
+
bits = []
|
|
1912
|
+
if matched_or:
|
|
1913
|
+
bits.append(f"OpenRouter benchmarks ({matched_or})")
|
|
1914
|
+
if matched_aa:
|
|
1915
|
+
bits.append(f"{aa_source} exact ({matched_aa})")
|
|
1916
|
+
if bits:
|
|
1917
|
+
quality_note = (
|
|
1918
|
+
f"quality via {' + '.join(bits)}: matched "
|
|
1919
|
+
f"{len(quality_by_id)}/{len(candidates)} candidates"
|
|
1920
|
+
)
|
|
1921
|
+
if unmatched:
|
|
1922
|
+
quality_note += "; unmatched candidates score 0 on quality"
|
|
1923
|
+
else:
|
|
1924
|
+
quality_note = "quality data unavailable, ranked on price/context/age"
|
|
1925
|
+
source_note = []
|
|
1926
|
+
if or_cached:
|
|
1927
|
+
source_note.append("catalog cached")
|
|
1928
|
+
if "discounts" in frontend_cache_hits:
|
|
1929
|
+
source_note.append("discounts cached")
|
|
1930
|
+
if "zdr" in frontend_cache_hits:
|
|
1931
|
+
source_note.append("ZDR cached")
|
|
1932
|
+
if "aa" in frontend_cache_hits:
|
|
1933
|
+
source_note.append("AA benchmarks cached")
|
|
1934
|
+
if aa_cached and aa_source:
|
|
1935
|
+
source_note.append("quality cached")
|
|
1936
|
+
if source_note:
|
|
1937
|
+
print(
|
|
1938
|
+
f"note: {', '.join(source_note)} (use --no-cache to refresh)",
|
|
1939
|
+
file=sys.stderr,
|
|
1940
|
+
)
|
|
1941
|
+
print(
|
|
1942
|
+
f"pool: {len(models)} listed -> {len(candidates)} candidates{drop_note}",
|
|
1943
|
+
file=sys.stderr,
|
|
1944
|
+
)
|
|
1945
|
+
print_table(top, len(candidates), weights, quality_note)
|
|
1946
|
+
return 0
|
|
1947
|
+
|
|
1948
|
+
|
|
1949
|
+
if __name__ == "__main__":
|
|
1950
|
+
sys.exit(main())
|