model-compare-opencode-plugin 0.1.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -7
- package/dist/tui.js +123 -10
- package/package.json +1 -1
- package/src/config.ts +5 -2
- package/src/core/catalog.ts +59 -12
- package/src/core/steering.ts +145 -0
- package/src/core/types.ts +28 -1
- package/src/index.ts +463 -15
- package/src/opencode/commands.ts +64 -3
- package/src/opencode/storage.ts +6 -1
- package/src/rpc.ts +45 -3
- package/src/sources/drops.ts +37 -0
- package/src/sources/index.ts +18 -6
- package/src/sources/local.ts +7 -3
- package/src/sources/remote.ts +68 -10
- package/src/tui.tsx +149 -9
- package/vendor/UPSTREAM.json +3 -3
- package/vendor/model_compare.py +1831 -106
package/vendor/model_compare.py
CHANGED
|
@@ -54,6 +54,7 @@ import time
|
|
|
54
54
|
import urllib.error
|
|
55
55
|
import urllib.parse
|
|
56
56
|
import urllib.request
|
|
57
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
57
58
|
from datetime import datetime, timezone
|
|
58
59
|
|
|
59
60
|
OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
|
|
@@ -67,7 +68,23 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
|
|
|
67
68
|
# (legacy endpoints 410 after 2026-11-04; see
|
|
68
69
|
# https://artificialanalysis.ai/data-api/migrate-v2-data).
|
|
69
70
|
AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
|
|
70
|
-
|
|
71
|
+
# Cache key for the AA API entries envelope. v3 adds the per-fetch `as_of`
|
|
72
|
+
# timestamp on every entry and the envelope's intelligence_index_version;
|
|
73
|
+
# payloads under the pre-v3 keys predate those fields and must never be read.
|
|
74
|
+
AA_ENTRIES_CACHE = "aa-intelligence-v3"
|
|
75
|
+
# The repo's own published catalog.json -- built by keyed CI runs, hydrated
|
|
76
|
+
# by unkeyed local runs (design: quality source reconciliation, source 2).
|
|
77
|
+
# The github.io host 301s to plain http://model-compare.dev/catalog.json, an
|
|
78
|
+
# unauthenticated downgrade on ranking-critical data; the https direct URL is
|
|
79
|
+
# verified 200 application/json. Do not point this back at github.io.
|
|
80
|
+
QUALITY_CATALOG_URL = "https://model-compare.dev/catalog.json"
|
|
81
|
+
# Cache key for the hydration envelope {"by_id", "generated_at",
|
|
82
|
+
# "schema_version"}. Only usable (non-empty by_id) results are ever written.
|
|
83
|
+
QUALITY_CATALOG_CACHE = "published-catalog"
|
|
84
|
+
# A published catalog generated more than this many days ago is stale:
|
|
85
|
+
# warn but use -- the records' as_of makes the staleness visible downstream.
|
|
86
|
+
QUALITY_CATALOG_MAX_AGE_DAYS = 14
|
|
87
|
+
VERSION = "0.5.0"
|
|
71
88
|
USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
|
|
72
89
|
|
|
73
90
|
# opencode expects provider-qualified model ids: openrouter/<provider/model>.
|
|
@@ -86,11 +103,15 @@ PRIORITY_WEIGHTS = {
|
|
|
86
103
|
"quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
|
|
87
104
|
}
|
|
88
105
|
|
|
89
|
-
CATALOG_SCHEMA_VERSION =
|
|
106
|
+
CATALOG_SCHEMA_VERSION = 4
|
|
90
107
|
|
|
91
108
|
# The drop-reason keys build_candidates counts, verbatim -- keep in sync with
|
|
92
109
|
# its drop() call sites. pool.dropped lists all of them zero-filled so the
|
|
93
|
-
# contract is self-describing.
|
|
110
|
+
# contract is self-describing. The last two come from the endpoint stage
|
|
111
|
+
# (apply_endpoint_pricing's drop sites), not build_candidates: snake_case
|
|
112
|
+
# while the older reasons are human phrases. The mixed naming convention is
|
|
113
|
+
# a deliberate, documented extension -- machine-readable reasons for the
|
|
114
|
+
# endpoint-stage drops the publish drop-rate gate keys on.
|
|
94
115
|
CATALOG_DROP_REASONS = (
|
|
95
116
|
"malformed id",
|
|
96
117
|
"context",
|
|
@@ -104,11 +125,74 @@ CATALOG_DROP_REASONS = (
|
|
|
104
125
|
"tool calling",
|
|
105
126
|
"expired",
|
|
106
127
|
"age",
|
|
128
|
+
"no_baseline_eligible_endpoint",
|
|
129
|
+
"endpoint_fetch_failed",
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
# The only reasons the endpoint stage itself produces (the last two
|
|
133
|
+
# CATALOG_DROP_REASONS entries) -- the empty-pool warning keys on them;
|
|
134
|
+
# web/publish.py keeps its own copy of this tuple by design.
|
|
135
|
+
ENDPOINT_DROP_REASONS = (
|
|
136
|
+
"no_baseline_eligible_endpoint",
|
|
137
|
+
"endpoint_fetch_failed",
|
|
107
138
|
)
|
|
108
139
|
|
|
109
140
|
PAREN_RE = re.compile(r"\([^)]*\)")
|
|
110
141
|
PROVIDER_PREFIX_RE = re.compile(r"^[^:\s]{1,30}:\s*")
|
|
111
142
|
|
|
143
|
+
# Quantization precision ladder: label -> rank, highest fidelity first. The
|
|
144
|
+
# ladder ordering in the design doc is authoritative: fp6 > int8 is strict,
|
|
145
|
+
# and formats within one fidelity tier share the same rank. "unknown" never
|
|
146
|
+
# enters the ladder -- it only routes through the first-party eligibility
|
|
147
|
+
# carve-out, never through rank comparison.
|
|
148
|
+
PRECISION_LADDER: dict[str, int] = {
|
|
149
|
+
"fp32": 4,
|
|
150
|
+
"bf16": 3,
|
|
151
|
+
"fp16": 3,
|
|
152
|
+
"fp8": 2,
|
|
153
|
+
"mxfp8": 2,
|
|
154
|
+
"fp6": 1,
|
|
155
|
+
"int8": 0,
|
|
156
|
+
"fp4": -1,
|
|
157
|
+
"mxfp4": -1,
|
|
158
|
+
"nvfp4": -1,
|
|
159
|
+
"int4": -2,
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
# Every quantization value an endpoint may report: the ladder keys plus the
|
|
163
|
+
# "unknown" sentinel. Whether a recorded value is recognized at all is
|
|
164
|
+
# membership here, not ladder rank.
|
|
165
|
+
QUANTIZATION_ENUM: frozenset[str] = frozenset(PRECISION_LADDER) | {"unknown"}
|
|
166
|
+
|
|
167
|
+
# Official partner hosts that serve closed labs' models: an endpoint on one
|
|
168
|
+
# of these providers counts as first-party for the eligibility carve-out
|
|
169
|
+
# even though it is not the maker itself. Entries are compared after
|
|
170
|
+
# normalize_provider (casefold + strip non-alphanumerics), so the slugs here
|
|
171
|
+
# are display forms -- "aws-bedrock" and "AWS Bedrock" both hit. Extend only
|
|
172
|
+
# with live-data evidence (e.g. an endpoint tag matching <lab>-official).
|
|
173
|
+
FIRST_PARTY_HOSTS = frozenset(
|
|
174
|
+
{"bedrock", "azure", "vertex", "azure-ai", "google-vertex", "aws-bedrock"}
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
# Explicit first-party alias pairs that normalization alone cannot bridge
|
|
178
|
+
# (design §4.2): makers that self-name differently than their OpenRouter
|
|
179
|
+
# author slug. Key = the exact live provider_name string, value = the
|
|
180
|
+
# author slug the pair is first-party FOR; both sides are compared
|
|
181
|
+
# normalized (case/punctuation variants match), and the alias only fires
|
|
182
|
+
# when the model's author equals the value. Verified against live data
|
|
183
|
+
# 2026-10-20 (task-13-fix-report.md section 8b; endpoints fetched fresh,
|
|
184
|
+
# every endpoint quantization "unknown", status 0):
|
|
185
|
+
# ("Amazon Bedrock", "amazon") -- amazon/nova-2-lite-v1, 2 endpoints
|
|
186
|
+
# ("Google AI Studio", "google") -- google/gemini-2.5-pro-preview, 3 endpoints
|
|
187
|
+
# ("Mistral", "mistralai") -- mistralai/mistral-large-4-0, 3 endpoints
|
|
188
|
+
# Add entries only after the same live verification -- never pin an
|
|
189
|
+
# unverified string.
|
|
190
|
+
FIRST_PARTY_ALIASES: dict[str, str] = {
|
|
191
|
+
"Amazon Bedrock": "amazon",
|
|
192
|
+
"Google AI Studio": "google",
|
|
193
|
+
"Mistral": "mistralai",
|
|
194
|
+
}
|
|
195
|
+
|
|
112
196
|
|
|
113
197
|
def warn(msg: str) -> None:
|
|
114
198
|
print(f"warning: {msg}", file=sys.stderr)
|
|
@@ -118,6 +202,44 @@ def norm_key(s: str) -> str:
|
|
|
118
202
|
return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
|
119
203
|
|
|
120
204
|
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
# Provider normalization + first-party detection
|
|
207
|
+
# ---------------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def normalize_provider(name: str) -> str:
|
|
211
|
+
"""Casefold a provider name and strip ALL non-alphanumeric characters.
|
|
212
|
+
|
|
213
|
+
"Z.AI", "z-ai" and "z ai" all normalize to "zai", so a live-recorded
|
|
214
|
+
maker name matches its author slug regardless of punctuation or case.
|
|
215
|
+
None or empty input normalizes to "".
|
|
216
|
+
"""
|
|
217
|
+
return re.sub(r"[^a-z0-9]+", "", (name or "").casefold())
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def is_first_party(provider_name: str, author: str) -> bool:
|
|
221
|
+
"""True iff the provider is the model's own maker or a partner host.
|
|
222
|
+
|
|
223
|
+
Criterion 1: the normalized provider name equals the normalized author
|
|
224
|
+
slug. Criterion 2: it matches a normalized FIRST_PARTY_HOSTS entry, or
|
|
225
|
+
a FIRST_PARTY_ALIASES key whose value equals the normalized author slug
|
|
226
|
+
(the §4.2 alias pairs: a maker self-naming differently than its author
|
|
227
|
+
slug is first-party only for that author). Everything else is
|
|
228
|
+
third-party.
|
|
229
|
+
"""
|
|
230
|
+
provider = normalize_provider(provider_name)
|
|
231
|
+
if provider == normalize_provider(author):
|
|
232
|
+
return True
|
|
233
|
+
if provider in {normalize_provider(host) for host in FIRST_PARTY_HOSTS}:
|
|
234
|
+
return True
|
|
235
|
+
author_slug = normalize_provider(author)
|
|
236
|
+
return any(
|
|
237
|
+
provider == normalize_provider(alias_provider)
|
|
238
|
+
and author_slug == normalize_provider(alias_author)
|
|
239
|
+
for alias_provider, alias_author in FIRST_PARTY_ALIASES.items()
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
121
243
|
# ---------------------------------------------------------------------------
|
|
122
244
|
# HTTP + cache
|
|
123
245
|
# ---------------------------------------------------------------------------
|
|
@@ -224,6 +346,1049 @@ def save_cache(name: str, payload) -> None:
|
|
|
224
346
|
warn(f"could not write cache: {exc}")
|
|
225
347
|
|
|
226
348
|
|
|
349
|
+
# ---------------------------------------------------------------------------
|
|
350
|
+
# OpenRouter endpoints (per-model provider list)
|
|
351
|
+
# ---------------------------------------------------------------------------
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
OPENROUTER_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{id}/endpoints"
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def load_cache_expired(name: str) -> dict | None:
|
|
358
|
+
"""Return the cached payload regardless of age; None when absent/corrupt.
|
|
359
|
+
|
|
360
|
+
load_cache drops payloads past their TTL; this variant serves them anyway
|
|
361
|
+
so a dropped model's last known endpoint annotations can still be shown
|
|
362
|
+
(marked stale by the caller). Never used for ranking.
|
|
363
|
+
"""
|
|
364
|
+
path = cache_path(name)
|
|
365
|
+
if not path:
|
|
366
|
+
return None
|
|
367
|
+
try:
|
|
368
|
+
with open(path) as fh:
|
|
369
|
+
blob = json.load(fh)
|
|
370
|
+
return blob["payload"]
|
|
371
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
372
|
+
return None
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def parse_endpoints(payload) -> list[dict]:
|
|
376
|
+
"""Keep the verified per-endpoint fields of an endpoints response.
|
|
377
|
+
|
|
378
|
+
The live GET /api/v1/models/{id}/endpoints response nests the list one
|
|
379
|
+
level below the usual envelope: `data` is the model object (id, name,
|
|
380
|
+
architecture, ...) and the per-provider entries live in
|
|
381
|
+
`data["endpoints"]`. `pricing` keeps prompt/completion (parsed through
|
|
382
|
+
parse_price) and the raw `overrides` list -- the tiered-pricing
|
|
383
|
+
machinery parses the entries itself. Fields beyond the verified
|
|
384
|
+
per-entry shape are dropped, and the parse keeps `status` as-is (the
|
|
385
|
+
caller filters).
|
|
386
|
+
"""
|
|
387
|
+
# Task 13 lesson: the first unit fixtures fed the models/find shape
|
|
388
|
+
# ({"data": [entry, ...]}) -- a list directly under "data" -- so the
|
|
389
|
+
# suite stayed green while every live parse returned [] and the whole
|
|
390
|
+
# pool died as no_baseline_eligible_endpoint. Pin the live envelope:
|
|
391
|
+
# the list is read ONLY from data["endpoints"], and every other shape
|
|
392
|
+
# parses to [] (no compat branch for the wrong fixture shape).
|
|
393
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
394
|
+
entries = data.get("endpoints") if isinstance(data, dict) else None
|
|
395
|
+
endpoints = []
|
|
396
|
+
for entry in entries or []:
|
|
397
|
+
if not isinstance(entry, dict):
|
|
398
|
+
continue
|
|
399
|
+
raw = entry.get("pricing")
|
|
400
|
+
raw_pricing = raw if isinstance(raw, dict) else {}
|
|
401
|
+
endpoints.append(
|
|
402
|
+
{
|
|
403
|
+
"provider_name": entry.get("provider_name"),
|
|
404
|
+
"tag": entry.get("tag"),
|
|
405
|
+
"quantization": entry.get("quantization"),
|
|
406
|
+
"pricing": {
|
|
407
|
+
"prompt": parse_price(raw_pricing.get("prompt")),
|
|
408
|
+
"completion": parse_price(raw_pricing.get("completion")),
|
|
409
|
+
"overrides": raw_pricing.get("overrides"),
|
|
410
|
+
},
|
|
411
|
+
"context_length": entry.get("context_length"),
|
|
412
|
+
"status": entry.get("status"),
|
|
413
|
+
"service_tier": entry.get("service_tier"),
|
|
414
|
+
}
|
|
415
|
+
)
|
|
416
|
+
return endpoints
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def fetch_model_endpoints(model_id: str, args) -> list[dict]:
|
|
420
|
+
"""Fetch + parse one model's endpoint list, cached per model.
|
|
421
|
+
|
|
422
|
+
The cache name flattens the model id ("/" -> "_") to keep the cache
|
|
423
|
+
files flat. Fetches retry 3 times (4 attempts total) via fetch_json,
|
|
424
|
+
which raises at once on 4xx and after the retries on 5xx/network
|
|
425
|
+
errors -- the caller owns the fail-closed policy.
|
|
426
|
+
"""
|
|
427
|
+
cache_name = f"endpoints:{model_id.replace('/', '_')}"
|
|
428
|
+
if not args.no_cache:
|
|
429
|
+
cached = load_cache(cache_name, args.cache_ttl)
|
|
430
|
+
if cached:
|
|
431
|
+
return cached
|
|
432
|
+
payload = fetch_json(
|
|
433
|
+
OPENROUTER_ENDPOINTS_URL.format(id=model_id), timeout=30, retries=3
|
|
434
|
+
)
|
|
435
|
+
endpoints = parse_endpoints(payload)
|
|
436
|
+
save_cache(cache_name, endpoints)
|
|
437
|
+
return endpoints
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def is_variant_endpoint(endpoint) -> bool:
|
|
441
|
+
"""True iff the endpoint is a :free/:batch variant or entirely free.
|
|
442
|
+
|
|
443
|
+
Variant iff its slug or tag carries a `:free`/`:batch` marker OR both
|
|
444
|
+
input and output prices are zero. `:batch` is cheaper-but-restricted,
|
|
445
|
+
not zero-priced -- only the marker test catches it. Missing or
|
|
446
|
+
unparseable prices are not zero.
|
|
447
|
+
"""
|
|
448
|
+
for field in ("slug", "tag"):
|
|
449
|
+
marker = endpoint.get(field)
|
|
450
|
+
if isinstance(marker, str) and (":free" in marker or ":batch" in marker):
|
|
451
|
+
return True
|
|
452
|
+
pricing = endpoint.get("pricing")
|
|
453
|
+
prices = pricing if isinstance(pricing, dict) else {}
|
|
454
|
+
return (
|
|
455
|
+
parse_price(prices.get("prompt")) == 0.0
|
|
456
|
+
and parse_price(prices.get("completion")) == 0.0
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def endpoint_schedule_ok(endpoint, args) -> bool:
|
|
461
|
+
"""False iff the endpoint's pricing.overrides fails the v2 schedule rule.
|
|
462
|
+
|
|
463
|
+
Reuses effective_pricing in full: the gate fires on malformed windows,
|
|
464
|
+
exactly-one-of utc_start/utc_end pairs, bad utc_days, combined
|
|
465
|
+
token+time entries and missing inheritance -- not just non-tiling
|
|
466
|
+
windows. An endpoint without overrides has no schedule to fail.
|
|
467
|
+
"""
|
|
468
|
+
result = effective_pricing(
|
|
469
|
+
endpoint.get("pricing"), args.min_context, args.input_share
|
|
470
|
+
)
|
|
471
|
+
return not result["schedule_error"]
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# Schedule gate for args-less callers: effective_pricing's schedule_error
|
|
475
|
+
# depends only on the SHAPE of pricing.overrides (malformed windows, bad
|
|
476
|
+
# utc_days, ...), never on min_context/input_share -- those only affect the
|
|
477
|
+
# resolved prices. derive_default_precision has no args parameter (pinned
|
|
478
|
+
# interface), so it gates through this fixed stand-in; any args would give
|
|
479
|
+
# the same schedule_error verdict.
|
|
480
|
+
_SCHEDULE_GATE_ARGS = argparse.Namespace(min_context=0, input_share=0.75)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _baseline_universe(endpoints: list[dict], args=None) -> list[dict]:
|
|
484
|
+
"""The shared derivation universe (design section 4.1).
|
|
485
|
+
|
|
486
|
+
Live (status == 0), not a variant endpoint (:free/:batch marker or
|
|
487
|
+
zero-priced), and passing the schedule gate. Schedule-broken endpoints
|
|
488
|
+
must not pin the default that governs healthy endpoints, same rationale
|
|
489
|
+
as the variant exclusion. args defaults to the structural stand-in
|
|
490
|
+
(see _SCHEDULE_GATE_ARGS); callers with real args pass them.
|
|
491
|
+
"""
|
|
492
|
+
if args is None:
|
|
493
|
+
args = _SCHEDULE_GATE_ARGS
|
|
494
|
+
return [
|
|
495
|
+
endpoint
|
|
496
|
+
for endpoint in endpoints
|
|
497
|
+
if endpoint.get("status") == 0
|
|
498
|
+
and not is_variant_endpoint(endpoint)
|
|
499
|
+
and endpoint_schedule_ok(endpoint, args)
|
|
500
|
+
]
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def derive_default_precision(endpoints: list[dict], author: str) -> str | None:
|
|
504
|
+
"""Derive a model's default precision from its endpoint list (section 4.1).
|
|
505
|
+
|
|
506
|
+
Universe: live non-variant schedule-passing endpoints (_baseline_universe).
|
|
507
|
+
Primary rule: the highest-ranked KNOWN quantization among the maker's
|
|
508
|
+
endpoints -- matched by normalized author-slug equality only, so partner
|
|
509
|
+
hosts never pin the maker's default (section 4.2 criterion 2 does not
|
|
510
|
+
apply here). Fallback: when the maker has no known quantization, the
|
|
511
|
+
highest-ranked known quantization across the FULL universe (partners
|
|
512
|
+
included -- best-available). None when no endpoint in the universe has a
|
|
513
|
+
known quantization (the all-unknown closed-model case): eligibility then
|
|
514
|
+
relies solely on the first-party carve-out.
|
|
515
|
+
|
|
516
|
+
Known means a PRECISION_LADDER key: "unknown" and unparseable/absent
|
|
517
|
+
values never enter the ladder. An endpoint whose provider_name is
|
|
518
|
+
absent/empty (or normalizes to empty) is skipped for maker matching
|
|
519
|
+
(never first-party -- fail-closed), but still feeds the fallback
|
|
520
|
+
universe.
|
|
521
|
+
"""
|
|
522
|
+
universe = _baseline_universe(endpoints, _SCHEDULE_GATE_ARGS)
|
|
523
|
+
maker = normalize_provider(author)
|
|
524
|
+
maker_best = None
|
|
525
|
+
fallback_best = None
|
|
526
|
+
for endpoint in universe:
|
|
527
|
+
quantization = endpoint.get("quantization")
|
|
528
|
+
if not isinstance(quantization, str) or quantization not in PRECISION_LADDER:
|
|
529
|
+
continue # "unknown", null, unrecognized: no rank, no pinning
|
|
530
|
+
rank = PRECISION_LADDER[quantization]
|
|
531
|
+
if fallback_best is None or rank > fallback_best[0]:
|
|
532
|
+
fallback_best = (rank, quantization)
|
|
533
|
+
# Maker matching only: a provider_name that normalizes to nothing
|
|
534
|
+
# must never match an (equally empty) author slug.
|
|
535
|
+
provider_slug = (
|
|
536
|
+
normalize_provider(endpoint.get("provider_name"))
|
|
537
|
+
if isinstance(endpoint.get("provider_name"), str)
|
|
538
|
+
else ""
|
|
539
|
+
)
|
|
540
|
+
if (
|
|
541
|
+
provider_slug
|
|
542
|
+
and provider_slug == maker
|
|
543
|
+
and (maker_best is None or rank > maker_best[0])
|
|
544
|
+
):
|
|
545
|
+
maker_best = (rank, quantization)
|
|
546
|
+
return (maker_best or fallback_best or (None, None))[1]
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def baseline_eligible(endpoint, default_precision, author, args) -> bool:
|
|
550
|
+
"""Baseline-eligibility rule (design section 4.3, fail-closed).
|
|
551
|
+
|
|
552
|
+
True iff the endpoint is live (status == 0), not a variant, passes the
|
|
553
|
+
schedule gate, and then either:
|
|
554
|
+
|
|
555
|
+
* its quantization is a known ladder label with rank >=
|
|
556
|
+
rank(default_precision) -- provider-blind (default_routing considers
|
|
557
|
+
every endpoint OpenRouter may send traffic to), or
|
|
558
|
+
* its quantization is "unknown" or null AND the endpoint is first-party
|
|
559
|
+
(the D3 carve-out: the maker or a partner host).
|
|
560
|
+
|
|
561
|
+
When default_precision is None (all-unknown model) the rank comparison is
|
|
562
|
+
meaningless -- only the carve-out applies. A provider_name that is
|
|
563
|
+
absent/empty (or normalizes to empty) is never first-party, so such an
|
|
564
|
+
endpoint can only qualify through the rank branch. An unrecognized
|
|
565
|
+
quantization string (neither a ladder label nor "unknown"/null) is a
|
|
566
|
+
schema anomaly: no rank comparison and no carve-out -- fail-closed
|
|
567
|
+
(fleet-wide new-label drift is what the publish drop-rate gate exists
|
|
568
|
+
to catch).
|
|
569
|
+
"""
|
|
570
|
+
if (
|
|
571
|
+
endpoint.get("status") != 0
|
|
572
|
+
or is_variant_endpoint(endpoint)
|
|
573
|
+
or not endpoint_schedule_ok(endpoint, args)
|
|
574
|
+
):
|
|
575
|
+
return False
|
|
576
|
+
quantization = endpoint.get("quantization")
|
|
577
|
+
if isinstance(quantization, str) and quantization in PRECISION_LADDER:
|
|
578
|
+
# Known quantization: the rank branch, provider-blind.
|
|
579
|
+
if default_precision is None:
|
|
580
|
+
return False
|
|
581
|
+
return PRECISION_LADDER[quantization] >= PRECISION_LADDER[default_precision]
|
|
582
|
+
# Unverifiable: the carve-out applies to "unknown" and null only --
|
|
583
|
+
# an unrecognized string fails closed. A provider_name that normalizes
|
|
584
|
+
# to nothing is never first-party (fail-closed; the literal
|
|
585
|
+
# is_first_party("", "") iff is guarded out here).
|
|
586
|
+
provider_name = endpoint.get("provider_name")
|
|
587
|
+
provider_slug = (
|
|
588
|
+
normalize_provider(provider_name) if isinstance(provider_name, str) else ""
|
|
589
|
+
)
|
|
590
|
+
return (
|
|
591
|
+
(quantization is None or quantization == "unknown")
|
|
592
|
+
and bool(provider_slug)
|
|
593
|
+
and is_first_party(provider_name, author)
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def pick_baseline(
|
|
598
|
+
endpoints: list[dict], eligible_flags: list[bool], args
|
|
599
|
+
) -> dict | None:
|
|
600
|
+
"""Pick the baseline endpoint among the eligible ones (design section 4.5).
|
|
601
|
+
|
|
602
|
+
Cheapest blended per-token price wins, computed from effective_pricing
|
|
603
|
+
with the args' min_context and input_share (the shared --input-share
|
|
604
|
+
convention). Ties broken by (normalized provider name, tag), then --
|
|
605
|
+
still tied -- first in list order (deterministic; Python's sort is
|
|
606
|
+
stable and the endpoints list order is the API's). Returns None when no
|
|
607
|
+
eligible endpoint has a usable price (the caller owns the drop).
|
|
608
|
+
"""
|
|
609
|
+
candidates = []
|
|
610
|
+
for endpoint, eligible in zip(endpoints, eligible_flags):
|
|
611
|
+
if not eligible:
|
|
612
|
+
continue
|
|
613
|
+
eff = effective_pricing(
|
|
614
|
+
endpoint.get("pricing"), args.min_context, args.input_share
|
|
615
|
+
)
|
|
616
|
+
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
617
|
+
if (
|
|
618
|
+
price_in is None
|
|
619
|
+
or price_out is None
|
|
620
|
+
or not math.isfinite(price_in)
|
|
621
|
+
or not math.isfinite(price_out)
|
|
622
|
+
):
|
|
623
|
+
continue # unpriceable: no executable blended price
|
|
624
|
+
blended = args.input_share * price_in + (1 - args.input_share) * price_out
|
|
625
|
+
provider_name = endpoint.get("provider_name")
|
|
626
|
+
candidates.append(
|
|
627
|
+
(
|
|
628
|
+
blended,
|
|
629
|
+
normalize_provider(provider_name)
|
|
630
|
+
if isinstance(provider_name, str)
|
|
631
|
+
else "",
|
|
632
|
+
endpoint.get("tag") or "",
|
|
633
|
+
endpoint,
|
|
634
|
+
)
|
|
635
|
+
)
|
|
636
|
+
if not candidates:
|
|
637
|
+
return None
|
|
638
|
+
return min(candidates, key=lambda candidate: candidate[:3])[3]
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _blended_price(endpoint, args) -> float | None:
|
|
642
|
+
"""Blended per-token price of one endpoint (pick_baseline's price rule).
|
|
643
|
+
|
|
644
|
+
effective_pricing with the args' min_context and input_share, then
|
|
645
|
+
input_share * price_in + (1 - input_share) * price_out. None when either
|
|
646
|
+
resolved price is missing or non-finite -- no executable blended price,
|
|
647
|
+
the same skip pick_baseline applies. A zero result is returned as-is:
|
|
648
|
+
zero-priced endpoints are variants (already excluded upstream), so the
|
|
649
|
+
callers apply their own zero guards defensively.
|
|
650
|
+
"""
|
|
651
|
+
eff = effective_pricing(endpoint.get("pricing"), args.min_context, args.input_share)
|
|
652
|
+
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
653
|
+
if (
|
|
654
|
+
price_in is None
|
|
655
|
+
or price_out is None
|
|
656
|
+
or not math.isfinite(price_in)
|
|
657
|
+
or not math.isfinite(price_out)
|
|
658
|
+
):
|
|
659
|
+
return None
|
|
660
|
+
blended = args.input_share * price_in + (1 - args.input_share) * price_out
|
|
661
|
+
return blended if math.isfinite(blended) else None
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _flag_list(endpoints: list[dict], eligible_flags: list[bool]):
|
|
665
|
+
"""Zip endpoints with their parallel eligibility flags, hardened.
|
|
666
|
+
|
|
667
|
+
eligible_flags is a parallel list to endpoints (the baseline_eligible
|
|
668
|
+
verdict per endpoint, same order -- Task 4 contract). A length mismatch
|
|
669
|
+
must raise, never truncate silently (Task 4 review ruling).
|
|
670
|
+
"""
|
|
671
|
+
if len(endpoints) != len(eligible_flags):
|
|
672
|
+
raise ValueError(
|
|
673
|
+
"eligible_flags must be a parallel list to endpoints: "
|
|
674
|
+
f"{len(endpoints)} endpoints, {len(eligible_flags)} flags"
|
|
675
|
+
)
|
|
676
|
+
return zip(endpoints, eligible_flags, strict=True)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def NON_ELIGIBLE_UNIVERSE(endpoints: list[dict], eligible_flags: list[bool]) -> list:
|
|
680
|
+
"""The shared annotations universe (design section 4.6).
|
|
681
|
+
|
|
682
|
+
Live (status == 0), non-variant endpoints that are NOT baseline-eligible:
|
|
683
|
+
sub-default known quantization, third-party unknown/null quantization, or
|
|
684
|
+
schedule-gate-failed. All annotations derive from this one universe.
|
|
685
|
+
Eligibility comes from the parallel eligible_flags list (the
|
|
686
|
+
baseline_eligible verdicts); variants and non-live endpoints are excluded
|
|
687
|
+
even though they too are not eligible -- unlike the steering `ignore`
|
|
688
|
+
list, which includes live variants (section 4.7).
|
|
689
|
+
|
|
690
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
691
|
+
"""
|
|
692
|
+
return [
|
|
693
|
+
endpoint
|
|
694
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags)
|
|
695
|
+
if endpoint.get("status") == 0
|
|
696
|
+
and not is_variant_endpoint(endpoint)
|
|
697
|
+
and not eligible
|
|
698
|
+
]
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def build_annotations(
|
|
702
|
+
endpoints: list[dict],
|
|
703
|
+
eligible_flags: list[bool],
|
|
704
|
+
baseline,
|
|
705
|
+
catalog_context,
|
|
706
|
+
args,
|
|
707
|
+
) -> dict:
|
|
708
|
+
"""Per-model endpoint annotations (design section 4.6).
|
|
709
|
+
|
|
710
|
+
All four annotations derive from one shared universe -- the live
|
|
711
|
+
non-variant endpoints that are not baseline-eligible
|
|
712
|
+
(NON_ELIGIBLE_UNIVERSE):
|
|
713
|
+
|
|
714
|
+
* price_spread: cheapest non-eligible blended price divided by the
|
|
715
|
+
cheapest eligible blended price, both blended with the args'
|
|
716
|
+
min_context and input_share (the shared --input-share convention).
|
|
717
|
+
None when the universe is empty -- generalized to None when either
|
|
718
|
+
side has no priceable member at all -- and None when either cheapest
|
|
719
|
+
price is 0 (zero-priced endpoints are variants by definition, so the
|
|
720
|
+
zero half of the guard is defensive only). Unpriceable endpoints are
|
|
721
|
+
skipped when looking for a cheapest price.
|
|
722
|
+
* cheapest_quantized: {provider, quantization, price} of the cheapest
|
|
723
|
+
member of the quantization-caused subset of the universe -- endpoints
|
|
724
|
+
non-eligible due to quantization only (sub-default known quantization
|
|
725
|
+
or third-party unknown/null), NOT schedule-gate-failed ones, which are
|
|
726
|
+
excluded even when cheaper. Unpriceable subset members are skipped
|
|
727
|
+
(defensively also non-positive ones, matching the repo-wide treatment
|
|
728
|
+
of negative prices as malformed); None when the subset has no
|
|
729
|
+
priceable member. Price ties keep the first in list order
|
|
730
|
+
(deterministic).
|
|
731
|
+
* risk_flag: True iff the sum of 1/blended**2 over the universe strictly
|
|
732
|
+
exceeds the sum over the baseline-eligible endpoints (exact tie ->
|
|
733
|
+
False). Any endpoint whose blended price is 0/None (defensively also
|
|
734
|
+
negative) is skipped from BOTH sums -- variants are already excluded,
|
|
735
|
+
so this only guards malformed data, and the flag stays deterministic.
|
|
736
|
+
* baseline_context_below_catalog: True iff the baseline endpoint's
|
|
737
|
+
context_length is smaller than the catalog context (the models-API
|
|
738
|
+
value). A None baseline -- or missing context on either side -- yields
|
|
739
|
+
False: the caveat marks a positive finding, never missing data.
|
|
740
|
+
|
|
741
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
742
|
+
"""
|
|
743
|
+
universe = NON_ELIGIBLE_UNIVERSE(endpoints, eligible_flags)
|
|
744
|
+
|
|
745
|
+
universe_prices = []
|
|
746
|
+
for endpoint in universe:
|
|
747
|
+
price = _blended_price(endpoint, args)
|
|
748
|
+
if price is not None:
|
|
749
|
+
universe_prices.append(price)
|
|
750
|
+
eligible_prices = []
|
|
751
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
752
|
+
if not eligible:
|
|
753
|
+
continue
|
|
754
|
+
price = _blended_price(endpoint, args)
|
|
755
|
+
if price is not None:
|
|
756
|
+
eligible_prices.append(price)
|
|
757
|
+
cheapest_non_eligible = min(universe_prices) if universe_prices else None
|
|
758
|
+
cheapest_eligible = min(eligible_prices) if eligible_prices else None
|
|
759
|
+
price_spread = (
|
|
760
|
+
cheapest_non_eligible / cheapest_eligible
|
|
761
|
+
if cheapest_non_eligible is not None
|
|
762
|
+
and cheapest_eligible is not None
|
|
763
|
+
and cheapest_non_eligible > 0
|
|
764
|
+
and cheapest_eligible > 0
|
|
765
|
+
else None
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
quantized_best = None
|
|
769
|
+
for endpoint in universe:
|
|
770
|
+
if not endpoint_schedule_ok(endpoint, args):
|
|
771
|
+
continue # schedule-gate-failed: not a quantization-caused exclusion
|
|
772
|
+
price = _blended_price(endpoint, args)
|
|
773
|
+
if price is None or price <= 0:
|
|
774
|
+
continue # unpriceable/non-positive: no comparable price
|
|
775
|
+
if quantized_best is None or price < quantized_best[0]:
|
|
776
|
+
quantized_best = (price, endpoint) # ties keep the first in list order
|
|
777
|
+
if quantized_best is None:
|
|
778
|
+
cheapest_quantized = None
|
|
779
|
+
else:
|
|
780
|
+
price, endpoint = quantized_best
|
|
781
|
+
cheapest_quantized = {
|
|
782
|
+
"provider": endpoint.get("provider_name"),
|
|
783
|
+
"quantization": endpoint.get("quantization"),
|
|
784
|
+
"price": price,
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
non_eligible_weight = 0.0
|
|
788
|
+
for endpoint in universe:
|
|
789
|
+
price = _blended_price(endpoint, args)
|
|
790
|
+
if price is None or price <= 0:
|
|
791
|
+
continue # skipped from BOTH sums, deterministically
|
|
792
|
+
non_eligible_weight += 1.0 / (price * price)
|
|
793
|
+
eligible_weight = 0.0
|
|
794
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
795
|
+
if not eligible:
|
|
796
|
+
continue
|
|
797
|
+
price = _blended_price(endpoint, args)
|
|
798
|
+
if price is None or price <= 0:
|
|
799
|
+
continue
|
|
800
|
+
eligible_weight += 1.0 / (price * price)
|
|
801
|
+
risk_flag = non_eligible_weight > eligible_weight
|
|
802
|
+
|
|
803
|
+
baseline_context = baseline.get("context_length") if baseline is not None else None
|
|
804
|
+
baseline_context_below_catalog = (
|
|
805
|
+
baseline_context is not None
|
|
806
|
+
and catalog_context is not None
|
|
807
|
+
and baseline_context < catalog_context
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
return {
|
|
811
|
+
"price_spread": price_spread,
|
|
812
|
+
"cheapest_quantized": cheapest_quantized,
|
|
813
|
+
"risk_flag": risk_flag,
|
|
814
|
+
"baseline_context_below_catalog": baseline_context_below_catalog,
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def build_steering(
|
|
819
|
+
model_id: str,
|
|
820
|
+
endpoints: list[dict],
|
|
821
|
+
eligible_flags: list[bool],
|
|
822
|
+
default_precision: str | None,
|
|
823
|
+
args,
|
|
824
|
+
) -> dict:
|
|
825
|
+
"""OpenRouter provider steering object (design section 4.7).
|
|
826
|
+
|
|
827
|
+
{"model": model_id, "provider": {"order": [...], "ignore": [...],
|
|
828
|
+
"quantizations": [...], "allow_fallbacks": True}}.
|
|
829
|
+
|
|
830
|
+
* order: tags of the baseline-eligible endpoints, cheapest blended first
|
|
831
|
+
(blended price asc, ties broken by (normalized provider name, tag) --
|
|
832
|
+
the same comparator as pick_baseline, stable so the final tie is list
|
|
833
|
+
order). Eligible endpoints whose blended price is unpriceable cannot
|
|
834
|
+
be price-ranked; they are appended after the ranked ones (still
|
|
835
|
+
(provider, tag)-ordered) so every eligible endpoint keeps a place --
|
|
836
|
+
ranked last, never silently left to default priority.
|
|
837
|
+
* ignore: tags of EVERY live endpoint that is not baseline-eligible --
|
|
838
|
+
sub-default quantization, third-party unknown/null, schedule-gate-
|
|
839
|
+
failed AND variants (variants are live but not eligible; the universal
|
|
840
|
+
quantifier includes them). Non-live (status != 0) endpoints are never
|
|
841
|
+
listed. List order (the API's).
|
|
842
|
+
* quantizations: sorted KNOWN quantizations (PRECISION_LADDER labels)
|
|
843
|
+
with rank >= rank(default_precision) among live non-variant endpoints,
|
|
844
|
+
deduplicated -- no schedule or eligibility filter. OMITTED from the
|
|
845
|
+
provider dict entirely when default_precision is None -- never an
|
|
846
|
+
empty list (OpenRouter semantics would filter out every endpoint).
|
|
847
|
+
* allow_fallbacks: True.
|
|
848
|
+
|
|
849
|
+
Signature deviation from the plan (controller ruling at Task 5 spawn):
|
|
850
|
+
args is a required parameter -- the order ranking is real pricing math
|
|
851
|
+
(effective_pricing via min_context/input_share), and reusing the
|
|
852
|
+
shape-only _SCHEDULE_GATE_ARGS stand-in for pricing was explicitly
|
|
853
|
+
ruled out in the Task 4 review.
|
|
854
|
+
|
|
855
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
856
|
+
"""
|
|
857
|
+
ignore = []
|
|
858
|
+
ranked = [] # (blended, normalized provider name, tag) -- pick_baseline order
|
|
859
|
+
unranked = [] # (normalized provider name, tag) -- eligible but unpriceable
|
|
860
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
861
|
+
if endpoint.get("status") == 0 and not eligible:
|
|
862
|
+
ignore.append(endpoint.get("tag") or "")
|
|
863
|
+
if not eligible:
|
|
864
|
+
continue
|
|
865
|
+
provider_name = endpoint.get("provider_name")
|
|
866
|
+
provider_key = (
|
|
867
|
+
normalize_provider(provider_name) if isinstance(provider_name, str) else ""
|
|
868
|
+
)
|
|
869
|
+
tag = endpoint.get("tag") or ""
|
|
870
|
+
price = _blended_price(endpoint, args)
|
|
871
|
+
if price is None:
|
|
872
|
+
unranked.append((provider_key, tag))
|
|
873
|
+
else:
|
|
874
|
+
ranked.append((price, provider_key, tag))
|
|
875
|
+
ranked.sort() # stable: the final tie is list order, as in pick_baseline
|
|
876
|
+
order = [tag for _, _, tag in ranked] + [tag for _, tag in sorted(unranked)]
|
|
877
|
+
|
|
878
|
+
provider = {"order": order, "ignore": ignore}
|
|
879
|
+
if default_precision is not None:
|
|
880
|
+
default_rank = PRECISION_LADDER[default_precision]
|
|
881
|
+
quants = set()
|
|
882
|
+
for endpoint in endpoints:
|
|
883
|
+
if endpoint.get("status") != 0 or is_variant_endpoint(endpoint):
|
|
884
|
+
continue
|
|
885
|
+
quantization = endpoint.get("quantization")
|
|
886
|
+
if (
|
|
887
|
+
not isinstance(quantization, str)
|
|
888
|
+
or quantization not in PRECISION_LADDER
|
|
889
|
+
):
|
|
890
|
+
continue # "unknown"/null/unrecognized: not a known quantization
|
|
891
|
+
if PRECISION_LADDER[quantization] >= default_rank:
|
|
892
|
+
quants.add(quantization)
|
|
893
|
+
provider["quantizations"] = sorted(quants)
|
|
894
|
+
provider["allow_fallbacks"] = True
|
|
895
|
+
return {"model": model_id, "provider": provider}
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def _stale_filtered_annotations(model_id, catalog_context, args):
|
|
899
|
+
"""Annotations for a dropped model's filtered entry, or None.
|
|
900
|
+
|
|
901
|
+
Built from the model's last cached endpoint list (load_cache_expired --
|
|
902
|
+
TTL ignored), never from a fresh fetch: the model is out of the ranking,
|
|
903
|
+
so the block is marked "stale": True for display in the filtered section
|
|
904
|
+
only (design section 4.9). Same builders as the live stage: eligibility
|
|
905
|
+
and baseline are re-derived from the cached list, so the shape matches a
|
|
906
|
+
ranked candidate's annotations. A model whose cache holds no endpoint
|
|
907
|
+
list at all yields None (the entry carries the drop reason alone).
|
|
908
|
+
"""
|
|
909
|
+
cached = load_cache_expired(f"endpoints:{model_id.replace('/', '_')}")
|
|
910
|
+
if not isinstance(cached, list):
|
|
911
|
+
return None
|
|
912
|
+
author = model_id.partition("/")[0]
|
|
913
|
+
try:
|
|
914
|
+
default_precision = derive_default_precision(cached, author)
|
|
915
|
+
eligible_flags = [
|
|
916
|
+
baseline_eligible(endpoint, default_precision, author, args)
|
|
917
|
+
for endpoint in cached
|
|
918
|
+
]
|
|
919
|
+
baseline = pick_baseline(cached, eligible_flags, args)
|
|
920
|
+
annotations = build_annotations(
|
|
921
|
+
cached, eligible_flags, baseline, catalog_context, args
|
|
922
|
+
)
|
|
923
|
+
except Exception:
|
|
924
|
+
# Display-only block for an already-dropped model: never fail the run.
|
|
925
|
+
return None
|
|
926
|
+
annotations["stale"] = True
|
|
927
|
+
return annotations
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _endpoint_stage_drop(extra_dropped, filtered, cand, reason, args):
|
|
931
|
+
"""Count one endpoint-stage drop and append its filtered entry.
|
|
932
|
+
|
|
933
|
+
Same validated entry shape as build_candidates' drop() plus the optional
|
|
934
|
+
stale annotations block (absent cache -> reasons only).
|
|
935
|
+
"""
|
|
936
|
+
extra_dropped[reason] = extra_dropped.get(reason, 0) + 1
|
|
937
|
+
entry = {"id": cand["id"], "name": cand["name"] or cand["id"], "reasons": [reason]}
|
|
938
|
+
annotations = _stale_filtered_annotations(cand["id"], cand.get("context"), args)
|
|
939
|
+
if annotations is not None:
|
|
940
|
+
entry["annotations"] = annotations
|
|
941
|
+
filtered.append(entry)
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def apply_endpoint_pricing(candidates, args, filtered):
|
|
945
|
+
"""Reprice every candidate at its baseline endpoint (design section 4.9).
|
|
946
|
+
|
|
947
|
+
The CLI stage between build_candidates and resolve_quality. Endpoint
|
|
948
|
+
fetches run concurrently (ThreadPoolExecutor, 8 workers) through
|
|
949
|
+
fetch_model_endpoints' per-model cache, but the results are applied PER
|
|
950
|
+
CANDIDATE ID in the original candidates order -- never in completion
|
|
951
|
+
order -- so the stage is deterministic.
|
|
952
|
+
|
|
953
|
+
Per candidate: endpoints = fetch. On fetch failure after fetch_json's
|
|
954
|
+
internal retries the model is dropped "endpoint_fetch_failed" and removed
|
|
955
|
+
from the returned list. With endpoints, default_precision is derived,
|
|
956
|
+
every endpoint scored for baseline eligibility and the baseline picked;
|
|
957
|
+
no usable baseline drops the model "no_baseline_eligible_endpoint"
|
|
958
|
+
(the stale-annotations filtered-entry rule covers both drop reasons).
|
|
959
|
+
Otherwise the candidate is mutated in place: the pricing block
|
|
960
|
+
(price_in/price_out/blended/base_*/tier/schedule/notes) is replaced with
|
|
961
|
+
the baseline endpoint's effective_pricing outputs scaled per-1M exactly
|
|
962
|
+
like build_candidates, and the additive keys "endpoints", "default_precision",
|
|
963
|
+
"annotations" (catalog context = the candidate's existing "context"),
|
|
964
|
+
"advertised_pricing" (the models-API scalars the candidate carried before
|
|
965
|
+
the mutation, source-tagged) and "steering" are attached.
|
|
966
|
+
|
|
967
|
+
A baseline whose prices are missing/non-finite counts as unusable
|
|
968
|
+
(pick_baseline's blended-price skip is authoritative for the top-level
|
|
969
|
+
effective prices). The same bar applies to the base prices the mutation
|
|
970
|
+
would publish: parse_price returns float | None, so a None base price
|
|
971
|
+
(tier-over-None-base endpoint) is reachable -- the isinstance(..., float)
|
|
972
|
+
half of the guard is load-bearing for None, not a redundant type check.
|
|
973
|
+
Returns (surviving candidates in the original order, dropped counts for
|
|
974
|
+
the two new reasons).
|
|
975
|
+
"""
|
|
976
|
+
extra_dropped: dict[str, int] = {}
|
|
977
|
+
if not candidates:
|
|
978
|
+
return candidates, extra_dropped
|
|
979
|
+
endpoints_by_id: dict[str, list] = {}
|
|
980
|
+
failed_ids: set[str] = set()
|
|
981
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
982
|
+
futures = {}
|
|
983
|
+
for cand in candidates:
|
|
984
|
+
# Duplicate ids fetch once and share the result.
|
|
985
|
+
futures.setdefault(
|
|
986
|
+
cand["id"], pool.submit(fetch_model_endpoints, cand["id"], args)
|
|
987
|
+
)
|
|
988
|
+
for cand_id, future in futures.items():
|
|
989
|
+
try:
|
|
990
|
+
endpoints_by_id[cand_id] = future.result()
|
|
991
|
+
except Exception:
|
|
992
|
+
# fetch_json already retried 3 times; the caller owns the
|
|
993
|
+
# fail-closed policy.
|
|
994
|
+
failed_ids.add(cand_id)
|
|
995
|
+
survivors = []
|
|
996
|
+
for cand in candidates:
|
|
997
|
+
cand_id = cand["id"]
|
|
998
|
+
if cand_id in failed_ids:
|
|
999
|
+
_endpoint_stage_drop(
|
|
1000
|
+
extra_dropped, filtered, cand, "endpoint_fetch_failed", args
|
|
1001
|
+
)
|
|
1002
|
+
continue
|
|
1003
|
+
endpoints = endpoints_by_id[cand_id]
|
|
1004
|
+
author = cand_id.partition("/")[0]
|
|
1005
|
+
default_precision = derive_default_precision(endpoints, author)
|
|
1006
|
+
eligible_flags = [
|
|
1007
|
+
baseline_eligible(endpoint, default_precision, author, args)
|
|
1008
|
+
for endpoint in endpoints
|
|
1009
|
+
]
|
|
1010
|
+
baseline = pick_baseline(endpoints, eligible_flags, args)
|
|
1011
|
+
eff = (
|
|
1012
|
+
effective_pricing(
|
|
1013
|
+
baseline.get("pricing"), args.min_context, args.input_share
|
|
1014
|
+
)
|
|
1015
|
+
if baseline is not None
|
|
1016
|
+
else None
|
|
1017
|
+
)
|
|
1018
|
+
if baseline is None or not all(
|
|
1019
|
+
isinstance(eff[key], float) and math.isfinite(eff[key])
|
|
1020
|
+
for key in ("price_in", "price_out", "base_price_in", "base_price_out")
|
|
1021
|
+
):
|
|
1022
|
+
_endpoint_stage_drop(
|
|
1023
|
+
extra_dropped, filtered, cand, "no_baseline_eligible_endpoint", args
|
|
1024
|
+
)
|
|
1025
|
+
continue
|
|
1026
|
+
# The advertised models-API scalars, captured before the mutation.
|
|
1027
|
+
advertised_pricing = {
|
|
1028
|
+
"input_per_1m": cand["price_in"],
|
|
1029
|
+
"output_per_1m": cand["price_out"],
|
|
1030
|
+
"blended_per_1m": cand["blended"],
|
|
1031
|
+
"source": "models-api",
|
|
1032
|
+
}
|
|
1033
|
+
# Replace the pricing block with the baseline endpoint's outputs,
|
|
1034
|
+
# scaled per-1M exactly like build_candidates.
|
|
1035
|
+
cand["price_in"] = eff["price_in"] * 1_000_000.0
|
|
1036
|
+
cand["price_out"] = eff["price_out"] * 1_000_000.0
|
|
1037
|
+
cand["blended"] = (
|
|
1038
|
+
args.input_share * cand["price_in"]
|
|
1039
|
+
+ (1.0 - args.input_share) * cand["price_out"]
|
|
1040
|
+
)
|
|
1041
|
+
cand["base_price_in"] = eff["base_price_in"] * 1_000_000.0
|
|
1042
|
+
cand["base_price_out"] = eff["base_price_out"] * 1_000_000.0
|
|
1043
|
+
cand["tier_prompt_tokens"] = eff["tier_prompt_tokens"]
|
|
1044
|
+
cand["tiers"] = eff["tiers"]
|
|
1045
|
+
cand["schedule"] = eff["schedule"]
|
|
1046
|
+
cand["max_discount"] = eff["max_discount"]
|
|
1047
|
+
cand["tier_note"] = fmt_tier_note(eff["tier_prompt_tokens"])
|
|
1048
|
+
cand["sched_note"] = fmt_sched_note(eff["max_discount"])
|
|
1049
|
+
cand["sched_detail"] = fmt_sched_detail(eff["schedule"], eff["peak_blended"])
|
|
1050
|
+
cand["endpoints"] = endpoints
|
|
1051
|
+
cand["default_precision"] = default_precision
|
|
1052
|
+
cand["annotations"] = build_annotations(
|
|
1053
|
+
endpoints, eligible_flags, baseline, cand["context"], args
|
|
1054
|
+
)
|
|
1055
|
+
cand["advertised_pricing"] = advertised_pricing
|
|
1056
|
+
cand["steering"] = build_steering(
|
|
1057
|
+
cand_id, endpoints, eligible_flags, default_precision, args
|
|
1058
|
+
)
|
|
1059
|
+
survivors.append(cand)
|
|
1060
|
+
return survivors, extra_dropped
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
# ---------------------------------------------------------------------------
|
|
1064
|
+
# OpenRouter session-cost dataset (per-task cost, outside the score)
|
|
1065
|
+
# ---------------------------------------------------------------------------
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
SESSION_COST_URL = "https://openrouter.ai/api/v1/datasets/session-cost"
|
|
1069
|
+
SESSION_COST_HARNESS_SLUG = "hermes-agent"
|
|
1070
|
+
# The turn range the catalog/CLI surfaces (Task 8).
|
|
1071
|
+
SESSION_COST_DISPLAY_TURN_RANGE = "10-49-turns"
|
|
1072
|
+
# The dataset's four turn ranges (brainstorm section 2.6) -- the catalog
|
|
1073
|
+
# contract's closed set (web/build_site_data.py pins the same tuple).
|
|
1074
|
+
# Cells naming anything else are dropped: an unknown range is dataset
|
|
1075
|
+
# drift, and the block must never fail the publish (design section 4.8).
|
|
1076
|
+
SESSION_COST_TURN_RANGES = ("1-turn", "2-9-turns", "10-49-turns", "50-plus-turns")
|
|
1077
|
+
SESSION_COST_ATTRIBUTION = "Data © OpenRouter, CC BY 4.0"
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def strip_permaslug_date(permaslug: str) -> str:
|
|
1081
|
+
"""Permaslug without its trailing -YYYYMMDD (undated ids map to themselves)."""
|
|
1082
|
+
match = re.match(r"^(.*)-(\d{8})$", permaslug)
|
|
1083
|
+
return match.group(1) if match else permaslug
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
def fetch_session_cost(args) -> dict | None:
|
|
1087
|
+
"""Per-task session-cost medians from OpenRouter's dataset, or None.
|
|
1088
|
+
|
|
1089
|
+
A separate published dimension (design section 4.8): the median USD
|
|
1090
|
+
cost of real sessions through the pinned published harness -- never
|
|
1091
|
+
part of the weighted score. One bulk paginated fetch per invocation
|
|
1092
|
+
(?app_slug=<harness>&limit=500&offset=k) regardless of the model list;
|
|
1093
|
+
pagination stops on a page shorter than the limit or once the next
|
|
1094
|
+
offset would exceed the API's 5000 cap. Cells key on dated permaslugs
|
|
1095
|
+
(z-ai/glm-5.3-flash-20260826), stripped to the bare catalog id with
|
|
1096
|
+
strip_permaslug_date; when several strip to the same id, the latest
|
|
1097
|
+
date wins each turn-range cell (same rule as build_aa_benchmarks).
|
|
1098
|
+
|
|
1099
|
+
Returns None (graceful degradation, design section 4.9) when no key is
|
|
1100
|
+
configured (the --openrouter-api-key flag, else OPENROUTER_API_KEY),
|
|
1101
|
+
when the fetch fails after fetch_json's retries, or when the response
|
|
1102
|
+
shape is unexpected -- a malformed payload must never crash the run.
|
|
1103
|
+
The merged result is cached under "session-cost" with the standard TTL.
|
|
1104
|
+
"""
|
|
1105
|
+
api_key = args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
|
|
1106
|
+
if not api_key:
|
|
1107
|
+
return None
|
|
1108
|
+
if not args.no_cache:
|
|
1109
|
+
cached = load_cache("session-cost", args.cache_ttl)
|
|
1110
|
+
if isinstance(cached, dict) and isinstance(cached.get("medians"), dict):
|
|
1111
|
+
return cached
|
|
1112
|
+
cells = []
|
|
1113
|
+
# The API caps limit at 500 and offset at 5000 (11 full pages max).
|
|
1114
|
+
offset = 0
|
|
1115
|
+
while offset <= 5000:
|
|
1116
|
+
try:
|
|
1117
|
+
payload = fetch_json(
|
|
1118
|
+
f"{SESSION_COST_URL}?app_slug={SESSION_COST_HARNESS_SLUG}"
|
|
1119
|
+
f"&limit=500&offset={offset}",
|
|
1120
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
1121
|
+
timeout=30,
|
|
1122
|
+
retries=3,
|
|
1123
|
+
)
|
|
1124
|
+
except Exception as exc:
|
|
1125
|
+
warn(f"session-cost dataset request failed ({exc})")
|
|
1126
|
+
return None
|
|
1127
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
1128
|
+
if not isinstance(data, list):
|
|
1129
|
+
warn("session-cost dataset returned an unexpected response shape")
|
|
1130
|
+
return None
|
|
1131
|
+
cells.extend(cell for cell in data if isinstance(cell, dict))
|
|
1132
|
+
if len(data) < 500:
|
|
1133
|
+
break
|
|
1134
|
+
offset += 500
|
|
1135
|
+
medians: dict[str, dict[str, float]] = {}
|
|
1136
|
+
seen_date: dict[tuple[str, str], str] = {}
|
|
1137
|
+
|
|
1138
|
+
def accept(bare, date, turn_range, median):
|
|
1139
|
+
key = (bare, turn_range)
|
|
1140
|
+
if key in seen_date and seen_date[key] >= date:
|
|
1141
|
+
return
|
|
1142
|
+
seen_date[key] = date
|
|
1143
|
+
medians.setdefault(bare, {})[turn_range] = median
|
|
1144
|
+
|
|
1145
|
+
for cell in cells:
|
|
1146
|
+
permaslug = cell.get("model_permaslug")
|
|
1147
|
+
turn_range = cell.get("turn_range")
|
|
1148
|
+
raw = cell.get("median_session_cost_usd")
|
|
1149
|
+
if not (
|
|
1150
|
+
isinstance(permaslug, str)
|
|
1151
|
+
and permaslug
|
|
1152
|
+
and isinstance(turn_range, str)
|
|
1153
|
+
and turn_range
|
|
1154
|
+
):
|
|
1155
|
+
continue
|
|
1156
|
+
# Closed contract set: a cell naming a new turn range is dropped,
|
|
1157
|
+
# never published (the catalog validator rejects unknown ranges,
|
|
1158
|
+
# and session cost must never fail the publish).
|
|
1159
|
+
if turn_range not in SESSION_COST_TURN_RANGES:
|
|
1160
|
+
continue
|
|
1161
|
+
# Non-finite medians would serialize as NaN and poison the catalog.
|
|
1162
|
+
if not isinstance(raw, (int, float)) or isinstance(raw, bool):
|
|
1163
|
+
continue
|
|
1164
|
+
try:
|
|
1165
|
+
median = float(raw) # huge ints raise OverflowError
|
|
1166
|
+
except OverflowError:
|
|
1167
|
+
continue
|
|
1168
|
+
if not math.isfinite(median):
|
|
1169
|
+
continue
|
|
1170
|
+
match = re.match(r"^(.*)-(\d{8})$", permaslug)
|
|
1171
|
+
accept(
|
|
1172
|
+
strip_permaslug_date(permaslug),
|
|
1173
|
+
match.group(2) if match else "",
|
|
1174
|
+
turn_range,
|
|
1175
|
+
median,
|
|
1176
|
+
)
|
|
1177
|
+
if not medians:
|
|
1178
|
+
warn("session-cost dataset returned no usable cells")
|
|
1179
|
+
return None
|
|
1180
|
+
meta = payload.get("meta") if isinstance(payload, dict) else None
|
|
1181
|
+
if not isinstance(meta, dict):
|
|
1182
|
+
warn("session-cost dataset returned an unexpected response shape")
|
|
1183
|
+
return None
|
|
1184
|
+
as_of = meta.get("as_of")
|
|
1185
|
+
window_days = meta.get("window_days")
|
|
1186
|
+
window_end_date = meta.get("window_end_date")
|
|
1187
|
+
# The catalog validator demands non-empty as_of/window_end_date strings
|
|
1188
|
+
# and a positive integer window_days; anything else is dataset drift.
|
|
1189
|
+
# Degrade to None (block omitted, "not available this refresh") rather
|
|
1190
|
+
# than emit a block the validator would reject -- session cost must
|
|
1191
|
+
# never fail the publish (design section 4.8).
|
|
1192
|
+
if not (
|
|
1193
|
+
isinstance(as_of, str)
|
|
1194
|
+
and as_of
|
|
1195
|
+
and isinstance(window_end_date, str)
|
|
1196
|
+
and window_end_date
|
|
1197
|
+
and isinstance(window_days, int)
|
|
1198
|
+
and not isinstance(window_days, bool)
|
|
1199
|
+
and window_days > 0
|
|
1200
|
+
):
|
|
1201
|
+
warn("session-cost dataset returned malformed meta; omitting the block")
|
|
1202
|
+
return None
|
|
1203
|
+
result = {
|
|
1204
|
+
"attribution": SESSION_COST_ATTRIBUTION,
|
|
1205
|
+
"app_slug": SESSION_COST_HARNESS_SLUG,
|
|
1206
|
+
"as_of": as_of,
|
|
1207
|
+
"window_days": window_days,
|
|
1208
|
+
"window_end_date": window_end_date,
|
|
1209
|
+
"medians": medians,
|
|
1210
|
+
}
|
|
1211
|
+
save_cache("session-cost", result)
|
|
1212
|
+
return result
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
# ---------------------------------------------------------------------------
|
|
1216
|
+
# Published catalog quality (hydration for unkeyed runs)
|
|
1217
|
+
# ---------------------------------------------------------------------------
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
# Pre-reconciliation published catalogs (schema v3, written before the AA
|
|
1221
|
+
# tier was relabeled) record the AA tier as "api"; hydration normalizes it
|
|
1222
|
+
# to the final "aa" label. Every other recorded label passes through as-is.
|
|
1223
|
+
_QUALITY_SOURCE_NORMALIZATION = {"api": "aa"}
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def catalog_quality_by_id(doc) -> dict:
|
|
1227
|
+
"""Pure parser: a published catalog document -> {id: quality record}.
|
|
1228
|
+
|
|
1229
|
+
No fetch, no cache, no args object -- the fetcher owns transport and
|
|
1230
|
+
freshness; this owns shape. Records are {"index", "source", "as_of"}
|
|
1231
|
+
and feed resolve_quality's catalog_quality parameter on unkeyed runs;
|
|
1232
|
+
the values were produced by a keyed CI run, so hydration is an exact
|
|
1233
|
+
catalog-id join with no re-matching.
|
|
1234
|
+
|
|
1235
|
+
Gates (design: failure modes):
|
|
1236
|
+
- schema_version must be an int >= 3; anything older warns
|
|
1237
|
+
"published catalog schema vX, expected >= 3; skipping hydration"
|
|
1238
|
+
and yields {}. Accepting v3 is the deploy-day skew case: the live
|
|
1239
|
+
artifact predates the v4 per-row quality_as_of.
|
|
1240
|
+
- Any shape surprise -- a non-dict document, a non-int schema_version,
|
|
1241
|
+
a non-list models, a row without a usable id, or a matched row whose
|
|
1242
|
+
quality is not a finite number or whose quality_match is not a
|
|
1243
|
+
non-empty string -- warns and yields {} for the WHOLE artifact.
|
|
1244
|
+
Never a partial hydration, never a raise: the catalog is
|
|
1245
|
+
machine-generated, so one malformed row means this is not the
|
|
1246
|
+
artifact we think it is.
|
|
1247
|
+
- Rows with quality None and quality_match None are legitimate
|
|
1248
|
+
unmatched models; they are skipped silently (nothing to hydrate).
|
|
1249
|
+
|
|
1250
|
+
Normalization: v3 rows map quality_match "api" -> "aa" and date every
|
|
1251
|
+
row with the catalog's generated_at -- the live v3 artifact carries
|
|
1252
|
+
quality_as_of: null, which must be ignored, not passed through.
|
|
1253
|
+
v4+ rows keep their recorded quality_as_of (None when absent) and
|
|
1254
|
+
source label as-is: that label is the row-level truth inside the
|
|
1255
|
+
published artifact (a keyed CI run emits "aa"/"openrouter", never
|
|
1256
|
+
"catalog"). resolve_quality's catalog tier owns the run-level label:
|
|
1257
|
+
it relabels every hydrated record to the chain label "catalog".
|
|
1258
|
+
"""
|
|
1259
|
+
if not isinstance(doc, dict):
|
|
1260
|
+
warn("published catalog returned an unexpected response shape")
|
|
1261
|
+
return {}
|
|
1262
|
+
schema_version = doc.get("schema_version")
|
|
1263
|
+
if type(schema_version) is not int:
|
|
1264
|
+
warn("published catalog returned an unexpected response shape")
|
|
1265
|
+
return {}
|
|
1266
|
+
if schema_version < 3:
|
|
1267
|
+
warn(
|
|
1268
|
+
f"published catalog schema v{schema_version}, expected >= 3; "
|
|
1269
|
+
"skipping hydration"
|
|
1270
|
+
)
|
|
1271
|
+
return {}
|
|
1272
|
+
models = doc.get("models")
|
|
1273
|
+
if not isinstance(models, list):
|
|
1274
|
+
warn("published catalog returned an unexpected response shape")
|
|
1275
|
+
return {}
|
|
1276
|
+
generated_at = doc.get("generated_at")
|
|
1277
|
+
v3_as_of = generated_at if isinstance(generated_at, str) and generated_at else None
|
|
1278
|
+
by_id: dict = {}
|
|
1279
|
+
for entry in models:
|
|
1280
|
+
if not isinstance(entry, dict):
|
|
1281
|
+
warn("published catalog returned an unexpected response shape")
|
|
1282
|
+
return {}
|
|
1283
|
+
model_id = entry.get("id")
|
|
1284
|
+
if not (isinstance(model_id, str) and model_id):
|
|
1285
|
+
warn("published catalog returned an unexpected response shape")
|
|
1286
|
+
return {}
|
|
1287
|
+
quality = entry.get("quality")
|
|
1288
|
+
match = entry.get("quality_match")
|
|
1289
|
+
if match is None and quality is None:
|
|
1290
|
+
continue # legitimate unmatched row: nothing to hydrate
|
|
1291
|
+
if not (isinstance(match, str) and match):
|
|
1292
|
+
warn("published catalog returned an unexpected response shape")
|
|
1293
|
+
return {}
|
|
1294
|
+
if not isinstance(quality, (int, float)) or isinstance(quality, bool):
|
|
1295
|
+
warn("published catalog returned an unexpected response shape")
|
|
1296
|
+
return {}
|
|
1297
|
+
try:
|
|
1298
|
+
index = float(quality) # huge ints raise OverflowError
|
|
1299
|
+
except OverflowError:
|
|
1300
|
+
warn("published catalog returned an unexpected response shape")
|
|
1301
|
+
return {}
|
|
1302
|
+
# Non-finite indexes would serialize as NaN and poison quality.
|
|
1303
|
+
if not math.isfinite(index):
|
|
1304
|
+
warn("published catalog returned an unexpected response shape")
|
|
1305
|
+
return {}
|
|
1306
|
+
# The AA indexes run 0..100 (the range the catalog validator
|
|
1307
|
+
# enforces on quality); an out-of-range value is poisoned data and
|
|
1308
|
+
# degrades the whole artifact. The surface is unkeyed runs: only
|
|
1309
|
+
# they hydrate from the published catalog.
|
|
1310
|
+
if not 0.0 <= index <= 100.0:
|
|
1311
|
+
warn("published catalog returned an unexpected response shape")
|
|
1312
|
+
return {}
|
|
1313
|
+
if schema_version == 3:
|
|
1314
|
+
as_of = v3_as_of
|
|
1315
|
+
else:
|
|
1316
|
+
recorded = entry.get("quality_as_of")
|
|
1317
|
+
as_of = recorded if isinstance(recorded, str) and recorded else None
|
|
1318
|
+
by_id[model_id] = {
|
|
1319
|
+
"index": index,
|
|
1320
|
+
"source": _QUALITY_SOURCE_NORMALIZATION.get(match, match),
|
|
1321
|
+
"as_of": as_of,
|
|
1322
|
+
}
|
|
1323
|
+
return by_id
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
def fetch_quality_catalog(args) -> dict | None:
|
|
1327
|
+
"""Quality records from the published catalog.json, or None.
|
|
1328
|
+
|
|
1329
|
+
The third quality source (design: source 2): the repo's own published
|
|
1330
|
+
catalog artifact, built by keyed CI runs and hydrated by unkeyed local
|
|
1331
|
+
runs. Never called on keyed runs -- a CI/site build hydrating from the
|
|
1332
|
+
published catalog would consume its own output (self-reference loop);
|
|
1333
|
+
Task 4 owns that wiring.
|
|
1334
|
+
|
|
1335
|
+
Returns None only on TRANSPORT failure (after fetch_json's retries);
|
|
1336
|
+
otherwise the envelope {"by_id": {id: record}, "generated_at":
|
|
1337
|
+
str|None, "schema_version": int}. An empty by_id means
|
|
1338
|
+
fetched-but-unusable (schema gate fail, malformed shape) -- Task 4's
|
|
1339
|
+
footer treats None and empty by_id identically ("catalog unavailable").
|
|
1340
|
+
|
|
1341
|
+
The envelope is cached under QUALITY_CATALOG_CACHE with the standard
|
|
1342
|
+
TTL, and only a usable (non-empty by_id) result is ever written: a
|
|
1343
|
+
degraded payload must self-heal on the next run. --no-cache neither
|
|
1344
|
+
reads nor writes. Staleness: generated_at older than
|
|
1345
|
+
QUALITY_CATALOG_MAX_AGE_DAYS warns but uses -- as_of makes the
|
|
1346
|
+
staleness visible downstream (the check runs only when rows actually
|
|
1347
|
+
hydrate -- warning about the age of a catalog we are not using is
|
|
1348
|
+
noise); an unusable generated_at warns that
|
|
1349
|
+
staleness cannot be checked and still uses the rows (their as_of then
|
|
1350
|
+
shows None rather than a wrong date).
|
|
1351
|
+
"""
|
|
1352
|
+
if not args.no_cache:
|
|
1353
|
+
cached = load_cache(QUALITY_CATALOG_CACHE, args.cache_ttl)
|
|
1354
|
+
if isinstance(cached, dict) and isinstance(cached.get("by_id"), dict):
|
|
1355
|
+
return cached
|
|
1356
|
+
try:
|
|
1357
|
+
doc = fetch_json(QUALITY_CATALOG_URL, timeout=30, retries=3)
|
|
1358
|
+
except Exception as exc:
|
|
1359
|
+
warn(f"published catalog request failed ({exc})")
|
|
1360
|
+
return None
|
|
1361
|
+
by_id = catalog_quality_by_id(doc)
|
|
1362
|
+
generated_at = doc.get("generated_at") if isinstance(doc, dict) else None
|
|
1363
|
+
if not isinstance(generated_at, str):
|
|
1364
|
+
generated_at = None
|
|
1365
|
+
schema_version = doc.get("schema_version") if isinstance(doc, dict) else 0
|
|
1366
|
+
if type(schema_version) is not int:
|
|
1367
|
+
schema_version = 0
|
|
1368
|
+
result = {
|
|
1369
|
+
"by_id": by_id,
|
|
1370
|
+
"generated_at": generated_at,
|
|
1371
|
+
"schema_version": schema_version,
|
|
1372
|
+
}
|
|
1373
|
+
if by_id:
|
|
1374
|
+
generated = parse_iso_datetime(generated_at)
|
|
1375
|
+
if generated is None:
|
|
1376
|
+
warn(
|
|
1377
|
+
"published catalog generated_at is missing or unparseable; "
|
|
1378
|
+
"cannot check staleness"
|
|
1379
|
+
)
|
|
1380
|
+
else:
|
|
1381
|
+
age_days = (datetime.now(timezone.utc) - generated).total_seconds() / 86400
|
|
1382
|
+
if age_days > QUALITY_CATALOG_MAX_AGE_DAYS:
|
|
1383
|
+
warn(
|
|
1384
|
+
f"published catalog is stale (generated {generated_at}, "
|
|
1385
|
+
f"{age_days:.0f} days old); using it anyway"
|
|
1386
|
+
)
|
|
1387
|
+
if not args.no_cache:
|
|
1388
|
+
save_cache(QUALITY_CATALOG_CACHE, result)
|
|
1389
|
+
return result
|
|
1390
|
+
|
|
1391
|
+
|
|
227
1392
|
# ---------------------------------------------------------------------------
|
|
228
1393
|
# OpenRouter catalog
|
|
229
1394
|
# ---------------------------------------------------------------------------
|
|
@@ -243,7 +1408,9 @@ def fetch_openrouter_models(args):
|
|
|
243
1408
|
|
|
244
1409
|
|
|
245
1410
|
FRONTEND_DISCOUNTS_CACHE = "openrouter-frontend-discounts"
|
|
246
|
-
|
|
1411
|
+
# v2: AA records are keyed by exact catalog slug and carry as_of (#17);
|
|
1412
|
+
# v1 cached the conflated date-stripped shape, so the old key is never read.
|
|
1413
|
+
FRONTEND_AA_CACHE = "openrouter-frontend-aa-v2"
|
|
247
1414
|
|
|
248
1415
|
|
|
249
1416
|
def fetch_openrouter_frontend(args):
|
|
@@ -301,7 +1468,8 @@ def fetch_openrouter_frontend(args):
|
|
|
301
1468
|
key = slug if variant in ("", "standard") else f"{slug}:{variant}"
|
|
302
1469
|
fresh_discounts.setdefault(key, float(raw))
|
|
303
1470
|
fresh_aa = build_aa_benchmarks(
|
|
304
|
-
data.get("benchmarks") if isinstance(data, dict) else None
|
|
1471
|
+
data.get("benchmarks") if isinstance(data, dict) else None,
|
|
1472
|
+
entries,
|
|
305
1473
|
)
|
|
306
1474
|
if fresh_discounts:
|
|
307
1475
|
discounts = fresh_discounts
|
|
@@ -792,7 +1960,7 @@ def parse_iso_datetime(value):
|
|
|
792
1960
|
# ---------------------------------------------------------------------------
|
|
793
1961
|
|
|
794
1962
|
|
|
795
|
-
def aa_api_entries(api_key: str) -> list:
|
|
1963
|
+
def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
|
|
796
1964
|
"""Intelligence entries from AA's V2 language models endpoint.
|
|
797
1965
|
|
|
798
1966
|
Documented envelope: {tier, intelligence_index_version, pagination,
|
|
@@ -803,8 +1971,16 @@ def aa_api_entries(api_key: str) -> list:
|
|
|
803
1971
|
has_more/total_pages contract with a 25-page hard cap; a failure past
|
|
804
1972
|
page 1 is best-effort (warn, keep earlier pages), a page-1 failure
|
|
805
1973
|
propagates.
|
|
1974
|
+
|
|
1975
|
+
Returns (entries, version): version is the envelope's
|
|
1976
|
+
intelligence_index_version (number-or-null) -- provenance for the whole
|
|
1977
|
+
fetch. Every surviving entry carries `as_of`, one ISO fetch timestamp
|
|
1978
|
+
shared by the whole fetch, so the cache and any downstream record can
|
|
1979
|
+
date the data even after the envelope's version is lost.
|
|
806
1980
|
"""
|
|
807
1981
|
entries = {}
|
|
1982
|
+
version = None
|
|
1983
|
+
as_of = datetime.now(timezone.utc).isoformat()
|
|
808
1984
|
page = 1
|
|
809
1985
|
while page <= 25: # hard cap: halt after processing page 25
|
|
810
1986
|
try:
|
|
@@ -819,6 +1995,15 @@ def aa_api_entries(api_key: str) -> list:
|
|
|
819
1995
|
warn(f"AA API page {page} failed ({exc}); using entries collected so far")
|
|
820
1996
|
break
|
|
821
1997
|
data = payload.get("data") if isinstance(payload, dict) else None
|
|
1998
|
+
# The envelope's version is one provenance fact for the whole
|
|
1999
|
+
# fetch; the last page's value wins (they are identical in practice).
|
|
2000
|
+
raw_version = (
|
|
2001
|
+
payload.get("intelligence_index_version")
|
|
2002
|
+
if isinstance(payload, dict)
|
|
2003
|
+
else None
|
|
2004
|
+
)
|
|
2005
|
+
if isinstance(raw_version, (int, float)) and not isinstance(raw_version, bool):
|
|
2006
|
+
version = raw_version
|
|
822
2007
|
for item in data if isinstance(data, list) else []:
|
|
823
2008
|
if not isinstance(item, dict):
|
|
824
2009
|
continue
|
|
@@ -851,6 +2036,10 @@ def aa_api_entries(api_key: str) -> list:
|
|
|
851
2036
|
"key": key,
|
|
852
2037
|
"name": name if isinstance(name, str) and name else key,
|
|
853
2038
|
"index": index,
|
|
2039
|
+
# One fetch timestamp for the whole fetch: every entry
|
|
2040
|
+
# from this run shares it, so the cache payload and any
|
|
2041
|
+
# downstream record can date the data exactly.
|
|
2042
|
+
"as_of": as_of,
|
|
854
2043
|
},
|
|
855
2044
|
)
|
|
856
2045
|
pag = payload.get("pagination") if isinstance(payload, dict) else None
|
|
@@ -862,37 +2051,53 @@ def aa_api_entries(api_key: str) -> list:
|
|
|
862
2051
|
if isinstance(total_pages, int) and page >= total_pages:
|
|
863
2052
|
break
|
|
864
2053
|
page += 1
|
|
865
|
-
return list(entries.values())
|
|
2054
|
+
return list(entries.values()), version
|
|
866
2055
|
|
|
867
2056
|
|
|
868
2057
|
def fetch_aa_entries(args):
|
|
869
2058
|
"""AA intelligence entries from the keyed AA API v2, or nothing.
|
|
870
2059
|
|
|
871
|
-
Returns (entries, source, cached). With a key, an API
|
|
872
|
-
empty result warns and yields ([], None, False).
|
|
873
|
-
silent and yields ([], None, False):
|
|
874
|
-
|
|
2060
|
+
Returns (entries, source, cached, index_version). With a key, an API
|
|
2061
|
+
failure or an empty result warns and yields ([], None, False, None).
|
|
2062
|
+
Without a key this is silent and yields ([], None, False, None):
|
|
2063
|
+
OpenRouter's benchmarks cover what they cover and the catalog records
|
|
2064
|
+
the absence (sources.aa).
|
|
2065
|
+
|
|
2066
|
+
`index_version` is the envelope's intelligence_index_version (number
|
|
2067
|
+
or null) -- provenance for the whole fetch. Entries carry `as_of`, one
|
|
2068
|
+
ISO fetch timestamp for the whole fetch; a cache hit reuses the
|
|
2069
|
+
stamped payload verbatim so cached rows stay dated exactly like fresh
|
|
2070
|
+
ones.
|
|
875
2071
|
"""
|
|
876
2072
|
if not args.no_cache:
|
|
877
|
-
#
|
|
878
|
-
#
|
|
879
|
-
# build_catalog rejects any other
|
|
880
|
-
|
|
2073
|
+
# v3: payloads cached under earlier keys predate the as_of stamp and
|
|
2074
|
+
# the intelligence_index_version capture and must never be read.
|
|
2075
|
+
# Only AA API entries are read; build_catalog rejects any other
|
|
2076
|
+
# source as unknown.
|
|
2077
|
+
cached = load_cache(AA_ENTRIES_CACHE, args.cache_ttl)
|
|
881
2078
|
if isinstance(cached, dict) and cached.get("source") == "AA API v2":
|
|
882
|
-
return
|
|
2079
|
+
return (
|
|
2080
|
+
cached.get("entries", []),
|
|
2081
|
+
cached.get("source"),
|
|
2082
|
+
True,
|
|
2083
|
+
cached.get("version"),
|
|
2084
|
+
)
|
|
883
2085
|
api_key = args.aa_api_key or os.environ.get("AA_API_KEY")
|
|
884
2086
|
if not api_key:
|
|
885
|
-
return [], None, False
|
|
2087
|
+
return [], None, False, None
|
|
886
2088
|
try:
|
|
887
|
-
entries = aa_api_entries(api_key)
|
|
2089
|
+
entries, version = aa_api_entries(api_key)
|
|
888
2090
|
except Exception as exc:
|
|
889
2091
|
warn(f"AA API request failed ({exc})")
|
|
890
|
-
return [], None, False
|
|
2092
|
+
return [], None, False, None
|
|
891
2093
|
if not entries:
|
|
892
2094
|
warn("AA API returned no intelligence scores")
|
|
893
|
-
return [], None, False
|
|
894
|
-
save_cache(
|
|
895
|
-
|
|
2095
|
+
return [], None, False, None
|
|
2096
|
+
save_cache(
|
|
2097
|
+
AA_ENTRIES_CACHE,
|
|
2098
|
+
{"entries": entries, "source": "AA API v2", "version": version},
|
|
2099
|
+
)
|
|
2100
|
+
return entries, "AA API v2", False, version
|
|
896
2101
|
|
|
897
2102
|
|
|
898
2103
|
# ---------------------------------------------------------------------------
|
|
@@ -901,6 +2106,13 @@ def fetch_aa_entries(args):
|
|
|
901
2106
|
|
|
902
2107
|
|
|
903
2108
|
def build_aa_lookup(entries):
|
|
2109
|
+
"""Index the AA entries into exact and fuzzy lookup maps.
|
|
2110
|
+
|
|
2111
|
+
Values are (index, as_of) pairs, not bare floats: the as_of travels
|
|
2112
|
+
with the match so resolve_quality can date every AA-sourced record
|
|
2113
|
+
without re-scanning the raw entry list. Entries without a usable key
|
|
2114
|
+
or a finite numeric index are skipped.
|
|
2115
|
+
"""
|
|
904
2116
|
exact = {}
|
|
905
2117
|
fuzzy = []
|
|
906
2118
|
for entry in entries:
|
|
@@ -913,11 +2125,12 @@ def build_aa_lookup(entries):
|
|
|
913
2125
|
# Non-finite indexes would serialize as NaN and poison quality.
|
|
914
2126
|
if not math.isfinite(index):
|
|
915
2127
|
continue
|
|
2128
|
+
as_of = entry.get("as_of") if isinstance(entry.get("as_of"), str) else None
|
|
916
2129
|
|
|
917
2130
|
def put(tier, raw):
|
|
918
2131
|
normalized = norm_key(raw)
|
|
919
2132
|
if normalized:
|
|
920
|
-
exact.setdefault((tier, normalized), index)
|
|
2133
|
+
exact.setdefault((tier, normalized), (index, as_of))
|
|
921
2134
|
|
|
922
2135
|
if "/" in key:
|
|
923
2136
|
put(0, key)
|
|
@@ -928,11 +2141,17 @@ def build_aa_lookup(entries):
|
|
|
928
2141
|
put(3, PAREN_RE.sub(" ", name))
|
|
929
2142
|
tokens = set(norm_key(key).split()) | set(norm_key(name).split())
|
|
930
2143
|
if tokens:
|
|
931
|
-
fuzzy.append((tokens, index))
|
|
2144
|
+
fuzzy.append((tokens, (index, as_of)))
|
|
932
2145
|
return exact, fuzzy
|
|
933
2146
|
|
|
934
2147
|
|
|
935
2148
|
def match_quality(model, exact, fuzzy, allow_fuzzy=False):
|
|
2149
|
+
"""Match a candidate to an AA entry; exact tiers only unless fuzzy is on.
|
|
2150
|
+
|
|
2151
|
+
Returns the winning entry's (index, as_of) pair, or None when no tier
|
|
2152
|
+
matches. Values come straight out of build_aa_lookup's maps, so the
|
|
2153
|
+
as_of is the entry's own fetch stamp, never a re-lookup.
|
|
2154
|
+
"""
|
|
936
2155
|
model_id = model["id"]
|
|
937
2156
|
name = model.get("name") or ""
|
|
938
2157
|
display = PROVIDER_PREFIX_RE.sub("", name).strip()
|
|
@@ -955,14 +2174,14 @@ def match_quality(model, exact, fuzzy, allow_fuzzy=False):
|
|
|
955
2174
|
own = set(norm_key(base).split()) | set(norm_key(display).split())
|
|
956
2175
|
if own:
|
|
957
2176
|
best = 0.0
|
|
958
|
-
|
|
959
|
-
for tokens,
|
|
2177
|
+
best_match = None
|
|
2178
|
+
for tokens, match in fuzzy:
|
|
960
2179
|
overlap = len(own & tokens) / len(own | tokens)
|
|
961
2180
|
if overlap > best:
|
|
962
2181
|
best = overlap
|
|
963
|
-
|
|
964
|
-
if best >= 0.5 and
|
|
965
|
-
return
|
|
2182
|
+
best_match = match
|
|
2183
|
+
if best >= 0.5 and best_match is not None:
|
|
2184
|
+
return best_match
|
|
966
2185
|
return None
|
|
967
2186
|
|
|
968
2187
|
|
|
@@ -974,24 +2193,37 @@ def base_model_id(model_id: str) -> str:
|
|
|
974
2193
|
AA_BENCHMARK_FIELDS = ("intelligence_index", "coding_index", "agentic_index")
|
|
975
2194
|
|
|
976
2195
|
|
|
977
|
-
def build_aa_benchmarks(benchmarks) -> dict:
|
|
978
|
-
"""Map
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
2196
|
+
def build_aa_benchmarks(benchmarks, models=None) -> dict:
|
|
2197
|
+
"""Map exact catalog slug -> OR-published AA trio from data.benchmarks.
|
|
2198
|
+
|
|
2199
|
+
Benchmark keys are dated permaslugs (deepseek/deepseek-v4.1-flash-
|
|
2200
|
+
20261017) whose date-stripped form can collide across listings of one
|
|
2201
|
+
model family: the snapshot listing deepseek/deepseek-v4.1-flash-1017
|
|
2202
|
+
and the base listing deepseek/deepseek-v4.1-flash both strip to
|
|
2203
|
+
deepseek/deepseek-v4.1-flash, so blind stripping with latest-date-wins
|
|
2204
|
+
made the base listing borrow the snapshot's score (#17). Keys resolve
|
|
2205
|
+
through three explicit passes over the surviving entries, and every
|
|
2206
|
+
accepted id is claimed by the pass that resolved it, so later passes
|
|
2207
|
+
never overwrite an earlier one:
|
|
2208
|
+
|
|
2209
|
+
1. Primary: a key equal to a models entry's permaslug maps to that
|
|
2210
|
+
entry's slug -- the catalog's own pairing, no guessing. Several
|
|
2211
|
+
dated permaslugs may legitimately pair with one slug (snapshot
|
|
2212
|
+
variants of one listing); among those the latest date wins.
|
|
2213
|
+
2. Undated: a key without a -YYYYMMDD suffix maps to itself, unless
|
|
2214
|
+
pass 1 claimed the id.
|
|
2215
|
+
3. Dated fallback: a dated key the catalog does not pair strips via
|
|
2216
|
+
strip_permaslug_date, accepted only when the stripped id is
|
|
2217
|
+
unclaimed and no other pass-3 key strips to the same id -- an
|
|
2218
|
+
ambiguous strip is the #17 conflation shape, so every candidate
|
|
2219
|
+
for that id drops rather than guess.
|
|
2220
|
+
|
|
2221
|
+
Entries whose values fail the finite-number guard are treated as
|
|
2222
|
+
absent before the passes run; entries with at least one valid field
|
|
2223
|
+
are kept. Records carry "as_of": the YYYYMMDD of the permaslug the
|
|
2224
|
+
trio came from, or None for undated keys.
|
|
985
2225
|
"""
|
|
986
|
-
|
|
987
|
-
seen_date = {}
|
|
988
|
-
|
|
989
|
-
def accept(bare, date, trio):
|
|
990
|
-
if bare in seen_date and seen_date[bare] >= date:
|
|
991
|
-
return
|
|
992
|
-
seen_date[bare] = date
|
|
993
|
-
aa_by_id[bare] = trio
|
|
994
|
-
|
|
2226
|
+
valid = {} # benchmark key -> (trio, permaslug date or "")
|
|
995
2227
|
for key, node in (benchmarks or {}).items():
|
|
996
2228
|
aa = node.get("aa") if isinstance(node, dict) else None
|
|
997
2229
|
if not isinstance(aa, dict):
|
|
@@ -1007,36 +2239,152 @@ def build_aa_benchmarks(benchmarks) -> dict:
|
|
|
1007
2239
|
trio[field] = float(value)
|
|
1008
2240
|
if not trio:
|
|
1009
2241
|
continue
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
else:
|
|
1014
|
-
accept(key, "", trio)
|
|
1015
|
-
return aa_by_id
|
|
2242
|
+
bare = strip_permaslug_date(key)
|
|
2243
|
+
# strip_permaslug_date removed a trailing -YYYYMMDD exactly when
|
|
2244
|
+
# the key was dated; the date is then the final 8 characters.
|
|
2245
|
+
valid[key] = (trio, "" if bare == key else key[-8:])
|
|
1016
2246
|
|
|
2247
|
+
permaslug_to_slug = {} # the catalog's own slug/permaslug pairing
|
|
2248
|
+
for entry in models or []:
|
|
2249
|
+
if not isinstance(entry, dict):
|
|
2250
|
+
continue
|
|
2251
|
+
slug = entry.get("slug") or ""
|
|
2252
|
+
permaslug = entry.get("permaslug") or ""
|
|
2253
|
+
if slug and permaslug:
|
|
2254
|
+
# standard and batch variants repeat an identical pair; the
|
|
2255
|
+
# overwrite is the dedup
|
|
2256
|
+
permaslug_to_slug[permaslug] = slug
|
|
2257
|
+
|
|
2258
|
+
aa_by_id = {}
|
|
2259
|
+
claimed = set()
|
|
1017
2260
|
|
|
1018
|
-
def
|
|
1019
|
-
|
|
2261
|
+
def record(trio, date):
|
|
2262
|
+
return {**trio, "as_of": date or None}
|
|
1020
2263
|
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
2264
|
+
# Pass 1: permaslug keys pair with their own slug; same-slug
|
|
2265
|
+
# snapshots resolve to the latest date (first wins on an equal date).
|
|
2266
|
+
seen_date = {}
|
|
2267
|
+
for key, (trio, date) in valid.items():
|
|
2268
|
+
slug = permaslug_to_slug.get(key)
|
|
2269
|
+
if slug is None or (slug in seen_date and seen_date[slug] >= date):
|
|
2270
|
+
continue
|
|
2271
|
+
seen_date[slug] = date
|
|
2272
|
+
aa_by_id[slug] = record(trio, date)
|
|
2273
|
+
claimed.add(slug)
|
|
2274
|
+
|
|
2275
|
+
# Pass 2: undated keys self-map unless pass 1 claimed the id.
|
|
2276
|
+
for key, (trio, date) in valid.items():
|
|
2277
|
+
if date or key in permaslug_to_slug or key in claimed:
|
|
2278
|
+
continue
|
|
2279
|
+
aa_by_id[key] = record(trio, date)
|
|
2280
|
+
claimed.add(key)
|
|
2281
|
+
|
|
2282
|
+
# Pass 3: unpaired dated keys strip, but only onto an unclaimed id
|
|
2283
|
+
# and only when the strip is unambiguous.
|
|
2284
|
+
strip_candidates = {}
|
|
2285
|
+
for key, (trio, date) in valid.items():
|
|
2286
|
+
if not date or key in permaslug_to_slug:
|
|
2287
|
+
continue
|
|
2288
|
+
stripped = strip_permaslug_date(key)
|
|
2289
|
+
strip_candidates.setdefault(stripped, []).append((date, trio))
|
|
2290
|
+
for bare, candidates in strip_candidates.items():
|
|
2291
|
+
if bare in claimed or len(candidates) > 1:
|
|
2292
|
+
continue
|
|
2293
|
+
date, trio = candidates[0]
|
|
2294
|
+
aa_by_id[bare] = record(trio, date)
|
|
2295
|
+
claimed.add(bare)
|
|
2296
|
+
|
|
2297
|
+
return aa_by_id
|
|
2298
|
+
|
|
2299
|
+
|
|
2300
|
+
def resolve_quality(
|
|
2301
|
+
candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality=None
|
|
2302
|
+
):
|
|
2303
|
+
"""Per-candidate quality records: AA API primary, catalog next, OR last.
|
|
2304
|
+
|
|
2305
|
+
Resolution order per candidate (inverted from the pre-reconciliation
|
|
2306
|
+
behavior where OpenRouter's benchmarks won):
|
|
2307
|
+
1. AA API entries -- match_quality's exact tiers (fuzzy stays off in
|
|
2308
|
+
production: it is the proven variant-conflation bug).
|
|
2309
|
+
2. catalog_quality -- records hydrated from the published catalog on
|
|
2310
|
+
unkeyed runs. Every winner is labeled "catalog": the run-level
|
|
2311
|
+
provenance is the source chain (read from our published catalog),
|
|
2312
|
+
overriding the row's own "aa"/"openrouter" label -- that label is
|
|
2313
|
+
the keyed CI run's row-level truth inside the artifact, and no
|
|
2314
|
+
real catalog row ever carries "catalog". as_of stays the row's
|
|
2315
|
+
own stamp.
|
|
2316
|
+
3. OpenRouter's benchmark trio -- per-slug, matched via
|
|
2317
|
+
base_model_id, so a :variant suffix inherits its base's trio.
|
|
2318
|
+
4. No match -- the candidate stays quality-blind downstream.
|
|
2319
|
+
|
|
2320
|
+
Every winner is a record {"index", "source", "as_of", "drift",
|
|
2321
|
+
"or_index", "or_as_of"}; compute_scores and build_schedules read
|
|
2322
|
+
record["index"], so cand["quality"] stays the float index they already
|
|
2323
|
+
expect. Drift is computed only when the winner is NOT OpenRouter: the
|
|
2324
|
+
OR trio for the same base model acts as the check value, and a
|
|
2325
|
+
disagreement of >= 3.0 points flags the record. When the winner IS
|
|
2326
|
+
OpenRouter (or no OR value exists) the drift pair is None -- there is
|
|
2327
|
+
no second source to disagree with.
|
|
1025
2328
|
"""
|
|
1026
|
-
fallback_source = {"AA API v2": "api"}.get(aa_source)
|
|
1027
2329
|
quality_by_id = {}
|
|
1028
2330
|
source_by_id = {}
|
|
1029
2331
|
for cand in candidates:
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
2332
|
+
record = None
|
|
2333
|
+
# Tier 1: AA API exact match. match_quality returns the winning
|
|
2334
|
+
# entry's (index, as_of) pair.
|
|
2335
|
+
match = match_quality({"id": cand["id"], "name": cand["name"]}, exact, fuzzy)
|
|
2336
|
+
if match is not None:
|
|
2337
|
+
index, as_of = match
|
|
2338
|
+
record = {"index": index, "source": "aa", "as_of": as_of}
|
|
2339
|
+
# Tier 2: catalog record (unkeyed runs hydrate from the published
|
|
2340
|
+
# catalog). The run-level provenance label is the source chain:
|
|
2341
|
+
# every hydrated record is tagged "catalog" -- read from our
|
|
2342
|
+
# published catalog -- overriding the row's own label, which is
|
|
2343
|
+
# the keyed CI run's row-level truth inside the artifact ("aa"/
|
|
2344
|
+
# "openrouter"; no real catalog row ever carries "catalog").
|
|
2345
|
+
# as_of stays the row's own stamp (v4 rows carry the CI run's
|
|
2346
|
+
# recorded quality_as_of; v3 rows the catalog's generated_at --
|
|
2347
|
+
# catalog_quality_by_id normalizes that).
|
|
2348
|
+
if record is None and catalog_quality is not None:
|
|
2349
|
+
catalog_record = catalog_quality.get(cand["id"])
|
|
2350
|
+
if catalog_record is not None:
|
|
2351
|
+
record = {
|
|
2352
|
+
"index": catalog_record["index"],
|
|
2353
|
+
"source": "catalog",
|
|
2354
|
+
"as_of": catalog_record["as_of"],
|
|
2355
|
+
}
|
|
2356
|
+
# Tier 3: OpenRouter's per-slug benchmark trio (last resort).
|
|
2357
|
+
if record is None:
|
|
2358
|
+
bench = aa_by_id.get(base_model_id(cand["id"]))
|
|
2359
|
+
intelligence = bench.get("intelligence_index") if bench else None
|
|
2360
|
+
if intelligence is not None:
|
|
2361
|
+
record = {
|
|
2362
|
+
"index": intelligence,
|
|
2363
|
+
"source": "openrouter",
|
|
2364
|
+
"as_of": bench.get("as_of"),
|
|
2365
|
+
}
|
|
2366
|
+
if record is None:
|
|
1035
2367
|
continue
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
2368
|
+
|
|
2369
|
+
# Drift: only meaningful when the winner is not OpenRouter itself.
|
|
2370
|
+
bench = aa_by_id.get(base_model_id(cand["id"])) or {}
|
|
2371
|
+
or_intelligence = bench.get("intelligence_index")
|
|
2372
|
+
if (
|
|
2373
|
+
record["source"] in ("aa", "catalog")
|
|
2374
|
+
and isinstance(or_intelligence, (int, float))
|
|
2375
|
+
and not isinstance(or_intelligence, bool)
|
|
2376
|
+
and math.isfinite(or_intelligence)
|
|
2377
|
+
):
|
|
2378
|
+
record["or_index"] = or_intelligence
|
|
2379
|
+
record["or_as_of"] = bench.get("as_of")
|
|
2380
|
+
record["drift"] = abs(record["index"] - or_intelligence) >= 3.0
|
|
2381
|
+
else:
|
|
2382
|
+
record["or_index"] = None
|
|
2383
|
+
record["or_as_of"] = None
|
|
2384
|
+
record["drift"] = None
|
|
2385
|
+
|
|
2386
|
+
quality_by_id[cand["id"]] = record
|
|
2387
|
+
source_by_id[cand["id"]] = record["source"]
|
|
1040
2388
|
return quality_by_id, source_by_id
|
|
1041
2389
|
|
|
1042
2390
|
|
|
@@ -1061,7 +2409,9 @@ def _per_1m(price_per_token: float) -> float:
|
|
|
1061
2409
|
return round(price_per_token * 1_000_000.0, 6)
|
|
1062
2410
|
|
|
1063
2411
|
|
|
1064
|
-
def build_schedules(
|
|
2412
|
+
def build_schedules(
|
|
2413
|
+
models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality=None
|
|
2414
|
+
):
|
|
1065
2415
|
"""Catalog-wide off-peak index built from the RAW model list (spec 5.4).
|
|
1066
2416
|
|
|
1067
2417
|
Deliberately unfiltered by the candidate pool: sub-floor-context and
|
|
@@ -1069,6 +2419,12 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
|
|
|
1069
2419
|
schedule models mostly sit below the 1M context floor). Router aliases
|
|
1070
2420
|
(~ids) and malformed ids are excluded; :batch variants are included as
|
|
1071
2421
|
distinct deals. Sorted by id.
|
|
2422
|
+
|
|
2423
|
+
Quality falls back through the same chain resolve_quality walks, minus
|
|
2424
|
+
its AA tier (the AA lookup only ever matched candidates): candidate
|
|
2425
|
+
records, then the hydrated catalog (unkeyed runs), then OpenRouter's
|
|
2426
|
+
benchmark trio. Every schedule row carries the winner's provenance
|
|
2427
|
+
(quality_source/quality_as_of) so the site's QUAL tooltip can render.
|
|
1072
2428
|
"""
|
|
1073
2429
|
entries = []
|
|
1074
2430
|
for model in models:
|
|
@@ -1093,10 +2449,38 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
|
|
|
1093
2449
|
continue
|
|
1094
2450
|
peak = max(windows, key=lambda window: window["blended"])
|
|
1095
2451
|
offpeak = min(windows, key=lambda window: window["blended"])
|
|
1096
|
-
|
|
2452
|
+
# quality_by_id values are records {"index", "source", ...}; the
|
|
2453
|
+
# schedule row keeps the bare float index verbatim (never a dict)
|
|
2454
|
+
# and echoes the winning record's provenance for the QUAL tooltip.
|
|
2455
|
+
# Fallback chain mirrors resolve_quality's minus its AA tier:
|
|
2456
|
+
# candidate record, hydrated catalog record (unkeyed runs; every
|
|
2457
|
+
# hydrated record is relabeled with the run-level chain label
|
|
2458
|
+
# "catalog", exactly like resolve_quality's catalog tier), then
|
|
2459
|
+
# OpenRouter's benchmark trio (row label "openrouter").
|
|
2460
|
+
record = quality_by_id.get(model_id)
|
|
2461
|
+
if isinstance(record, dict):
|
|
2462
|
+
quality = record.get("index")
|
|
2463
|
+
quality_source = record.get("source")
|
|
2464
|
+
quality_as_of = record.get("as_of")
|
|
2465
|
+
else:
|
|
2466
|
+
quality = record
|
|
2467
|
+
quality_source = None
|
|
2468
|
+
quality_as_of = None
|
|
2469
|
+
if quality is None and catalog_quality is not None:
|
|
2470
|
+
catalog_record = catalog_quality.get(model_id)
|
|
2471
|
+
if catalog_record is not None:
|
|
2472
|
+
quality = catalog_record["index"]
|
|
2473
|
+
quality_source = "catalog"
|
|
2474
|
+
quality_as_of = catalog_record["as_of"]
|
|
1097
2475
|
if quality is None:
|
|
1098
2476
|
benchmark = aa_by_id.get(base_model_id(model_id)) or {}
|
|
1099
2477
|
quality = benchmark.get("intelligence_index")
|
|
2478
|
+
if quality is not None:
|
|
2479
|
+
quality_source = "openrouter"
|
|
2480
|
+
quality_as_of = benchmark.get("as_of")
|
|
2481
|
+
else:
|
|
2482
|
+
quality_source = None
|
|
2483
|
+
quality_as_of = None
|
|
1100
2484
|
name = model.get("name") or model_id
|
|
1101
2485
|
entries.append(
|
|
1102
2486
|
{
|
|
@@ -1104,6 +2488,10 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
|
|
|
1104
2488
|
"name": PROVIDER_PREFIX_RE.sub("", name).strip() or name,
|
|
1105
2489
|
"context": coerce_int(model.get("context_length"), 0),
|
|
1106
2490
|
"quality": quality,
|
|
2491
|
+
# Provenance for the site's QUAL tooltip (null when the
|
|
2492
|
+
# quality came without any -- e.g. a bare-float test map).
|
|
2493
|
+
"quality_source": quality_source,
|
|
2494
|
+
"quality_as_of": quality_as_of,
|
|
1107
2495
|
"score": balanced_by_id.get(model_id),
|
|
1108
2496
|
"max_discount": eff["max_discount"],
|
|
1109
2497
|
"sched_note": fmt_sched_note(eff["max_discount"]),
|
|
@@ -1313,11 +2701,19 @@ def compute_scores(candidates, args, quality_by_id):
|
|
|
1313
2701
|
0.0, min(1.0, 1.0 - (math.log10(price + 0.01) - low) / price_span)
|
|
1314
2702
|
)
|
|
1315
2703
|
quality = quality_by_id.get(cand["id"])
|
|
1316
|
-
|
|
1317
|
-
|
|
2704
|
+
# quality_by_id values are records {"index", "source", ...}; the
|
|
2705
|
+
# candidate keeps the bare float index (that is what ranking_key,
|
|
2706
|
+
# build_catalog's quality field and print_json's quality_index all
|
|
2707
|
+
# expect) and the full record travels alongside for provenance.
|
|
2708
|
+
record = quality if isinstance(quality, dict) else None
|
|
2709
|
+
cand["quality"] = record["index"] if record else quality
|
|
2710
|
+
cand["quality_record"] = record
|
|
2711
|
+
if cand["quality"] is None:
|
|
1318
2712
|
cand["quality_score"] = 0.0
|
|
1319
2713
|
else:
|
|
1320
|
-
cand["quality_score"] = max(
|
|
2714
|
+
cand["quality_score"] = max(
|
|
2715
|
+
0.0, min(1.0, cand["quality"] / args.quality_ref)
|
|
2716
|
+
)
|
|
1321
2717
|
context = cand["context"]
|
|
1322
2718
|
cand["context_score"] = (
|
|
1323
2719
|
max(0.0, min(1.0, math.log(context / floor_ctx) / ctx_span))
|
|
@@ -1413,6 +2809,151 @@ def fmt_discount(value) -> str:
|
|
|
1413
2809
|
return f"{value:.0%}" if has_discount(value) else "--"
|
|
1414
2810
|
|
|
1415
2811
|
|
|
2812
|
+
def fmt_prec(cand) -> str:
|
|
2813
|
+
"""PREC cell: the precision the ranked price is quoted at.
|
|
2814
|
+
|
|
2815
|
+
The maker's default precision when disclosed, the best available
|
|
2816
|
+
otherwise, "n/a" when none could be derived; "!" flags default routing
|
|
2817
|
+
that favors cheaper sub-default endpoints (annotations.risk_flag).
|
|
2818
|
+
Candidates that never went through the endpoint stage degrade to "n/a".
|
|
2819
|
+
"""
|
|
2820
|
+
cell = cand.get("default_precision") or "n/a"
|
|
2821
|
+
if (cand.get("annotations") or {}).get("risk_flag"):
|
|
2822
|
+
cell += "!"
|
|
2823
|
+
return cell
|
|
2824
|
+
|
|
2825
|
+
|
|
2826
|
+
def fmt_quant_note(annotations: dict) -> str | None:
|
|
2827
|
+
"""One-line note on the cheapest sub-default endpoint (PREC tooltip).
|
|
2828
|
+
|
|
2829
|
+
The exact string the site shows (Task 11 renders it verbatim), so the
|
|
2830
|
+
format is pinned by test. None when there is no non-eligible live
|
|
2831
|
+
endpoint -- nothing cheaper to route to. cheapest_quantized.price is
|
|
2832
|
+
stored per-token, the way build_annotations leaves it; the note shows
|
|
2833
|
+
USD per 1M like every other price the CLI prints. A missing spread
|
|
2834
|
+
(defensive) drops the parenthetical. A null quantization (third-party
|
|
2835
|
+
null == "unknown" for eligibility) and a null provider_name on
|
|
2836
|
+
malformed data render the design's "unknown" sentinel, never Python's
|
|
2837
|
+
None repr.
|
|
2838
|
+
"""
|
|
2839
|
+
quant = (annotations or {}).get("cheapest_quantized")
|
|
2840
|
+
if not quant:
|
|
2841
|
+
return None
|
|
2842
|
+
quantization = quant["quantization"] or "unknown"
|
|
2843
|
+
provider = quant["provider"] or "unknown"
|
|
2844
|
+
note = (
|
|
2845
|
+
f"cheapest non-baseline: {quantization} @ {provider}, "
|
|
2846
|
+
f"${fmt_price(quant['price'] * 1_000_000)}/M"
|
|
2847
|
+
)
|
|
2848
|
+
spread = (annotations or {}).get("price_spread")
|
|
2849
|
+
if spread is not None:
|
|
2850
|
+
note += f" in ({spread:.2f}x baseline)"
|
|
2851
|
+
return note
|
|
2852
|
+
|
|
2853
|
+
|
|
2854
|
+
def _footer_as_of(source, value):
|
|
2855
|
+
"""Format a record's as_of for its footer block (None when unusable).
|
|
2856
|
+
|
|
2857
|
+
Two stamp shapes circulate. A YYYYMMDD date (OpenRouter's benchmark
|
|
2858
|
+
date, and catalog rows hydrated from OR rows) renders as a plain date
|
|
2859
|
+
("as of 2026-10-17") for ANY source -- a date-only stamp never
|
|
2860
|
+
fabricates a time of day. ISO stamps: live AA records carry one fetch
|
|
2861
|
+
timestamp for the whole fetch (checked today), so their block shows
|
|
2862
|
+
the time of day only; other rows carry the full stamp ("as of
|
|
2863
|
+
2026-10-23 13:15Z").
|
|
2864
|
+
"""
|
|
2865
|
+
if not value:
|
|
2866
|
+
return None
|
|
2867
|
+
text = str(value)
|
|
2868
|
+
if len(text) == 8 and text.isdigit():
|
|
2869
|
+
return f"{text[:4]}-{text[4:6]}-{text[6:]}"
|
|
2870
|
+
parsed = parse_iso_datetime(value)
|
|
2871
|
+
if parsed is None:
|
|
2872
|
+
return None
|
|
2873
|
+
if source == "aa":
|
|
2874
|
+
return parsed.strftime("%H:%MZ")
|
|
2875
|
+
return parsed.strftime("%Y-%m-%d %H:%MZ")
|
|
2876
|
+
|
|
2877
|
+
|
|
2878
|
+
def _as_of_instant(value):
|
|
2879
|
+
"""Parse an as_of stamp to a comparable UTC instant (None when unusable).
|
|
2880
|
+
|
|
2881
|
+
The pipeline carries ISO stamps and the YYYYMMDD benchmark date; a
|
|
2882
|
+
date-only stamp means "that day" and compares as midnight UTC.
|
|
2883
|
+
"""
|
|
2884
|
+
text = str(value)
|
|
2885
|
+
if len(text) == 8 and text.isdigit():
|
|
2886
|
+
return parse_iso_datetime(f"{text[:4]}-{text[4:6]}-{text[6:]}")
|
|
2887
|
+
return parse_iso_datetime(value)
|
|
2888
|
+
|
|
2889
|
+
|
|
2890
|
+
def _later_as_of(value, current) -> bool:
|
|
2891
|
+
"""True when the value stamp names a later instant than current.
|
|
2892
|
+
|
|
2893
|
+
Parsed instants, never raw strings: a YYYYMMDD stamp must not
|
|
2894
|
+
lexicographically beat a fresher ISO stamp ("20261017" sorts above
|
|
2895
|
+
every "2026-..." string). Ties keep the earlier pick; an unparseable
|
|
2896
|
+
stamp never displaces anything.
|
|
2897
|
+
"""
|
|
2898
|
+
new, old = _as_of_instant(value), _as_of_instant(current)
|
|
2899
|
+
if new is None or old is None:
|
|
2900
|
+
return new is not None
|
|
2901
|
+
return new > old
|
|
2902
|
+
|
|
2903
|
+
|
|
2904
|
+
def quality_footer_note(quality_by_id, aa_index_version, unmatched):
|
|
2905
|
+
"""Compose the footer's quality note from the resolved records.
|
|
2906
|
+
|
|
2907
|
+
Lists per-source counts, each source's own as_of and the AA index
|
|
2908
|
+
version when known, e.g. "AA index v4.3 - live AA (58, checked 13:15Z)
|
|
2909
|
+
| our catalog (12, as of 2026-10-23 13:15Z) | OpenRouter snapshot (2,
|
|
2910
|
+
as of 2026-10-17)". A trailing "unmatched score 0" names candidates
|
|
2911
|
+
with no quality record, and a drift footnote appears when any resolved
|
|
2912
|
+
record disagrees with the OpenRouter check value. Returns None when no
|
|
2913
|
+
record resolved -- run() falls back to the quality-blind note.
|
|
2914
|
+
"""
|
|
2915
|
+
labels = {
|
|
2916
|
+
"aa": "live AA",
|
|
2917
|
+
"catalog": "our catalog",
|
|
2918
|
+
"openrouter": "OpenRouter snapshot",
|
|
2919
|
+
}
|
|
2920
|
+
stamp_words = {"aa": "checked", "catalog": "as of", "openrouter": "as of"}
|
|
2921
|
+
counts = {}
|
|
2922
|
+
latest = {}
|
|
2923
|
+
for record in quality_by_id.values():
|
|
2924
|
+
source = record.get("source")
|
|
2925
|
+
if source not in labels:
|
|
2926
|
+
continue
|
|
2927
|
+
counts[source] = counts.get(source, 0) + 1
|
|
2928
|
+
value = record.get("as_of")
|
|
2929
|
+
if value:
|
|
2930
|
+
current = latest.get(source)
|
|
2931
|
+
if current is None or _later_as_of(value, current):
|
|
2932
|
+
latest[source] = value
|
|
2933
|
+
blocks = []
|
|
2934
|
+
for source in ("aa", "catalog", "openrouter"):
|
|
2935
|
+
count = counts.get(source, 0)
|
|
2936
|
+
if not count:
|
|
2937
|
+
continue
|
|
2938
|
+
stamp = _footer_as_of(source, latest.get(source))
|
|
2939
|
+
if stamp:
|
|
2940
|
+
blocks.append(f"{labels[source]} ({count}, {stamp_words[source]} {stamp})")
|
|
2941
|
+
else:
|
|
2942
|
+
blocks.append(f"{labels[source]} ({count})")
|
|
2943
|
+
if not blocks:
|
|
2944
|
+
return None
|
|
2945
|
+
note = "quality: "
|
|
2946
|
+
if aa_index_version is not None:
|
|
2947
|
+
note += f"AA index v{aa_index_version} - "
|
|
2948
|
+
note += " | ".join(blocks)
|
|
2949
|
+
if unmatched:
|
|
2950
|
+
note += "; unmatched score 0"
|
|
2951
|
+
drift = sum(1 for record in quality_by_id.values() if record.get("drift"))
|
|
2952
|
+
if drift:
|
|
2953
|
+
note += f"; drift vs OpenRouter snapshot ({drift})"
|
|
2954
|
+
return note
|
|
2955
|
+
|
|
2956
|
+
|
|
1416
2957
|
def print_table(top, total_candidates, weights, quality_note):
|
|
1417
2958
|
headers = [
|
|
1418
2959
|
"RANK",
|
|
@@ -1421,6 +2962,7 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1421
2962
|
"$IN/M",
|
|
1422
2963
|
"$OUT/M",
|
|
1423
2964
|
"DISC",
|
|
2965
|
+
"PREC",
|
|
1424
2966
|
"TIER",
|
|
1425
2967
|
"CTX",
|
|
1426
2968
|
"AGE",
|
|
@@ -1436,6 +2978,7 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1436
2978
|
fmt_price(cand["price_in"]),
|
|
1437
2979
|
fmt_price(cand["price_out"]),
|
|
1438
2980
|
fmt_discount(cand["discount"]),
|
|
2981
|
+
fmt_prec(cand),
|
|
1439
2982
|
cand["tier_note"] or cand["sched_note"] or "--",
|
|
1440
2983
|
fmt_context(cand["context"]),
|
|
1441
2984
|
fmt_age(cand["age_days"]),
|
|
@@ -1482,6 +3025,12 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1482
3025
|
"schedule whose discount is under 1%. --json lists each model's "
|
|
1483
3026
|
"schedule."
|
|
1484
3027
|
)
|
|
3028
|
+
print(
|
|
3029
|
+
"PREC = precision the ranked price is quoted at (maker's default when "
|
|
3030
|
+
"disclosed, best available otherwise; ! = default routing favors "
|
|
3031
|
+
"cheaper sub-default endpoints); prices are the cheapest live endpoint "
|
|
3032
|
+
"at that precision."
|
|
3033
|
+
)
|
|
1485
3034
|
|
|
1486
3035
|
|
|
1487
3036
|
def print_json(top):
|
|
@@ -1492,6 +3041,15 @@ def print_json(top):
|
|
|
1492
3041
|
"name": cand["name"],
|
|
1493
3042
|
"score": round(cand["score"], 4),
|
|
1494
3043
|
"quality_index": cand["quality"],
|
|
3044
|
+
# Provenance block, verbatim from the quality record resolved
|
|
3045
|
+
# in compute_scores: the winning tier's source and as_of plus
|
|
3046
|
+
# the drift check against OpenRouter's benchmarks. Rows without
|
|
3047
|
+
# a record (pre-endpoint test paths) degrade to None.
|
|
3048
|
+
"quality_source": (cand.get("quality_record") or {}).get("source"),
|
|
3049
|
+
"quality_as_of": (cand.get("quality_record") or {}).get("as_of"),
|
|
3050
|
+
"quality_drift": (cand.get("quality_record") or {}).get("drift"),
|
|
3051
|
+
"quality_or_index": (cand.get("quality_record") or {}).get("or_index"),
|
|
3052
|
+
"quality_or_as_of": (cand.get("quality_record") or {}).get("or_as_of"),
|
|
1495
3053
|
"input_usd_per_m": round(cand["price_in"], 6),
|
|
1496
3054
|
"output_usd_per_m": round(cand["price_out"], 6),
|
|
1497
3055
|
"blended_usd_per_m": round(cand["blended"], 6),
|
|
@@ -1518,6 +3076,31 @@ def print_json(top):
|
|
|
1518
3076
|
"age_days": round(cand["age_days"], 1)
|
|
1519
3077
|
if cand["age_days"] is not None
|
|
1520
3078
|
else None,
|
|
3079
|
+
# Additive keys (Task 9's ROW_KEYS extension pins this exact
|
|
3080
|
+
# key set and order). Risk/spread come from the annotations
|
|
3081
|
+
# block; session_cost_usd is run()'s per-candidate slice of the
|
|
3082
|
+
# Task 7 envelope, shown for the display turn range (run()
|
|
3083
|
+
# attaches it before any output path). Candidates that never
|
|
3084
|
+
# went through either stage (historical test paths) degrade to
|
|
3085
|
+
# null, as does a model the dataset covers only in other turn
|
|
3086
|
+
# ranges.
|
|
3087
|
+
"default_precision": cand.get("default_precision"),
|
|
3088
|
+
"risk_flag": (cand.get("annotations") or {}).get("risk_flag"),
|
|
3089
|
+
"price_spread": (cand.get("annotations") or {}).get("price_spread"),
|
|
3090
|
+
"session_cost_usd": (cand.get("session_cost") or {}).get(
|
|
3091
|
+
SESSION_COST_DISPLAY_TURN_RANGE
|
|
3092
|
+
),
|
|
3093
|
+
# The cheapest-sub-default note, exactly as the CLI prints it
|
|
3094
|
+
# (Task 11 shows it verbatim). Emit-first: Task 9's ROW_KEYS
|
|
3095
|
+
# pins this key last; pre-endpoint candidates degrade to None.
|
|
3096
|
+
"quant_note": fmt_quant_note(cand.get("annotations")),
|
|
3097
|
+
# One extra row key AFTER quant_note -- steering is NOT in
|
|
3098
|
+
# ROW_KEYS. build_site_data's row check is missing-key-only, so
|
|
3099
|
+
# extras pass through (tier_note precedent) and the site
|
|
3100
|
+
# ignores it. The OpenRouter provider routing object (design
|
|
3101
|
+
# section 4.7) the plugin contract reads; pre-endpoint
|
|
3102
|
+
# candidates degrade to None.
|
|
3103
|
+
"steering": cand.get("steering"),
|
|
1521
3104
|
}
|
|
1522
3105
|
for cand in top
|
|
1523
3106
|
]
|
|
@@ -1535,6 +3118,11 @@ def build_catalog(
|
|
|
1535
3118
|
aa_source,
|
|
1536
3119
|
aa_by_id,
|
|
1537
3120
|
quality_source_by_id,
|
|
3121
|
+
session_cost=None,
|
|
3122
|
+
session_cost_mode="skipped",
|
|
3123
|
+
index_version=None,
|
|
3124
|
+
catalog_generated_at=None,
|
|
3125
|
+
catalog_quality=None,
|
|
1538
3126
|
):
|
|
1539
3127
|
"""Assemble the full-catalog document (see README "Catalog output").
|
|
1540
3128
|
|
|
@@ -1542,21 +3130,56 @@ def build_catalog(
|
|
|
1542
3130
|
component scores) and never mutates them. Deterministic apart from
|
|
1543
3131
|
generated_at: models sort by (-overall.balanced, id), filtered by id,
|
|
1544
3132
|
and age uses date precision so two runs in the same UTC day match.
|
|
3133
|
+
|
|
3134
|
+
session_cost is the Task 7 fetch envelope (verbatim, medians included)
|
|
3135
|
+
or None; session_cost_mode is run()'s resolution of the key-presence
|
|
3136
|
+
fact ("ok" / "unavailable" / "skipped") -- build_catalog stays pure and
|
|
3137
|
+
never fetches or inspects the environment itself. index_version is the
|
|
3138
|
+
AA envelope's intelligence_index_version (number or null) and
|
|
3139
|
+
catalog_generated_at the published catalog's generated_at when this
|
|
3140
|
+
run hydrated from it (else None); both pass verbatim into sources.aa.
|
|
3141
|
+
catalog_quality is the hydrated record map (or None): candidates read
|
|
3142
|
+
it through resolve_quality, and build_schedules falls back to it for
|
|
3143
|
+
non-candidates.
|
|
1545
3144
|
"""
|
|
1546
3145
|
now = datetime.now(timezone.utc)
|
|
1547
3146
|
weights = catalog_weights(candidates, quality_by_id)
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
#
|
|
1551
|
-
# would contradict itself (mode none with matched quality scores).
|
|
3147
|
+
if aa_source is not None and aa_source != "AA API v2":
|
|
3148
|
+
# Never publish a document whose AA source we cannot name: the
|
|
3149
|
+
# consumers' mode enum has no slot for it.
|
|
1552
3150
|
raise ValueError(f"unknown AA source: {aa_source!r}")
|
|
3151
|
+
# The three counts are disjoint label buckets: resolve_quality's strict
|
|
3152
|
+
# rank gives every candidate at most one record, so their sum is exactly
|
|
3153
|
+
# the number of entries with a non-null quality_match -- the invariant
|
|
3154
|
+
# the site validator enforces. A label outside the buckets would break
|
|
3155
|
+
# it silently, so it fails loudly here.
|
|
3156
|
+
matched_aa = sum(1 for s in quality_source_by_id.values() if s == "aa")
|
|
3157
|
+
matched_catalog = sum(1 for s in quality_source_by_id.values() if s == "catalog")
|
|
1553
3158
|
matched_openrouter = sum(
|
|
1554
3159
|
1 for s in quality_source_by_id.values() if s == "openrouter"
|
|
1555
3160
|
)
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
3161
|
+
unknown = sorted(
|
|
3162
|
+
{
|
|
3163
|
+
s
|
|
3164
|
+
for s in quality_source_by_id.values()
|
|
3165
|
+
if s not in ("aa", "catalog", "openrouter")
|
|
3166
|
+
}
|
|
3167
|
+
)
|
|
3168
|
+
if unknown:
|
|
3169
|
+
raise ValueError(f"unknown quality source: {unknown[0]!r}")
|
|
3170
|
+
# mode is the run's quality chain, not a match outcome: "aa" when the
|
|
3171
|
+
# AA API returned usable entries, "catalog" when unkeyed runs hydrated
|
|
3172
|
+
# the published catalog, "none" when neither tier produced data.
|
|
3173
|
+
# OpenRouter's benchmarks are the chain's shared last resort, never a
|
|
3174
|
+
# mode of their own -- matched_openrouter carries their count. The
|
|
3175
|
+
# catalog branch also fires on the counts alone: hydrated records are
|
|
3176
|
+
# proof of a usable catalog even when its generated_at was unusable.
|
|
3177
|
+
if aa_source is not None:
|
|
3178
|
+
aa_mode = "aa"
|
|
3179
|
+
elif catalog_generated_at is not None or matched_catalog or matched_aa:
|
|
3180
|
+
aa_mode = "catalog"
|
|
3181
|
+
else:
|
|
3182
|
+
aa_mode = "none"
|
|
1560
3183
|
|
|
1561
3184
|
entries = []
|
|
1562
3185
|
for cand in candidates:
|
|
@@ -1587,6 +3210,12 @@ def build_catalog(
|
|
|
1587
3210
|
4,
|
|
1588
3211
|
)
|
|
1589
3212
|
bench = aa_by_id.get(base_model_id(cand["id"])) or {}
|
|
3213
|
+
# The quality record map is already a parameter; the winning
|
|
3214
|
+
# record's as_of and drift verdict become entry fields verbatim.
|
|
3215
|
+
# Unmatched candidates degrade to null -- same shape as the v3
|
|
3216
|
+
# additive blocks below.
|
|
3217
|
+
record = quality_by_id.get(cand["id"])
|
|
3218
|
+
record = record if isinstance(record, dict) else None
|
|
1590
3219
|
entries.append(
|
|
1591
3220
|
{
|
|
1592
3221
|
"id": cand["id"],
|
|
@@ -1654,7 +3283,25 @@ def build_catalog(
|
|
|
1654
3283
|
},
|
|
1655
3284
|
"quality": cand["quality"],
|
|
1656
3285
|
"quality_match": quality_source_by_id.get(cand["id"]),
|
|
3286
|
+
# Provenance (schema v4), verbatim from the record: the
|
|
3287
|
+
# winning source's own as_of and the drift verdict against
|
|
3288
|
+
# OpenRouter's check value.
|
|
3289
|
+
"quality_as_of": record["as_of"] if record else None,
|
|
3290
|
+
"quality_drift": record["drift"] if record else None,
|
|
1657
3291
|
"scores": {**scores, "overall": overall},
|
|
3292
|
+
# Additive provider-aware blocks (schema v3), verbatim from
|
|
3293
|
+
# the candidate. Candidates that never went through the
|
|
3294
|
+
# endpoint stage (historical test paths) degrade to null --
|
|
3295
|
+
# .get() with null defaults for these NEW keys only; the
|
|
3296
|
+
# pre-existing keys above stay direct-indexed.
|
|
3297
|
+
"endpoints": cand.get("endpoints"),
|
|
3298
|
+
"default_precision": cand.get("default_precision"),
|
|
3299
|
+
"advertised_pricing": cand.get("advertised_pricing"),
|
|
3300
|
+
"annotations": cand.get("annotations"),
|
|
3301
|
+
# The OpenRouter provider routing object (design section
|
|
3302
|
+
# 4.7) the plugin contract reads -- same null-default
|
|
3303
|
+
# degradation shape as the blocks above.
|
|
3304
|
+
"steering": cand.get("steering"),
|
|
1658
3305
|
}
|
|
1659
3306
|
)
|
|
1660
3307
|
entries.sort(key=lambda e: (-e["scores"]["overall"]["balanced"], e["id"]))
|
|
@@ -1665,7 +3312,7 @@ def build_catalog(
|
|
|
1665
3312
|
|
|
1666
3313
|
return {
|
|
1667
3314
|
"schedules": build_schedules(
|
|
1668
|
-
models, args, quality_by_id, aa_by_id, balanced_by_id
|
|
3315
|
+
models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality
|
|
1669
3316
|
),
|
|
1670
3317
|
"schema_version": CATALOG_SCHEMA_VERSION,
|
|
1671
3318
|
"tool": "model-compare",
|
|
@@ -1686,13 +3333,31 @@ def build_catalog(
|
|
|
1686
3333
|
"openrouter": "ok",
|
|
1687
3334
|
"aa": {
|
|
1688
3335
|
"mode": aa_mode,
|
|
1689
|
-
"
|
|
1690
|
-
"
|
|
3336
|
+
"matched_aa": matched_aa,
|
|
3337
|
+
"matched_catalog": matched_catalog,
|
|
1691
3338
|
"matched_openrouter": matched_openrouter,
|
|
3339
|
+
# Provenance meta (schema v4): the published catalog's stamp
|
|
3340
|
+
# when this run hydrated from it, and the AA envelope's
|
|
3341
|
+
# intelligence index version when the AA API ran. Both null
|
|
3342
|
+
# when their source was not consulted.
|
|
3343
|
+
"catalog_generated_at": catalog_generated_at,
|
|
3344
|
+
"intelligence_index_version": index_version,
|
|
1692
3345
|
},
|
|
1693
3346
|
"zdr": "skipped" if args.no_zdr else "ok",
|
|
1694
3347
|
"discounts": "ok" if discounts else "unavailable",
|
|
3348
|
+
# The endpoint stage always ran before build_catalog is reached;
|
|
3349
|
+
# its failures surface as pool.dropped counts
|
|
3350
|
+
# (endpoint_fetch_failed / no_baseline_eligible_endpoint), never
|
|
3351
|
+
# as source state -- so this is deterministic "ok".
|
|
3352
|
+
"endpoints": "ok",
|
|
3353
|
+
# "ok" (envelope fetched) / "unavailable" (key configured, fetch
|
|
3354
|
+
# failed) / "skipped" (no key) -- resolved in run().
|
|
3355
|
+
"session_cost": session_cost_mode,
|
|
1695
3356
|
},
|
|
3357
|
+
# Per-task session-cost medians (Task 7), verbatim from the fetch
|
|
3358
|
+
# envelope. Omitted entirely when unavailable: consumers branch on
|
|
3359
|
+
# key presence, never on a null shape.
|
|
3360
|
+
**({"session_cost": session_cost} if session_cost is not None else {}),
|
|
1696
3361
|
"pool": {
|
|
1697
3362
|
"listed": len(models),
|
|
1698
3363
|
"candidates": len(candidates),
|
|
@@ -1787,6 +3452,11 @@ def parse_args(argv=None):
|
|
|
1787
3452
|
default=None,
|
|
1788
3453
|
help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
|
|
1789
3454
|
)
|
|
3455
|
+
parser.add_argument(
|
|
3456
|
+
"--openrouter-api-key",
|
|
3457
|
+
default=None,
|
|
3458
|
+
help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
|
|
3459
|
+
)
|
|
1790
3460
|
parser.add_argument(
|
|
1791
3461
|
"--no-require-tools",
|
|
1792
3462
|
action="store_true",
|
|
@@ -1857,23 +3527,82 @@ def run(args, models, or_cached) -> int:
|
|
|
1857
3527
|
"(--no-zdr to override)"
|
|
1858
3528
|
)
|
|
1859
3529
|
return 1
|
|
1860
|
-
|
|
3530
|
+
# Mode-scoped quality chains: a keyed run consults the AA API only; an
|
|
3531
|
+
# unkeyed run hydrates from the published catalog only. The catalog
|
|
3532
|
+
# fetcher must NEVER run on keyed builds -- a CI/site build hydrating
|
|
3533
|
+
# from the published catalog would consume its own output (a
|
|
3534
|
+
# self-reference loop). Both chains end in OpenRouter's benchmarks as
|
|
3535
|
+
# the last resort, then nothing.
|
|
3536
|
+
if args.aa_api_key or os.environ.get("AA_API_KEY"):
|
|
3537
|
+
aa_entries, aa_source, aa_cached, aa_index_version = fetch_aa_entries(args)
|
|
3538
|
+
catalog_quality = None
|
|
3539
|
+
catalog_generated_at = None
|
|
3540
|
+
else:
|
|
3541
|
+
aa_entries, aa_source, aa_cached, aa_index_version = [], None, False, None
|
|
3542
|
+
# None (transport failure) and an empty by_id (fetched-but-unusable
|
|
3543
|
+
# artifact) both degrade to no catalog tier -- resolve_quality and
|
|
3544
|
+
# the footer treat them identically as "catalog unavailable".
|
|
3545
|
+
catalog_doc = fetch_quality_catalog(args)
|
|
3546
|
+
catalog_quality = (catalog_doc or {}).get("by_id") or None
|
|
3547
|
+
# The catalog meta only means something when hydration actually
|
|
3548
|
+
# produced rows: an unusable artifact contributes no meta, so the
|
|
3549
|
+
# sources block and the mode never credit a catalog that yielded
|
|
3550
|
+
# nothing.
|
|
3551
|
+
catalog_generated_at = (
|
|
3552
|
+
(catalog_doc or {}).get("generated_at") if catalog_quality else None
|
|
3553
|
+
)
|
|
1861
3554
|
exact, fuzzy = build_aa_lookup(aa_entries)
|
|
1862
3555
|
|
|
1863
3556
|
filtered = []
|
|
1864
3557
|
candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
|
|
3558
|
+
# The provider-aware stage: every surviving candidate is repriced at its
|
|
3559
|
+
# baseline endpoint (or dropped fail-closed) before any scoring runs.
|
|
3560
|
+
candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
|
|
3561
|
+
dropped.update(extra_dropped)
|
|
1865
3562
|
if not candidates:
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
)
|
|
3563
|
+
# When the endpoint stage wiped the pool, naming the filter knobs is
|
|
3564
|
+
# misleading (they passed; the drops came after) -- warn with the
|
|
3565
|
+
# endpoint-stage drop counts instead (both reasons, zero-filled).
|
|
3566
|
+
counts = {r: extra_dropped.get(r, 0) for r in ENDPOINT_DROP_REASONS}
|
|
3567
|
+
if any(counts.values()):
|
|
3568
|
+
bits = ", ".join(f"{k}={v}" for k, v in counts.items())
|
|
3569
|
+
warn(f"no models after the endpoint stage: {bits}")
|
|
3570
|
+
else:
|
|
3571
|
+
warn(
|
|
3572
|
+
f"no models satisfy the filters (min-context={args.min_context}, "
|
|
3573
|
+
f"tools={'off' if args.no_require_tools else 'required'}); "
|
|
3574
|
+
"relax them and retry"
|
|
3575
|
+
)
|
|
1870
3576
|
return 2
|
|
1871
3577
|
|
|
1872
3578
|
quality_by_id, quality_source_by_id = resolve_quality(
|
|
1873
|
-
candidates, aa_by_id, exact, fuzzy, aa_source
|
|
3579
|
+
candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality
|
|
1874
3580
|
)
|
|
1875
3581
|
weights = compute_scores(candidates, args, quality_by_id)
|
|
1876
3582
|
|
|
3583
|
+
# Per-task session-cost medians (Task 7): one fetch per invocation, but
|
|
3584
|
+
# only on the output paths that surface it (--catalog block, --json
|
|
3585
|
+
# rows; publish runs --json per priority, so the cache makes the repeat
|
|
3586
|
+
# cost free). A keyed --best/plain run skips the paginated fetch
|
|
3587
|
+
# entirely -- the medians never reach those outputs. Keyless,
|
|
3588
|
+
# fetch_session_cost returns None before any network or cache access.
|
|
3589
|
+
if args.catalog or args.json:
|
|
3590
|
+
session_cost = fetch_session_cost(args)
|
|
3591
|
+
# The key-presence fact distinguishes "skipped" from "unavailable";
|
|
3592
|
+
# same expression fetch_session_cost uses for its own guard.
|
|
3593
|
+
session_cost_mode = (
|
|
3594
|
+
"skipped"
|
|
3595
|
+
if not (args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY"))
|
|
3596
|
+
else ("ok" if session_cost else "unavailable")
|
|
3597
|
+
)
|
|
3598
|
+
else:
|
|
3599
|
+
session_cost = None
|
|
3600
|
+
session_cost_mode = "skipped"
|
|
3601
|
+
# run() owns the per-candidate attachment (build_catalog stays pure);
|
|
3602
|
+
# print_json reads it for the session_cost_usd display cell.
|
|
3603
|
+
for cand in candidates:
|
|
3604
|
+
cand["session_cost"] = (session_cost or {}).get("medians", {}).get(cand["id"])
|
|
3605
|
+
|
|
1877
3606
|
if args.catalog:
|
|
1878
3607
|
print_catalog(
|
|
1879
3608
|
build_catalog(
|
|
@@ -1887,6 +3616,11 @@ def run(args, models, or_cached) -> int:
|
|
|
1887
3616
|
aa_source,
|
|
1888
3617
|
aa_by_id,
|
|
1889
3618
|
quality_source_by_id,
|
|
3619
|
+
session_cost,
|
|
3620
|
+
session_cost_mode,
|
|
3621
|
+
aa_index_version,
|
|
3622
|
+
catalog_generated_at,
|
|
3623
|
+
catalog_quality,
|
|
1890
3624
|
)
|
|
1891
3625
|
)
|
|
1892
3626
|
return 0
|
|
@@ -1905,23 +3639,14 @@ def run(args, models, or_cached) -> int:
|
|
|
1905
3639
|
f"{v} {k}" for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
|
|
1906
3640
|
)
|
|
1907
3641
|
drop_note = f" (dropped: {bits})"
|
|
1908
|
-
|
|
1909
|
-
|
|
3642
|
+
# The footer lists per-source counts, each source's own as_of and
|
|
3643
|
+
# the AA index version when known; unmatched candidates and drift
|
|
3644
|
+
# get trailing footnotes. No record resolved -> the quality-blind
|
|
3645
|
+
# fallback note.
|
|
1910
3646
|
unmatched = len(candidates) - len(quality_by_id)
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
if matched_aa:
|
|
1915
|
-
bits.append(f"{aa_source} exact ({matched_aa})")
|
|
1916
|
-
if bits:
|
|
1917
|
-
quality_note = (
|
|
1918
|
-
f"quality via {' + '.join(bits)}: matched "
|
|
1919
|
-
f"{len(quality_by_id)}/{len(candidates)} candidates"
|
|
1920
|
-
)
|
|
1921
|
-
if unmatched:
|
|
1922
|
-
quality_note += "; unmatched candidates score 0 on quality"
|
|
1923
|
-
else:
|
|
1924
|
-
quality_note = "quality data unavailable, ranked on price/context/age"
|
|
3647
|
+
quality_note = quality_footer_note(
|
|
3648
|
+
quality_by_id, aa_index_version, unmatched
|
|
3649
|
+
) or ("quality data unavailable, ranked on price/context/age")
|
|
1925
3650
|
source_note = []
|
|
1926
3651
|
if or_cached:
|
|
1927
3652
|
source_note.append("catalog cached")
|