model-compare-opencode-plugin 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -8
- package/dist/tui.js +415 -67
- package/package.json +1 -1
- package/src/config.ts +5 -2
- package/src/core/catalog.ts +57 -12
- package/src/core/steering.ts +145 -0
- package/src/core/types.ts +28 -1
- package/src/index.ts +463 -15
- package/src/opencode/commands.ts +64 -3
- package/src/opencode/storage.ts +6 -1
- package/src/rpc.ts +45 -3
- package/src/sources/drops.ts +37 -0
- package/src/sources/index.ts +18 -6
- package/src/sources/local.ts +7 -3
- package/src/sources/remote.ts +68 -10
- package/src/tui.tsx +455 -86
- package/vendor/UPSTREAM.json +3 -3
- package/vendor/model_compare.py +1129 -7
package/vendor/model_compare.py
CHANGED
|
@@ -54,6 +54,7 @@ import time
|
|
|
54
54
|
import urllib.error
|
|
55
55
|
import urllib.parse
|
|
56
56
|
import urllib.request
|
|
57
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
57
58
|
from datetime import datetime, timezone
|
|
58
59
|
|
|
59
60
|
OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
|
|
@@ -67,7 +68,7 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
|
|
|
67
68
|
# (legacy endpoints 410 after 2026-11-04; see
|
|
68
69
|
# https://artificialanalysis.ai/data-api/migrate-v2-data).
|
|
69
70
|
AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
|
|
70
|
-
VERSION = "0.
|
|
71
|
+
VERSION = "0.4.0"
|
|
71
72
|
USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
|
|
72
73
|
|
|
73
74
|
# opencode expects provider-qualified model ids: openrouter/<provider/model>.
|
|
@@ -86,11 +87,15 @@ PRIORITY_WEIGHTS = {
|
|
|
86
87
|
"quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
|
|
87
88
|
}
|
|
88
89
|
|
|
89
|
-
CATALOG_SCHEMA_VERSION =
|
|
90
|
+
CATALOG_SCHEMA_VERSION = 3
|
|
90
91
|
|
|
91
92
|
# The drop-reason keys build_candidates counts, verbatim -- keep in sync with
|
|
92
93
|
# its drop() call sites. pool.dropped lists all of them zero-filled so the
|
|
93
|
-
# contract is self-describing.
|
|
94
|
+
# contract is self-describing. The last two come from the endpoint stage
|
|
95
|
+
# (apply_endpoint_pricing's drop sites), not build_candidates: snake_case
|
|
96
|
+
# while the older reasons are human phrases. The mixed naming convention is
|
|
97
|
+
# a deliberate, documented extension -- machine-readable reasons for the
|
|
98
|
+
# endpoint-stage drops the publish drop-rate gate keys on.
|
|
94
99
|
CATALOG_DROP_REASONS = (
|
|
95
100
|
"malformed id",
|
|
96
101
|
"context",
|
|
@@ -104,11 +109,74 @@ CATALOG_DROP_REASONS = (
|
|
|
104
109
|
"tool calling",
|
|
105
110
|
"expired",
|
|
106
111
|
"age",
|
|
112
|
+
"no_baseline_eligible_endpoint",
|
|
113
|
+
"endpoint_fetch_failed",
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# The only reasons the endpoint stage itself produces (the last two
|
|
117
|
+
# CATALOG_DROP_REASONS entries) -- the empty-pool warning keys on them;
|
|
118
|
+
# web/publish.py keeps its own copy of this tuple by design.
|
|
119
|
+
ENDPOINT_DROP_REASONS = (
|
|
120
|
+
"no_baseline_eligible_endpoint",
|
|
121
|
+
"endpoint_fetch_failed",
|
|
107
122
|
)
|
|
108
123
|
|
|
109
124
|
PAREN_RE = re.compile(r"\([^)]*\)")
|
|
110
125
|
PROVIDER_PREFIX_RE = re.compile(r"^[^:\s]{1,30}:\s*")
|
|
111
126
|
|
|
127
|
+
# Quantization precision ladder: label -> rank, highest fidelity first. The
|
|
128
|
+
# ladder ordering in the design doc is authoritative: fp6 > int8 is strict,
|
|
129
|
+
# and formats within one fidelity tier share the same rank. "unknown" never
|
|
130
|
+
# enters the ladder -- it only routes through the first-party eligibility
|
|
131
|
+
# carve-out, never through rank comparison.
|
|
132
|
+
PRECISION_LADDER: dict[str, int] = {
|
|
133
|
+
"fp32": 4,
|
|
134
|
+
"bf16": 3,
|
|
135
|
+
"fp16": 3,
|
|
136
|
+
"fp8": 2,
|
|
137
|
+
"mxfp8": 2,
|
|
138
|
+
"fp6": 1,
|
|
139
|
+
"int8": 0,
|
|
140
|
+
"fp4": -1,
|
|
141
|
+
"mxfp4": -1,
|
|
142
|
+
"nvfp4": -1,
|
|
143
|
+
"int4": -2,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
# Every quantization value an endpoint may report: the ladder keys plus the
|
|
147
|
+
# "unknown" sentinel. Whether a recorded value is recognized at all is
|
|
148
|
+
# membership here, not ladder rank.
|
|
149
|
+
QUANTIZATION_ENUM: frozenset[str] = frozenset(PRECISION_LADDER) | {"unknown"}
|
|
150
|
+
|
|
151
|
+
# Official partner hosts that serve closed labs' models: an endpoint on one
|
|
152
|
+
# of these providers counts as first-party for the eligibility carve-out
|
|
153
|
+
# even though it is not the maker itself. Entries are compared after
|
|
154
|
+
# normalize_provider (casefold + strip non-alphanumerics), so the slugs here
|
|
155
|
+
# are display forms -- "aws-bedrock" and "AWS Bedrock" both hit. Extend only
|
|
156
|
+
# with live-data evidence (e.g. an endpoint tag matching <lab>-official).
|
|
157
|
+
FIRST_PARTY_HOSTS = frozenset(
|
|
158
|
+
{"bedrock", "azure", "vertex", "azure-ai", "google-vertex", "aws-bedrock"}
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Explicit first-party alias pairs that normalization alone cannot bridge
|
|
162
|
+
# (design §4.2): makers that self-name differently than their OpenRouter
|
|
163
|
+
# author slug. Key = the exact live provider_name string, value = the
|
|
164
|
+
# author slug the pair is first-party FOR; both sides are compared
|
|
165
|
+
# normalized (case/punctuation variants match), and the alias only fires
|
|
166
|
+
# when the model's author equals the value. Verified against live data
|
|
167
|
+
# 2026-10-20 (task-13-fix-report.md section 8b; endpoints fetched fresh,
|
|
168
|
+
# every endpoint quantization "unknown", status 0):
|
|
169
|
+
# ("Amazon Bedrock", "amazon") -- amazon/nova-2-lite-v1, 2 endpoints
|
|
170
|
+
# ("Google AI Studio", "google") -- google/gemini-2.5-pro-preview, 3 endpoints
|
|
171
|
+
# ("Mistral", "mistralai") -- mistralai/mistral-large-4-0, 3 endpoints
|
|
172
|
+
# Add entries only after the same live verification -- never pin an
|
|
173
|
+
# unverified string.
|
|
174
|
+
FIRST_PARTY_ALIASES: dict[str, str] = {
|
|
175
|
+
"Amazon Bedrock": "amazon",
|
|
176
|
+
"Google AI Studio": "google",
|
|
177
|
+
"Mistral": "mistralai",
|
|
178
|
+
}
|
|
179
|
+
|
|
112
180
|
|
|
113
181
|
def warn(msg: str) -> None:
|
|
114
182
|
print(f"warning: {msg}", file=sys.stderr)
|
|
@@ -118,6 +186,44 @@ def norm_key(s: str) -> str:
|
|
|
118
186
|
return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
|
119
187
|
|
|
120
188
|
|
|
189
|
+
# ---------------------------------------------------------------------------
|
|
190
|
+
# Provider normalization + first-party detection
|
|
191
|
+
# ---------------------------------------------------------------------------
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def normalize_provider(name: str) -> str:
|
|
195
|
+
"""Casefold a provider name and strip ALL non-alphanumeric characters.
|
|
196
|
+
|
|
197
|
+
"Z.AI", "z-ai" and "z ai" all normalize to "zai", so a live-recorded
|
|
198
|
+
maker name matches its author slug regardless of punctuation or case.
|
|
199
|
+
None or empty input normalizes to "".
|
|
200
|
+
"""
|
|
201
|
+
return re.sub(r"[^a-z0-9]+", "", (name or "").casefold())
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def is_first_party(provider_name: str, author: str) -> bool:
|
|
205
|
+
"""True iff the provider is the model's own maker or a partner host.
|
|
206
|
+
|
|
207
|
+
Criterion 1: the normalized provider name equals the normalized author
|
|
208
|
+
slug. Criterion 2: it matches a normalized FIRST_PARTY_HOSTS entry, or
|
|
209
|
+
a FIRST_PARTY_ALIASES key whose value equals the normalized author slug
|
|
210
|
+
(the §4.2 alias pairs: a maker self-naming differently than its author
|
|
211
|
+
slug is first-party only for that author). Everything else is
|
|
212
|
+
third-party.
|
|
213
|
+
"""
|
|
214
|
+
provider = normalize_provider(provider_name)
|
|
215
|
+
if provider == normalize_provider(author):
|
|
216
|
+
return True
|
|
217
|
+
if provider in {normalize_provider(host) for host in FIRST_PARTY_HOSTS}:
|
|
218
|
+
return True
|
|
219
|
+
author_slug = normalize_provider(author)
|
|
220
|
+
return any(
|
|
221
|
+
provider == normalize_provider(alias_provider)
|
|
222
|
+
and author_slug == normalize_provider(alias_author)
|
|
223
|
+
for alias_provider, alias_author in FIRST_PARTY_ALIASES.items()
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
121
227
|
# ---------------------------------------------------------------------------
|
|
122
228
|
# HTTP + cache
|
|
123
229
|
# ---------------------------------------------------------------------------
|
|
@@ -224,6 +330,872 @@ def save_cache(name: str, payload) -> None:
|
|
|
224
330
|
warn(f"could not write cache: {exc}")
|
|
225
331
|
|
|
226
332
|
|
|
333
|
+
# ---------------------------------------------------------------------------
|
|
334
|
+
# OpenRouter endpoints (per-model provider list)
|
|
335
|
+
# ---------------------------------------------------------------------------
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
OPENROUTER_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{id}/endpoints"
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def load_cache_expired(name: str) -> dict | None:
|
|
342
|
+
"""Return the cached payload regardless of age; None when absent/corrupt.
|
|
343
|
+
|
|
344
|
+
load_cache drops payloads past their TTL; this variant serves them anyway
|
|
345
|
+
so a dropped model's last known endpoint annotations can still be shown
|
|
346
|
+
(marked stale by the caller). Never used for ranking.
|
|
347
|
+
"""
|
|
348
|
+
path = cache_path(name)
|
|
349
|
+
if not path:
|
|
350
|
+
return None
|
|
351
|
+
try:
|
|
352
|
+
with open(path) as fh:
|
|
353
|
+
blob = json.load(fh)
|
|
354
|
+
return blob["payload"]
|
|
355
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
356
|
+
return None
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def parse_endpoints(payload) -> list[dict]:
|
|
360
|
+
"""Keep the verified per-endpoint fields of an endpoints response.
|
|
361
|
+
|
|
362
|
+
The live GET /api/v1/models/{id}/endpoints response nests the list one
|
|
363
|
+
level below the usual envelope: `data` is the model object (id, name,
|
|
364
|
+
architecture, ...) and the per-provider entries live in
|
|
365
|
+
`data["endpoints"]`. `pricing` keeps prompt/completion (parsed through
|
|
366
|
+
parse_price) and the raw `overrides` list -- the tiered-pricing
|
|
367
|
+
machinery parses the entries itself. Fields beyond the verified
|
|
368
|
+
per-entry shape are dropped, and the parse keeps `status` as-is (the
|
|
369
|
+
caller filters).
|
|
370
|
+
"""
|
|
371
|
+
# Task 13 lesson: the first unit fixtures fed the models/find shape
|
|
372
|
+
# ({"data": [entry, ...]}) -- a list directly under "data" -- so the
|
|
373
|
+
# suite stayed green while every live parse returned [] and the whole
|
|
374
|
+
# pool died as no_baseline_eligible_endpoint. Pin the live envelope:
|
|
375
|
+
# the list is read ONLY from data["endpoints"], and every other shape
|
|
376
|
+
# parses to [] (no compat branch for the wrong fixture shape).
|
|
377
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
378
|
+
entries = data.get("endpoints") if isinstance(data, dict) else None
|
|
379
|
+
endpoints = []
|
|
380
|
+
for entry in entries or []:
|
|
381
|
+
if not isinstance(entry, dict):
|
|
382
|
+
continue
|
|
383
|
+
raw = entry.get("pricing")
|
|
384
|
+
raw_pricing = raw if isinstance(raw, dict) else {}
|
|
385
|
+
endpoints.append(
|
|
386
|
+
{
|
|
387
|
+
"provider_name": entry.get("provider_name"),
|
|
388
|
+
"tag": entry.get("tag"),
|
|
389
|
+
"quantization": entry.get("quantization"),
|
|
390
|
+
"pricing": {
|
|
391
|
+
"prompt": parse_price(raw_pricing.get("prompt")),
|
|
392
|
+
"completion": parse_price(raw_pricing.get("completion")),
|
|
393
|
+
"overrides": raw_pricing.get("overrides"),
|
|
394
|
+
},
|
|
395
|
+
"context_length": entry.get("context_length"),
|
|
396
|
+
"status": entry.get("status"),
|
|
397
|
+
"service_tier": entry.get("service_tier"),
|
|
398
|
+
}
|
|
399
|
+
)
|
|
400
|
+
return endpoints
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def fetch_model_endpoints(model_id: str, args) -> list[dict]:
|
|
404
|
+
"""Fetch + parse one model's endpoint list, cached per model.
|
|
405
|
+
|
|
406
|
+
The cache name flattens the model id ("/" -> "_") to keep the cache
|
|
407
|
+
files flat. Fetches retry 3 times (4 attempts total) via fetch_json,
|
|
408
|
+
which raises at once on 4xx and after the retries on 5xx/network
|
|
409
|
+
errors -- the caller owns the fail-closed policy.
|
|
410
|
+
"""
|
|
411
|
+
cache_name = f"endpoints:{model_id.replace('/', '_')}"
|
|
412
|
+
if not args.no_cache:
|
|
413
|
+
cached = load_cache(cache_name, args.cache_ttl)
|
|
414
|
+
if cached:
|
|
415
|
+
return cached
|
|
416
|
+
payload = fetch_json(
|
|
417
|
+
OPENROUTER_ENDPOINTS_URL.format(id=model_id), timeout=30, retries=3
|
|
418
|
+
)
|
|
419
|
+
endpoints = parse_endpoints(payload)
|
|
420
|
+
save_cache(cache_name, endpoints)
|
|
421
|
+
return endpoints
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def is_variant_endpoint(endpoint) -> bool:
|
|
425
|
+
"""True iff the endpoint is a :free/:batch variant or entirely free.
|
|
426
|
+
|
|
427
|
+
Variant iff its slug or tag carries a `:free`/`:batch` marker OR both
|
|
428
|
+
input and output prices are zero. `:batch` is cheaper-but-restricted,
|
|
429
|
+
not zero-priced -- only the marker test catches it. Missing or
|
|
430
|
+
unparseable prices are not zero.
|
|
431
|
+
"""
|
|
432
|
+
for field in ("slug", "tag"):
|
|
433
|
+
marker = endpoint.get(field)
|
|
434
|
+
if isinstance(marker, str) and (":free" in marker or ":batch" in marker):
|
|
435
|
+
return True
|
|
436
|
+
pricing = endpoint.get("pricing")
|
|
437
|
+
prices = pricing if isinstance(pricing, dict) else {}
|
|
438
|
+
return (
|
|
439
|
+
parse_price(prices.get("prompt")) == 0.0
|
|
440
|
+
and parse_price(prices.get("completion")) == 0.0
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def endpoint_schedule_ok(endpoint, args) -> bool:
|
|
445
|
+
"""False iff the endpoint's pricing.overrides fails the v2 schedule rule.
|
|
446
|
+
|
|
447
|
+
Reuses effective_pricing in full: the gate fires on malformed windows,
|
|
448
|
+
exactly-one-of utc_start/utc_end pairs, bad utc_days, combined
|
|
449
|
+
token+time entries and missing inheritance -- not just non-tiling
|
|
450
|
+
windows. An endpoint without overrides has no schedule to fail.
|
|
451
|
+
"""
|
|
452
|
+
result = effective_pricing(
|
|
453
|
+
endpoint.get("pricing"), args.min_context, args.input_share
|
|
454
|
+
)
|
|
455
|
+
return not result["schedule_error"]
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
# Schedule gate for args-less callers: effective_pricing's schedule_error
|
|
459
|
+
# depends only on the SHAPE of pricing.overrides (malformed windows, bad
|
|
460
|
+
# utc_days, ...), never on min_context/input_share -- those only affect the
|
|
461
|
+
# resolved prices. derive_default_precision has no args parameter (pinned
|
|
462
|
+
# interface), so it gates through this fixed stand-in; any args would give
|
|
463
|
+
# the same schedule_error verdict.
|
|
464
|
+
_SCHEDULE_GATE_ARGS = argparse.Namespace(min_context=0, input_share=0.75)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _baseline_universe(endpoints: list[dict], args=None) -> list[dict]:
|
|
468
|
+
"""The shared derivation universe (design section 4.1).
|
|
469
|
+
|
|
470
|
+
Live (status == 0), not a variant endpoint (:free/:batch marker or
|
|
471
|
+
zero-priced), and passing the schedule gate. Schedule-broken endpoints
|
|
472
|
+
must not pin the default that governs healthy endpoints, same rationale
|
|
473
|
+
as the variant exclusion. args defaults to the structural stand-in
|
|
474
|
+
(see _SCHEDULE_GATE_ARGS); callers with real args pass them.
|
|
475
|
+
"""
|
|
476
|
+
if args is None:
|
|
477
|
+
args = _SCHEDULE_GATE_ARGS
|
|
478
|
+
return [
|
|
479
|
+
endpoint
|
|
480
|
+
for endpoint in endpoints
|
|
481
|
+
if endpoint.get("status") == 0
|
|
482
|
+
and not is_variant_endpoint(endpoint)
|
|
483
|
+
and endpoint_schedule_ok(endpoint, args)
|
|
484
|
+
]
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def derive_default_precision(endpoints: list[dict], author: str) -> str | None:
|
|
488
|
+
"""Derive a model's default precision from its endpoint list (section 4.1).
|
|
489
|
+
|
|
490
|
+
Universe: live non-variant schedule-passing endpoints (_baseline_universe).
|
|
491
|
+
Primary rule: the highest-ranked KNOWN quantization among the maker's
|
|
492
|
+
endpoints -- matched by normalized author-slug equality only, so partner
|
|
493
|
+
hosts never pin the maker's default (section 4.2 criterion 2 does not
|
|
494
|
+
apply here). Fallback: when the maker has no known quantization, the
|
|
495
|
+
highest-ranked known quantization across the FULL universe (partners
|
|
496
|
+
included -- best-available). None when no endpoint in the universe has a
|
|
497
|
+
known quantization (the all-unknown closed-model case): eligibility then
|
|
498
|
+
relies solely on the first-party carve-out.
|
|
499
|
+
|
|
500
|
+
Known means a PRECISION_LADDER key: "unknown" and unparseable/absent
|
|
501
|
+
values never enter the ladder. An endpoint whose provider_name is
|
|
502
|
+
absent/empty (or normalizes to empty) is skipped for maker matching
|
|
503
|
+
(never first-party -- fail-closed), but still feeds the fallback
|
|
504
|
+
universe.
|
|
505
|
+
"""
|
|
506
|
+
universe = _baseline_universe(endpoints, _SCHEDULE_GATE_ARGS)
|
|
507
|
+
maker = normalize_provider(author)
|
|
508
|
+
maker_best = None
|
|
509
|
+
fallback_best = None
|
|
510
|
+
for endpoint in universe:
|
|
511
|
+
quantization = endpoint.get("quantization")
|
|
512
|
+
if not isinstance(quantization, str) or quantization not in PRECISION_LADDER:
|
|
513
|
+
continue # "unknown", null, unrecognized: no rank, no pinning
|
|
514
|
+
rank = PRECISION_LADDER[quantization]
|
|
515
|
+
if fallback_best is None or rank > fallback_best[0]:
|
|
516
|
+
fallback_best = (rank, quantization)
|
|
517
|
+
# Maker matching only: a provider_name that normalizes to nothing
|
|
518
|
+
# must never match an (equally empty) author slug.
|
|
519
|
+
provider_slug = (
|
|
520
|
+
normalize_provider(endpoint.get("provider_name"))
|
|
521
|
+
if isinstance(endpoint.get("provider_name"), str)
|
|
522
|
+
else ""
|
|
523
|
+
)
|
|
524
|
+
if (
|
|
525
|
+
provider_slug
|
|
526
|
+
and provider_slug == maker
|
|
527
|
+
and (maker_best is None or rank > maker_best[0])
|
|
528
|
+
):
|
|
529
|
+
maker_best = (rank, quantization)
|
|
530
|
+
return (maker_best or fallback_best or (None, None))[1]
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def baseline_eligible(endpoint, default_precision, author, args) -> bool:
|
|
534
|
+
"""Baseline-eligibility rule (design section 4.3, fail-closed).
|
|
535
|
+
|
|
536
|
+
True iff the endpoint is live (status == 0), not a variant, passes the
|
|
537
|
+
schedule gate, and then either:
|
|
538
|
+
|
|
539
|
+
* its quantization is a known ladder label with rank >=
|
|
540
|
+
rank(default_precision) -- provider-blind (default_routing considers
|
|
541
|
+
every endpoint OpenRouter may send traffic to), or
|
|
542
|
+
* its quantization is "unknown" or null AND the endpoint is first-party
|
|
543
|
+
(the D3 carve-out: the maker or a partner host).
|
|
544
|
+
|
|
545
|
+
When default_precision is None (all-unknown model) the rank comparison is
|
|
546
|
+
meaningless -- only the carve-out applies. A provider_name that is
|
|
547
|
+
absent/empty (or normalizes to empty) is never first-party, so such an
|
|
548
|
+
endpoint can only qualify through the rank branch. An unrecognized
|
|
549
|
+
quantization string (neither a ladder label nor "unknown"/null) is a
|
|
550
|
+
schema anomaly: no rank comparison and no carve-out -- fail-closed
|
|
551
|
+
(fleet-wide new-label drift is what the publish drop-rate gate exists
|
|
552
|
+
to catch).
|
|
553
|
+
"""
|
|
554
|
+
if (
|
|
555
|
+
endpoint.get("status") != 0
|
|
556
|
+
or is_variant_endpoint(endpoint)
|
|
557
|
+
or not endpoint_schedule_ok(endpoint, args)
|
|
558
|
+
):
|
|
559
|
+
return False
|
|
560
|
+
quantization = endpoint.get("quantization")
|
|
561
|
+
if isinstance(quantization, str) and quantization in PRECISION_LADDER:
|
|
562
|
+
# Known quantization: the rank branch, provider-blind.
|
|
563
|
+
if default_precision is None:
|
|
564
|
+
return False
|
|
565
|
+
return PRECISION_LADDER[quantization] >= PRECISION_LADDER[default_precision]
|
|
566
|
+
# Unverifiable: the carve-out applies to "unknown" and null only --
|
|
567
|
+
# an unrecognized string fails closed. A provider_name that normalizes
|
|
568
|
+
# to nothing is never first-party (fail-closed; the literal
|
|
569
|
+
# is_first_party("", "") iff is guarded out here).
|
|
570
|
+
provider_name = endpoint.get("provider_name")
|
|
571
|
+
provider_slug = (
|
|
572
|
+
normalize_provider(provider_name) if isinstance(provider_name, str) else ""
|
|
573
|
+
)
|
|
574
|
+
return (
|
|
575
|
+
(quantization is None or quantization == "unknown")
|
|
576
|
+
and bool(provider_slug)
|
|
577
|
+
and is_first_party(provider_name, author)
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def pick_baseline(
|
|
582
|
+
endpoints: list[dict], eligible_flags: list[bool], args
|
|
583
|
+
) -> dict | None:
|
|
584
|
+
"""Pick the baseline endpoint among the eligible ones (design section 4.5).
|
|
585
|
+
|
|
586
|
+
Cheapest blended per-token price wins, computed from effective_pricing
|
|
587
|
+
with the args' min_context and input_share (the shared --input-share
|
|
588
|
+
convention). Ties broken by (normalized provider name, tag), then --
|
|
589
|
+
still tied -- first in list order (deterministic; Python's sort is
|
|
590
|
+
stable and the endpoints list order is the API's). Returns None when no
|
|
591
|
+
eligible endpoint has a usable price (the caller owns the drop).
|
|
592
|
+
"""
|
|
593
|
+
candidates = []
|
|
594
|
+
for endpoint, eligible in zip(endpoints, eligible_flags):
|
|
595
|
+
if not eligible:
|
|
596
|
+
continue
|
|
597
|
+
eff = effective_pricing(
|
|
598
|
+
endpoint.get("pricing"), args.min_context, args.input_share
|
|
599
|
+
)
|
|
600
|
+
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
601
|
+
if (
|
|
602
|
+
price_in is None
|
|
603
|
+
or price_out is None
|
|
604
|
+
or not math.isfinite(price_in)
|
|
605
|
+
or not math.isfinite(price_out)
|
|
606
|
+
):
|
|
607
|
+
continue # unpriceable: no executable blended price
|
|
608
|
+
blended = args.input_share * price_in + (1 - args.input_share) * price_out
|
|
609
|
+
provider_name = endpoint.get("provider_name")
|
|
610
|
+
candidates.append(
|
|
611
|
+
(
|
|
612
|
+
blended,
|
|
613
|
+
normalize_provider(provider_name)
|
|
614
|
+
if isinstance(provider_name, str)
|
|
615
|
+
else "",
|
|
616
|
+
endpoint.get("tag") or "",
|
|
617
|
+
endpoint,
|
|
618
|
+
)
|
|
619
|
+
)
|
|
620
|
+
if not candidates:
|
|
621
|
+
return None
|
|
622
|
+
return min(candidates, key=lambda candidate: candidate[:3])[3]
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _blended_price(endpoint, args) -> float | None:
|
|
626
|
+
"""Blended per-token price of one endpoint (pick_baseline's price rule).
|
|
627
|
+
|
|
628
|
+
effective_pricing with the args' min_context and input_share, then
|
|
629
|
+
input_share * price_in + (1 - input_share) * price_out. None when either
|
|
630
|
+
resolved price is missing or non-finite -- no executable blended price,
|
|
631
|
+
the same skip pick_baseline applies. A zero result is returned as-is:
|
|
632
|
+
zero-priced endpoints are variants (already excluded upstream), so the
|
|
633
|
+
callers apply their own zero guards defensively.
|
|
634
|
+
"""
|
|
635
|
+
eff = effective_pricing(endpoint.get("pricing"), args.min_context, args.input_share)
|
|
636
|
+
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
637
|
+
if (
|
|
638
|
+
price_in is None
|
|
639
|
+
or price_out is None
|
|
640
|
+
or not math.isfinite(price_in)
|
|
641
|
+
or not math.isfinite(price_out)
|
|
642
|
+
):
|
|
643
|
+
return None
|
|
644
|
+
blended = args.input_share * price_in + (1 - args.input_share) * price_out
|
|
645
|
+
return blended if math.isfinite(blended) else None
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _flag_list(endpoints: list[dict], eligible_flags: list[bool]):
|
|
649
|
+
"""Zip endpoints with their parallel eligibility flags, hardened.
|
|
650
|
+
|
|
651
|
+
eligible_flags is a parallel list to endpoints (the baseline_eligible
|
|
652
|
+
verdict per endpoint, same order -- Task 4 contract). A length mismatch
|
|
653
|
+
must raise, never truncate silently (Task 4 review ruling).
|
|
654
|
+
"""
|
|
655
|
+
if len(endpoints) != len(eligible_flags):
|
|
656
|
+
raise ValueError(
|
|
657
|
+
"eligible_flags must be a parallel list to endpoints: "
|
|
658
|
+
f"{len(endpoints)} endpoints, {len(eligible_flags)} flags"
|
|
659
|
+
)
|
|
660
|
+
return zip(endpoints, eligible_flags, strict=True)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def NON_ELIGIBLE_UNIVERSE(endpoints: list[dict], eligible_flags: list[bool]) -> list:
|
|
664
|
+
"""The shared annotations universe (design section 4.6).
|
|
665
|
+
|
|
666
|
+
Live (status == 0), non-variant endpoints that are NOT baseline-eligible:
|
|
667
|
+
sub-default known quantization, third-party unknown/null quantization, or
|
|
668
|
+
schedule-gate-failed. All annotations derive from this one universe.
|
|
669
|
+
Eligibility comes from the parallel eligible_flags list (the
|
|
670
|
+
baseline_eligible verdicts); variants and non-live endpoints are excluded
|
|
671
|
+
even though they too are not eligible -- unlike the steering `ignore`
|
|
672
|
+
list, which includes live variants (section 4.7).
|
|
673
|
+
|
|
674
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
675
|
+
"""
|
|
676
|
+
return [
|
|
677
|
+
endpoint
|
|
678
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags)
|
|
679
|
+
if endpoint.get("status") == 0
|
|
680
|
+
and not is_variant_endpoint(endpoint)
|
|
681
|
+
and not eligible
|
|
682
|
+
]
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def build_annotations(
|
|
686
|
+
endpoints: list[dict],
|
|
687
|
+
eligible_flags: list[bool],
|
|
688
|
+
baseline,
|
|
689
|
+
catalog_context,
|
|
690
|
+
args,
|
|
691
|
+
) -> dict:
|
|
692
|
+
"""Per-model endpoint annotations (design section 4.6).
|
|
693
|
+
|
|
694
|
+
All four annotations derive from one shared universe -- the live
|
|
695
|
+
non-variant endpoints that are not baseline-eligible
|
|
696
|
+
(NON_ELIGIBLE_UNIVERSE):
|
|
697
|
+
|
|
698
|
+
* price_spread: cheapest non-eligible blended price divided by the
|
|
699
|
+
cheapest eligible blended price, both blended with the args'
|
|
700
|
+
min_context and input_share (the shared --input-share convention).
|
|
701
|
+
None when the universe is empty -- generalized to None when either
|
|
702
|
+
side has no priceable member at all -- and None when either cheapest
|
|
703
|
+
price is 0 (zero-priced endpoints are variants by definition, so the
|
|
704
|
+
zero half of the guard is defensive only). Unpriceable endpoints are
|
|
705
|
+
skipped when looking for a cheapest price.
|
|
706
|
+
* cheapest_quantized: {provider, quantization, price} of the cheapest
|
|
707
|
+
member of the quantization-caused subset of the universe -- endpoints
|
|
708
|
+
non-eligible due to quantization only (sub-default known quantization
|
|
709
|
+
or third-party unknown/null), NOT schedule-gate-failed ones, which are
|
|
710
|
+
excluded even when cheaper. Unpriceable subset members are skipped
|
|
711
|
+
(defensively also non-positive ones, matching the repo-wide treatment
|
|
712
|
+
of negative prices as malformed); None when the subset has no
|
|
713
|
+
priceable member. Price ties keep the first in list order
|
|
714
|
+
(deterministic).
|
|
715
|
+
* risk_flag: True iff the sum of 1/blended**2 over the universe strictly
|
|
716
|
+
exceeds the sum over the baseline-eligible endpoints (exact tie ->
|
|
717
|
+
False). Any endpoint whose blended price is 0/None (defensively also
|
|
718
|
+
negative) is skipped from BOTH sums -- variants are already excluded,
|
|
719
|
+
so this only guards malformed data, and the flag stays deterministic.
|
|
720
|
+
* baseline_context_below_catalog: True iff the baseline endpoint's
|
|
721
|
+
context_length is smaller than the catalog context (the models-API
|
|
722
|
+
value). A None baseline -- or missing context on either side -- yields
|
|
723
|
+
False: the caveat marks a positive finding, never missing data.
|
|
724
|
+
|
|
725
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
726
|
+
"""
|
|
727
|
+
universe = NON_ELIGIBLE_UNIVERSE(endpoints, eligible_flags)
|
|
728
|
+
|
|
729
|
+
universe_prices = []
|
|
730
|
+
for endpoint in universe:
|
|
731
|
+
price = _blended_price(endpoint, args)
|
|
732
|
+
if price is not None:
|
|
733
|
+
universe_prices.append(price)
|
|
734
|
+
eligible_prices = []
|
|
735
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
736
|
+
if not eligible:
|
|
737
|
+
continue
|
|
738
|
+
price = _blended_price(endpoint, args)
|
|
739
|
+
if price is not None:
|
|
740
|
+
eligible_prices.append(price)
|
|
741
|
+
cheapest_non_eligible = min(universe_prices) if universe_prices else None
|
|
742
|
+
cheapest_eligible = min(eligible_prices) if eligible_prices else None
|
|
743
|
+
price_spread = (
|
|
744
|
+
cheapest_non_eligible / cheapest_eligible
|
|
745
|
+
if cheapest_non_eligible is not None
|
|
746
|
+
and cheapest_eligible is not None
|
|
747
|
+
and cheapest_non_eligible > 0
|
|
748
|
+
and cheapest_eligible > 0
|
|
749
|
+
else None
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
quantized_best = None
|
|
753
|
+
for endpoint in universe:
|
|
754
|
+
if not endpoint_schedule_ok(endpoint, args):
|
|
755
|
+
continue # schedule-gate-failed: not a quantization-caused exclusion
|
|
756
|
+
price = _blended_price(endpoint, args)
|
|
757
|
+
if price is None or price <= 0:
|
|
758
|
+
continue # unpriceable/non-positive: no comparable price
|
|
759
|
+
if quantized_best is None or price < quantized_best[0]:
|
|
760
|
+
quantized_best = (price, endpoint) # ties keep the first in list order
|
|
761
|
+
if quantized_best is None:
|
|
762
|
+
cheapest_quantized = None
|
|
763
|
+
else:
|
|
764
|
+
price, endpoint = quantized_best
|
|
765
|
+
cheapest_quantized = {
|
|
766
|
+
"provider": endpoint.get("provider_name"),
|
|
767
|
+
"quantization": endpoint.get("quantization"),
|
|
768
|
+
"price": price,
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
non_eligible_weight = 0.0
|
|
772
|
+
for endpoint in universe:
|
|
773
|
+
price = _blended_price(endpoint, args)
|
|
774
|
+
if price is None or price <= 0:
|
|
775
|
+
continue # skipped from BOTH sums, deterministically
|
|
776
|
+
non_eligible_weight += 1.0 / (price * price)
|
|
777
|
+
eligible_weight = 0.0
|
|
778
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
779
|
+
if not eligible:
|
|
780
|
+
continue
|
|
781
|
+
price = _blended_price(endpoint, args)
|
|
782
|
+
if price is None or price <= 0:
|
|
783
|
+
continue
|
|
784
|
+
eligible_weight += 1.0 / (price * price)
|
|
785
|
+
risk_flag = non_eligible_weight > eligible_weight
|
|
786
|
+
|
|
787
|
+
baseline_context = baseline.get("context_length") if baseline is not None else None
|
|
788
|
+
baseline_context_below_catalog = (
|
|
789
|
+
baseline_context is not None
|
|
790
|
+
and catalog_context is not None
|
|
791
|
+
and baseline_context < catalog_context
|
|
792
|
+
)
|
|
793
|
+
|
|
794
|
+
return {
|
|
795
|
+
"price_spread": price_spread,
|
|
796
|
+
"cheapest_quantized": cheapest_quantized,
|
|
797
|
+
"risk_flag": risk_flag,
|
|
798
|
+
"baseline_context_below_catalog": baseline_context_below_catalog,
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def build_steering(
|
|
803
|
+
model_id: str,
|
|
804
|
+
endpoints: list[dict],
|
|
805
|
+
eligible_flags: list[bool],
|
|
806
|
+
default_precision: str | None,
|
|
807
|
+
args,
|
|
808
|
+
) -> dict:
|
|
809
|
+
"""OpenRouter provider steering object (design section 4.7).
|
|
810
|
+
|
|
811
|
+
{"model": model_id, "provider": {"order": [...], "ignore": [...],
|
|
812
|
+
"quantizations": [...], "allow_fallbacks": True}}.
|
|
813
|
+
|
|
814
|
+
* order: tags of the baseline-eligible endpoints, cheapest blended first
|
|
815
|
+
(blended price asc, ties broken by (normalized provider name, tag) --
|
|
816
|
+
the same comparator as pick_baseline, stable so the final tie is list
|
|
817
|
+
order). Eligible endpoints whose blended price is unpriceable cannot
|
|
818
|
+
be price-ranked; they are appended after the ranked ones (still
|
|
819
|
+
(provider, tag)-ordered) so every eligible endpoint keeps a place --
|
|
820
|
+
ranked last, never silently left to default priority.
|
|
821
|
+
* ignore: tags of EVERY live endpoint that is not baseline-eligible --
|
|
822
|
+
sub-default quantization, third-party unknown/null, schedule-gate-
|
|
823
|
+
failed AND variants (variants are live but not eligible; the universal
|
|
824
|
+
quantifier includes them). Non-live (status != 0) endpoints are never
|
|
825
|
+
listed. List order (the API's).
|
|
826
|
+
* quantizations: sorted KNOWN quantizations (PRECISION_LADDER labels)
|
|
827
|
+
with rank >= rank(default_precision) among live non-variant endpoints,
|
|
828
|
+
deduplicated -- no schedule or eligibility filter. OMITTED from the
|
|
829
|
+
provider dict entirely when default_precision is None -- never an
|
|
830
|
+
empty list (OpenRouter semantics would filter out every endpoint).
|
|
831
|
+
* allow_fallbacks: True.
|
|
832
|
+
|
|
833
|
+
Signature deviation from the plan (controller ruling at Task 5 spawn):
|
|
834
|
+
args is a required parameter -- the order ranking is real pricing math
|
|
835
|
+
(effective_pricing via min_context/input_share), and reusing the
|
|
836
|
+
shape-only _SCHEDULE_GATE_ARGS stand-in for pricing was explicitly
|
|
837
|
+
ruled out in the Task 4 review.
|
|
838
|
+
|
|
839
|
+
Raises ValueError when eligible_flags does not match endpoints in length.
|
|
840
|
+
"""
|
|
841
|
+
ignore = []
|
|
842
|
+
ranked = [] # (blended, normalized provider name, tag) -- pick_baseline order
|
|
843
|
+
unranked = [] # (normalized provider name, tag) -- eligible but unpriceable
|
|
844
|
+
for endpoint, eligible in _flag_list(endpoints, eligible_flags):
|
|
845
|
+
if endpoint.get("status") == 0 and not eligible:
|
|
846
|
+
ignore.append(endpoint.get("tag") or "")
|
|
847
|
+
if not eligible:
|
|
848
|
+
continue
|
|
849
|
+
provider_name = endpoint.get("provider_name")
|
|
850
|
+
provider_key = (
|
|
851
|
+
normalize_provider(provider_name) if isinstance(provider_name, str) else ""
|
|
852
|
+
)
|
|
853
|
+
tag = endpoint.get("tag") or ""
|
|
854
|
+
price = _blended_price(endpoint, args)
|
|
855
|
+
if price is None:
|
|
856
|
+
unranked.append((provider_key, tag))
|
|
857
|
+
else:
|
|
858
|
+
ranked.append((price, provider_key, tag))
|
|
859
|
+
ranked.sort() # stable: the final tie is list order, as in pick_baseline
|
|
860
|
+
order = [tag for _, _, tag in ranked] + [tag for _, tag in sorted(unranked)]
|
|
861
|
+
|
|
862
|
+
provider = {"order": order, "ignore": ignore}
|
|
863
|
+
if default_precision is not None:
|
|
864
|
+
default_rank = PRECISION_LADDER[default_precision]
|
|
865
|
+
quants = set()
|
|
866
|
+
for endpoint in endpoints:
|
|
867
|
+
if endpoint.get("status") != 0 or is_variant_endpoint(endpoint):
|
|
868
|
+
continue
|
|
869
|
+
quantization = endpoint.get("quantization")
|
|
870
|
+
if (
|
|
871
|
+
not isinstance(quantization, str)
|
|
872
|
+
or quantization not in PRECISION_LADDER
|
|
873
|
+
):
|
|
874
|
+
continue # "unknown"/null/unrecognized: not a known quantization
|
|
875
|
+
if PRECISION_LADDER[quantization] >= default_rank:
|
|
876
|
+
quants.add(quantization)
|
|
877
|
+
provider["quantizations"] = sorted(quants)
|
|
878
|
+
provider["allow_fallbacks"] = True
|
|
879
|
+
return {"model": model_id, "provider": provider}
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def _stale_filtered_annotations(model_id, catalog_context, args):
|
|
883
|
+
"""Annotations for a dropped model's filtered entry, or None.
|
|
884
|
+
|
|
885
|
+
Built from the model's last cached endpoint list (load_cache_expired --
|
|
886
|
+
TTL ignored), never from a fresh fetch: the model is out of the ranking,
|
|
887
|
+
so the block is marked "stale": True for display in the filtered section
|
|
888
|
+
only (design section 4.9). Same builders as the live stage: eligibility
|
|
889
|
+
and baseline are re-derived from the cached list, so the shape matches a
|
|
890
|
+
ranked candidate's annotations. A model whose cache holds no endpoint
|
|
891
|
+
list at all yields None (the entry carries the drop reason alone).
|
|
892
|
+
"""
|
|
893
|
+
cached = load_cache_expired(f"endpoints:{model_id.replace('/', '_')}")
|
|
894
|
+
if not isinstance(cached, list):
|
|
895
|
+
return None
|
|
896
|
+
author = model_id.partition("/")[0]
|
|
897
|
+
try:
|
|
898
|
+
default_precision = derive_default_precision(cached, author)
|
|
899
|
+
eligible_flags = [
|
|
900
|
+
baseline_eligible(endpoint, default_precision, author, args)
|
|
901
|
+
for endpoint in cached
|
|
902
|
+
]
|
|
903
|
+
baseline = pick_baseline(cached, eligible_flags, args)
|
|
904
|
+
annotations = build_annotations(
|
|
905
|
+
cached, eligible_flags, baseline, catalog_context, args
|
|
906
|
+
)
|
|
907
|
+
except Exception:
|
|
908
|
+
# Display-only block for an already-dropped model: never fail the run.
|
|
909
|
+
return None
|
|
910
|
+
annotations["stale"] = True
|
|
911
|
+
return annotations
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _endpoint_stage_drop(extra_dropped, filtered, cand, reason, args):
|
|
915
|
+
"""Count one endpoint-stage drop and append its filtered entry.
|
|
916
|
+
|
|
917
|
+
Same validated entry shape as build_candidates' drop() plus the optional
|
|
918
|
+
stale annotations block (absent cache -> reasons only).
|
|
919
|
+
"""
|
|
920
|
+
extra_dropped[reason] = extra_dropped.get(reason, 0) + 1
|
|
921
|
+
entry = {"id": cand["id"], "name": cand["name"] or cand["id"], "reasons": [reason]}
|
|
922
|
+
annotations = _stale_filtered_annotations(cand["id"], cand.get("context"), args)
|
|
923
|
+
if annotations is not None:
|
|
924
|
+
entry["annotations"] = annotations
|
|
925
|
+
filtered.append(entry)
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def apply_endpoint_pricing(candidates, args, filtered):
|
|
929
|
+
"""Reprice every candidate at its baseline endpoint (design section 4.9).
|
|
930
|
+
|
|
931
|
+
The CLI stage between build_candidates and resolve_quality. Endpoint
|
|
932
|
+
fetches run concurrently (ThreadPoolExecutor, 8 workers) through
|
|
933
|
+
fetch_model_endpoints' per-model cache, but the results are applied PER
|
|
934
|
+
CANDIDATE ID in the original candidates order -- never in completion
|
|
935
|
+
order -- so the stage is deterministic.
|
|
936
|
+
|
|
937
|
+
Per candidate: endpoints = fetch. On fetch failure after fetch_json's
|
|
938
|
+
internal retries the model is dropped "endpoint_fetch_failed" and removed
|
|
939
|
+
from the returned list. With endpoints, default_precision is derived,
|
|
940
|
+
every endpoint scored for baseline eligibility and the baseline picked;
|
|
941
|
+
no usable baseline drops the model "no_baseline_eligible_endpoint"
|
|
942
|
+
(the stale-annotations filtered-entry rule covers both drop reasons).
|
|
943
|
+
Otherwise the candidate is mutated in place: the pricing block
|
|
944
|
+
(price_in/price_out/blended/base_*/tier/schedule/notes) is replaced with
|
|
945
|
+
the baseline endpoint's effective_pricing outputs scaled per-1M exactly
|
|
946
|
+
like build_candidates, and the additive keys "endpoints", "default_precision",
|
|
947
|
+
"annotations" (catalog context = the candidate's existing "context"),
|
|
948
|
+
"advertised_pricing" (the models-API scalars the candidate carried before
|
|
949
|
+
the mutation, source-tagged) and "steering" are attached.
|
|
950
|
+
|
|
951
|
+
A baseline whose prices are missing/non-finite counts as unusable
|
|
952
|
+
(pick_baseline's blended-price skip is authoritative for the top-level
|
|
953
|
+
effective prices). The same bar applies to the base prices the mutation
|
|
954
|
+
would publish: parse_price returns float | None, so a None base price
|
|
955
|
+
(tier-over-None-base endpoint) is reachable -- the isinstance(..., float)
|
|
956
|
+
half of the guard is load-bearing for None, not a redundant type check.
|
|
957
|
+
Returns (surviving candidates in the original order, dropped counts for
|
|
958
|
+
the two new reasons).
|
|
959
|
+
"""
|
|
960
|
+
extra_dropped: dict[str, int] = {}
|
|
961
|
+
if not candidates:
|
|
962
|
+
return candidates, extra_dropped
|
|
963
|
+
endpoints_by_id: dict[str, list] = {}
|
|
964
|
+
failed_ids: set[str] = set()
|
|
965
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
966
|
+
futures = {}
|
|
967
|
+
for cand in candidates:
|
|
968
|
+
# Duplicate ids fetch once and share the result.
|
|
969
|
+
futures.setdefault(
|
|
970
|
+
cand["id"], pool.submit(fetch_model_endpoints, cand["id"], args)
|
|
971
|
+
)
|
|
972
|
+
for cand_id, future in futures.items():
|
|
973
|
+
try:
|
|
974
|
+
endpoints_by_id[cand_id] = future.result()
|
|
975
|
+
except Exception:
|
|
976
|
+
# fetch_json already retried 3 times; the caller owns the
|
|
977
|
+
# fail-closed policy.
|
|
978
|
+
failed_ids.add(cand_id)
|
|
979
|
+
survivors = []
|
|
980
|
+
for cand in candidates:
|
|
981
|
+
cand_id = cand["id"]
|
|
982
|
+
if cand_id in failed_ids:
|
|
983
|
+
_endpoint_stage_drop(
|
|
984
|
+
extra_dropped, filtered, cand, "endpoint_fetch_failed", args
|
|
985
|
+
)
|
|
986
|
+
continue
|
|
987
|
+
endpoints = endpoints_by_id[cand_id]
|
|
988
|
+
author = cand_id.partition("/")[0]
|
|
989
|
+
default_precision = derive_default_precision(endpoints, author)
|
|
990
|
+
eligible_flags = [
|
|
991
|
+
baseline_eligible(endpoint, default_precision, author, args)
|
|
992
|
+
for endpoint in endpoints
|
|
993
|
+
]
|
|
994
|
+
baseline = pick_baseline(endpoints, eligible_flags, args)
|
|
995
|
+
eff = (
|
|
996
|
+
effective_pricing(
|
|
997
|
+
baseline.get("pricing"), args.min_context, args.input_share
|
|
998
|
+
)
|
|
999
|
+
if baseline is not None
|
|
1000
|
+
else None
|
|
1001
|
+
)
|
|
1002
|
+
if baseline is None or not all(
|
|
1003
|
+
isinstance(eff[key], float) and math.isfinite(eff[key])
|
|
1004
|
+
for key in ("price_in", "price_out", "base_price_in", "base_price_out")
|
|
1005
|
+
):
|
|
1006
|
+
_endpoint_stage_drop(
|
|
1007
|
+
extra_dropped, filtered, cand, "no_baseline_eligible_endpoint", args
|
|
1008
|
+
)
|
|
1009
|
+
continue
|
|
1010
|
+
# The advertised models-API scalars, captured before the mutation.
|
|
1011
|
+
advertised_pricing = {
|
|
1012
|
+
"input_per_1m": cand["price_in"],
|
|
1013
|
+
"output_per_1m": cand["price_out"],
|
|
1014
|
+
"blended_per_1m": cand["blended"],
|
|
1015
|
+
"source": "models-api",
|
|
1016
|
+
}
|
|
1017
|
+
# Replace the pricing block with the baseline endpoint's outputs,
|
|
1018
|
+
# scaled per-1M exactly like build_candidates.
|
|
1019
|
+
cand["price_in"] = eff["price_in"] * 1_000_000.0
|
|
1020
|
+
cand["price_out"] = eff["price_out"] * 1_000_000.0
|
|
1021
|
+
cand["blended"] = (
|
|
1022
|
+
args.input_share * cand["price_in"]
|
|
1023
|
+
+ (1.0 - args.input_share) * cand["price_out"]
|
|
1024
|
+
)
|
|
1025
|
+
cand["base_price_in"] = eff["base_price_in"] * 1_000_000.0
|
|
1026
|
+
cand["base_price_out"] = eff["base_price_out"] * 1_000_000.0
|
|
1027
|
+
cand["tier_prompt_tokens"] = eff["tier_prompt_tokens"]
|
|
1028
|
+
cand["tiers"] = eff["tiers"]
|
|
1029
|
+
cand["schedule"] = eff["schedule"]
|
|
1030
|
+
cand["max_discount"] = eff["max_discount"]
|
|
1031
|
+
cand["tier_note"] = fmt_tier_note(eff["tier_prompt_tokens"])
|
|
1032
|
+
cand["sched_note"] = fmt_sched_note(eff["max_discount"])
|
|
1033
|
+
cand["sched_detail"] = fmt_sched_detail(eff["schedule"], eff["peak_blended"])
|
|
1034
|
+
cand["endpoints"] = endpoints
|
|
1035
|
+
cand["default_precision"] = default_precision
|
|
1036
|
+
cand["annotations"] = build_annotations(
|
|
1037
|
+
endpoints, eligible_flags, baseline, cand["context"], args
|
|
1038
|
+
)
|
|
1039
|
+
cand["advertised_pricing"] = advertised_pricing
|
|
1040
|
+
cand["steering"] = build_steering(
|
|
1041
|
+
cand_id, endpoints, eligible_flags, default_precision, args
|
|
1042
|
+
)
|
|
1043
|
+
survivors.append(cand)
|
|
1044
|
+
return survivors, extra_dropped
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
# ---------------------------------------------------------------------------
|
|
1048
|
+
# OpenRouter session-cost dataset (per-task cost, outside the score)
|
|
1049
|
+
# ---------------------------------------------------------------------------
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
SESSION_COST_URL = "https://openrouter.ai/api/v1/datasets/session-cost"
|
|
1053
|
+
SESSION_COST_HARNESS_SLUG = "hermes-agent"
|
|
1054
|
+
# The turn range the catalog/CLI surfaces (Task 8).
|
|
1055
|
+
SESSION_COST_DISPLAY_TURN_RANGE = "10-49-turns"
|
|
1056
|
+
# The dataset's four turn ranges (brainstorm section 2.6) -- the catalog
|
|
1057
|
+
# contract's closed set (web/build_site_data.py pins the same tuple).
|
|
1058
|
+
# Cells naming anything else are dropped: an unknown range is dataset
|
|
1059
|
+
# drift, and the block must never fail the publish (design section 4.8).
|
|
1060
|
+
SESSION_COST_TURN_RANGES = ("1-turn", "2-9-turns", "10-49-turns", "50-plus-turns")
|
|
1061
|
+
SESSION_COST_ATTRIBUTION = "Data © OpenRouter, CC BY 4.0"
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def strip_permaslug_date(permaslug: str) -> str:
|
|
1065
|
+
"""Permaslug without its trailing -YYYYMMDD (undated ids map to themselves)."""
|
|
1066
|
+
match = re.match(r"^(.*)-(\d{8})$", permaslug)
|
|
1067
|
+
return match.group(1) if match else permaslug
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def fetch_session_cost(args) -> dict | None:
|
|
1071
|
+
"""Per-task session-cost medians from OpenRouter's dataset, or None.
|
|
1072
|
+
|
|
1073
|
+
A separate published dimension (design section 4.8): the median USD
|
|
1074
|
+
cost of real sessions through the pinned published harness -- never
|
|
1075
|
+
part of the weighted score. One bulk paginated fetch per invocation
|
|
1076
|
+
(?app_slug=<harness>&limit=500&offset=k) regardless of the model list;
|
|
1077
|
+
pagination stops on a page shorter than the limit or once the next
|
|
1078
|
+
offset would exceed the API's 5000 cap. Cells key on dated permaslugs
|
|
1079
|
+
(z-ai/glm-5.3-flash-20260826), stripped to the bare catalog id with
|
|
1080
|
+
strip_permaslug_date; when several strip to the same id, the latest
|
|
1081
|
+
date wins each turn-range cell (same rule as build_aa_benchmarks).
|
|
1082
|
+
|
|
1083
|
+
Returns None (graceful degradation, design section 4.9) when no key is
|
|
1084
|
+
configured (the --openrouter-api-key flag, else OPENROUTER_API_KEY),
|
|
1085
|
+
when the fetch fails after fetch_json's retries, or when the response
|
|
1086
|
+
shape is unexpected -- a malformed payload must never crash the run.
|
|
1087
|
+
The merged result is cached under "session-cost" with the standard TTL.
|
|
1088
|
+
"""
|
|
1089
|
+
api_key = args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
|
|
1090
|
+
if not api_key:
|
|
1091
|
+
return None
|
|
1092
|
+
if not args.no_cache:
|
|
1093
|
+
cached = load_cache("session-cost", args.cache_ttl)
|
|
1094
|
+
if isinstance(cached, dict) and isinstance(cached.get("medians"), dict):
|
|
1095
|
+
return cached
|
|
1096
|
+
cells = []
|
|
1097
|
+
# The API caps limit at 500 and offset at 5000 (11 full pages max).
|
|
1098
|
+
offset = 0
|
|
1099
|
+
while offset <= 5000:
|
|
1100
|
+
try:
|
|
1101
|
+
payload = fetch_json(
|
|
1102
|
+
f"{SESSION_COST_URL}?app_slug={SESSION_COST_HARNESS_SLUG}"
|
|
1103
|
+
f"&limit=500&offset={offset}",
|
|
1104
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
1105
|
+
timeout=30,
|
|
1106
|
+
retries=3,
|
|
1107
|
+
)
|
|
1108
|
+
except Exception as exc:
|
|
1109
|
+
warn(f"session-cost dataset request failed ({exc})")
|
|
1110
|
+
return None
|
|
1111
|
+
data = payload.get("data") if isinstance(payload, dict) else None
|
|
1112
|
+
if not isinstance(data, list):
|
|
1113
|
+
warn("session-cost dataset returned an unexpected response shape")
|
|
1114
|
+
return None
|
|
1115
|
+
cells.extend(cell for cell in data if isinstance(cell, dict))
|
|
1116
|
+
if len(data) < 500:
|
|
1117
|
+
break
|
|
1118
|
+
offset += 500
|
|
1119
|
+
medians: dict[str, dict[str, float]] = {}
|
|
1120
|
+
seen_date: dict[tuple[str, str], str] = {}
|
|
1121
|
+
|
|
1122
|
+
def accept(bare, date, turn_range, median):
|
|
1123
|
+
key = (bare, turn_range)
|
|
1124
|
+
if key in seen_date and seen_date[key] >= date:
|
|
1125
|
+
return
|
|
1126
|
+
seen_date[key] = date
|
|
1127
|
+
medians.setdefault(bare, {})[turn_range] = median
|
|
1128
|
+
|
|
1129
|
+
for cell in cells:
|
|
1130
|
+
permaslug = cell.get("model_permaslug")
|
|
1131
|
+
turn_range = cell.get("turn_range")
|
|
1132
|
+
raw = cell.get("median_session_cost_usd")
|
|
1133
|
+
if not (
|
|
1134
|
+
isinstance(permaslug, str)
|
|
1135
|
+
and permaslug
|
|
1136
|
+
and isinstance(turn_range, str)
|
|
1137
|
+
and turn_range
|
|
1138
|
+
):
|
|
1139
|
+
continue
|
|
1140
|
+
# Closed contract set: a cell naming a new turn range is dropped,
|
|
1141
|
+
# never published (the catalog validator rejects unknown ranges,
|
|
1142
|
+
# and session cost must never fail the publish).
|
|
1143
|
+
if turn_range not in SESSION_COST_TURN_RANGES:
|
|
1144
|
+
continue
|
|
1145
|
+
# Non-finite medians would serialize as NaN and poison the catalog.
|
|
1146
|
+
if not isinstance(raw, (int, float)) or isinstance(raw, bool):
|
|
1147
|
+
continue
|
|
1148
|
+
try:
|
|
1149
|
+
median = float(raw) # huge ints raise OverflowError
|
|
1150
|
+
except OverflowError:
|
|
1151
|
+
continue
|
|
1152
|
+
if not math.isfinite(median):
|
|
1153
|
+
continue
|
|
1154
|
+
match = re.match(r"^(.*)-(\d{8})$", permaslug)
|
|
1155
|
+
accept(
|
|
1156
|
+
strip_permaslug_date(permaslug),
|
|
1157
|
+
match.group(2) if match else "",
|
|
1158
|
+
turn_range,
|
|
1159
|
+
median,
|
|
1160
|
+
)
|
|
1161
|
+
if not medians:
|
|
1162
|
+
warn("session-cost dataset returned no usable cells")
|
|
1163
|
+
return None
|
|
1164
|
+
meta = payload.get("meta") if isinstance(payload, dict) else None
|
|
1165
|
+
if not isinstance(meta, dict):
|
|
1166
|
+
warn("session-cost dataset returned an unexpected response shape")
|
|
1167
|
+
return None
|
|
1168
|
+
as_of = meta.get("as_of")
|
|
1169
|
+
window_days = meta.get("window_days")
|
|
1170
|
+
window_end_date = meta.get("window_end_date")
|
|
1171
|
+
# The catalog validator demands non-empty as_of/window_end_date strings
|
|
1172
|
+
# and a positive integer window_days; anything else is dataset drift.
|
|
1173
|
+
# Degrade to None (block omitted, "not available this refresh") rather
|
|
1174
|
+
# than emit a block the validator would reject -- session cost must
|
|
1175
|
+
# never fail the publish (design section 4.8).
|
|
1176
|
+
if not (
|
|
1177
|
+
isinstance(as_of, str)
|
|
1178
|
+
and as_of
|
|
1179
|
+
and isinstance(window_end_date, str)
|
|
1180
|
+
and window_end_date
|
|
1181
|
+
and isinstance(window_days, int)
|
|
1182
|
+
and not isinstance(window_days, bool)
|
|
1183
|
+
and window_days > 0
|
|
1184
|
+
):
|
|
1185
|
+
warn("session-cost dataset returned malformed meta; omitting the block")
|
|
1186
|
+
return None
|
|
1187
|
+
result = {
|
|
1188
|
+
"attribution": SESSION_COST_ATTRIBUTION,
|
|
1189
|
+
"app_slug": SESSION_COST_HARNESS_SLUG,
|
|
1190
|
+
"as_of": as_of,
|
|
1191
|
+
"window_days": window_days,
|
|
1192
|
+
"window_end_date": window_end_date,
|
|
1193
|
+
"medians": medians,
|
|
1194
|
+
}
|
|
1195
|
+
save_cache("session-cost", result)
|
|
1196
|
+
return result
|
|
1197
|
+
|
|
1198
|
+
|
|
227
1199
|
# ---------------------------------------------------------------------------
|
|
228
1200
|
# OpenRouter catalog
|
|
229
1201
|
# ---------------------------------------------------------------------------
|
|
@@ -1413,6 +2385,48 @@ def fmt_discount(value) -> str:
|
|
|
1413
2385
|
return f"{value:.0%}" if has_discount(value) else "--"
|
|
1414
2386
|
|
|
1415
2387
|
|
|
2388
|
+
def fmt_prec(cand) -> str:
|
|
2389
|
+
"""PREC cell: the precision the ranked price is quoted at.
|
|
2390
|
+
|
|
2391
|
+
The maker's default precision when disclosed, the best available
|
|
2392
|
+
otherwise, "n/a" when none could be derived; "!" flags default routing
|
|
2393
|
+
that favors cheaper sub-default endpoints (annotations.risk_flag).
|
|
2394
|
+
Candidates that never went through the endpoint stage degrade to "n/a".
|
|
2395
|
+
"""
|
|
2396
|
+
cell = cand.get("default_precision") or "n/a"
|
|
2397
|
+
if (cand.get("annotations") or {}).get("risk_flag"):
|
|
2398
|
+
cell += "!"
|
|
2399
|
+
return cell
|
|
2400
|
+
|
|
2401
|
+
|
|
2402
|
+
def fmt_quant_note(annotations: dict) -> str | None:
|
|
2403
|
+
"""One-line note on the cheapest sub-default endpoint (PREC tooltip).
|
|
2404
|
+
|
|
2405
|
+
The exact string the site shows (Task 11 renders it verbatim), so the
|
|
2406
|
+
format is pinned by test. None when there is no non-eligible live
|
|
2407
|
+
endpoint -- nothing cheaper to route to. cheapest_quantized.price is
|
|
2408
|
+
stored per-token, the way build_annotations leaves it; the note shows
|
|
2409
|
+
USD per 1M like every other price the CLI prints. A missing spread
|
|
2410
|
+
(defensive) drops the parenthetical. A null quantization (third-party
|
|
2411
|
+
null == "unknown" for eligibility) and a null provider_name on
|
|
2412
|
+
malformed data render the design's "unknown" sentinel, never Python's
|
|
2413
|
+
None repr.
|
|
2414
|
+
"""
|
|
2415
|
+
quant = (annotations or {}).get("cheapest_quantized")
|
|
2416
|
+
if not quant:
|
|
2417
|
+
return None
|
|
2418
|
+
quantization = quant["quantization"] or "unknown"
|
|
2419
|
+
provider = quant["provider"] or "unknown"
|
|
2420
|
+
note = (
|
|
2421
|
+
f"cheapest non-baseline: {quantization} @ {provider}, "
|
|
2422
|
+
f"${fmt_price(quant['price'] * 1_000_000)}/M"
|
|
2423
|
+
)
|
|
2424
|
+
spread = (annotations or {}).get("price_spread")
|
|
2425
|
+
if spread is not None:
|
|
2426
|
+
note += f" in ({spread:.2f}x baseline)"
|
|
2427
|
+
return note
|
|
2428
|
+
|
|
2429
|
+
|
|
1416
2430
|
def print_table(top, total_candidates, weights, quality_note):
|
|
1417
2431
|
headers = [
|
|
1418
2432
|
"RANK",
|
|
@@ -1421,6 +2435,7 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1421
2435
|
"$IN/M",
|
|
1422
2436
|
"$OUT/M",
|
|
1423
2437
|
"DISC",
|
|
2438
|
+
"PREC",
|
|
1424
2439
|
"TIER",
|
|
1425
2440
|
"CTX",
|
|
1426
2441
|
"AGE",
|
|
@@ -1436,6 +2451,7 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1436
2451
|
fmt_price(cand["price_in"]),
|
|
1437
2452
|
fmt_price(cand["price_out"]),
|
|
1438
2453
|
fmt_discount(cand["discount"]),
|
|
2454
|
+
fmt_prec(cand),
|
|
1439
2455
|
cand["tier_note"] or cand["sched_note"] or "--",
|
|
1440
2456
|
fmt_context(cand["context"]),
|
|
1441
2457
|
fmt_age(cand["age_days"]),
|
|
@@ -1482,6 +2498,12 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
1482
2498
|
"schedule whose discount is under 1%. --json lists each model's "
|
|
1483
2499
|
"schedule."
|
|
1484
2500
|
)
|
|
2501
|
+
print(
|
|
2502
|
+
"PREC = precision the ranked price is quoted at (maker's default when "
|
|
2503
|
+
"disclosed, best available otherwise; ! = default routing favors "
|
|
2504
|
+
"cheaper sub-default endpoints); prices are the cheapest live endpoint "
|
|
2505
|
+
"at that precision."
|
|
2506
|
+
)
|
|
1485
2507
|
|
|
1486
2508
|
|
|
1487
2509
|
def print_json(top):
|
|
@@ -1518,6 +2540,31 @@ def print_json(top):
|
|
|
1518
2540
|
"age_days": round(cand["age_days"], 1)
|
|
1519
2541
|
if cand["age_days"] is not None
|
|
1520
2542
|
else None,
|
|
2543
|
+
# Additive keys (Task 9's ROW_KEYS extension pins this exact
|
|
2544
|
+
# key set and order). Risk/spread come from the annotations
|
|
2545
|
+
# block; session_cost_usd is run()'s per-candidate slice of the
|
|
2546
|
+
# Task 7 envelope, shown for the display turn range (run()
|
|
2547
|
+
# attaches it before any output path). Candidates that never
|
|
2548
|
+
# went through either stage (historical test paths) degrade to
|
|
2549
|
+
# null, as does a model the dataset covers only in other turn
|
|
2550
|
+
# ranges.
|
|
2551
|
+
"default_precision": cand.get("default_precision"),
|
|
2552
|
+
"risk_flag": (cand.get("annotations") or {}).get("risk_flag"),
|
|
2553
|
+
"price_spread": (cand.get("annotations") or {}).get("price_spread"),
|
|
2554
|
+
"session_cost_usd": (cand.get("session_cost") or {}).get(
|
|
2555
|
+
SESSION_COST_DISPLAY_TURN_RANGE
|
|
2556
|
+
),
|
|
2557
|
+
# The cheapest-sub-default note, exactly as the CLI prints it
|
|
2558
|
+
# (Task 11 shows it verbatim). Emit-first: Task 9's ROW_KEYS
|
|
2559
|
+
# pins this key last; pre-endpoint candidates degrade to None.
|
|
2560
|
+
"quant_note": fmt_quant_note(cand.get("annotations")),
|
|
2561
|
+
# One extra row key AFTER quant_note -- steering is NOT in
|
|
2562
|
+
# ROW_KEYS. build_site_data's row check is missing-key-only, so
|
|
2563
|
+
# extras pass through (tier_note precedent) and the site
|
|
2564
|
+
# ignores it. The OpenRouter provider routing object (design
|
|
2565
|
+
# section 4.7) the plugin contract reads; pre-endpoint
|
|
2566
|
+
# candidates degrade to None.
|
|
2567
|
+
"steering": cand.get("steering"),
|
|
1521
2568
|
}
|
|
1522
2569
|
for cand in top
|
|
1523
2570
|
]
|
|
@@ -1535,6 +2582,8 @@ def build_catalog(
|
|
|
1535
2582
|
aa_source,
|
|
1536
2583
|
aa_by_id,
|
|
1537
2584
|
quality_source_by_id,
|
|
2585
|
+
session_cost=None,
|
|
2586
|
+
session_cost_mode="skipped",
|
|
1538
2587
|
):
|
|
1539
2588
|
"""Assemble the full-catalog document (see README "Catalog output").
|
|
1540
2589
|
|
|
@@ -1542,6 +2591,11 @@ def build_catalog(
|
|
|
1542
2591
|
component scores) and never mutates them. Deterministic apart from
|
|
1543
2592
|
generated_at: models sort by (-overall.balanced, id), filtered by id,
|
|
1544
2593
|
and age uses date precision so two runs in the same UTC day match.
|
|
2594
|
+
|
|
2595
|
+
session_cost is the Task 7 fetch envelope (verbatim, medians included)
|
|
2596
|
+
or None; session_cost_mode is run()'s resolution of the key-presence
|
|
2597
|
+
fact ("ok" / "unavailable" / "skipped") -- build_catalog stays pure and
|
|
2598
|
+
never fetches or inspects the environment itself.
|
|
1545
2599
|
"""
|
|
1546
2600
|
now = datetime.now(timezone.utc)
|
|
1547
2601
|
weights = catalog_weights(candidates, quality_by_id)
|
|
@@ -1655,6 +2709,19 @@ def build_catalog(
|
|
|
1655
2709
|
"quality": cand["quality"],
|
|
1656
2710
|
"quality_match": quality_source_by_id.get(cand["id"]),
|
|
1657
2711
|
"scores": {**scores, "overall": overall},
|
|
2712
|
+
# Additive provider-aware blocks (schema v3), verbatim from
|
|
2713
|
+
# the candidate. Candidates that never went through the
|
|
2714
|
+
# endpoint stage (historical test paths) degrade to null --
|
|
2715
|
+
# .get() with null defaults for these NEW keys only; the
|
|
2716
|
+
# pre-existing keys above stay direct-indexed.
|
|
2717
|
+
"endpoints": cand.get("endpoints"),
|
|
2718
|
+
"default_precision": cand.get("default_precision"),
|
|
2719
|
+
"advertised_pricing": cand.get("advertised_pricing"),
|
|
2720
|
+
"annotations": cand.get("annotations"),
|
|
2721
|
+
# The OpenRouter provider routing object (design section
|
|
2722
|
+
# 4.7) the plugin contract reads -- same null-default
|
|
2723
|
+
# degradation shape as the blocks above.
|
|
2724
|
+
"steering": cand.get("steering"),
|
|
1658
2725
|
}
|
|
1659
2726
|
)
|
|
1660
2727
|
entries.sort(key=lambda e: (-e["scores"]["overall"]["balanced"], e["id"]))
|
|
@@ -1692,7 +2759,19 @@ def build_catalog(
|
|
|
1692
2759
|
},
|
|
1693
2760
|
"zdr": "skipped" if args.no_zdr else "ok",
|
|
1694
2761
|
"discounts": "ok" if discounts else "unavailable",
|
|
2762
|
+
# The endpoint stage always ran before build_catalog is reached;
|
|
2763
|
+
# its failures surface as pool.dropped counts
|
|
2764
|
+
# (endpoint_fetch_failed / no_baseline_eligible_endpoint), never
|
|
2765
|
+
# as source state -- so this is deterministic "ok".
|
|
2766
|
+
"endpoints": "ok",
|
|
2767
|
+
# "ok" (envelope fetched) / "unavailable" (key configured, fetch
|
|
2768
|
+
# failed) / "skipped" (no key) -- resolved in run().
|
|
2769
|
+
"session_cost": session_cost_mode,
|
|
1695
2770
|
},
|
|
2771
|
+
# Per-task session-cost medians (Task 7), verbatim from the fetch
|
|
2772
|
+
# envelope. Omitted entirely when unavailable: consumers branch on
|
|
2773
|
+
# key presence, never on a null shape.
|
|
2774
|
+
**({"session_cost": session_cost} if session_cost is not None else {}),
|
|
1696
2775
|
"pool": {
|
|
1697
2776
|
"listed": len(models),
|
|
1698
2777
|
"candidates": len(candidates),
|
|
@@ -1787,6 +2866,11 @@ def parse_args(argv=None):
|
|
|
1787
2866
|
default=None,
|
|
1788
2867
|
help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
|
|
1789
2868
|
)
|
|
2869
|
+
parser.add_argument(
|
|
2870
|
+
"--openrouter-api-key",
|
|
2871
|
+
default=None,
|
|
2872
|
+
help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
|
|
2873
|
+
)
|
|
1790
2874
|
parser.add_argument(
|
|
1791
2875
|
"--no-require-tools",
|
|
1792
2876
|
action="store_true",
|
|
@@ -1862,11 +2946,24 @@ def run(args, models, or_cached) -> int:
|
|
|
1862
2946
|
|
|
1863
2947
|
filtered = []
|
|
1864
2948
|
candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
|
|
2949
|
+
# The provider-aware stage: every surviving candidate is repriced at its
|
|
2950
|
+
# baseline endpoint (or dropped fail-closed) before any scoring runs.
|
|
2951
|
+
candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
|
|
2952
|
+
dropped.update(extra_dropped)
|
|
1865
2953
|
if not candidates:
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
)
|
|
2954
|
+
# When the endpoint stage wiped the pool, naming the filter knobs is
|
|
2955
|
+
# misleading (they passed; the drops came after) -- warn with the
|
|
2956
|
+
# endpoint-stage drop counts instead (both reasons, zero-filled).
|
|
2957
|
+
counts = {r: extra_dropped.get(r, 0) for r in ENDPOINT_DROP_REASONS}
|
|
2958
|
+
if any(counts.values()):
|
|
2959
|
+
bits = ", ".join(f"{k}={v}" for k, v in counts.items())
|
|
2960
|
+
warn(f"no models after the endpoint stage: {bits}")
|
|
2961
|
+
else:
|
|
2962
|
+
warn(
|
|
2963
|
+
f"no models satisfy the filters (min-context={args.min_context}, "
|
|
2964
|
+
f"tools={'off' if args.no_require_tools else 'required'}); "
|
|
2965
|
+
"relax them and retry"
|
|
2966
|
+
)
|
|
1870
2967
|
return 2
|
|
1871
2968
|
|
|
1872
2969
|
quality_by_id, quality_source_by_id = resolve_quality(
|
|
@@ -1874,6 +2971,29 @@ def run(args, models, or_cached) -> int:
|
|
|
1874
2971
|
)
|
|
1875
2972
|
weights = compute_scores(candidates, args, quality_by_id)
|
|
1876
2973
|
|
|
2974
|
+
# Per-task session-cost medians (Task 7): one fetch per invocation, but
|
|
2975
|
+
# only on the output paths that surface it (--catalog block, --json
|
|
2976
|
+
# rows; publish runs --json per priority, so the cache makes the repeat
|
|
2977
|
+
# cost free). A keyed --best/plain run skips the paginated fetch
|
|
2978
|
+
# entirely -- the medians never reach those outputs. Keyless,
|
|
2979
|
+
# fetch_session_cost returns None before any network or cache access.
|
|
2980
|
+
if args.catalog or args.json:
|
|
2981
|
+
session_cost = fetch_session_cost(args)
|
|
2982
|
+
# The key-presence fact distinguishes "skipped" from "unavailable";
|
|
2983
|
+
# same expression fetch_session_cost uses for its own guard.
|
|
2984
|
+
session_cost_mode = (
|
|
2985
|
+
"skipped"
|
|
2986
|
+
if not (args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY"))
|
|
2987
|
+
else ("ok" if session_cost else "unavailable")
|
|
2988
|
+
)
|
|
2989
|
+
else:
|
|
2990
|
+
session_cost = None
|
|
2991
|
+
session_cost_mode = "skipped"
|
|
2992
|
+
# run() owns the per-candidate attachment (build_catalog stays pure);
|
|
2993
|
+
# print_json reads it for the session_cost_usd display cell.
|
|
2994
|
+
for cand in candidates:
|
|
2995
|
+
cand["session_cost"] = (session_cost or {}).get("medians", {}).get(cand["id"])
|
|
2996
|
+
|
|
1877
2997
|
if args.catalog:
|
|
1878
2998
|
print_catalog(
|
|
1879
2999
|
build_catalog(
|
|
@@ -1887,6 +3007,8 @@ def run(args, models, or_cached) -> int:
|
|
|
1887
3007
|
aa_source,
|
|
1888
3008
|
aa_by_id,
|
|
1889
3009
|
quality_source_by_id,
|
|
3010
|
+
session_cost,
|
|
3011
|
+
session_cost_mode,
|
|
1890
3012
|
)
|
|
1891
3013
|
)
|
|
1892
3014
|
return 0
|