model-compare-opencode-plugin 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/tui.js +18 -6
- package/package.json +7 -6
- package/src/core/catalog.ts +98 -16
- package/src/core/reasoning.ts +96 -0
- package/src/core/steering.ts +36 -0
- package/src/core/types.ts +34 -1
- package/src/index.ts +41 -14
- package/src/opencode/aliases.ts +19 -5
- package/src/opencode/commands.ts +41 -3
- package/src/opencode/guard.ts +4 -2
- package/src/opencode/usebest.ts +13 -3
- package/src/rpc.ts +30 -2
- package/src/tui.tsx +26 -6
- package/vendor/UPSTREAM.json +3 -3
- package/vendor/model_compare.py +2140 -430
package/vendor/model_compare.py
CHANGED
|
@@ -37,6 +37,9 @@ Examples:
|
|
|
37
37
|
model_compare.py --best print only "#1 opencode id" (openrouter/<model>)
|
|
38
38
|
model_compare.py --priority price --top 3 three cheapest-sensible picks
|
|
39
39
|
model_compare.py --discount only currently-discounted models
|
|
40
|
+
model_compare.py --off-peak also list scheduled off-peak deals
|
|
41
|
+
model_compare.py --detail simple fewer columns, friendlier labels
|
|
42
|
+
model_compare.py --bare just the table(s): no summary or legend
|
|
40
43
|
model_compare.py --catalog full catalog (ranked + filtered) as one JSON document
|
|
41
44
|
MODEL=$(model_compare.py --best) shell integration (opencode --model $MODEL)
|
|
42
45
|
"""
|
|
@@ -68,10 +71,27 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
|
|
|
68
71
|
# (legacy endpoints 410 after 2026-11-04; see
|
|
69
72
|
# https://artificialanalysis.ai/data-api/migrate-v2-data).
|
|
70
73
|
AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
|
|
71
|
-
# Cache key for the AA API entries envelope.
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
AA_ENTRIES_CACHE = "aa-intelligence-
|
|
74
|
+
# Cache key for the AA API entries envelope. v4 adds the per-effort fields
|
|
75
|
+
# (id, slug, base, effort, creator, cost_per_task) parsed from each row's
|
|
76
|
+
# name/slug; v3 payloads predate them and must never be read.
|
|
77
|
+
AA_ENTRIES_CACHE = "aa-intelligence-v4"
|
|
78
|
+
# Reasoning-effort labels AA encodes in a row's name parenthetical or slug
|
|
79
|
+
# suffix. AA_EFFORT_ORDER is the display order (deepest first);
|
|
80
|
+
# REASONING_GATEWAY_EFFORTS is what OpenRouter accepts as a discrete effort.
|
|
81
|
+
AA_EFFORT_ORDER = (
|
|
82
|
+
"max",
|
|
83
|
+
"xhigh",
|
|
84
|
+
"high",
|
|
85
|
+
"medium",
|
|
86
|
+
"low",
|
|
87
|
+
"minimal",
|
|
88
|
+
"none",
|
|
89
|
+
"reasoning",
|
|
90
|
+
)
|
|
91
|
+
REASONING_GATEWAY_EFFORTS = ("max", "xhigh", "high", "medium", "low", "minimal", "none")
|
|
92
|
+
# --reasoning-effort values: "default" keeps each model's effective default
|
|
93
|
+
# and does not filter; the rest select and filter (spec §7).
|
|
94
|
+
REASONING_EFFORT_CHOICES = ("default",) + tuple(reversed(REASONING_GATEWAY_EFFORTS))
|
|
75
95
|
# The repo's own published catalog.json -- built by keyed CI runs, hydrated
|
|
76
96
|
# by unkeyed local runs (design: quality source reconciliation, source 2).
|
|
77
97
|
# The github.io host 301s to plain http://model-compare.dev/catalog.json, an
|
|
@@ -84,7 +104,7 @@ QUALITY_CATALOG_CACHE = "published-catalog"
|
|
|
84
104
|
# A published catalog generated more than this many days ago is stale:
|
|
85
105
|
# warn but use -- the records' as_of makes the staleness visible downstream.
|
|
86
106
|
QUALITY_CATALOG_MAX_AGE_DAYS = 14
|
|
87
|
-
VERSION = "0.
|
|
107
|
+
VERSION = "0.6.0"
|
|
88
108
|
USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
|
|
89
109
|
|
|
90
110
|
# opencode expects provider-qualified model ids: openrouter/<provider/model>.
|
|
@@ -103,7 +123,7 @@ PRIORITY_WEIGHTS = {
|
|
|
103
123
|
"quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
|
|
104
124
|
}
|
|
105
125
|
|
|
106
|
-
CATALOG_SCHEMA_VERSION =
|
|
126
|
+
CATALOG_SCHEMA_VERSION = 5
|
|
107
127
|
|
|
108
128
|
# The drop-reason keys build_candidates counts, verbatim -- keep in sync with
|
|
109
129
|
# its drop() call sites. pool.dropped lists all of them zero-filled so the
|
|
@@ -125,6 +145,7 @@ CATALOG_DROP_REASONS = (
|
|
|
125
145
|
"tool calling",
|
|
126
146
|
"expired",
|
|
127
147
|
"age",
|
|
148
|
+
"reasoning effort",
|
|
128
149
|
"no_baseline_eligible_endpoint",
|
|
129
150
|
"endpoint_fetch_failed",
|
|
130
151
|
)
|
|
@@ -1217,110 +1238,229 @@ def fetch_session_cost(args) -> dict | None:
|
|
|
1217
1238
|
# ---------------------------------------------------------------------------
|
|
1218
1239
|
|
|
1219
1240
|
|
|
1220
|
-
# Pre-reconciliation published catalogs (schema v3
|
|
1221
|
-
#
|
|
1222
|
-
|
|
1223
|
-
_QUALITY_SOURCE_NORMALIZATION = {"api": "aa"}
|
|
1241
|
+
# Pre-reconciliation published catalogs (schema v3/v4) are retired: v5 is
|
|
1242
|
+
# the only shape whose `quality` means "the index at the requested effort".
|
|
1243
|
+
CATALOG_QUALITY_SCHEMA_VERSION = 5
|
|
1224
1244
|
|
|
1225
1245
|
|
|
1226
|
-
def
|
|
1227
|
-
"""
|
|
1246
|
+
def _catalog_index(value) -> float | None:
|
|
1247
|
+
"""A finite in-range catalog index, or None for poisoned/absent input."""
|
|
1248
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
|
1249
|
+
return None
|
|
1250
|
+
try:
|
|
1251
|
+
index = float(value) # huge ints raise OverflowError
|
|
1252
|
+
except OverflowError:
|
|
1253
|
+
return None
|
|
1254
|
+
if not math.isfinite(index) or not 0.0 <= index <= 100.0:
|
|
1255
|
+
return None
|
|
1256
|
+
return index
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
def _parse_catalog_v5(doc) -> tuple[dict, dict]:
|
|
1260
|
+
"""Pure parser: a published catalog v5 document -> (by_id, depths_by_id).
|
|
1228
1261
|
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
the
|
|
1233
|
-
|
|
1262
|
+
by_id maps id -> {"index", "source", "as_of", "effort"} for the model's
|
|
1263
|
+
main quality (the index at the keyed run's requested effort). The
|
|
1264
|
+
depth map maps id -> {"default_effort", "depths": {effort: record}} for
|
|
1265
|
+
the model's full measured depth list; resolve_quality re-selects from
|
|
1266
|
+
it at the unkeyed run's live effective default.
|
|
1234
1267
|
|
|
1235
1268
|
Gates (design: failure modes):
|
|
1236
|
-
- schema_version must be an int
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1269
|
+
- schema_version must be an int == 5. v3/v4 predate the depth contract
|
|
1270
|
+
(their `quality` means max effort), so they warn
|
|
1271
|
+
"published catalog schema vX, expected 5; skipping hydration" and
|
|
1272
|
+
yield empty maps. Mixing semantics is never allowed.
|
|
1240
1273
|
- Any shape surprise -- a non-dict document, a non-int schema_version,
|
|
1241
|
-
a non-list models, a row without a usable id,
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
artifact we think it is.
|
|
1274
|
+
a non-list models, a row without a usable id, a main quality that is
|
|
1275
|
+
not finite/in-range, or malformed depth data -- warns and yields
|
|
1276
|
+
empty maps for the WHOLE artifact. Never a partial hydration, never
|
|
1277
|
+
a raise: the catalog is machine-generated, so one malformed row
|
|
1278
|
+
means this is not the artifact we think it is.
|
|
1247
1279
|
- Rows with quality None and quality_match None are legitimate
|
|
1248
|
-
unmatched models; they are skipped
|
|
1280
|
+
unmatched models; they are skipped in by_id but their depth block is
|
|
1281
|
+
still parsed (a model can have measured depths without a main score).
|
|
1249
1282
|
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
v4+ rows keep their recorded quality_as_of (None when absent) and
|
|
1254
|
-
source label as-is: that label is the row-level truth inside the
|
|
1255
|
-
published artifact (a keyed CI run emits "aa"/"openrouter", never
|
|
1256
|
-
"catalog"). resolve_quality's catalog tier owns the run-level label:
|
|
1257
|
-
it relabels every hydrated record to the chain label "catalog".
|
|
1283
|
+
Per-depth records use the published shape {effort, index,
|
|
1284
|
+
cost_per_task_usd, source, as_of} -- the quality record's own shape
|
|
1285
|
+
(`index`, never `quality_index`).
|
|
1258
1286
|
"""
|
|
1259
1287
|
if not isinstance(doc, dict):
|
|
1260
1288
|
warn("published catalog returned an unexpected response shape")
|
|
1261
|
-
return {}
|
|
1289
|
+
return {}, {}
|
|
1262
1290
|
schema_version = doc.get("schema_version")
|
|
1263
1291
|
if type(schema_version) is not int:
|
|
1264
1292
|
warn("published catalog returned an unexpected response shape")
|
|
1265
|
-
return {}
|
|
1266
|
-
if schema_version
|
|
1293
|
+
return {}, {}
|
|
1294
|
+
if schema_version != CATALOG_QUALITY_SCHEMA_VERSION:
|
|
1267
1295
|
warn(
|
|
1268
|
-
f"published catalog schema v{schema_version}, expected
|
|
1269
|
-
"skipping hydration"
|
|
1296
|
+
f"published catalog schema v{schema_version}, expected "
|
|
1297
|
+
f"{CATALOG_QUALITY_SCHEMA_VERSION}; skipping hydration"
|
|
1270
1298
|
)
|
|
1271
|
-
return {}
|
|
1299
|
+
return {}, {}
|
|
1272
1300
|
models = doc.get("models")
|
|
1273
1301
|
if not isinstance(models, list):
|
|
1274
1302
|
warn("published catalog returned an unexpected response shape")
|
|
1275
|
-
return {}
|
|
1276
|
-
generated_at = doc.get("generated_at")
|
|
1277
|
-
v3_as_of = generated_at if isinstance(generated_at, str) and generated_at else None
|
|
1303
|
+
return {}, {}
|
|
1278
1304
|
by_id: dict = {}
|
|
1305
|
+
depths_by_id: dict = {}
|
|
1279
1306
|
for entry in models:
|
|
1280
1307
|
if not isinstance(entry, dict):
|
|
1281
1308
|
warn("published catalog returned an unexpected response shape")
|
|
1282
|
-
return {}
|
|
1309
|
+
return {}, {}
|
|
1283
1310
|
model_id = entry.get("id")
|
|
1284
1311
|
if not (isinstance(model_id, str) and model_id):
|
|
1285
1312
|
warn("published catalog returned an unexpected response shape")
|
|
1286
|
-
return {}
|
|
1313
|
+
return {}, {}
|
|
1287
1314
|
quality = entry.get("quality")
|
|
1288
1315
|
match = entry.get("quality_match")
|
|
1289
|
-
if match is None and quality is None:
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1316
|
+
if not (match is None and quality is None):
|
|
1317
|
+
if not (isinstance(match, str) and match):
|
|
1318
|
+
warn("published catalog returned an unexpected response shape")
|
|
1319
|
+
return {}, {}
|
|
1320
|
+
index = _catalog_index(quality)
|
|
1321
|
+
if index is None:
|
|
1322
|
+
warn("published catalog returned an unexpected response shape")
|
|
1323
|
+
return {}, {}
|
|
1324
|
+
effort = entry.get("quality_effort")
|
|
1325
|
+
if effort is not None and not (isinstance(effort, str) and effort):
|
|
1326
|
+
warn("published catalog returned an unexpected response shape")
|
|
1327
|
+
return {}, {}
|
|
1328
|
+
recorded = entry.get("quality_as_of")
|
|
1329
|
+
by_id[model_id] = {
|
|
1330
|
+
"index": index,
|
|
1331
|
+
"source": match,
|
|
1332
|
+
"as_of": recorded if isinstance(recorded, str) and recorded else None,
|
|
1333
|
+
"effort": effort,
|
|
1334
|
+
}
|
|
1335
|
+
reasoning = entry.get("reasoning")
|
|
1336
|
+
if reasoning is None:
|
|
1337
|
+
continue
|
|
1338
|
+
if not isinstance(reasoning, dict):
|
|
1300
1339
|
warn("published catalog returned an unexpected response shape")
|
|
1301
|
-
return {}
|
|
1302
|
-
|
|
1303
|
-
if not
|
|
1340
|
+
return {}, {}
|
|
1341
|
+
default_effort = reasoning.get("default_effort")
|
|
1342
|
+
if default_effort is not None and not isinstance(default_effort, str):
|
|
1304
1343
|
warn("published catalog returned an unexpected response shape")
|
|
1305
|
-
return {}
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
if not 0.0 <= index <= 100.0:
|
|
1344
|
+
return {}, {}
|
|
1345
|
+
raw_depths = reasoning.get("depths")
|
|
1346
|
+
if raw_depths is None:
|
|
1347
|
+
raw_depths = []
|
|
1348
|
+
if not isinstance(raw_depths, list):
|
|
1311
1349
|
warn("published catalog returned an unexpected response shape")
|
|
1312
|
-
return {}
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1350
|
+
return {}, {}
|
|
1351
|
+
depths: dict = {}
|
|
1352
|
+
for depth in raw_depths:
|
|
1353
|
+
if not isinstance(depth, dict):
|
|
1354
|
+
warn("published catalog returned an unexpected response shape")
|
|
1355
|
+
return {}, {}
|
|
1356
|
+
effort = depth.get("effort")
|
|
1357
|
+
if not (isinstance(effort, str) and effort):
|
|
1358
|
+
warn("published catalog returned an unexpected response shape")
|
|
1359
|
+
return {}, {}
|
|
1360
|
+
index = _catalog_index(depth.get("index"))
|
|
1361
|
+
if index is None:
|
|
1362
|
+
warn("published catalog returned an unexpected response shape")
|
|
1363
|
+
return {}, {}
|
|
1364
|
+
cost = depth.get("cost_per_task_usd")
|
|
1365
|
+
if cost is not None:
|
|
1366
|
+
cost = _finite_cost(cost)
|
|
1367
|
+
if cost is None:
|
|
1368
|
+
warn("published catalog returned an unexpected response shape")
|
|
1369
|
+
return {}, {}
|
|
1370
|
+
source = depth.get("source")
|
|
1371
|
+
if not (isinstance(source, str) and source):
|
|
1372
|
+
warn("published catalog returned an unexpected response shape")
|
|
1373
|
+
return {}, {}
|
|
1374
|
+
as_of = depth.get("as_of")
|
|
1375
|
+
if as_of is not None and not isinstance(as_of, str):
|
|
1376
|
+
warn("published catalog returned an unexpected response shape")
|
|
1377
|
+
return {}, {}
|
|
1378
|
+
if effort in depths:
|
|
1379
|
+
# Duplicate efforts are ambiguous: fail closed, never guess.
|
|
1380
|
+
warn("published catalog returned an unexpected response shape")
|
|
1381
|
+
return {}, {}
|
|
1382
|
+
depths[effort] = {
|
|
1383
|
+
"effort": effort,
|
|
1384
|
+
"index": index,
|
|
1385
|
+
"cost_per_task_usd": cost,
|
|
1386
|
+
"source": source,
|
|
1387
|
+
"as_of": as_of if as_of else None,
|
|
1388
|
+
}
|
|
1389
|
+
depths_by_id[model_id] = {
|
|
1390
|
+
"default_effort": default_effort if default_effort else None,
|
|
1391
|
+
"depths": depths,
|
|
1392
|
+
}
|
|
1393
|
+
return by_id, depths_by_id
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def catalog_quality_by_id(doc) -> dict:
|
|
1397
|
+
"""The v5 main-quality map {id: {"index", "source", "as_of", "effort"}}.
|
|
1398
|
+
|
|
1399
|
+
Pure parser; see _parse_catalog_v5 for the gates and the depth map.
|
|
1400
|
+
"""
|
|
1401
|
+
return _parse_catalog_v5(doc)[0]
|
|
1402
|
+
|
|
1403
|
+
|
|
1404
|
+
def _catalog_schedule_entries(doc) -> dict:
|
|
1405
|
+
"""Pure parser: published catalog v5 schedules -> id -> recorded provenance.
|
|
1406
|
+
|
|
1407
|
+
Spec 6.5: unkeyed runs rebuild the Deals table from the published
|
|
1408
|
+
catalog's schedules array, because schedule-only models (sub-1M context,
|
|
1409
|
+
non-ZDR) never appear in the models array. Each value is
|
|
1410
|
+
{"quality", "quality_effort", "quality_as_of", "quality_source"},
|
|
1411
|
+
copied verbatim so the rebuild publishes the keyed run's recorded
|
|
1412
|
+
provenance unchanged.
|
|
1413
|
+
|
|
1414
|
+
A non-list schedules block yields an empty map. Individual malformed
|
|
1415
|
+
rows (not an object, no usable id, a non-finite/out-of-range quality, or
|
|
1416
|
+
a wrong-typed provenance field) are skipped rather than guessed; the
|
|
1417
|
+
model then falls through the existing tiers.
|
|
1418
|
+
|
|
1419
|
+
Schema gate: only a v5 catalog contributes schedule entries. v4/v3
|
|
1420
|
+
schedules predate ``quality_effort``, so hydrating them would mix the
|
|
1421
|
+
old max-effort semantics into a default-effort run (spec 9.1).
|
|
1422
|
+
"""
|
|
1423
|
+
if not isinstance(doc, dict):
|
|
1424
|
+
return {}
|
|
1425
|
+
schema_version = doc.get("schema_version")
|
|
1426
|
+
if (
|
|
1427
|
+
type(schema_version) is not int
|
|
1428
|
+
or schema_version != CATALOG_QUALITY_SCHEMA_VERSION
|
|
1429
|
+
):
|
|
1430
|
+
return {}
|
|
1431
|
+
schedules = doc.get("schedules")
|
|
1432
|
+
if not isinstance(schedules, list):
|
|
1433
|
+
return {}
|
|
1434
|
+
entries = {}
|
|
1435
|
+
for entry in schedules:
|
|
1436
|
+
if not isinstance(entry, dict):
|
|
1437
|
+
continue
|
|
1438
|
+
model_id = entry.get("id")
|
|
1439
|
+
if not (isinstance(model_id, str) and model_id):
|
|
1440
|
+
continue
|
|
1441
|
+
quality = entry.get("quality")
|
|
1442
|
+
if quality is not None:
|
|
1443
|
+
quality = _catalog_index(quality)
|
|
1444
|
+
if quality is None:
|
|
1445
|
+
continue
|
|
1446
|
+
quality_effort = entry.get("quality_effort")
|
|
1447
|
+
if quality_effort is not None and not (
|
|
1448
|
+
isinstance(quality_effort, str) and quality_effort
|
|
1449
|
+
):
|
|
1450
|
+
continue
|
|
1451
|
+
quality_as_of = entry.get("quality_as_of")
|
|
1452
|
+
if quality_as_of is not None and not isinstance(quality_as_of, str):
|
|
1453
|
+
continue
|
|
1454
|
+
quality_source = entry.get("quality_source")
|
|
1455
|
+
if quality_source is not None and not isinstance(quality_source, str):
|
|
1456
|
+
continue
|
|
1457
|
+
entries[model_id] = {
|
|
1458
|
+
"quality": quality,
|
|
1459
|
+
"quality_effort": quality_effort,
|
|
1460
|
+
"quality_as_of": quality_as_of,
|
|
1461
|
+
"quality_source": quality_source,
|
|
1322
1462
|
}
|
|
1323
|
-
return
|
|
1463
|
+
return entries
|
|
1324
1464
|
|
|
1325
1465
|
|
|
1326
1466
|
def fetch_quality_catalog(args) -> dict | None:
|
|
@@ -1330,47 +1470,60 @@ def fetch_quality_catalog(args) -> dict | None:
|
|
|
1330
1470
|
catalog artifact, built by keyed CI runs and hydrated by unkeyed local
|
|
1331
1471
|
runs. Never called on keyed runs -- a CI/site build hydrating from the
|
|
1332
1472
|
published catalog would consume its own output (self-reference loop);
|
|
1333
|
-
|
|
1473
|
+
run() owns that wiring.
|
|
1334
1474
|
|
|
1335
1475
|
Returns None only on TRANSPORT failure (after fetch_json's retries);
|
|
1336
|
-
otherwise the envelope {"by_id": {id: record}, "
|
|
1337
|
-
|
|
1476
|
+
otherwise the envelope {"by_id": {id: record}, "depths_by_id":
|
|
1477
|
+
{id: {"default_effort", "depths": {effort: record}}}, "generated_at":
|
|
1478
|
+
str|None, "schedules_by_id": {id: recorded schedule provenance}}. An
|
|
1479
|
+
empty by_id AND empty depths_by_id AND empty schedules_by_id means
|
|
1338
1480
|
fetched-but-unusable (schema gate fail, malformed shape) -- Task 4's
|
|
1339
|
-
footer treats None and empty
|
|
1481
|
+
footer treats None and empty maps identically ("catalog unavailable").
|
|
1340
1482
|
|
|
1341
1483
|
The envelope is cached under QUALITY_CATALOG_CACHE with the standard
|
|
1342
|
-
TTL
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1484
|
+
TTL. A usable result (non-empty by_id, depths_by_id or schedules_by_id)
|
|
1485
|
+
is written; so is a v4 catalog's deliberately empty envelope -- that is
|
|
1486
|
+
the stable deploy-skew state, and caching it avoids a refetch loop.
|
|
1487
|
+
Older/unknown schemas and malformed v5 payloads stay uncached so a
|
|
1488
|
+
degraded payload self-heals on the next run. A cached envelope from
|
|
1489
|
+
before the v5 depth contract lacks depths_by_id, and one from before
|
|
1490
|
+
schedule hydration lacks schedules_by_id; both are refetched rather
|
|
1491
|
+
than serving depth/schedule-blind quality. --no-cache neither reads nor
|
|
1492
|
+
writes. Staleness: generated_at older than QUALITY_CATALOG_MAX_AGE_DAYS
|
|
1493
|
+
warns but uses -- as_of makes the staleness visible downstream (the
|
|
1494
|
+
check runs only when rows actually hydrate -- warning about the age of
|
|
1495
|
+
a catalog we are not using is noise); an unusable generated_at warns
|
|
1496
|
+
that staleness cannot be checked and still uses the rows (their as_of
|
|
1497
|
+
then shows None rather than a wrong date).
|
|
1351
1498
|
"""
|
|
1352
1499
|
if not args.no_cache:
|
|
1353
1500
|
cached = load_cache(QUALITY_CATALOG_CACHE, args.cache_ttl)
|
|
1354
|
-
if
|
|
1501
|
+
if (
|
|
1502
|
+
isinstance(cached, dict)
|
|
1503
|
+
and isinstance(cached.get("by_id"), dict)
|
|
1504
|
+
and isinstance(cached.get("depths_by_id"), dict)
|
|
1505
|
+
and isinstance(cached.get("schedules_by_id"), dict)
|
|
1506
|
+
):
|
|
1355
1507
|
return cached
|
|
1356
1508
|
try:
|
|
1357
1509
|
doc = fetch_json(QUALITY_CATALOG_URL, timeout=30, retries=3)
|
|
1358
1510
|
except Exception as exc:
|
|
1359
1511
|
warn(f"published catalog request failed ({exc})")
|
|
1360
1512
|
return None
|
|
1361
|
-
by_id =
|
|
1513
|
+
by_id, depths_by_id = _parse_catalog_v5(doc)
|
|
1514
|
+
schedules_by_id = _catalog_schedule_entries(doc)
|
|
1362
1515
|
generated_at = doc.get("generated_at") if isinstance(doc, dict) else None
|
|
1363
1516
|
if not isinstance(generated_at, str):
|
|
1364
1517
|
generated_at = None
|
|
1365
|
-
schema_version = doc.get("schema_version") if isinstance(doc, dict) else
|
|
1366
|
-
if type(schema_version) is not int:
|
|
1367
|
-
schema_version = 0
|
|
1518
|
+
schema_version = doc.get("schema_version") if isinstance(doc, dict) else None
|
|
1368
1519
|
result = {
|
|
1369
1520
|
"by_id": by_id,
|
|
1521
|
+
"depths_by_id": depths_by_id,
|
|
1370
1522
|
"generated_at": generated_at,
|
|
1371
|
-
"
|
|
1523
|
+
"schedules_by_id": schedules_by_id,
|
|
1372
1524
|
}
|
|
1373
|
-
|
|
1525
|
+
usable = bool(by_id or depths_by_id or schedules_by_id)
|
|
1526
|
+
if usable:
|
|
1374
1527
|
generated = parse_iso_datetime(generated_at)
|
|
1375
1528
|
if generated is None:
|
|
1376
1529
|
warn(
|
|
@@ -1384,6 +1537,11 @@ def fetch_quality_catalog(args) -> dict | None:
|
|
|
1384
1537
|
f"published catalog is stale (generated {generated_at}, "
|
|
1385
1538
|
f"{age_days:.0f} days old); using it anyway"
|
|
1386
1539
|
)
|
|
1540
|
+
# A v4 catalog is the stable deploy-skew state: its envelope is
|
|
1541
|
+
# deliberately empty (no hydration), but caching it prevents a
|
|
1542
|
+
# refetch-every-run loop. Older/unknown schemas and malformed v5
|
|
1543
|
+
# payloads still self-heal by staying uncached.
|
|
1544
|
+
if usable or (type(schema_version) is int and schema_version == 4):
|
|
1387
1545
|
if not args.no_cache:
|
|
1388
1546
|
save_cache(QUALITY_CATALOG_CACHE, result)
|
|
1389
1547
|
return result
|
|
@@ -1411,23 +1569,31 @@ FRONTEND_DISCOUNTS_CACHE = "openrouter-frontend-discounts"
|
|
|
1411
1569
|
# v2: AA records are keyed by exact catalog slug and carry as_of (#17);
|
|
1412
1570
|
# v1 cached the conflated date-stripped shape, so the old key is never read.
|
|
1413
1571
|
FRONTEND_AA_CACHE = "openrouter-frontend-aa-v2"
|
|
1572
|
+
# The frontend listing projected to the reasoning fields (slug, permaslug,
|
|
1573
|
+
# supports_reasoning, reasoning_config). Cached separately so a fully cached
|
|
1574
|
+
# run keeps the reasoning fallback instead of degrading it to unknown.
|
|
1575
|
+
FRONTEND_REASONING_CACHE = "openrouter-frontend-reasoning-v1"
|
|
1414
1576
|
|
|
1415
1577
|
|
|
1416
1578
|
def fetch_openrouter_frontend(args):
|
|
1417
|
-
"""Derive discounts, ZDR ids
|
|
1418
|
-
|
|
1419
|
-
Two loads: the base frontend URL serves discounts
|
|
1420
|
-
response; the per-endpoint ZDR list
|
|
1421
|
-
under --no-zdr. Each payload caches
|
|
1422
|
-
cached only when non-empty, so a
|
|
1423
|
-
next run. A base-URL outage never
|
|
1424
|
-
|
|
1579
|
+
"""Derive discounts, ZDR ids, OR-published AA benchmarks and reasoning.
|
|
1580
|
+
|
|
1581
|
+
Two loads: the base frontend URL serves discounts, benchmarks and the
|
|
1582
|
+
per-model reasoning config in one response; the per-endpoint ZDR list
|
|
1583
|
+
serves ZDR and is skipped entirely under --no-zdr. Each payload caches
|
|
1584
|
+
independently under its own key and is cached only when non-empty, so a
|
|
1585
|
+
degraded payload self-heals on the next run. A base-URL outage never
|
|
1586
|
+
blocks the ZDR fetch, and vice versa. Returns
|
|
1587
|
+
(discounts, zdr_ids, aa_by_id, cache_hits, frontend_models) where
|
|
1425
1588
|
cache_hits names the payloads served from cache ("discounts", "zdr",
|
|
1426
|
-
"aa")
|
|
1589
|
+
"aa"), and frontend_models is the frontend listing projected to the
|
|
1590
|
+
fields build_reasoning_metadata needs (slug, permaslug,
|
|
1591
|
+
supports_reasoning, reasoning_config).
|
|
1427
1592
|
"""
|
|
1428
1593
|
discounts = {}
|
|
1429
1594
|
aa_by_id = {}
|
|
1430
1595
|
zdr_ids = set()
|
|
1596
|
+
frontend_models = []
|
|
1431
1597
|
cache_hits = set()
|
|
1432
1598
|
|
|
1433
1599
|
aa_from_cache = False
|
|
@@ -1441,6 +1607,9 @@ def fetch_openrouter_frontend(args):
|
|
|
1441
1607
|
aa_by_id = cached
|
|
1442
1608
|
aa_from_cache = True
|
|
1443
1609
|
cache_hits.add("aa")
|
|
1610
|
+
cached = load_cache(FRONTEND_REASONING_CACHE, args.cache_ttl)
|
|
1611
|
+
if cached:
|
|
1612
|
+
frontend_models = cached
|
|
1444
1613
|
|
|
1445
1614
|
if not discounts or not aa_from_cache:
|
|
1446
1615
|
try:
|
|
@@ -1490,21 +1659,31 @@ def fetch_openrouter_frontend(args):
|
|
|
1490
1659
|
"no AA benchmark entries found; treating OpenRouter "
|
|
1491
1660
|
"benchmarks as unavailable"
|
|
1492
1661
|
)
|
|
1662
|
+
fresh_reasoning = _frontend_reasoning_entries(entries)
|
|
1663
|
+
if fresh_reasoning:
|
|
1664
|
+
frontend_models = fresh_reasoning
|
|
1665
|
+
save_cache(FRONTEND_REASONING_CACHE, frontend_models)
|
|
1493
1666
|
|
|
1494
1667
|
if args.no_zdr:
|
|
1495
|
-
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
1668
|
+
return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
|
|
1496
1669
|
if not args.no_cache:
|
|
1497
1670
|
# v3: the id set now comes from the per-endpoint ZDR list; v1/v2
|
|
1498
1671
|
# sets were derived from the models/find response and must never
|
|
1499
1672
|
# be read.
|
|
1500
1673
|
cached = load_cache("openrouter-zdr-v3", args.cache_ttl)
|
|
1501
1674
|
if cached:
|
|
1502
|
-
return
|
|
1675
|
+
return (
|
|
1676
|
+
discounts,
|
|
1677
|
+
set(cached),
|
|
1678
|
+
aa_by_id,
|
|
1679
|
+
cache_hits | {"zdr"},
|
|
1680
|
+
frontend_models,
|
|
1681
|
+
)
|
|
1503
1682
|
try:
|
|
1504
1683
|
payload = fetch_json(OPENROUTER_ZDR_URL, timeout=30)
|
|
1505
1684
|
except Exception as exc:
|
|
1506
1685
|
warn(f"could not fetch ZDR data: {exc}")
|
|
1507
|
-
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
1686
|
+
return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
|
|
1508
1687
|
# One entry per ZDR endpoint, keyed by the exact model id (variants
|
|
1509
1688
|
# such as :free are listed individually). An id is ZDR iff it has at
|
|
1510
1689
|
# least one ZDR endpoint. The models/find ?zdr=true filter cannot be
|
|
@@ -1530,9 +1709,9 @@ def fetch_openrouter_frontend(args):
|
|
|
1530
1709
|
ids.add(model_id)
|
|
1531
1710
|
if not ids:
|
|
1532
1711
|
warn("no ZDR entries found; treating ZDR data as unavailable")
|
|
1533
|
-
return discounts, zdr_ids, aa_by_id, cache_hits
|
|
1712
|
+
return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
|
|
1534
1713
|
save_cache("openrouter-zdr-v3", sorted(ids))
|
|
1535
|
-
return discounts, ids, aa_by_id, cache_hits
|
|
1714
|
+
return discounts, ids, aa_by_id, cache_hits, frontend_models
|
|
1536
1715
|
|
|
1537
1716
|
|
|
1538
1717
|
def parse_price(value) -> float | None:
|
|
@@ -1959,18 +2138,146 @@ def parse_iso_datetime(value):
|
|
|
1959
2138
|
# Artificial Analysis intelligence index
|
|
1960
2139
|
# ---------------------------------------------------------------------------
|
|
1961
2140
|
|
|
2141
|
+
_AA_DISCRETE_EFFORTS = frozenset(("max", "xhigh", "high", "medium", "low", "minimal"))
|
|
2142
|
+
_AA_TRAILING_PAREN_RE = re.compile(r"\s*\(([^()]*)\)\s*$")
|
|
2143
|
+
_AA_SLUG_EFFORT_RE = re.compile(
|
|
2144
|
+
r"-(max|xhigh|high|medium|low|minimal|non-reasoning|reasoning)$", re.IGNORECASE
|
|
2145
|
+
)
|
|
2146
|
+
_AA_SLUG_EFFORT_LABELS = {
|
|
2147
|
+
"max": "max",
|
|
2148
|
+
"xhigh": "xhigh",
|
|
2149
|
+
"high": "high",
|
|
2150
|
+
"medium": "medium",
|
|
2151
|
+
"low": "low",
|
|
2152
|
+
"minimal": "minimal",
|
|
2153
|
+
"non-reasoning": "none",
|
|
2154
|
+
"reasoning": "reasoning",
|
|
2155
|
+
}
|
|
2156
|
+
# Shapes (normalized parenthetical text, or the no-label case) already
|
|
2157
|
+
# warned about. Spec §6.1: one warning per unknown row shape, not per row.
|
|
2158
|
+
_AA_WARNED_UNKNOWN_SHAPES: set[str] = set()
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
def _classify_aa_effort_tokens(raw: str) -> tuple[str | None, bool]:
|
|
2162
|
+
"""Classify one name parenthetical's comma-separated tokens.
|
|
2163
|
+
|
|
2164
|
+
"Default Fallback" is a qualifier, not a label. A parenthetical that
|
|
2165
|
+
names two different states -- two discrete efforts, or non-reasoning
|
|
2166
|
+
plus an effort -- is ambiguous: no effort, never guessed.
|
|
2167
|
+
"""
|
|
2168
|
+
labels = set()
|
|
2169
|
+
for token in (raw or "").split(","):
|
|
2170
|
+
token = token.strip().lower()
|
|
2171
|
+
if not token or token == "default fallback":
|
|
2172
|
+
continue
|
|
2173
|
+
if token in _AA_DISCRETE_EFFORTS:
|
|
2174
|
+
labels.add(token)
|
|
2175
|
+
elif token == "non-reasoning":
|
|
2176
|
+
labels.add("none")
|
|
2177
|
+
elif token == "reasoning":
|
|
2178
|
+
labels.add("reasoning")
|
|
2179
|
+
if len(labels) > 1:
|
|
2180
|
+
return None, True
|
|
2181
|
+
if labels:
|
|
2182
|
+
return labels.pop(), False
|
|
2183
|
+
return None, False
|
|
2184
|
+
|
|
2185
|
+
|
|
2186
|
+
def _warn_unknown_aa_shape(parenthetical: str | None) -> None:
|
|
2187
|
+
"""Warn once per distinct unrecognized effort shape (spec §6.1)."""
|
|
2188
|
+
if parenthetical is not None:
|
|
2189
|
+
detail = f"parenthetical {parenthetical!r}"
|
|
2190
|
+
shape = "parenthetical:" + norm_key(parenthetical)
|
|
2191
|
+
else:
|
|
2192
|
+
detail = "no effort label"
|
|
2193
|
+
shape = "no-effort-label"
|
|
2194
|
+
if shape in _AA_WARNED_UNKNOWN_SHAPES:
|
|
2195
|
+
return
|
|
2196
|
+
_AA_WARNED_UNKNOWN_SHAPES.add(shape)
|
|
2197
|
+
warn(f"AA row has an unrecognized effort shape ({detail}); row is display-only")
|
|
2198
|
+
|
|
2199
|
+
|
|
2200
|
+
def parse_aa_effort(name: str, slug: str) -> dict:
|
|
2201
|
+
"""Split an AA row's name/slug into its base model and effort label.
|
|
2202
|
+
|
|
2203
|
+
The name's trailing parenthetical is authoritative; when it names no
|
|
2204
|
+
effort (and is not ambiguous) the slug suffix is the fallback. Returns
|
|
2205
|
+
base_name/base_slug with the label stripped, effort (None when unknown
|
|
2206
|
+
or ambiguous), and ambiguous=True for contradictory labels such as
|
|
2207
|
+
"(Non-reasoning, High)" or "(High, Max)". Ambiguous and unknown rows
|
|
2208
|
+
are display-only: callers must never guess an effort for them.
|
|
2209
|
+
"""
|
|
2210
|
+
name = name if isinstance(name, str) else ""
|
|
2211
|
+
slug = slug if isinstance(slug, str) else ""
|
|
2212
|
+
|
|
2213
|
+
name_match = _AA_TRAILING_PAREN_RE.search(name)
|
|
2214
|
+
if name_match:
|
|
2215
|
+
base_name = name[: name_match.start()].strip()
|
|
2216
|
+
effort, ambiguous = _classify_aa_effort_tokens(name_match.group(1))
|
|
2217
|
+
else:
|
|
2218
|
+
base_name = name.strip()
|
|
2219
|
+
effort, ambiguous = None, False
|
|
2220
|
+
|
|
2221
|
+
slug_match = _AA_SLUG_EFFORT_RE.search(slug)
|
|
2222
|
+
if slug_match:
|
|
2223
|
+
base_slug = slug[: slug_match.start()]
|
|
2224
|
+
slug_effort = _AA_SLUG_EFFORT_LABELS[slug_match.group(1).lower()]
|
|
2225
|
+
else:
|
|
2226
|
+
base_slug = slug
|
|
2227
|
+
slug_effort = None
|
|
2228
|
+
|
|
2229
|
+
if effort is None and not ambiguous:
|
|
2230
|
+
effort = slug_effort
|
|
2231
|
+
|
|
2232
|
+
if effort is None and not ambiguous:
|
|
2233
|
+
_warn_unknown_aa_shape(name_match.group(1) if name_match is not None else None)
|
|
2234
|
+
|
|
2235
|
+
return {
|
|
2236
|
+
"base_name": base_name,
|
|
2237
|
+
"base_slug": base_slug,
|
|
2238
|
+
"effort": effort,
|
|
2239
|
+
"ambiguous": ambiguous,
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
def _finite_cost(value) -> float | None:
|
|
2244
|
+
"""A finite float cost, or None for absent/non-numeric/non-finite input."""
|
|
2245
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
|
2246
|
+
return None
|
|
2247
|
+
try:
|
|
2248
|
+
value = float(value) # huge ints raise OverflowError
|
|
2249
|
+
except OverflowError:
|
|
2250
|
+
return None
|
|
2251
|
+
return value if math.isfinite(value) else None
|
|
2252
|
+
|
|
2253
|
+
|
|
2254
|
+
def _aa_cost_per_task(item: dict) -> float | None:
|
|
2255
|
+
"""AA's measured cost per benchmark task, or None when not reported."""
|
|
2256
|
+
block = item.get("artificial_analysis_intelligence_index_cost")
|
|
2257
|
+
if not isinstance(block, dict):
|
|
2258
|
+
return None
|
|
2259
|
+
cost_per_task = block.get("cost_per_task")
|
|
2260
|
+
if not isinstance(cost_per_task, dict):
|
|
2261
|
+
return None
|
|
2262
|
+
return _finite_cost(cost_per_task.get("total_cost"))
|
|
2263
|
+
|
|
1962
2264
|
|
|
1963
2265
|
def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
|
|
1964
2266
|
"""Intelligence entries from AA's V2 language models endpoint.
|
|
1965
2267
|
|
|
1966
2268
|
Documented envelope: {tier, intelligence_index_version, pagination,
|
|
1967
2269
|
data[]} with snake_case fields; the index lives at
|
|
1968
|
-
data[].evaluations.artificial_analysis_intelligence_index
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
2270
|
+
data[].evaluations.artificial_analysis_intelligence_index and the
|
|
2271
|
+
measured benchmark cost at
|
|
2272
|
+
data[].artificial_analysis_intelligence_index_cost.cost_per_task.total_cost.
|
|
2273
|
+
AA returns one row per measured reasoning-effort variant; the effort is
|
|
2274
|
+
parsed from the row's name/slug by parse_aa_effort. Items without a
|
|
2275
|
+
usable slug/name or without a finite numeric index are skipped (nulls
|
|
2276
|
+
mean "not measured"). Rows are deduped by their AA `id` (fallback
|
|
2277
|
+
(slug, name)), never by slug alone: variants often share a slug.
|
|
2278
|
+
Pagination follows the documented has_more/total_pages contract with a
|
|
2279
|
+
25-page hard cap; a failure past page 1 is best-effort (warn, keep
|
|
2280
|
+
earlier pages), a page-1 failure propagates.
|
|
1974
2281
|
|
|
1975
2282
|
Returns (entries, version): version is the envelope's
|
|
1976
2283
|
intelligence_index_version (number-or-null) -- provenance for the whole
|
|
@@ -2030,18 +2337,36 @@ def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
|
|
|
2030
2337
|
continue
|
|
2031
2338
|
if not math.isfinite(index):
|
|
2032
2339
|
continue
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
)
|
|
2340
|
+
entry_id = item.get("id")
|
|
2341
|
+
entry_id = entry_id if isinstance(entry_id, str) and entry_id else None
|
|
2342
|
+
slug = slug if isinstance(slug, str) and slug else None
|
|
2343
|
+
raw_name = name if isinstance(name, str) and name else ""
|
|
2344
|
+
parsed = parse_aa_effort(raw_name, slug or "")
|
|
2345
|
+
name = raw_name or key
|
|
2346
|
+
dedupe_key = entry_id if entry_id else (slug, name)
|
|
2347
|
+
if dedupe_key in entries:
|
|
2348
|
+
continue
|
|
2349
|
+
creator = None
|
|
2350
|
+
raw_creator = item.get("model_creator")
|
|
2351
|
+
if isinstance(raw_creator, dict):
|
|
2352
|
+
raw_creator_name = raw_creator.get("name")
|
|
2353
|
+
if isinstance(raw_creator_name, str) and raw_creator_name:
|
|
2354
|
+
creator = raw_creator_name
|
|
2355
|
+
entries[dedupe_key] = {
|
|
2356
|
+
"key": key,
|
|
2357
|
+
"name": name,
|
|
2358
|
+
"index": index,
|
|
2359
|
+
# One fetch timestamp for the whole fetch: every entry
|
|
2360
|
+
# from this run shares it, so the cache payload and any
|
|
2361
|
+
# downstream record can date the data exactly.
|
|
2362
|
+
"as_of": as_of,
|
|
2363
|
+
"id": entry_id,
|
|
2364
|
+
"slug": slug,
|
|
2365
|
+
"base": parsed["base_slug"] or parsed["base_name"],
|
|
2366
|
+
"effort": parsed["effort"],
|
|
2367
|
+
"creator": creator,
|
|
2368
|
+
"cost_per_task": _aa_cost_per_task(item),
|
|
2369
|
+
}
|
|
2045
2370
|
pag = payload.get("pagination") if isinstance(payload, dict) else None
|
|
2046
2371
|
if not isinstance(pag, dict):
|
|
2047
2372
|
break # undocumented shape: single-page degrade
|
|
@@ -2070,10 +2395,10 @@ def fetch_aa_entries(args):
|
|
|
2070
2395
|
ones.
|
|
2071
2396
|
"""
|
|
2072
2397
|
if not args.no_cache:
|
|
2073
|
-
#
|
|
2074
|
-
#
|
|
2075
|
-
# Only AA API entries are read; build_catalog rejects any
|
|
2076
|
-
# source as unknown.
|
|
2398
|
+
# v4: payloads cached under earlier keys predate the per-effort
|
|
2399
|
+
# fields (id, base, effort, creator, cost_per_task) and must never
|
|
2400
|
+
# be read. Only AA API entries are read; build_catalog rejects any
|
|
2401
|
+
# other source as unknown.
|
|
2077
2402
|
cached = load_cache(AA_ENTRIES_CACHE, args.cache_ttl)
|
|
2078
2403
|
if isinstance(cached, dict) and cached.get("source") == "AA API v2":
|
|
2079
2404
|
return (
|
|
@@ -2105,6 +2430,26 @@ def fetch_aa_entries(args):
|
|
|
2105
2430
|
# ---------------------------------------------------------------------------
|
|
2106
2431
|
|
|
2107
2432
|
|
|
2433
|
+
def dedupe_aa_entries_by_key(entries):
|
|
2434
|
+
"""Restore the legacy lookup's one-entry-per-key contract.
|
|
2435
|
+
|
|
2436
|
+
The depth path keeps every effort variant (aa_api_entries dedupes by AA
|
|
2437
|
+
id), but build_aa_lookup expects the old slug-keyed selection: for each
|
|
2438
|
+
`key`, the first occurrence in list order wins. This is the seam between
|
|
2439
|
+
the per-effort ingestion and the unchanged legacy matcher.
|
|
2440
|
+
"""
|
|
2441
|
+
unique = []
|
|
2442
|
+
seen = set()
|
|
2443
|
+
for entry in entries:
|
|
2444
|
+
key = entry.get("key")
|
|
2445
|
+
if key:
|
|
2446
|
+
if key in seen:
|
|
2447
|
+
continue
|
|
2448
|
+
seen.add(key)
|
|
2449
|
+
unique.append(entry)
|
|
2450
|
+
return unique
|
|
2451
|
+
|
|
2452
|
+
|
|
2108
2453
|
def build_aa_lookup(entries):
|
|
2109
2454
|
"""Index the AA entries into exact and fuzzy lookup maps.
|
|
2110
2455
|
|
|
@@ -2145,6 +2490,167 @@ def build_aa_lookup(entries):
|
|
|
2145
2490
|
return exact, fuzzy
|
|
2146
2491
|
|
|
2147
2492
|
|
|
2493
|
+
def build_aa_depth_lookup(entries):
|
|
2494
|
+
"""Index AA entries as base key -> effort -> depth record.
|
|
2495
|
+
|
|
2496
|
+
Each record is normalized for downstream publication:
|
|
2497
|
+
{"effort", "index", "cost_per_task_usd", "source": "aa", "as_of"}.
|
|
2498
|
+
Rows whose effort is absent or ambiguous are display-only and skipped.
|
|
2499
|
+
Duplicate (base key, effort) records from different AA rows are stored
|
|
2500
|
+
as None (ambiguous: matched as absent), never guessed. Base keys come
|
|
2501
|
+
from the name and slug with the effort label stripped, both
|
|
2502
|
+
unqualified and creator-qualified ("<creator_norm>:<base_key>"), where
|
|
2503
|
+
creator_norm is normalize_provider(model_creator.name).
|
|
2504
|
+
"""
|
|
2505
|
+
lookup = {}
|
|
2506
|
+
seen = set()
|
|
2507
|
+
for entry in entries:
|
|
2508
|
+
if not isinstance(entry, dict):
|
|
2509
|
+
continue
|
|
2510
|
+
entry_id = entry.get("id")
|
|
2511
|
+
dedupe_key = (
|
|
2512
|
+
entry_id
|
|
2513
|
+
if isinstance(entry_id, str) and entry_id
|
|
2514
|
+
else (entry.get("slug"), entry.get("name"))
|
|
2515
|
+
)
|
|
2516
|
+
if dedupe_key in seen:
|
|
2517
|
+
continue
|
|
2518
|
+
parsed = parse_aa_effort(entry.get("name") or "", entry.get("slug") or "")
|
|
2519
|
+
effort = parsed["effort"]
|
|
2520
|
+
if not effort or parsed["ambiguous"]:
|
|
2521
|
+
continue
|
|
2522
|
+
raw_index = entry.get("index")
|
|
2523
|
+
if not isinstance(raw_index, (int, float)) or isinstance(raw_index, bool):
|
|
2524
|
+
continue
|
|
2525
|
+
index = float(raw_index)
|
|
2526
|
+
if not math.isfinite(index):
|
|
2527
|
+
continue
|
|
2528
|
+
seen.add(dedupe_key)
|
|
2529
|
+
record = {
|
|
2530
|
+
"effort": effort,
|
|
2531
|
+
"index": index,
|
|
2532
|
+
"cost_per_task_usd": _finite_cost(entry.get("cost_per_task")),
|
|
2533
|
+
"source": "aa",
|
|
2534
|
+
"as_of": (
|
|
2535
|
+
entry.get("as_of") if isinstance(entry.get("as_of"), str) else None
|
|
2536
|
+
),
|
|
2537
|
+
}
|
|
2538
|
+
base_keys = []
|
|
2539
|
+
for base in (parsed["base_name"], parsed["base_slug"]):
|
|
2540
|
+
normalized = norm_key(base)
|
|
2541
|
+
if normalized and normalized not in base_keys:
|
|
2542
|
+
base_keys.append(normalized)
|
|
2543
|
+
creator = entry.get("creator")
|
|
2544
|
+
if not isinstance(creator, str) or not creator:
|
|
2545
|
+
raw_creator = entry.get("model_creator")
|
|
2546
|
+
if isinstance(raw_creator, dict) and isinstance(
|
|
2547
|
+
raw_creator.get("name"), str
|
|
2548
|
+
):
|
|
2549
|
+
creator = raw_creator["name"]
|
|
2550
|
+
creator_norm = normalize_provider(creator) if isinstance(creator, str) else ""
|
|
2551
|
+
keys = list(base_keys)
|
|
2552
|
+
if creator_norm:
|
|
2553
|
+
keys.extend(f"{creator_norm}:{base_key}" for base_key in base_keys)
|
|
2554
|
+
for base_key in keys:
|
|
2555
|
+
efforts = lookup.setdefault(base_key, {})
|
|
2556
|
+
if effort in efforts:
|
|
2557
|
+
efforts[effort] = None
|
|
2558
|
+
else:
|
|
2559
|
+
efforts[effort] = record
|
|
2560
|
+
return lookup
|
|
2561
|
+
|
|
2562
|
+
|
|
2563
|
+
def _depth_group(model, lookup):
|
|
2564
|
+
"""The candidate's matching base group (effort -> record), or None.
|
|
2565
|
+
|
|
2566
|
+
Exact tiers only, in the legacy order: full id, base id, display name,
|
|
2567
|
+
display name sans parentheticals. At each tier the creator-qualified
|
|
2568
|
+
key (the OpenRouter author prefix normalized with normalize_provider)
|
|
2569
|
+
is tried before the unqualified key. The first tier that resolves a
|
|
2570
|
+
base group wins; a tier whose group lacks the requested effort yields
|
|
2571
|
+
nothing rather than falling through to a lower tier. Ambiguous
|
|
2572
|
+
(base, effort) records are stored as None by build_aa_depth_lookup and
|
|
2573
|
+
match as absent; exact-key lookup means one tier cannot otherwise
|
|
2574
|
+
resolve to multiple base groups. The fuzzy pass is never used here --
|
|
2575
|
+
it is the #17 conflation bug.
|
|
2576
|
+
"""
|
|
2577
|
+
if not isinstance(lookup, dict):
|
|
2578
|
+
return None
|
|
2579
|
+
model_id = model.get("id") or ""
|
|
2580
|
+
name = model.get("name") or ""
|
|
2581
|
+
display = PROVIDER_PREFIX_RE.sub("", name).strip()
|
|
2582
|
+
without_variant = model_id.split(":", 1)[0]
|
|
2583
|
+
_, _, base = without_variant.partition("/")
|
|
2584
|
+
creator = normalize_provider(without_variant.partition("/")[0])
|
|
2585
|
+
for raw in (model_id, base, display, PAREN_RE.sub(" ", display)):
|
|
2586
|
+
normalized = norm_key(raw)
|
|
2587
|
+
if not normalized:
|
|
2588
|
+
continue
|
|
2589
|
+
if creator:
|
|
2590
|
+
qualified = lookup.get(f"{creator}:{normalized}")
|
|
2591
|
+
if qualified is not None:
|
|
2592
|
+
return qualified
|
|
2593
|
+
group = lookup.get(normalized)
|
|
2594
|
+
if group is not None:
|
|
2595
|
+
return group
|
|
2596
|
+
return None
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
def match_depth_quality(model, lookup, effort):
|
|
2600
|
+
"""The depth record at `effort` for the candidate, or None.
|
|
2601
|
+
|
|
2602
|
+
Strict selection: a matched base group with no record at this exact
|
|
2603
|
+
effort yields None. Another effort is never substituted, and the
|
|
2604
|
+
caller must not fall through to a lower tier.
|
|
2605
|
+
"""
|
|
2606
|
+
if not effort:
|
|
2607
|
+
return None
|
|
2608
|
+
group = _depth_group(model, lookup)
|
|
2609
|
+
if group is None:
|
|
2610
|
+
return None
|
|
2611
|
+
return group.get(effort)
|
|
2612
|
+
|
|
2613
|
+
|
|
2614
|
+
def _ordered_depth_records(group):
|
|
2615
|
+
"""Measured depth records from an effort-keyed group, display-ordered.
|
|
2616
|
+
|
|
2617
|
+
Skips ambiguous/absent entries (a None record) and any record without a
|
|
2618
|
+
finite index. Every record carries the internal shape resolve_quality
|
|
2619
|
+
attaches to cand["reasoning_depths"]:
|
|
2620
|
+
{"effort", "index", "cost_per_task_usd", "source", "as_of"}.
|
|
2621
|
+
"""
|
|
2622
|
+
records = []
|
|
2623
|
+
for effort, record in (group or {}).items():
|
|
2624
|
+
if not isinstance(record, dict):
|
|
2625
|
+
continue
|
|
2626
|
+
index = record.get("index")
|
|
2627
|
+
if (
|
|
2628
|
+
not isinstance(index, (int, float))
|
|
2629
|
+
or isinstance(index, bool)
|
|
2630
|
+
or not math.isfinite(index)
|
|
2631
|
+
):
|
|
2632
|
+
continue
|
|
2633
|
+
records.append(
|
|
2634
|
+
{
|
|
2635
|
+
"effort": record.get("effort") or effort,
|
|
2636
|
+
"index": float(index),
|
|
2637
|
+
"cost_per_task_usd": record.get("cost_per_task_usd"),
|
|
2638
|
+
"source": record.get("source"),
|
|
2639
|
+
"as_of": record.get("as_of"),
|
|
2640
|
+
}
|
|
2641
|
+
)
|
|
2642
|
+
|
|
2643
|
+
def order(record):
|
|
2644
|
+
effort = record["effort"]
|
|
2645
|
+
try:
|
|
2646
|
+
return (AA_EFFORT_ORDER.index(effort), effort)
|
|
2647
|
+
except ValueError:
|
|
2648
|
+
return (len(AA_EFFORT_ORDER), effort or "")
|
|
2649
|
+
|
|
2650
|
+
records.sort(key=order)
|
|
2651
|
+
return records
|
|
2652
|
+
|
|
2653
|
+
|
|
2148
2654
|
def match_quality(model, exact, fuzzy, allow_fuzzy=False):
|
|
2149
2655
|
"""Match a candidate to an AA entry; exact tiers only unless fuzzy is on.
|
|
2150
2656
|
|
|
@@ -2193,6 +2699,67 @@ def base_model_id(model_id: str) -> str:
|
|
|
2193
2699
|
AA_BENCHMARK_FIELDS = ("intelligence_index", "coding_index", "agentic_index")
|
|
2194
2700
|
|
|
2195
2701
|
|
|
2702
|
+
def _pair_permaslug_payloads(valid, models):
|
|
2703
|
+
"""Resolve permaslug-keyed payloads to exact catalog slugs.
|
|
2704
|
+
|
|
2705
|
+
``valid`` maps a permaslug (or an undated catalog id) to
|
|
2706
|
+
``(payload, date)``, where ``date`` is the key's trailing -YYYYMMDD or
|
|
2707
|
+
"" for an undated key. ``models`` carries the catalog's own
|
|
2708
|
+
slug/permaslug pairing. The three passes are the #17-safe pairing
|
|
2709
|
+
documented on build_aa_benchmarks: permaslug pairs first (latest date
|
|
2710
|
+
wins per slug), then undated keys self-map, then an unambiguous dated
|
|
2711
|
+
strip. Returns ``{slug: (payload, date)}``.
|
|
2712
|
+
"""
|
|
2713
|
+
permaslug_to_slug = {}
|
|
2714
|
+
for entry in models or []:
|
|
2715
|
+
if not isinstance(entry, dict):
|
|
2716
|
+
continue
|
|
2717
|
+
slug = entry.get("slug") or ""
|
|
2718
|
+
permaslug = entry.get("permaslug") or ""
|
|
2719
|
+
if slug and permaslug:
|
|
2720
|
+
# standard and batch variants repeat an identical pair; the
|
|
2721
|
+
# overwrite is the dedup
|
|
2722
|
+
permaslug_to_slug[permaslug] = slug
|
|
2723
|
+
|
|
2724
|
+
paired = {}
|
|
2725
|
+
claimed = set()
|
|
2726
|
+
|
|
2727
|
+
# Pass 1: permaslug keys pair with their own slug; same-slug
|
|
2728
|
+
# snapshots resolve to the latest date (first wins on an equal date).
|
|
2729
|
+
seen_date = {}
|
|
2730
|
+
for key, (payload, date) in valid.items():
|
|
2731
|
+
slug = permaslug_to_slug.get(key)
|
|
2732
|
+
if slug is None or (slug in seen_date and seen_date[slug] >= date):
|
|
2733
|
+
continue
|
|
2734
|
+
seen_date[slug] = date
|
|
2735
|
+
paired[slug] = (payload, date)
|
|
2736
|
+
claimed.add(slug)
|
|
2737
|
+
|
|
2738
|
+
# Pass 2: undated keys self-map unless pass 1 claimed the id.
|
|
2739
|
+
for key, (payload, date) in valid.items():
|
|
2740
|
+
if date or key in permaslug_to_slug or key in claimed:
|
|
2741
|
+
continue
|
|
2742
|
+
paired[key] = (payload, date)
|
|
2743
|
+
claimed.add(key)
|
|
2744
|
+
|
|
2745
|
+
# Pass 3: unpaired dated keys strip, but only onto an unclaimed id
|
|
2746
|
+
# and only when the strip is unambiguous.
|
|
2747
|
+
strip_candidates = {}
|
|
2748
|
+
for key, (payload, date) in valid.items():
|
|
2749
|
+
if not date or key in permaslug_to_slug:
|
|
2750
|
+
continue
|
|
2751
|
+
stripped = strip_permaslug_date(key)
|
|
2752
|
+
strip_candidates.setdefault(stripped, []).append((date, payload))
|
|
2753
|
+
for bare, candidates in strip_candidates.items():
|
|
2754
|
+
if bare in claimed or len(candidates) > 1:
|
|
2755
|
+
continue
|
|
2756
|
+
date, payload = candidates[0]
|
|
2757
|
+
paired[bare] = (payload, date)
|
|
2758
|
+
claimed.add(bare)
|
|
2759
|
+
|
|
2760
|
+
return paired
|
|
2761
|
+
|
|
2762
|
+
|
|
2196
2763
|
def build_aa_benchmarks(benchmarks, models=None) -> dict:
|
|
2197
2764
|
"""Map exact catalog slug -> OR-published AA trio from data.benchmarks.
|
|
2198
2765
|
|
|
@@ -2244,131 +2811,421 @@ def build_aa_benchmarks(benchmarks, models=None) -> dict:
|
|
|
2244
2811
|
# the key was dated; the date is then the final 8 characters.
|
|
2245
2812
|
valid[key] = (trio, "" if bare == key else key[-8:])
|
|
2246
2813
|
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
slug = entry.get("slug") or ""
|
|
2252
|
-
permaslug = entry.get("permaslug") or ""
|
|
2253
|
-
if slug and permaslug:
|
|
2254
|
-
# standard and batch variants repeat an identical pair; the
|
|
2255
|
-
# overwrite is the dedup
|
|
2256
|
-
permaslug_to_slug[permaslug] = slug
|
|
2814
|
+
paired = _pair_permaslug_payloads(valid, models)
|
|
2815
|
+
return {
|
|
2816
|
+
slug: {**trio, "as_of": date or None} for slug, (trio, date) in paired.items()
|
|
2817
|
+
}
|
|
2257
2818
|
|
|
2258
|
-
aa_by_id = {}
|
|
2259
|
-
claimed = set()
|
|
2260
2819
|
|
|
2261
|
-
|
|
2262
|
-
|
|
2820
|
+
FRONTEND_REASONING_FIELDS = (
|
|
2821
|
+
"slug",
|
|
2822
|
+
"permaslug",
|
|
2823
|
+
"supports_reasoning",
|
|
2824
|
+
"reasoning_config",
|
|
2825
|
+
)
|
|
2263
2826
|
|
|
2264
|
-
# Pass 1: permaslug keys pair with their own slug; same-slug
|
|
2265
|
-
# snapshots resolve to the latest date (first wins on an equal date).
|
|
2266
|
-
seen_date = {}
|
|
2267
|
-
for key, (trio, date) in valid.items():
|
|
2268
|
-
slug = permaslug_to_slug.get(key)
|
|
2269
|
-
if slug is None or (slug in seen_date and seen_date[slug] >= date):
|
|
2270
|
-
continue
|
|
2271
|
-
seen_date[slug] = date
|
|
2272
|
-
aa_by_id[slug] = record(trio, date)
|
|
2273
|
-
claimed.add(slug)
|
|
2274
2827
|
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2828
|
+
def _frontend_reasoning_entries(entries):
|
|
2829
|
+
"""Project frontend model entries to the fields reasoning metadata needs."""
|
|
2830
|
+
projected = []
|
|
2831
|
+
for entry in entries or []:
|
|
2832
|
+
if not isinstance(entry, dict):
|
|
2278
2833
|
continue
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
# Pass 3: unpaired dated keys strip, but only onto an unclaimed id
|
|
2283
|
-
# and only when the strip is unambiguous.
|
|
2284
|
-
strip_candidates = {}
|
|
2285
|
-
for key, (trio, date) in valid.items():
|
|
2286
|
-
if not date or key in permaslug_to_slug:
|
|
2834
|
+
slug = entry.get("slug") or ""
|
|
2835
|
+
if slug.startswith("~"):
|
|
2287
2836
|
continue
|
|
2288
|
-
|
|
2289
|
-
strip_candidates.setdefault(stripped, []).append((date, trio))
|
|
2290
|
-
for bare, candidates in strip_candidates.items():
|
|
2291
|
-
if bare in claimed or len(candidates) > 1:
|
|
2837
|
+
if not (slug or entry.get("permaslug")):
|
|
2292
2838
|
continue
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2839
|
+
projected.append(
|
|
2840
|
+
{field: entry.get(field) for field in FRONTEND_REASONING_FIELDS}
|
|
2841
|
+
)
|
|
2842
|
+
return projected
|
|
2296
2843
|
|
|
2297
|
-
return aa_by_id
|
|
2298
2844
|
|
|
2845
|
+
def _effort_list(value):
|
|
2846
|
+
"""Expand a supported-efforts field; null means all gateway efforts."""
|
|
2847
|
+
if value is None:
|
|
2848
|
+
return list(REASONING_GATEWAY_EFFORTS)
|
|
2849
|
+
if not isinstance(value, (list, tuple)):
|
|
2850
|
+
return []
|
|
2851
|
+
return [effort for effort in value if isinstance(effort, str)]
|
|
2852
|
+
|
|
2853
|
+
|
|
2854
|
+
def _reasoning_field(primary, config, primary_key, frontend_key):
|
|
2855
|
+
"""Field precedence: models API field, then frontend config, then unknown."""
|
|
2856
|
+
if isinstance(primary, dict) and primary_key in primary:
|
|
2857
|
+
return primary[primary_key]
|
|
2858
|
+
if isinstance(config, dict) and frontend_key in config:
|
|
2859
|
+
return config[frontend_key]
|
|
2860
|
+
return None
|
|
2299
2861
|
|
|
2300
|
-
def resolve_quality(
|
|
2301
|
-
candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality=None
|
|
2302
|
-
):
|
|
2303
|
-
"""Per-candidate quality records: AA API primary, catalog next, OR last.
|
|
2304
2862
|
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
overriding the row's own "aa"/"openrouter" label -- that label is
|
|
2313
|
-
the keyed CI run's row-level truth inside the artifact, and no
|
|
2314
|
-
real catalog row ever carries "catalog". as_of stays the row's
|
|
2315
|
-
own stamp.
|
|
2316
|
-
3. OpenRouter's benchmark trio -- per-slug, matched via
|
|
2317
|
-
base_model_id, so a :variant suffix inherits its base's trio.
|
|
2318
|
-
4. No match -- the candidate stays quality-blind downstream.
|
|
2319
|
-
|
|
2320
|
-
Every winner is a record {"index", "source", "as_of", "drift",
|
|
2321
|
-
"or_index", "or_as_of"}; compute_scores and build_schedules read
|
|
2322
|
-
record["index"], so cand["quality"] stays the float index they already
|
|
2323
|
-
expect. Drift is computed only when the winner is NOT OpenRouter: the
|
|
2324
|
-
OR trio for the same base model acts as the check value, and a
|
|
2325
|
-
disagreement of >= 3.0 points flags the record. When the winner IS
|
|
2326
|
-
OpenRouter (or no OR value exists) the drift pair is None -- there is
|
|
2327
|
-
no second source to disagree with.
|
|
2863
|
+
def _optional_bool(value, field):
|
|
2864
|
+
"""A published boolean field: bool, or null when the source is unknown.
|
|
2865
|
+
|
|
2866
|
+
Spec 9.1 publishes ``supports_max_tokens`` as bool or null. The old
|
|
2867
|
+
bool() coercion conflated an omitted/unknown field with false. Explicit
|
|
2868
|
+
booleans pass through; common string forms are coerced; anything else
|
|
2869
|
+
warns and degrades to null (fail closed, never guess).
|
|
2328
2870
|
"""
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
# every hydrated record is tagged "catalog" -- read from our
|
|
2342
|
-
# published catalog -- overriding the row's own label, which is
|
|
2343
|
-
# the keyed CI run's row-level truth inside the artifact ("aa"/
|
|
2344
|
-
# "openrouter"; no real catalog row ever carries "catalog").
|
|
2345
|
-
# as_of stays the row's own stamp (v4 rows carry the CI run's
|
|
2346
|
-
# recorded quality_as_of; v3 rows the catalog's generated_at --
|
|
2347
|
-
# catalog_quality_by_id normalizes that).
|
|
2348
|
-
if record is None and catalog_quality is not None:
|
|
2349
|
-
catalog_record = catalog_quality.get(cand["id"])
|
|
2350
|
-
if catalog_record is not None:
|
|
2351
|
-
record = {
|
|
2352
|
-
"index": catalog_record["index"],
|
|
2353
|
-
"source": "catalog",
|
|
2354
|
-
"as_of": catalog_record["as_of"],
|
|
2355
|
-
}
|
|
2356
|
-
# Tier 3: OpenRouter's per-slug benchmark trio (last resort).
|
|
2357
|
-
if record is None:
|
|
2358
|
-
bench = aa_by_id.get(base_model_id(cand["id"]))
|
|
2359
|
-
intelligence = bench.get("intelligence_index") if bench else None
|
|
2360
|
-
if intelligence is not None:
|
|
2361
|
-
record = {
|
|
2362
|
-
"index": intelligence,
|
|
2363
|
-
"source": "openrouter",
|
|
2364
|
-
"as_of": bench.get("as_of"),
|
|
2365
|
-
}
|
|
2366
|
-
if record is None:
|
|
2367
|
-
continue
|
|
2871
|
+
if value is None:
|
|
2872
|
+
return None
|
|
2873
|
+
if isinstance(value, bool):
|
|
2874
|
+
return value
|
|
2875
|
+
if isinstance(value, str):
|
|
2876
|
+
lowered = value.strip().lower()
|
|
2877
|
+
if lowered in ("true", "1", "yes"):
|
|
2878
|
+
return True
|
|
2879
|
+
if lowered in ("false", "0", "no"):
|
|
2880
|
+
return False
|
|
2881
|
+
warn(f"unexpected reasoning.{field} value {value!r}; treating it as unknown")
|
|
2882
|
+
return None
|
|
2368
2883
|
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2884
|
+
|
|
2885
|
+
def _optional_effort(value):
|
|
2886
|
+
"""A published effort label: a non-empty string, or null when unknown.
|
|
2887
|
+
|
|
2888
|
+
Malformed external metadata must degrade to unknown rather than poison
|
|
2889
|
+
the published catalog (the catalog validator rejects a non-string
|
|
2890
|
+
default_effort, which would fail the whole artifact).
|
|
2891
|
+
"""
|
|
2892
|
+
if value is None:
|
|
2893
|
+
return None
|
|
2894
|
+
if isinstance(value, str) and value.strip():
|
|
2895
|
+
return value
|
|
2896
|
+
warn(f"unexpected reasoning.default_effort value {value!r}; treating it as unknown")
|
|
2897
|
+
return None
|
|
2898
|
+
|
|
2899
|
+
|
|
2900
|
+
def _supported_efforts(primary, config):
|
|
2901
|
+
"""The expanded effort list; an omitted field stays empty.
|
|
2902
|
+
|
|
2903
|
+
The models API object is authoritative for effort selection: when it is
|
|
2904
|
+
present but omits supported_efforts, the model has no discrete efforts
|
|
2905
|
+
(single-state), even if the frontend's null "all efforts" marker would
|
|
2906
|
+
expand. The frontend list is only consulted when the primary object is
|
|
2907
|
+
absent entirely.
|
|
2908
|
+
"""
|
|
2909
|
+
if isinstance(primary, dict):
|
|
2910
|
+
if "supported_efforts" in primary:
|
|
2911
|
+
return _effort_list(primary["supported_efforts"])
|
|
2912
|
+
return []
|
|
2913
|
+
if isinstance(config, dict) and "supported_reasoning_efforts" in config:
|
|
2914
|
+
return _effort_list(config["supported_reasoning_efforts"])
|
|
2915
|
+
return []
|
|
2916
|
+
|
|
2917
|
+
|
|
2918
|
+
def effective_default_effort(reasoning: dict | None) -> str | None:
|
|
2919
|
+
"""The effort a request with no ``reasoning`` field actually runs at (§5.2).
|
|
2920
|
+
|
|
2921
|
+
Single-state/token-budget models (no discrete efforts after expansion)
|
|
2922
|
+
resolve to ``reasoning`` or ``none`` by ``default_enabled``; that branch
|
|
2923
|
+
runs before the discrete-effort rules. Non-reasoning models are ``none``;
|
|
2924
|
+
missing metadata is unknown (None).
|
|
2925
|
+
"""
|
|
2926
|
+
if not isinstance(reasoning, dict):
|
|
2927
|
+
return None
|
|
2928
|
+
if reasoning.get("is_reasoning") is False:
|
|
2929
|
+
return "none"
|
|
2930
|
+
if reasoning.get("is_reasoning") is not True:
|
|
2931
|
+
return None
|
|
2932
|
+
efforts = reasoning.get("supported_efforts") or []
|
|
2933
|
+
mandatory = reasoning.get("mandatory")
|
|
2934
|
+
default_enabled = reasoning.get("default_enabled")
|
|
2935
|
+
if mandatory is True and default_enabled is False:
|
|
2936
|
+
warn(
|
|
2937
|
+
"contradictory reasoning metadata: mandatory=true with "
|
|
2938
|
+
"default_enabled=false; treating the effective default as unknown"
|
|
2939
|
+
)
|
|
2940
|
+
return None
|
|
2941
|
+
if not efforts:
|
|
2942
|
+
# Single-state or token-budget model: reasoning is one state.
|
|
2943
|
+
return "reasoning" if default_enabled is not False else "none"
|
|
2944
|
+
if default_enabled is False:
|
|
2945
|
+
return "none"
|
|
2946
|
+
default_effort = reasoning.get("default_effort")
|
|
2947
|
+
if default_effort == "none":
|
|
2948
|
+
return "none"
|
|
2949
|
+
if isinstance(default_effort, str) and default_effort:
|
|
2950
|
+
return default_effort
|
|
2951
|
+
return None
|
|
2952
|
+
|
|
2953
|
+
|
|
2954
|
+
def supports_effort(reasoning: dict | None, effort: str) -> bool:
|
|
2955
|
+
"""Whether a model can run ``effort`` (the §7 filter rule)."""
|
|
2956
|
+
if not isinstance(reasoning, dict):
|
|
2957
|
+
return False
|
|
2958
|
+
efforts = reasoning.get("supported_efforts") or []
|
|
2959
|
+
if effort in efforts:
|
|
2960
|
+
return True
|
|
2961
|
+
if effort == "none":
|
|
2962
|
+
if reasoning.get("is_reasoning") is False:
|
|
2963
|
+
return True
|
|
2964
|
+
# A non-mandatory single-state model exposes a reasoning toggle.
|
|
2965
|
+
return (
|
|
2966
|
+
not efforts
|
|
2967
|
+
and reasoning.get("is_reasoning") is True
|
|
2968
|
+
and reasoning.get("mandatory") is not True
|
|
2969
|
+
)
|
|
2970
|
+
if effort == "reasoning":
|
|
2971
|
+
return not efforts and reasoning.get("is_reasoning") is True
|
|
2972
|
+
return False
|
|
2973
|
+
|
|
2974
|
+
|
|
2975
|
+
def build_reasoning_metadata(
|
|
2976
|
+
models: list[dict], frontend_models: list[dict] | None = None
|
|
2977
|
+
) -> dict[str, dict]:
|
|
2978
|
+
"""Per-model reasoning metadata, models API first and frontend second.
|
|
2979
|
+
|
|
2980
|
+
Each value carries the raw flags for provenance plus the *effective*
|
|
2981
|
+
default effort (§5.2) and the expanded supported-effort list. Frontend
|
|
2982
|
+
entries are keyed by slug/permaslug, so they attach to a candidate id
|
|
2983
|
+
through the same three-pass pairing build_aa_benchmarks uses.
|
|
2984
|
+
"""
|
|
2985
|
+
models = [
|
|
2986
|
+
model for model in models or [] if isinstance(model, dict) and model.get("id")
|
|
2987
|
+
]
|
|
2988
|
+
frontend_models = _frontend_reasoning_entries(frontend_models)
|
|
2989
|
+
|
|
2990
|
+
valid = {}
|
|
2991
|
+
for entry in frontend_models:
|
|
2992
|
+
slug = entry.get("slug") or ""
|
|
2993
|
+
permaslug = entry.get("permaslug") or ""
|
|
2994
|
+
key = permaslug or slug
|
|
2995
|
+
if not key:
|
|
2996
|
+
continue
|
|
2997
|
+
bare = strip_permaslug_date(key)
|
|
2998
|
+
valid[key] = (entry, "" if bare == key else key[-8:])
|
|
2999
|
+
if slug and slug != key:
|
|
3000
|
+
# The undated slug is the exact-id self-map (pass 2). Keep it
|
|
3001
|
+
# even when a dated permaslug is present, so a listing whose
|
|
3002
|
+
# permaslug is unpaired still resolves by its own id.
|
|
3003
|
+
valid[slug] = (entry, "")
|
|
3004
|
+
|
|
3005
|
+
catalog_pairs = [
|
|
3006
|
+
{"slug": model["id"], "permaslug": model.get("canonical_slug") or ""}
|
|
3007
|
+
for model in models
|
|
3008
|
+
]
|
|
3009
|
+
frontend_by_id = {
|
|
3010
|
+
candidate_id: entry
|
|
3011
|
+
for candidate_id, (entry, _date) in _pair_permaslug_payloads(
|
|
3012
|
+
valid, catalog_pairs
|
|
3013
|
+
).items()
|
|
3014
|
+
}
|
|
3015
|
+
# Exact-slug self-map. The permaslug pairing is authoritative when it
|
|
3016
|
+
# resolves a listing, but a shared canonical slug (base and :batch) is
|
|
3017
|
+
# claimed by only one listing, so the other still needs its own slug.
|
|
3018
|
+
frontend_by_slug = {
|
|
3019
|
+
entry["slug"]: entry for entry in frontend_models if entry.get("slug")
|
|
3020
|
+
}
|
|
3021
|
+
|
|
3022
|
+
metadata = {}
|
|
3023
|
+
for model in models:
|
|
3024
|
+
model_id = model["id"]
|
|
3025
|
+
primary = model.get("reasoning")
|
|
3026
|
+
if not isinstance(primary, dict):
|
|
3027
|
+
primary = None
|
|
3028
|
+
entry = frontend_by_id.get(model_id) or frontend_by_slug.get(model_id)
|
|
3029
|
+
config = None
|
|
3030
|
+
supports_reasoning = None
|
|
3031
|
+
if isinstance(entry, dict):
|
|
3032
|
+
config = entry.get("reasoning_config")
|
|
3033
|
+
if not isinstance(config, dict):
|
|
3034
|
+
config = None
|
|
3035
|
+
supports_reasoning = entry.get("supports_reasoning")
|
|
3036
|
+
|
|
3037
|
+
if primary is not None:
|
|
3038
|
+
is_reasoning = True
|
|
3039
|
+
elif isinstance(supports_reasoning, bool):
|
|
3040
|
+
is_reasoning = supports_reasoning
|
|
3041
|
+
else:
|
|
3042
|
+
is_reasoning = None
|
|
3043
|
+
|
|
3044
|
+
reasoning = {
|
|
3045
|
+
"default_effort": _optional_effort(
|
|
3046
|
+
_reasoning_field(
|
|
3047
|
+
primary, config, "default_effort", "default_reasoning_effort"
|
|
3048
|
+
)
|
|
3049
|
+
),
|
|
3050
|
+
"supported_efforts": _supported_efforts(primary, config),
|
|
3051
|
+
"default_enabled": _optional_bool(
|
|
3052
|
+
_reasoning_field(
|
|
3053
|
+
primary, config, "default_enabled", "default_reasoning_enabled"
|
|
3054
|
+
),
|
|
3055
|
+
"default_enabled",
|
|
3056
|
+
),
|
|
3057
|
+
"mandatory": _optional_bool(
|
|
3058
|
+
_reasoning_field(
|
|
3059
|
+
primary, config, "mandatory", "is_mandatory_reasoning"
|
|
3060
|
+
),
|
|
3061
|
+
"mandatory",
|
|
3062
|
+
),
|
|
3063
|
+
"supports_max_tokens": _optional_bool(
|
|
3064
|
+
_reasoning_field(
|
|
3065
|
+
primary,
|
|
3066
|
+
config,
|
|
3067
|
+
"supports_max_tokens",
|
|
3068
|
+
"supports_reasoning_max_tokens",
|
|
3069
|
+
),
|
|
3070
|
+
"supports_max_tokens",
|
|
3071
|
+
),
|
|
3072
|
+
"is_reasoning": is_reasoning,
|
|
3073
|
+
}
|
|
3074
|
+
reasoning["default_effort"] = effective_default_effort(reasoning)
|
|
3075
|
+
metadata[model_id] = reasoning
|
|
3076
|
+
return metadata
|
|
3077
|
+
|
|
3078
|
+
|
|
3079
|
+
def resolve_quality(
|
|
3080
|
+
candidates,
|
|
3081
|
+
aa_by_id,
|
|
3082
|
+
exact,
|
|
3083
|
+
fuzzy,
|
|
3084
|
+
aa_source,
|
|
3085
|
+
catalog_quality=None,
|
|
3086
|
+
*,
|
|
3087
|
+
depth_lookup=None,
|
|
3088
|
+
catalog_depths=None,
|
|
3089
|
+
requested_effort="default",
|
|
3090
|
+
):
|
|
3091
|
+
"""Per-candidate quality records: AA API primary, catalog next, OR last.
|
|
3092
|
+
|
|
3093
|
+
Depth-aware mode (either new input is not None; run() always passes the
|
|
3094
|
+
AA depth lookup) selects the record at the requested effort -- the
|
|
3095
|
+
candidate's effective default when requested_effort is "default", else
|
|
3096
|
+
the explicit --reasoning-effort value. Resolution order per candidate:
|
|
3097
|
+
1. AA depth lookup -- match_depth_quality's exact tiers. source "aa",
|
|
3098
|
+
effort from the record.
|
|
3099
|
+
2. catalog_depths -- the published catalog's per-model depth map,
|
|
3100
|
+
re-selected at the LIVE effective default (which may differ from
|
|
3101
|
+
the keyed run's recorded default). source "catalog", the record's
|
|
3102
|
+
own as_of.
|
|
3103
|
+
3. OpenRouter's benchmark trio -- only for a model known to be
|
|
3104
|
+
non-reasoning (is_reasoning False) at effort "none". The trio has
|
|
3105
|
+
no effort label, so a reasoning model never uses it as its main
|
|
3106
|
+
score; unknown reasoning is not non-reasoning.
|
|
3107
|
+
4. No record -- the candidate stays quality-blind.
|
|
3108
|
+
|
|
3109
|
+
Strict selection: a matched base group with no record at the requested
|
|
3110
|
+
effort yields nothing at that tier. Another effort is never
|
|
3111
|
+
substituted, and a reasoning model never falls through to the OR trio.
|
|
3112
|
+
|
|
3113
|
+
The legacy path (both new inputs None) keeps the pre-depth behavior:
|
|
3114
|
+
match_quality's exact tiers, then the hydrated catalog_quality by id,
|
|
3115
|
+
then the OR trio for any model.
|
|
3116
|
+
|
|
3117
|
+
Every depth-mode record is {"index", "source", "as_of", "effort",
|
|
3118
|
+
"drift", "or_index", "or_as_of"}; legacy records omit "effort".
|
|
3119
|
+
Drift is computed only when the winner is not OpenRouter and only
|
|
3120
|
+
like-for-like: the old |index - or_index| >= 3.0 rule applies at effort
|
|
3121
|
+
"none"; any other effort keeps the OR reference but sets drift None.
|
|
3122
|
+
cand["reasoning_depths"] carries the full measured depth list (AA or
|
|
3123
|
+
catalog group, or the single OR "none" record). The list is single-source:
|
|
3124
|
+
it comes from whichever tier supplied the main record, and sources are
|
|
3125
|
+
never merged.
|
|
3126
|
+
"""
|
|
3127
|
+
quality_by_id = {}
|
|
3128
|
+
source_by_id = {}
|
|
3129
|
+
depth_mode = depth_lookup is not None or catalog_depths is not None
|
|
3130
|
+
for cand in candidates:
|
|
3131
|
+
record = None
|
|
3132
|
+
if depth_mode:
|
|
3133
|
+
effort = (
|
|
3134
|
+
requested_effort
|
|
3135
|
+
if requested_effort != "default"
|
|
3136
|
+
else cand.get("default_effort")
|
|
3137
|
+
)
|
|
3138
|
+
measured = []
|
|
3139
|
+
group = _depth_group(cand, depth_lookup) if depth_lookup else None
|
|
3140
|
+
if group is not None:
|
|
3141
|
+
measured = _ordered_depth_records(group)
|
|
3142
|
+
depth_record = group.get(effort) if effort else None
|
|
3143
|
+
if isinstance(depth_record, dict):
|
|
3144
|
+
record = {
|
|
3145
|
+
"index": depth_record["index"],
|
|
3146
|
+
"source": "aa",
|
|
3147
|
+
"as_of": depth_record.get("as_of"),
|
|
3148
|
+
"effort": depth_record.get("effort") or effort,
|
|
3149
|
+
}
|
|
3150
|
+
if record is None and catalog_depths is not None:
|
|
3151
|
+
catalog_entry = catalog_depths.get(cand["id"])
|
|
3152
|
+
if isinstance(catalog_entry, dict):
|
|
3153
|
+
catalog_group = catalog_entry.get("depths")
|
|
3154
|
+
if isinstance(catalog_group, dict):
|
|
3155
|
+
measured = _ordered_depth_records(catalog_group)
|
|
3156
|
+
depth_record = catalog_group.get(effort) if effort else None
|
|
3157
|
+
if isinstance(depth_record, dict):
|
|
3158
|
+
record = {
|
|
3159
|
+
"index": depth_record["index"],
|
|
3160
|
+
"source": "catalog",
|
|
3161
|
+
"as_of": depth_record.get("as_of"),
|
|
3162
|
+
"effort": depth_record.get("effort") or effort,
|
|
3163
|
+
}
|
|
3164
|
+
reasoning = cand.get("reasoning")
|
|
3165
|
+
is_non_reasoning = (
|
|
3166
|
+
isinstance(reasoning, dict) and reasoning.get("is_reasoning") is False
|
|
3167
|
+
)
|
|
3168
|
+
if record is None and is_non_reasoning and effort == "none":
|
|
3169
|
+
bench = aa_by_id.get(base_model_id(cand["id"]))
|
|
3170
|
+
intelligence = bench.get("intelligence_index") if bench else None
|
|
3171
|
+
if intelligence is not None:
|
|
3172
|
+
record = {
|
|
3173
|
+
"index": intelligence,
|
|
3174
|
+
"source": "openrouter",
|
|
3175
|
+
"as_of": bench.get("as_of"),
|
|
3176
|
+
"effort": "none",
|
|
3177
|
+
}
|
|
3178
|
+
measured = [
|
|
3179
|
+
{
|
|
3180
|
+
"effort": "none",
|
|
3181
|
+
"index": intelligence,
|
|
3182
|
+
"cost_per_task_usd": None,
|
|
3183
|
+
"source": "openrouter",
|
|
3184
|
+
"as_of": bench.get("as_of"),
|
|
3185
|
+
}
|
|
3186
|
+
]
|
|
3187
|
+
cand["reasoning_depths"] = measured
|
|
3188
|
+
else:
|
|
3189
|
+
# Tier 1: AA API exact match. match_quality returns the winning
|
|
3190
|
+
# entry's (index, as_of) pair.
|
|
3191
|
+
match = match_quality(
|
|
3192
|
+
{"id": cand["id"], "name": cand["name"]}, exact, fuzzy
|
|
3193
|
+
)
|
|
3194
|
+
if match is not None:
|
|
3195
|
+
index, as_of = match
|
|
3196
|
+
record = {"index": index, "source": "aa", "as_of": as_of}
|
|
3197
|
+
# Tier 2: catalog record (unkeyed runs hydrate from the published
|
|
3198
|
+
# catalog). The run-level provenance label is the source chain:
|
|
3199
|
+
# every hydrated record is tagged "catalog" -- read from our
|
|
3200
|
+
# published catalog -- overriding the row's own label, which is
|
|
3201
|
+
# the keyed CI run's row-level truth inside the artifact ("aa"/
|
|
3202
|
+
# "openrouter"; no real catalog row ever carries "catalog").
|
|
3203
|
+
# as_of stays the row's own stamp.
|
|
3204
|
+
if record is None and catalog_quality is not None:
|
|
3205
|
+
catalog_record = catalog_quality.get(cand["id"])
|
|
3206
|
+
if catalog_record is not None:
|
|
3207
|
+
record = {
|
|
3208
|
+
"index": catalog_record["index"],
|
|
3209
|
+
"source": "catalog",
|
|
3210
|
+
"as_of": catalog_record["as_of"],
|
|
3211
|
+
}
|
|
3212
|
+
# Tier 3: OpenRouter's per-slug benchmark trio (last resort).
|
|
3213
|
+
if record is None:
|
|
3214
|
+
bench = aa_by_id.get(base_model_id(cand["id"]))
|
|
3215
|
+
intelligence = bench.get("intelligence_index") if bench else None
|
|
3216
|
+
if intelligence is not None:
|
|
3217
|
+
record = {
|
|
3218
|
+
"index": intelligence,
|
|
3219
|
+
"source": "openrouter",
|
|
3220
|
+
"as_of": bench.get("as_of"),
|
|
3221
|
+
}
|
|
3222
|
+
if record is None:
|
|
3223
|
+
continue
|
|
3224
|
+
|
|
3225
|
+
# Drift: only meaningful when the winner is not OpenRouter itself.
|
|
3226
|
+
bench = aa_by_id.get(base_model_id(cand["id"])) or {}
|
|
3227
|
+
or_intelligence = bench.get("intelligence_index")
|
|
3228
|
+
reasoning = cand.get("reasoning") or {}
|
|
2372
3229
|
if (
|
|
2373
3230
|
record["source"] in ("aa", "catalog")
|
|
2374
3231
|
and isinstance(or_intelligence, (int, float))
|
|
@@ -2377,7 +3234,18 @@ def resolve_quality(
|
|
|
2377
3234
|
):
|
|
2378
3235
|
record["or_index"] = or_intelligence
|
|
2379
3236
|
record["or_as_of"] = bench.get("as_of")
|
|
2380
|
-
|
|
3237
|
+
# The comparable case is a non-reasoning MODEL at effort "none",
|
|
3238
|
+
# not merely an effort label: a reasoning model whose effective
|
|
3239
|
+
# default is "none" still has an OR snapshot measured at its
|
|
3240
|
+
# reasoning state, so a difference may be a depth difference.
|
|
3241
|
+
comparable = (not depth_mode) or (
|
|
3242
|
+
reasoning.get("is_reasoning") is False
|
|
3243
|
+
and record.get("effort") == "none"
|
|
3244
|
+
)
|
|
3245
|
+
if comparable:
|
|
3246
|
+
record["drift"] = abs(record["index"] - or_intelligence) >= 3.0
|
|
3247
|
+
else:
|
|
3248
|
+
record["drift"] = None
|
|
2381
3249
|
else:
|
|
2382
3250
|
record["or_index"] = None
|
|
2383
3251
|
record["or_as_of"] = None
|
|
@@ -2410,7 +3278,18 @@ def _per_1m(price_per_token: float) -> float:
|
|
|
2410
3278
|
|
|
2411
3279
|
|
|
2412
3280
|
def build_schedules(
|
|
2413
|
-
models,
|
|
3281
|
+
models,
|
|
3282
|
+
args,
|
|
3283
|
+
quality_by_id,
|
|
3284
|
+
aa_by_id,
|
|
3285
|
+
balanced_by_id,
|
|
3286
|
+
catalog_quality=None,
|
|
3287
|
+
*,
|
|
3288
|
+
depth_lookup=None,
|
|
3289
|
+
catalog_depths=None,
|
|
3290
|
+
reasoning_by_id=None,
|
|
3291
|
+
requested_effort="default",
|
|
3292
|
+
schedule_entries=None,
|
|
2414
3293
|
):
|
|
2415
3294
|
"""Catalog-wide off-peak index built from the RAW model list (spec 5.4).
|
|
2416
3295
|
|
|
@@ -2420,12 +3299,39 @@ def build_schedules(
|
|
|
2420
3299
|
(~ids) and malformed ids are excluded; :batch variants are included as
|
|
2421
3300
|
distinct deals. Sorted by id.
|
|
2422
3301
|
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
3302
|
+
Depth-aware mode (run() always supplies the AA depth lookup; an unkeyed
|
|
3303
|
+
run supplies the catalog depth map and the published schedule entries)
|
|
3304
|
+
follows spec 6.5:
|
|
3305
|
+
0. ``schedule_entries`` (unkeyed runs): the published catalog's v5
|
|
3306
|
+
schedules array is the source of truth for Deals. Whenever an
|
|
3307
|
+
entry exists with a non-null quality AND a non-empty
|
|
3308
|
+
``quality_effort``, its quality and all three provenance fields
|
|
3309
|
+
are copied VERBATIM (no run-level relabeling); requested_effort is
|
|
3310
|
+
ignored because there is no depth list to re-select from. An entry
|
|
3311
|
+
with no effort label is legacy/unknown (spec 6.5): its quality is
|
|
3312
|
+
null and the lower tiers are skipped. A null-quality or absent
|
|
3313
|
+
entry falls through to the tiers below.
|
|
3314
|
+
1. AA depth lookup at the effective default effort (or the explicit
|
|
3315
|
+
``requested_effort``) -- keyed runs only.
|
|
3316
|
+
2. Catalog depth map at the entry's recorded default effort -- unkeyed
|
|
3317
|
+
runs use that provenance as-is, never re-selecting at the live
|
|
3318
|
+
default.
|
|
3319
|
+
3. Catalog main-quality map, only when the record carries an effort
|
|
3320
|
+
(a record without one is legacy/unknown: quality stays null).
|
|
3321
|
+
4. OpenRouter's benchmark trio, only for a model known to be
|
|
3322
|
+
non-reasoning (``is_reasoning`` False) at effort "none". A
|
|
3323
|
+
reasoning model never falls through to the trio: it has no effort
|
|
3324
|
+
label, so using it would publish a max-effort snapshot as the
|
|
3325
|
+
default.
|
|
3326
|
+
Strict selection at tiers 1-2: a matched group with no record at the
|
|
3327
|
+
selected effort yields nothing; another effort is never substituted.
|
|
3328
|
+
|
|
3329
|
+
The legacy path (both new inputs None) keeps the pre-depth fallback
|
|
3330
|
+
chain: candidate record, hydrated catalog, then the OR trio. Every row
|
|
3331
|
+
carries ``quality_effort`` (string or null) alongside the winning
|
|
3332
|
+
record's provenance (quality_source/quality_as_of).
|
|
2428
3333
|
"""
|
|
3334
|
+
depth_mode = depth_lookup is not None or catalog_depths is not None
|
|
2429
3335
|
entries = []
|
|
2430
3336
|
for model in models:
|
|
2431
3337
|
model_id = model.get("id") or ""
|
|
@@ -2452,35 +3358,107 @@ def build_schedules(
|
|
|
2452
3358
|
# quality_by_id values are records {"index", "source", ...}; the
|
|
2453
3359
|
# schedule row keeps the bare float index verbatim (never a dict)
|
|
2454
3360
|
# and echoes the winning record's provenance for the QUAL tooltip.
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
3361
|
+
quality = None
|
|
3362
|
+
quality_source = None
|
|
3363
|
+
quality_as_of = None
|
|
3364
|
+
quality_effort = None
|
|
3365
|
+
schedule_entry = (schedule_entries or {}).get(model_id)
|
|
3366
|
+
if (
|
|
3367
|
+
isinstance(schedule_entry, dict)
|
|
3368
|
+
and schedule_entry.get("quality") is not None
|
|
3369
|
+
):
|
|
3370
|
+
effort_label = schedule_entry.get("quality_effort")
|
|
3371
|
+
if isinstance(effort_label, str) and effort_label:
|
|
3372
|
+
# Unkeyed source of truth: recorded provenance verbatim.
|
|
3373
|
+
quality = schedule_entry.get("quality")
|
|
3374
|
+
quality_source = schedule_entry.get("quality_source")
|
|
3375
|
+
quality_as_of = schedule_entry.get("quality_as_of")
|
|
3376
|
+
quality_effort = effort_label
|
|
3377
|
+
# else: a recorded entry with no effort label is legacy/unknown
|
|
3378
|
+
# (spec 6.5): its quality is null, and the lower tiers are
|
|
3379
|
+
# skipped rather than re-deriving a max-effort OR snapshot.
|
|
3380
|
+
elif depth_mode:
|
|
3381
|
+
reasoning = (reasoning_by_id or {}).get(model_id)
|
|
3382
|
+
if not isinstance(reasoning, dict):
|
|
3383
|
+
reasoning = None
|
|
3384
|
+
effort = (
|
|
3385
|
+
requested_effort
|
|
3386
|
+
if requested_effort != "default"
|
|
3387
|
+
else (reasoning or {}).get("default_effort")
|
|
3388
|
+
)
|
|
3389
|
+
# Tier 1: AA depth lookup (keyed runs).
|
|
3390
|
+
group = _depth_group(model, depth_lookup) if depth_lookup else None
|
|
3391
|
+
record = group.get(effort) if group is not None and effort else None
|
|
3392
|
+
if isinstance(record, dict):
|
|
3393
|
+
quality = record.get("index")
|
|
3394
|
+
quality_source = "aa"
|
|
3395
|
+
quality_as_of = record.get("as_of")
|
|
3396
|
+
quality_effort = record.get("effort") or effort
|
|
3397
|
+
# Tier 2: catalog depth map (unkeyed runs), recorded default
|
|
3398
|
+
# effort as-is -- the keyed run's provenance, not a live
|
|
3399
|
+
# re-selection.
|
|
3400
|
+
if quality is None and catalog_depths is not None:
|
|
3401
|
+
catalog_entry = catalog_depths.get(model_id)
|
|
3402
|
+
if isinstance(catalog_entry, dict):
|
|
3403
|
+
recorded_default = catalog_entry.get("default_effort")
|
|
3404
|
+
group = catalog_entry.get("depths")
|
|
3405
|
+
record = (
|
|
3406
|
+
group.get(recorded_default)
|
|
3407
|
+
if isinstance(group, dict) and recorded_default
|
|
3408
|
+
else None
|
|
3409
|
+
)
|
|
3410
|
+
if isinstance(record, dict):
|
|
3411
|
+
quality = record.get("index")
|
|
3412
|
+
quality_source = "catalog"
|
|
3413
|
+
quality_as_of = record.get("as_of")
|
|
3414
|
+
quality_effort = record.get("effort") or recorded_default
|
|
3415
|
+
# Tier 3: catalog main-quality map (unkeyed runs). A record with
|
|
3416
|
+
# no quality_effort is legacy/unknown: null, never a guess.
|
|
3417
|
+
if quality is None and catalog_quality is not None:
|
|
3418
|
+
catalog_record = catalog_quality.get(model_id)
|
|
3419
|
+
if isinstance(catalog_record, dict) and catalog_record.get("effort"):
|
|
3420
|
+
quality = catalog_record.get("index")
|
|
3421
|
+
quality_source = "catalog"
|
|
3422
|
+
quality_as_of = catalog_record.get("as_of")
|
|
3423
|
+
quality_effort = catalog_record.get("effort")
|
|
3424
|
+
# Tier 4: OR trio only for known non-reasoning models.
|
|
3425
|
+
if (
|
|
3426
|
+
quality is None
|
|
3427
|
+
and reasoning is not None
|
|
3428
|
+
and reasoning.get("is_reasoning") is False
|
|
3429
|
+
):
|
|
3430
|
+
benchmark = aa_by_id.get(base_model_id(model_id)) or {}
|
|
3431
|
+
quality = benchmark.get("intelligence_index")
|
|
3432
|
+
if quality is not None:
|
|
3433
|
+
quality_source = "openrouter"
|
|
3434
|
+
quality_as_of = benchmark.get("as_of")
|
|
3435
|
+
quality_effort = "none"
|
|
2465
3436
|
else:
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
quality_source = "catalog"
|
|
2474
|
-
quality_as_of = catalog_record["as_of"]
|
|
2475
|
-
if quality is None:
|
|
2476
|
-
benchmark = aa_by_id.get(base_model_id(model_id)) or {}
|
|
2477
|
-
quality = benchmark.get("intelligence_index")
|
|
2478
|
-
if quality is not None:
|
|
2479
|
-
quality_source = "openrouter"
|
|
2480
|
-
quality_as_of = benchmark.get("as_of")
|
|
3437
|
+
# Legacy path: candidate record, hydrated catalog, OR trio.
|
|
3438
|
+
record = quality_by_id.get(model_id)
|
|
3439
|
+
if isinstance(record, dict):
|
|
3440
|
+
quality = record.get("index")
|
|
3441
|
+
quality_source = record.get("source")
|
|
3442
|
+
quality_as_of = record.get("as_of")
|
|
3443
|
+
quality_effort = record.get("effort")
|
|
2481
3444
|
else:
|
|
2482
|
-
|
|
2483
|
-
|
|
3445
|
+
quality = record
|
|
3446
|
+
if quality is None and catalog_quality is not None:
|
|
3447
|
+
catalog_record = catalog_quality.get(model_id)
|
|
3448
|
+
if catalog_record is not None:
|
|
3449
|
+
quality = catalog_record["index"]
|
|
3450
|
+
quality_source = "catalog"
|
|
3451
|
+
quality_as_of = catalog_record["as_of"]
|
|
3452
|
+
quality_effort = catalog_record.get("effort")
|
|
3453
|
+
if quality is None:
|
|
3454
|
+
benchmark = aa_by_id.get(base_model_id(model_id)) or {}
|
|
3455
|
+
quality = benchmark.get("intelligence_index")
|
|
3456
|
+
if quality is not None:
|
|
3457
|
+
quality_source = "openrouter"
|
|
3458
|
+
quality_as_of = benchmark.get("as_of")
|
|
3459
|
+
else:
|
|
3460
|
+
quality_source = None
|
|
3461
|
+
quality_as_of = None
|
|
2484
3462
|
name = model.get("name") or model_id
|
|
2485
3463
|
entries.append(
|
|
2486
3464
|
{
|
|
@@ -2492,6 +3470,7 @@ def build_schedules(
|
|
|
2492
3470
|
# quality came without any -- e.g. a bare-float test map).
|
|
2493
3471
|
"quality_source": quality_source,
|
|
2494
3472
|
"quality_as_of": quality_as_of,
|
|
3473
|
+
"quality_effort": quality_effort,
|
|
2495
3474
|
"score": balanced_by_id.get(model_id),
|
|
2496
3475
|
"max_discount": eff["max_discount"],
|
|
2497
3476
|
"sched_note": fmt_sched_note(eff["max_discount"]),
|
|
@@ -2543,14 +3522,54 @@ def catalog_weights(candidates, quality_by_id):
|
|
|
2543
3522
|
return weights
|
|
2544
3523
|
|
|
2545
3524
|
|
|
3525
|
+
def overall_scores(cand, weights=None) -> dict:
|
|
3526
|
+
"""Per-priority overall scores from rounded components (catalog parity).
|
|
3527
|
+
|
|
3528
|
+
weights is the per-priority map from catalog_weights; callers that must
|
|
3529
|
+
match the catalog on quality-blind runs pass it, otherwise the base
|
|
3530
|
+
PRIORITY_WEIGHTS apply.
|
|
3531
|
+
"""
|
|
3532
|
+
weights = weights or PRIORITY_WEIGHTS
|
|
3533
|
+
scores = {
|
|
3534
|
+
"price": round(cand["price_score"], 4),
|
|
3535
|
+
"quality": round(cand["quality_score"], 4),
|
|
3536
|
+
"context": round(cand["context_score"], 4),
|
|
3537
|
+
"age": round(cand["age_score"], 4),
|
|
3538
|
+
}
|
|
3539
|
+
return {
|
|
3540
|
+
priority: round(
|
|
3541
|
+
weights[priority].get("quality", 0.0) * scores["quality"]
|
|
3542
|
+
+ weights[priority].get("price", 0.0) * scores["price"]
|
|
3543
|
+
+ weights[priority].get("context", 0.0) * scores["context"]
|
|
3544
|
+
+ weights[priority].get("age", 0.0) * scores["age"],
|
|
3545
|
+
4,
|
|
3546
|
+
)
|
|
3547
|
+
for priority in PRIORITY_WEIGHTS
|
|
3548
|
+
}
|
|
3549
|
+
|
|
3550
|
+
|
|
3551
|
+
def off_peak_deals(schedules, limit=10):
|
|
3552
|
+
rows = [row for row in schedules if f"{row['max_discount']:.0%}" != "0%"]
|
|
3553
|
+
rows.sort(key=lambda row: (-row["max_discount"], row["id"]))
|
|
3554
|
+
return rows[:limit]
|
|
3555
|
+
|
|
3556
|
+
|
|
2546
3557
|
# ---------------------------------------------------------------------------
|
|
2547
3558
|
# Filtering and scoring
|
|
2548
3559
|
# ---------------------------------------------------------------------------
|
|
2549
3560
|
|
|
2550
3561
|
|
|
2551
|
-
def build_candidates(
|
|
3562
|
+
def build_candidates(
|
|
3563
|
+
models,
|
|
3564
|
+
args,
|
|
3565
|
+
discounts,
|
|
3566
|
+
zdr_ids,
|
|
3567
|
+
filtered_out=None,
|
|
3568
|
+
reasoning_by_id: dict[str, dict] | None = None,
|
|
3569
|
+
):
|
|
2552
3570
|
now = time.time()
|
|
2553
3571
|
require_tools = not args.no_require_tools
|
|
3572
|
+
requested_effort = getattr(args, "reasoning_effort", "default")
|
|
2554
3573
|
dropped = {}
|
|
2555
3574
|
candidates = []
|
|
2556
3575
|
|
|
@@ -2598,6 +3617,22 @@ def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
|
|
|
2598
3617
|
# schedule-only AND mixed models fail closed here.
|
|
2599
3618
|
drop("schedule", model_id, model.get("name"))
|
|
2600
3619
|
continue
|
|
3620
|
+
# The effort filter runs before the ZDR/discount gates: it is the
|
|
3621
|
+
# caller's explicit request, so it is the most useful drop reason
|
|
3622
|
+
# when several gates would reject the model.
|
|
3623
|
+
reasoning = (reasoning_by_id or {}).get(model_id)
|
|
3624
|
+
if not isinstance(reasoning, dict):
|
|
3625
|
+
reasoning = None
|
|
3626
|
+
# A model can always run at its own effective default (§7 equality
|
|
3627
|
+
# carve-out), even when that effort is absent from supported_efforts
|
|
3628
|
+
# -- e.g. default_enabled false resolves to "none" while the list
|
|
3629
|
+
# omits it.
|
|
3630
|
+
if requested_effort != "default" and not (
|
|
3631
|
+
supports_effort(reasoning, requested_effort)
|
|
3632
|
+
or requested_effort == (reasoning or {}).get("default_effort")
|
|
3633
|
+
):
|
|
3634
|
+
drop("reasoning effort", model_id, model.get("name"))
|
|
3635
|
+
continue
|
|
2601
3636
|
price_in, price_out = eff["price_in"], eff["price_out"]
|
|
2602
3637
|
if args.exclude_free and price_in == 0 and price_out == 0:
|
|
2603
3638
|
drop("free", model_id, model.get("name"))
|
|
@@ -2666,6 +3701,11 @@ def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
|
|
|
2666
3701
|
"tool_calling": tool_calling,
|
|
2667
3702
|
"zdr": None if args.no_zdr else model_id in (zdr_ids or set()),
|
|
2668
3703
|
"expired": expired,
|
|
3704
|
+
"reasoning": reasoning,
|
|
3705
|
+
"default_effort": (reasoning or {}).get("default_effort"),
|
|
3706
|
+
"supported_efforts": list(
|
|
3707
|
+
(reasoning or {}).get("supported_efforts") or []
|
|
3708
|
+
),
|
|
2669
3709
|
}
|
|
2670
3710
|
)
|
|
2671
3711
|
|
|
@@ -2809,6 +3849,42 @@ def fmt_discount(value) -> str:
|
|
|
2809
3849
|
return f"{value:.0%}" if has_discount(value) else "--"
|
|
2810
3850
|
|
|
2811
3851
|
|
|
3852
|
+
def _display_default_effort(cand):
|
|
3853
|
+
"""The effort a default run uses, for the DEPTH cell and the depth block.
|
|
3854
|
+
|
|
3855
|
+
Production candidates carry it on both ``cand["default_effort"]`` and the
|
|
3856
|
+
reasoning block; partial rows (tests, legacy paths) may carry only one.
|
|
3857
|
+
"""
|
|
3858
|
+
return cand.get("default_effort") or (cand.get("reasoning") or {}).get(
|
|
3859
|
+
"default_effort"
|
|
3860
|
+
)
|
|
3861
|
+
|
|
3862
|
+
|
|
3863
|
+
def _published_reasoning_block(cand):
|
|
3864
|
+
"""The six published reasoning keys plus the measured depth records.
|
|
3865
|
+
|
|
3866
|
+
``is_reasoning`` is an internal flag: neither catalog v5 nor spec §8.4
|
|
3867
|
+
publishes it, so it never appears here.
|
|
3868
|
+
"""
|
|
3869
|
+
reasoning = cand.get("reasoning") or {}
|
|
3870
|
+
return {
|
|
3871
|
+
"default_effort": reasoning.get("default_effort"),
|
|
3872
|
+
"supported_efforts": reasoning.get("supported_efforts") or [],
|
|
3873
|
+
"default_enabled": reasoning.get("default_enabled"),
|
|
3874
|
+
"mandatory": reasoning.get("mandatory"),
|
|
3875
|
+
"supports_max_tokens": reasoning.get("supports_max_tokens"),
|
|
3876
|
+
"depths": cand.get("reasoning_depths") or [],
|
|
3877
|
+
}
|
|
3878
|
+
|
|
3879
|
+
|
|
3880
|
+
def fmt_depth(cand) -> str:
|
|
3881
|
+
"""DEPTH cell: the effort QUAL was measured at, ``*`` on the default."""
|
|
3882
|
+
effort = (cand.get("quality_record") or {}).get("effort")
|
|
3883
|
+
if not effort:
|
|
3884
|
+
return "-"
|
|
3885
|
+
return f"{effort}*" if effort == _display_default_effort(cand) else effort
|
|
3886
|
+
|
|
3887
|
+
|
|
2812
3888
|
def fmt_prec(cand) -> str:
|
|
2813
3889
|
"""PREC cell: the precision the ranked price is quoted at.
|
|
2814
3890
|
|
|
@@ -2901,20 +3977,24 @@ def _later_as_of(value, current) -> bool:
|
|
|
2901
3977
|
return new > old
|
|
2902
3978
|
|
|
2903
3979
|
|
|
3980
|
+
QUALITY_SOURCE_LABEL = "Quality source: "
|
|
3981
|
+
|
|
3982
|
+
|
|
2904
3983
|
def quality_footer_note(quality_by_id, aa_index_version, unmatched):
|
|
2905
3984
|
"""Compose the footer's quality note from the resolved records.
|
|
2906
3985
|
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
record resolved -- run() falls back to the quality-blind
|
|
3986
|
+
Returns {"sources": [...], "clauses": [...], "warn": bool}. Each source
|
|
3987
|
+
names its model count, its own as_of and the AA index version when
|
|
3988
|
+
known, e.g. "AA index v4.3 - live AA (58 models, checked 13:15Z)".
|
|
3989
|
+
Clauses carry the unmatched-candidates note and the drift footnote when
|
|
3990
|
+
any resolved record disagrees with the OpenRouter check value; the drift
|
|
3991
|
+
count covers only comparable (non-reasoning) rows and says so. Returns
|
|
3992
|
+
None when no record resolved -- run() falls back to the quality-blind
|
|
3993
|
+
note.
|
|
2914
3994
|
"""
|
|
2915
3995
|
labels = {
|
|
2916
3996
|
"aa": "live AA",
|
|
2917
|
-
"catalog": "our catalog",
|
|
3997
|
+
"catalog": "our published catalog",
|
|
2918
3998
|
"openrouter": "OpenRouter snapshot",
|
|
2919
3999
|
}
|
|
2920
4000
|
stamp_words = {"aa": "checked", "catalog": "as of", "openrouter": "as of"}
|
|
@@ -2935,29 +4015,61 @@ def quality_footer_note(quality_by_id, aa_index_version, unmatched):
|
|
|
2935
4015
|
count = counts.get(source, 0)
|
|
2936
4016
|
if not count:
|
|
2937
4017
|
continue
|
|
4018
|
+
plural = "" if count == 1 else "s"
|
|
2938
4019
|
stamp = _footer_as_of(source, latest.get(source))
|
|
2939
4020
|
if stamp:
|
|
2940
|
-
blocks.append(
|
|
4021
|
+
blocks.append(
|
|
4022
|
+
f"{labels[source]} ({count} model{plural}, {stamp_words[source]} {stamp})"
|
|
4023
|
+
)
|
|
2941
4024
|
else:
|
|
2942
|
-
blocks.append(f"{labels[source]} ({count})")
|
|
4025
|
+
blocks.append(f"{labels[source]} ({count} model{plural})")
|
|
2943
4026
|
if not blocks:
|
|
2944
4027
|
return None
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
4028
|
+
if aa_index_version is not None and counts.get("aa"):
|
|
4029
|
+
# Only the AA block gets the index-version prefix; a run whose first
|
|
4030
|
+
# (or only) source is the OpenRouter snapshot must not be mislabelled
|
|
4031
|
+
# as an AA index version.
|
|
4032
|
+
blocks[0] = f"AA index v{aa_index_version} - {blocks[0]}"
|
|
4033
|
+
clauses = []
|
|
2949
4034
|
if unmatched:
|
|
2950
|
-
|
|
4035
|
+
plural = "" if unmatched == 1 else "s"
|
|
4036
|
+
clauses.append(
|
|
4037
|
+
f"{unmatched} model{plural} unmatched (no quality data; quality score 0)"
|
|
4038
|
+
)
|
|
2951
4039
|
drift = sum(1 for record in quality_by_id.values() if record.get("drift"))
|
|
2952
4040
|
if drift:
|
|
2953
|
-
|
|
2954
|
-
|
|
4041
|
+
rows = "row" if drift == 1 else "rows"
|
|
4042
|
+
clauses.append(
|
|
4043
|
+
f"drift vs OpenRouter snapshot: {drift} comparable (non-reasoning) {rows}"
|
|
4044
|
+
)
|
|
4045
|
+
return {"sources": blocks, "clauses": clauses, "warn": bool(unmatched)}
|
|
4046
|
+
|
|
4047
|
+
|
|
4048
|
+
def quality_source_lines(note) -> list[str]:
|
|
4049
|
+
"""Render the footer as one source per line, 16-space continuation."""
|
|
4050
|
+
if not note or not note.get("sources"):
|
|
4051
|
+
return [QUALITY_SOURCE_LABEL + "unavailable; ranked on price/context/age"]
|
|
4052
|
+
indent = " " * len(QUALITY_SOURCE_LABEL)
|
|
4053
|
+
sources = note["sources"]
|
|
4054
|
+
clauses = note.get("clauses") or []
|
|
4055
|
+
lines = []
|
|
4056
|
+
for index, source in enumerate(sources):
|
|
4057
|
+
prefix = QUALITY_SOURCE_LABEL if index == 0 else indent
|
|
4058
|
+
if index < len(sources) - 1:
|
|
4059
|
+
lines.append(prefix + source + " |")
|
|
4060
|
+
elif clauses:
|
|
4061
|
+
lines.append(prefix + source + ";")
|
|
4062
|
+
else:
|
|
4063
|
+
lines.append(prefix + source)
|
|
4064
|
+
lines.extend(indent + clause for clause in clauses)
|
|
4065
|
+
return lines
|
|
2955
4066
|
|
|
2956
4067
|
|
|
2957
|
-
|
|
2958
|
-
|
|
4068
|
+
TABLE_COLUMNS = {
|
|
4069
|
+
"expert": (
|
|
2959
4070
|
"RANK",
|
|
2960
4071
|
"MODEL",
|
|
4072
|
+
"DEPTH",
|
|
2961
4073
|
"QUAL",
|
|
2962
4074
|
"$IN/M",
|
|
2963
4075
|
"$OUT/M",
|
|
@@ -2967,72 +4079,448 @@ def print_table(top, total_candidates, weights, quality_note):
|
|
|
2967
4079
|
"CTX",
|
|
2968
4080
|
"AGE",
|
|
2969
4081
|
"SCORE",
|
|
4082
|
+
),
|
|
4083
|
+
"simple": ("RANK", "MODEL", "QUALITY", "IN $/M", "OUT $/M", "DISCOUNT", "AGE"),
|
|
4084
|
+
}
|
|
4085
|
+
|
|
4086
|
+
LEGENDS = {
|
|
4087
|
+
"expert": (
|
|
4088
|
+
(
|
|
4089
|
+
"QUALITY",
|
|
4090
|
+
['Artificial Analysis intelligence index; "-" = unknown (scores 0)'],
|
|
4091
|
+
),
|
|
4092
|
+
(
|
|
4093
|
+
"DEPTH",
|
|
4094
|
+
[
|
|
4095
|
+
"reasoning effort the QUAL score was measured at; * = OpenRouter's",
|
|
4096
|
+
"default; - = no known effort; --no-depths hides the depth detail",
|
|
4097
|
+
],
|
|
4098
|
+
),
|
|
4099
|
+
(
|
|
4100
|
+
"$IN/$OUT",
|
|
4101
|
+
[
|
|
4102
|
+
"USD per 1M tokens at the cheapest eligible endpoint, blended",
|
|
4103
|
+
"{in_pct} input / {out_pct} output (--input-share); differs from the",
|
|
4104
|
+
"openrouter.ai default-endpoint price by design. A higher rate",
|
|
4105
|
+
"may apply once the prompt outgrows a model's short-context tier.",
|
|
4106
|
+
],
|
|
4107
|
+
),
|
|
4108
|
+
("DISC", ["active discount"]),
|
|
4109
|
+
(
|
|
4110
|
+
"PREC",
|
|
4111
|
+
[
|
|
4112
|
+
'quoted precision ("!" = default routing favors cheaper',
|
|
4113
|
+
"sub-default endpoints)",
|
|
4114
|
+
],
|
|
4115
|
+
),
|
|
4116
|
+
(
|
|
4117
|
+
"TIER",
|
|
4118
|
+
[
|
|
4119
|
+
"how prices vary: >Nk = pricier above that prompt size; -N% =",
|
|
4120
|
+
"scheduled off-peak discount (price shown: the standard rate);",
|
|
4121
|
+
"SCHED = schedule with a discount under 1% (--off-peak lists them)",
|
|
4122
|
+
],
|
|
4123
|
+
),
|
|
4124
|
+
("CTX", ["context window; AGE = days since listed on OpenRouter"]),
|
|
4125
|
+
("SCORE", ["weighted blend of the four criteria above"]),
|
|
4126
|
+
),
|
|
4127
|
+
"simple": (
|
|
4128
|
+
("RANK", ["position in this ranking"]),
|
|
4129
|
+
("MODEL", ["provider/model id as listed; --best prefixes openrouter/"]),
|
|
4130
|
+
(
|
|
4131
|
+
"QUALITY",
|
|
4132
|
+
[
|
|
4133
|
+
'Artificial Analysis intelligence index; "-" = unknown (scores 0)',
|
|
4134
|
+
"the suffix after · is the reasoning effort the score was measured at",
|
|
4135
|
+
],
|
|
4136
|
+
),
|
|
4137
|
+
(
|
|
4138
|
+
"IN/OUT $/M",
|
|
4139
|
+
[
|
|
4140
|
+
"USD per 1M tokens at the cheapest eligible endpoint, blended",
|
|
4141
|
+
"{in_pct} input / {out_pct} output (--input-share)",
|
|
4142
|
+
],
|
|
4143
|
+
),
|
|
4144
|
+
("DISCOUNT", ["active discount, if any"]),
|
|
4145
|
+
("AGE", ["days since listed on OpenRouter"]),
|
|
4146
|
+
),
|
|
4147
|
+
}
|
|
4148
|
+
|
|
4149
|
+
|
|
4150
|
+
def fmt_quality(value) -> str:
|
|
4151
|
+
return "-" if value is None else f"{value:.1f}"
|
|
4152
|
+
|
|
4153
|
+
|
|
4154
|
+
_CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]")
|
|
4155
|
+
|
|
4156
|
+
|
|
4157
|
+
def display_text(value) -> str:
|
|
4158
|
+
"""Return untrusted text safe to print: strip C0/C1 control characters.
|
|
4159
|
+
|
|
4160
|
+
Model ids come from the OpenRouter API. An id carrying ESC (SGR/CSI
|
|
4161
|
+
injection) or a newline (forged output lines) must never reach the
|
|
4162
|
+
terminal raw. Normal ids pass through unchanged.
|
|
4163
|
+
"""
|
|
4164
|
+
return _CONTROL_CHARS_RE.sub("", str(value))
|
|
4165
|
+
|
|
4166
|
+
|
|
4167
|
+
def table_cells(cand, rank):
|
|
4168
|
+
effort = (cand.get("quality_record") or {}).get("effort")
|
|
4169
|
+
quality_cell = fmt_quality(cand["quality"])
|
|
4170
|
+
if effort and cand["quality"] is not None:
|
|
4171
|
+
# Simple view has no DEPTH column; the effort rides on QUALITY.
|
|
4172
|
+
quality_cell += f" · {effort}"
|
|
4173
|
+
return {
|
|
4174
|
+
"RANK": str(rank),
|
|
4175
|
+
"MODEL": display_text(cand["id"]),
|
|
4176
|
+
"DEPTH": fmt_depth(cand),
|
|
4177
|
+
"QUAL": fmt_quality(cand["quality"]),
|
|
4178
|
+
"QUALITY": quality_cell,
|
|
4179
|
+
"$IN/M": fmt_price(cand["price_in"]),
|
|
4180
|
+
"IN $/M": fmt_price(cand["price_in"]),
|
|
4181
|
+
"$OUT/M": fmt_price(cand["price_out"]),
|
|
4182
|
+
"OUT $/M": fmt_price(cand["price_out"]),
|
|
4183
|
+
"DISC": fmt_discount(cand["discount"]),
|
|
4184
|
+
"DISCOUNT": fmt_discount(cand["discount"]),
|
|
4185
|
+
"PREC": fmt_prec(cand),
|
|
4186
|
+
"TIER": cand["tier_note"] or cand["sched_note"] or "--",
|
|
4187
|
+
"CTX": fmt_context(cand["context"]),
|
|
4188
|
+
"AGE": fmt_age(cand["age_days"]),
|
|
4189
|
+
"SCORE": f"{cand['score']:.3f}",
|
|
4190
|
+
}
|
|
4191
|
+
|
|
4192
|
+
|
|
4193
|
+
def format_weights(weights) -> str:
|
|
4194
|
+
"""Render weights as integer percentages that sum to exactly 100.
|
|
4195
|
+
|
|
4196
|
+
Naive per-item ``:.0%`` rounding can print 101% (or 99%) because each
|
|
4197
|
+
share rounds independently. Largest-remainder rounding floors every
|
|
4198
|
+
share, then hands the leftover points to the largest fractional parts
|
|
4199
|
+
(deterministic name tie-break) so the line is always self-consistent.
|
|
4200
|
+
"""
|
|
4201
|
+
items = sorted(weights.items(), reverse=True)
|
|
4202
|
+
total = sum(value for _, value in items)
|
|
4203
|
+
if total <= 0:
|
|
4204
|
+
return ", ".join(f"{name} 0%" for name, _ in items)
|
|
4205
|
+
exact = [(name, value / total * 100) for name, value in items]
|
|
4206
|
+
floors = [
|
|
4207
|
+
(name, math.floor(share), share - math.floor(share)) for name, share in exact
|
|
2970
4208
|
]
|
|
2971
|
-
|
|
4209
|
+
remainder = 100 - sum(floor for _, floor, _ in floors)
|
|
4210
|
+
order = sorted(range(len(floors)), key=lambda i: (-floors[i][2], floors[i][0]))
|
|
4211
|
+
bumped = set(order[:remainder])
|
|
4212
|
+
return ", ".join(
|
|
4213
|
+
f"{name} {floor + (1 if index in bumped else 0)}%"
|
|
4214
|
+
for index, (name, floor, _) in enumerate(floors)
|
|
4215
|
+
)
|
|
4216
|
+
|
|
4217
|
+
|
|
4218
|
+
def print_table(
|
|
4219
|
+
top,
|
|
4220
|
+
weights,
|
|
4221
|
+
quality_note,
|
|
4222
|
+
detail="expert",
|
|
4223
|
+
style=None,
|
|
4224
|
+
bare=False,
|
|
4225
|
+
input_share=0.75,
|
|
4226
|
+
depth_detail=None,
|
|
4227
|
+
):
|
|
4228
|
+
style = style or Style(False)
|
|
4229
|
+
columns = TABLE_COLUMNS[detail]
|
|
4230
|
+
rows = [table_cells(cand, rank) for rank, cand in enumerate(top, 1)]
|
|
4231
|
+
widths = {col: max([len(col)] + [len(row[col]) for row in rows]) for col in columns}
|
|
4232
|
+
|
|
4233
|
+
def pad(col, value):
|
|
4234
|
+
if col == "MODEL":
|
|
4235
|
+
return value.ljust(widths[col])
|
|
4236
|
+
return value.rjust(widths[col])
|
|
4237
|
+
|
|
4238
|
+
header = []
|
|
4239
|
+
for col in columns:
|
|
4240
|
+
text = col.ljust(widths[col])
|
|
4241
|
+
header.append(text if col == "SCORE" else style.bold(text))
|
|
4242
|
+
print(" ".join(header))
|
|
4243
|
+
print(style.dim(" ".join("-" * widths[col] for col in columns)))
|
|
4244
|
+
for rank, row in enumerate(rows, 1):
|
|
4245
|
+
cells = []
|
|
4246
|
+
for col in columns:
|
|
4247
|
+
text = pad(col, row[col])
|
|
4248
|
+
if col == "RANK":
|
|
4249
|
+
text = style.bold(text)
|
|
4250
|
+
if rank == 1:
|
|
4251
|
+
text = style.green(text)
|
|
4252
|
+
elif rank == 1:
|
|
4253
|
+
text = style.green(text)
|
|
4254
|
+
elif col in ("QUAL", "QUALITY") and row[col] == "-":
|
|
4255
|
+
text = style.dim(text)
|
|
4256
|
+
cells.append(text)
|
|
4257
|
+
print(" ".join(cells))
|
|
4258
|
+
if bare:
|
|
4259
|
+
return
|
|
4260
|
+
print()
|
|
4261
|
+
print("Weights: " + format_weights(weights))
|
|
4262
|
+
warn = not quality_note or quality_note.get("warn")
|
|
4263
|
+
for line in quality_source_lines(quality_note):
|
|
4264
|
+
print(style.yellow(line) if warn else line)
|
|
4265
|
+
if depth_detail is not None:
|
|
4266
|
+
print()
|
|
4267
|
+
depth_detail(top)
|
|
4268
|
+
print()
|
|
4269
|
+
legend = LEGENDS[detail]
|
|
4270
|
+
width = max(len(term) for term, _ in legend)
|
|
4271
|
+
in_pct = f"{input_share:.0%}"
|
|
4272
|
+
out_pct = f"{1 - input_share:.0%}"
|
|
4273
|
+
for term, lines in legend:
|
|
4274
|
+
lines = [
|
|
4275
|
+
line.replace("{in_pct}", in_pct).replace("{out_pct}", out_pct)
|
|
4276
|
+
for line in lines
|
|
4277
|
+
]
|
|
4278
|
+
print(style.bold(term.ljust(width)) + " " + lines[0])
|
|
4279
|
+
for extra in lines[1:]:
|
|
4280
|
+
print(" " * (width + 2) + extra)
|
|
4281
|
+
|
|
4282
|
+
|
|
4283
|
+
DEALS_COLUMNS = (
|
|
4284
|
+
"MODEL",
|
|
4285
|
+
"OFF-PEAK",
|
|
4286
|
+
"SCHEDULE",
|
|
4287
|
+
"QUAL",
|
|
4288
|
+
"$IN/M",
|
|
4289
|
+
"$OUT/M",
|
|
4290
|
+
"CTX",
|
|
4291
|
+
"SCORE",
|
|
4292
|
+
)
|
|
4293
|
+
|
|
4294
|
+
DEALS_LEGEND = (
|
|
4295
|
+
("OFF-PEAK", "largest scheduled discount vs the standard rate"),
|
|
4296
|
+
("SCHEDULE", "when the discounted rate applies (UTC)"),
|
|
4297
|
+
)
|
|
4298
|
+
|
|
4299
|
+
|
|
4300
|
+
def print_deals(rows, style=None, bare=False):
|
|
4301
|
+
style = style or Style(False)
|
|
4302
|
+
if not rows:
|
|
4303
|
+
if not bare:
|
|
4304
|
+
print()
|
|
4305
|
+
print("No scheduled off-peak discounts in the current data.")
|
|
4306
|
+
return
|
|
4307
|
+
cells = [
|
|
4308
|
+
{
|
|
4309
|
+
"MODEL": display_text(row["id"]),
|
|
4310
|
+
"OFF-PEAK": row["sched_note"],
|
|
4311
|
+
"SCHEDULE": row["sched_detail"] or "--",
|
|
4312
|
+
"QUAL": fmt_quality(row["quality"]),
|
|
4313
|
+
"$IN/M": fmt_price(row["offpeak"]["input_per_1m"]),
|
|
4314
|
+
"$OUT/M": fmt_price(row["offpeak"]["output_per_1m"]),
|
|
4315
|
+
"CTX": fmt_context(row["context"]),
|
|
4316
|
+
"SCORE": "-" if row["score"] is None else f"{row['score']:.3f}",
|
|
4317
|
+
}
|
|
4318
|
+
for row in rows
|
|
4319
|
+
]
|
|
4320
|
+
widths = {
|
|
4321
|
+
col: max([len(col)] + [len(cell[col]) for cell in cells])
|
|
4322
|
+
for col in DEALS_COLUMNS
|
|
4323
|
+
}
|
|
4324
|
+
|
|
4325
|
+
def pad(col, value):
|
|
4326
|
+
if col in ("MODEL", "SCHEDULE"):
|
|
4327
|
+
return value.ljust(widths[col])
|
|
4328
|
+
return value.rjust(widths[col])
|
|
4329
|
+
|
|
4330
|
+
if not bare:
|
|
4331
|
+
print()
|
|
4332
|
+
print(
|
|
4333
|
+
style.bold(
|
|
4334
|
+
"Off-peak deals (cheapest off-peak window's prices; "
|
|
4335
|
+
"the standard rate applies at all other times)"
|
|
4336
|
+
)
|
|
4337
|
+
)
|
|
4338
|
+
print()
|
|
4339
|
+
header = []
|
|
4340
|
+
for col in DEALS_COLUMNS:
|
|
4341
|
+
# Every deals header is left-aligned (matching the main table and
|
|
4342
|
+
# the frozen mockup); only the data cells keep numeric right-align.
|
|
4343
|
+
text = col.ljust(widths[col])
|
|
4344
|
+
header.append(text if col == "SCORE" else style.bold(text))
|
|
4345
|
+
print(" ".join(header))
|
|
4346
|
+
print(style.dim(" ".join("-" * widths[col] for col in DEALS_COLUMNS)))
|
|
4347
|
+
for cell in cells:
|
|
4348
|
+
out = []
|
|
4349
|
+
for col in DEALS_COLUMNS:
|
|
4350
|
+
text = pad(col, cell[col])
|
|
4351
|
+
if col == "OFF-PEAK":
|
|
4352
|
+
text = style.bold(text)
|
|
4353
|
+
elif col == "QUAL" and cell[col] == "-":
|
|
4354
|
+
text = style.dim(text)
|
|
4355
|
+
out.append(text)
|
|
4356
|
+
print(" ".join(out))
|
|
4357
|
+
if bare:
|
|
4358
|
+
return
|
|
4359
|
+
print()
|
|
4360
|
+
width = max(len(term) for term, _ in DEALS_LEGEND)
|
|
4361
|
+
for term, text in DEALS_LEGEND:
|
|
4362
|
+
print(style.bold(term.ljust(width)) + " " + text)
|
|
4363
|
+
|
|
4364
|
+
|
|
4365
|
+
def print_depth_detail(top):
|
|
4366
|
+
"""Per-model AA depth detail: measured scores and selectable variants.
|
|
4367
|
+
|
|
4368
|
+
One block per model in the given (rank) order, listing the union of
|
|
4369
|
+
OpenRouter-supported and AA-measured efforts, deepest first. Models with
|
|
4370
|
+
no measured depths get a single ``no AA depth data`` line. Display only:
|
|
4371
|
+
AA $/task never enters the weighted score.
|
|
4372
|
+
"""
|
|
4373
|
+
if not top:
|
|
4374
|
+
return
|
|
4375
|
+
|
|
4376
|
+
headers = ["effort", "AA idx", "AA $/task", "vs default", "select"]
|
|
4377
|
+
|
|
4378
|
+
def finite(value):
|
|
4379
|
+
if (
|
|
4380
|
+
isinstance(value, (int, float))
|
|
4381
|
+
and not isinstance(value, bool)
|
|
4382
|
+
and math.isfinite(value)
|
|
4383
|
+
):
|
|
4384
|
+
return value
|
|
4385
|
+
return None
|
|
4386
|
+
|
|
4387
|
+
def order(effort):
|
|
4388
|
+
try:
|
|
4389
|
+
return (AA_EFFORT_ORDER.index(effort), effort)
|
|
4390
|
+
except ValueError:
|
|
4391
|
+
return (len(AA_EFFORT_ORDER), effort or "")
|
|
4392
|
+
|
|
4393
|
+
blocks = []
|
|
2972
4394
|
for rank, cand in enumerate(top, 1):
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
4395
|
+
reasoning = cand.get("reasoning") or {}
|
|
4396
|
+
supported = list(reasoning.get("supported_efforts") or [])
|
|
4397
|
+
records = [
|
|
4398
|
+
record
|
|
4399
|
+
for record in (cand.get("reasoning_depths") or [])
|
|
4400
|
+
if isinstance(record, dict) and record.get("effort")
|
|
4401
|
+
]
|
|
4402
|
+
measured = {record["effort"]: record for record in records}
|
|
4403
|
+
default = _display_default_effort(cand)
|
|
4404
|
+
is_non_reasoning = reasoning.get("is_reasoning") is False
|
|
4405
|
+
# A non-mandatory single-state model exposes only the "none" toggle.
|
|
4406
|
+
single_state_none = (
|
|
4407
|
+
reasoning.get("is_reasoning") is True
|
|
4408
|
+
and reasoning.get("mandatory") is not True
|
|
4409
|
+
and not supported
|
|
2987
4410
|
)
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
4411
|
+
default_cost = (
|
|
4412
|
+
finite((measured.get(default) or {}).get("cost_per_task_usd"))
|
|
4413
|
+
if default
|
|
4414
|
+
else None
|
|
4415
|
+
)
|
|
4416
|
+
efforts = sorted(set(supported) | set(measured), key=order)
|
|
4417
|
+
rows = []
|
|
4418
|
+
for effort in efforts:
|
|
4419
|
+
record = measured.get(effort) or {}
|
|
4420
|
+
index = finite(record.get("index"))
|
|
4421
|
+
cost = finite(record.get("cost_per_task_usd"))
|
|
4422
|
+
if effort == "reasoning":
|
|
4423
|
+
# Single-state reasoning row: the bare id is the only form.
|
|
4424
|
+
select = "-"
|
|
4425
|
+
elif effort == "none" and is_non_reasoning:
|
|
4426
|
+
# A non-reasoning model's bare id is the only form.
|
|
4427
|
+
select = "-"
|
|
4428
|
+
elif effort in supported:
|
|
4429
|
+
select = f"#{effort}"
|
|
4430
|
+
elif effort == "none" and single_state_none:
|
|
4431
|
+
select = "#none"
|
|
4432
|
+
else:
|
|
4433
|
+
select = "- not selectable"
|
|
4434
|
+
ratio = "-"
|
|
4435
|
+
if cost and default_cost:
|
|
4436
|
+
value = cost / default_cost
|
|
4437
|
+
ratio = f"{value:.1f}x" if value >= 1 else f"{value:.2f}x"
|
|
4438
|
+
rows.append(
|
|
4439
|
+
[
|
|
4440
|
+
f"{effort}*" if effort == default else effort,
|
|
4441
|
+
"-" if index is None else f"{index:.1f}",
|
|
4442
|
+
"-" if cost is None else f"{cost:.2f}",
|
|
4443
|
+
ratio,
|
|
4444
|
+
select,
|
|
4445
|
+
]
|
|
4446
|
+
)
|
|
4447
|
+
has_measured = any(
|
|
4448
|
+
finite(record.get("index")) is not None for record in measured.values()
|
|
4449
|
+
)
|
|
4450
|
+
blocks.append(
|
|
4451
|
+
{
|
|
4452
|
+
"rank": rank,
|
|
4453
|
+
"cand": cand,
|
|
4454
|
+
"default": default,
|
|
4455
|
+
"rows": rows,
|
|
4456
|
+
"has_records": bool(records),
|
|
4457
|
+
"missing_default": (
|
|
4458
|
+
bool(default)
|
|
4459
|
+
and default in efforts
|
|
4460
|
+
and finite((measured.get(default) or {}).get("index")) is None
|
|
4461
|
+
and has_measured
|
|
4462
|
+
),
|
|
4463
|
+
}
|
|
4464
|
+
)
|
|
4465
|
+
|
|
4466
|
+
widths = [len(header) for header in headers]
|
|
4467
|
+
for block in blocks:
|
|
4468
|
+
for row in block["rows"]:
|
|
4469
|
+
for i, cell in enumerate(row):
|
|
4470
|
+
widths[i] = max(widths[i], len(cell))
|
|
4471
|
+
|
|
4472
|
+
def emit(cells):
|
|
4473
|
+
padded = [cell.ljust(width) for cell, width in zip(cells[:-1], widths[:-1])]
|
|
4474
|
+
print(" " + " ".join(padded + [cells[-1]]))
|
|
4475
|
+
|
|
4476
|
+
print("depth detail (AA index / AA $ per task; OR $/M is constant across depths)")
|
|
4477
|
+
for position, block in enumerate(blocks):
|
|
4478
|
+
if position:
|
|
4479
|
+
print()
|
|
4480
|
+
cand = block["cand"]
|
|
4481
|
+
print(
|
|
4482
|
+
f"#{block['rank']} {cand['id']} "
|
|
4483
|
+
f"OR ${cand['price_in']:.2f} in / ${cand['price_out']:.2f} out per 1M "
|
|
4484
|
+
f"default: {block['default'] or '-'}"
|
|
4485
|
+
)
|
|
4486
|
+
if not block["has_records"]:
|
|
4487
|
+
print(" no AA depth data")
|
|
4488
|
+
continue
|
|
4489
|
+
if block["missing_default"]:
|
|
4490
|
+
print(f" AA has no {block['default']} record for this model.")
|
|
4491
|
+
emit(headers)
|
|
4492
|
+
for row in block["rows"]:
|
|
4493
|
+
emit(row)
|
|
4494
|
+
|
|
3009
4495
|
print()
|
|
3010
4496
|
print(
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
"
|
|
3015
|
-
"DISC = active discount; AGE = time since listed on OpenRouter"
|
|
3016
|
-
)
|
|
3017
|
-
print(
|
|
3018
|
-
"Prices shown are what a long session pays: several models bill at a "
|
|
3019
|
-
"higher rate once the prompt outgrows their cheap short-context tier."
|
|
3020
|
-
)
|
|
3021
|
-
print(
|
|
3022
|
-
"TIER marks how prices vary: a token threshold is the prompt size "
|
|
3023
|
-
"above which the higher rate applies; a percentage is a scheduled "
|
|
3024
|
-
"off-peak discount (price shown: the standard rate); SCHED marks a "
|
|
3025
|
-
"schedule whose discount is under 1%. --json lists each model's "
|
|
3026
|
-
"schedule."
|
|
3027
|
-
)
|
|
3028
|
-
print(
|
|
3029
|
-
"PREC = precision the ranked price is quoted at (maker's default when "
|
|
3030
|
-
"disclosed, best available otherwise; ! = default routing favors "
|
|
3031
|
-
"cheaper sub-default endpoints); prices are the cheapest live endpoint "
|
|
3032
|
-
"at that precision."
|
|
4497
|
+
"* = OpenRouter default; append #effort to the model id in OpenCode; "
|
|
4498
|
+
"- in select = not selectable; AA $/task is AA's measured benchmark "
|
|
4499
|
+
"cost at AA's reference price, not the OR per-token price; it never "
|
|
4500
|
+
"enters the weighted score."
|
|
3033
4501
|
)
|
|
3034
4502
|
|
|
3035
4503
|
|
|
4504
|
+
def print_best(top, args):
|
|
4505
|
+
"""Print the top candidate as an OpenCode model id.
|
|
4506
|
+
|
|
4507
|
+
A non-default ``--reasoning-effort`` appends ``#<effort>`` only when the
|
|
4508
|
+
model exposes that variant; otherwise the bare id selects the default.
|
|
4509
|
+
"""
|
|
4510
|
+
cand = top[0]
|
|
4511
|
+
model_id = opencode_model_id(cand["id"])
|
|
4512
|
+
effort = args.reasoning_effort
|
|
4513
|
+
reasoning = cand.get("reasoning")
|
|
4514
|
+
if (
|
|
4515
|
+
effort != "default"
|
|
4516
|
+
and isinstance(reasoning, dict)
|
|
4517
|
+
and reasoning.get("is_reasoning") is not False
|
|
4518
|
+
and supports_effort(reasoning, effort)
|
|
4519
|
+
):
|
|
4520
|
+
model_id += f"#{effort}"
|
|
4521
|
+
print(display_text(model_id))
|
|
4522
|
+
|
|
4523
|
+
|
|
3036
4524
|
def print_json(top):
|
|
3037
4525
|
payload = [
|
|
3038
4526
|
{
|
|
@@ -3050,6 +4538,18 @@ def print_json(top):
|
|
|
3050
4538
|
"quality_drift": (cand.get("quality_record") or {}).get("drift"),
|
|
3051
4539
|
"quality_or_index": (cand.get("quality_record") or {}).get("or_index"),
|
|
3052
4540
|
"quality_or_as_of": (cand.get("quality_record") or {}).get("or_as_of"),
|
|
4541
|
+
# The effort the quality record was measured at. Legacy records
|
|
4542
|
+
# (pre-depth paths) carry no "effort" key, so this degrades to
|
|
4543
|
+
# None exactly like a missing record.
|
|
4544
|
+
"quality_effort": (cand.get("quality_record") or {}).get("effort"),
|
|
4545
|
+
# The model's effective default effort, duplicated at top level:
|
|
4546
|
+
# data.json rows must keep it after the nested reasoning block is
|
|
4547
|
+
# dropped downstream. Same resolver as the DEPTH cell.
|
|
4548
|
+
"default_effort": _display_default_effort(cand),
|
|
4549
|
+
# The six published reasoning fields plus the measured depth
|
|
4550
|
+
# records; the internal is_reasoning flag never leaves the
|
|
4551
|
+
# process (catalog v5 and spec §8.4 publish only these keys).
|
|
4552
|
+
"reasoning": _published_reasoning_block(cand),
|
|
3053
4553
|
"input_usd_per_m": round(cand["price_in"], 6),
|
|
3054
4554
|
"output_usd_per_m": round(cand["price_out"], 6),
|
|
3055
4555
|
"blended_usd_per_m": round(cand["blended"], 6),
|
|
@@ -3123,6 +4623,12 @@ def build_catalog(
|
|
|
3123
4623
|
index_version=None,
|
|
3124
4624
|
catalog_generated_at=None,
|
|
3125
4625
|
catalog_quality=None,
|
|
4626
|
+
*,
|
|
4627
|
+
depth_lookup=None,
|
|
4628
|
+
catalog_depths=None,
|
|
4629
|
+
reasoning_by_id=None,
|
|
4630
|
+
requested_effort="default",
|
|
4631
|
+
schedule_entries=None,
|
|
3126
4632
|
):
|
|
3127
4633
|
"""Assemble the full-catalog document (see README "Catalog output").
|
|
3128
4634
|
|
|
@@ -3141,6 +4647,8 @@ def build_catalog(
|
|
|
3141
4647
|
catalog_quality is the hydrated record map (or None): candidates read
|
|
3142
4648
|
it through resolve_quality, and build_schedules falls back to it for
|
|
3143
4649
|
non-candidates.
|
|
4650
|
+
The keyword-only depth inputs feed build_schedules (spec 6.5); run()
|
|
4651
|
+
always supplies them, direct callers may omit them for the legacy path.
|
|
3144
4652
|
"""
|
|
3145
4653
|
now = datetime.now(timezone.utc)
|
|
3146
4654
|
weights = catalog_weights(candidates, quality_by_id)
|
|
@@ -3167,6 +4675,14 @@ def build_catalog(
|
|
|
3167
4675
|
)
|
|
3168
4676
|
if unknown:
|
|
3169
4677
|
raise ValueError(f"unknown quality source: {unknown[0]!r}")
|
|
4678
|
+
# Depth counts are candidate-scoped and separate from the disjoint label
|
|
4679
|
+
# buckets above (spec 9.1): schedule entries carry their own provenance
|
|
4680
|
+
# and never enter these counts or the pool invariants. matched_default
|
|
4681
|
+
# is the number of main quality records under strict selection -- exactly
|
|
4682
|
+
# the label buckets' sum, kept explicit for the publish gate.
|
|
4683
|
+
matched_default = matched_aa + matched_catalog + matched_openrouter
|
|
4684
|
+
depth_rows = sum(len(cand.get("reasoning_depths") or []) for cand in candidates)
|
|
4685
|
+
matched_depth = sum(1 for cand in candidates if cand.get("reasoning_depths"))
|
|
3170
4686
|
# mode is the run's quality chain, not a match outcome: "aa" when the
|
|
3171
4687
|
# AA API returned usable entries, "catalog" when unkeyed runs hydrated
|
|
3172
4688
|
# the published catalog, "none" when neither tier produced data.
|
|
@@ -3199,23 +4715,31 @@ def build_catalog(
|
|
|
3199
4715
|
"context": round(cand["context_score"], 4),
|
|
3200
4716
|
"age": round(cand["age_score"], 4),
|
|
3201
4717
|
}
|
|
3202
|
-
overall =
|
|
3203
|
-
for priority in PRIORITY_WEIGHTS:
|
|
3204
|
-
w = weights[priority]
|
|
3205
|
-
overall[priority] = round(
|
|
3206
|
-
w.get("quality", 0.0) * scores["quality"]
|
|
3207
|
-
+ w.get("price", 0.0) * scores["price"]
|
|
3208
|
-
+ w.get("context", 0.0) * scores["context"]
|
|
3209
|
-
+ w.get("age", 0.0) * scores["age"],
|
|
3210
|
-
4,
|
|
3211
|
-
)
|
|
4718
|
+
overall = overall_scores(cand, weights)
|
|
3212
4719
|
bench = aa_by_id.get(base_model_id(cand["id"])) or {}
|
|
3213
4720
|
# The quality record map is already a parameter; the winning
|
|
3214
4721
|
# record's as_of and drift verdict become entry fields verbatim.
|
|
3215
4722
|
# Unmatched candidates degrade to null -- same shape as the v3
|
|
3216
|
-
# additive blocks below.
|
|
3217
|
-
|
|
3218
|
-
|
|
4723
|
+
# additive blocks below. The candidate's own record wins (it is the
|
|
4724
|
+
# same object compute_scores attached), with the map as the fallback
|
|
4725
|
+
# for direct callers that skip compute_scores.
|
|
4726
|
+
record = cand.get("quality_record")
|
|
4727
|
+
if not isinstance(record, dict):
|
|
4728
|
+
record = quality_by_id.get(cand["id"])
|
|
4729
|
+
record = record if isinstance(record, dict) else None
|
|
4730
|
+
reasoning = cand.get("reasoning")
|
|
4731
|
+
reasoning = reasoning if isinstance(reasoning, dict) else {}
|
|
4732
|
+
depths = [
|
|
4733
|
+
{
|
|
4734
|
+
"effort": depth.get("effort"),
|
|
4735
|
+
"index": depth.get("index"),
|
|
4736
|
+
"cost_per_task_usd": depth.get("cost_per_task_usd"),
|
|
4737
|
+
"source": depth.get("source"),
|
|
4738
|
+
"as_of": depth.get("as_of"),
|
|
4739
|
+
}
|
|
4740
|
+
for depth in (cand.get("reasoning_depths") or [])
|
|
4741
|
+
if isinstance(depth, dict)
|
|
4742
|
+
]
|
|
3219
4743
|
entries.append(
|
|
3220
4744
|
{
|
|
3221
4745
|
"id": cand["id"],
|
|
@@ -3286,8 +4810,22 @@ def build_catalog(
|
|
|
3286
4810
|
# Provenance (schema v4), verbatim from the record: the
|
|
3287
4811
|
# winning source's own as_of and the drift verdict against
|
|
3288
4812
|
# OpenRouter's check value.
|
|
3289
|
-
"quality_as_of": record
|
|
3290
|
-
"quality_drift": record
|
|
4813
|
+
"quality_as_of": record.get("as_of") if record else None,
|
|
4814
|
+
"quality_drift": record.get("drift") if record else None,
|
|
4815
|
+
# Schema v5 depth contract: the effort the main quality was
|
|
4816
|
+
# measured at, and the model's reasoning metadata plus every
|
|
4817
|
+
# measured depth record (record shape: effort/index/
|
|
4818
|
+
# cost_per_task_usd/source/as_of -- `index`, never
|
|
4819
|
+
# `quality_index`).
|
|
4820
|
+
"quality_effort": record.get("effort") if record else None,
|
|
4821
|
+
"reasoning": {
|
|
4822
|
+
"default_effort": reasoning.get("default_effort"),
|
|
4823
|
+
"supported_efforts": reasoning.get("supported_efforts") or [],
|
|
4824
|
+
"default_enabled": reasoning.get("default_enabled"),
|
|
4825
|
+
"mandatory": reasoning.get("mandatory"),
|
|
4826
|
+
"supports_max_tokens": reasoning.get("supports_max_tokens"),
|
|
4827
|
+
"depths": depths,
|
|
4828
|
+
},
|
|
3291
4829
|
"scores": {**scores, "overall": overall},
|
|
3292
4830
|
# Additive provider-aware blocks (schema v3), verbatim from
|
|
3293
4831
|
# the candidate. Candidates that never went through the
|
|
@@ -3312,7 +4850,17 @@ def build_catalog(
|
|
|
3312
4850
|
|
|
3313
4851
|
return {
|
|
3314
4852
|
"schedules": build_schedules(
|
|
3315
|
-
models,
|
|
4853
|
+
models,
|
|
4854
|
+
args,
|
|
4855
|
+
quality_by_id,
|
|
4856
|
+
aa_by_id,
|
|
4857
|
+
balanced_by_id,
|
|
4858
|
+
catalog_quality,
|
|
4859
|
+
depth_lookup=depth_lookup,
|
|
4860
|
+
catalog_depths=catalog_depths,
|
|
4861
|
+
reasoning_by_id=reasoning_by_id,
|
|
4862
|
+
requested_effort=requested_effort,
|
|
4863
|
+
schedule_entries=schedule_entries,
|
|
3316
4864
|
),
|
|
3317
4865
|
"schema_version": CATALOG_SCHEMA_VERSION,
|
|
3318
4866
|
"tool": "model-compare",
|
|
@@ -3327,6 +4875,9 @@ def build_catalog(
|
|
|
3327
4875
|
"require_tools": not args.no_require_tools,
|
|
3328
4876
|
"exclude_free": args.exclude_free,
|
|
3329
4877
|
"include_batch": args.include_batch,
|
|
4878
|
+
# The requested effort documents the semantic of every model's
|
|
4879
|
+
# quality value (default = each model's effective default).
|
|
4880
|
+
"reasoning_effort": getattr(args, "reasoning_effort", "default"),
|
|
3330
4881
|
"weights": weights,
|
|
3331
4882
|
},
|
|
3332
4883
|
"sources": {
|
|
@@ -3336,6 +4887,11 @@ def build_catalog(
|
|
|
3336
4887
|
"matched_aa": matched_aa,
|
|
3337
4888
|
"matched_catalog": matched_catalog,
|
|
3338
4889
|
"matched_openrouter": matched_openrouter,
|
|
4890
|
+
# Depth coverage (schema v5), candidate-scoped like the
|
|
4891
|
+
# label buckets above. Schedule entries never contribute.
|
|
4892
|
+
"depth_rows": depth_rows,
|
|
4893
|
+
"matched_depth": matched_depth,
|
|
4894
|
+
"matched_default": matched_default,
|
|
3339
4895
|
# Provenance meta (schema v4): the published catalog's stamp
|
|
3340
4896
|
# when this run hydrated from it, and the AA envelope's
|
|
3341
4897
|
# intelligence index version when the AA API ran. Both null
|
|
@@ -3378,6 +4934,49 @@ def print_catalog(document):
|
|
|
3378
4934
|
print(json.dumps(document, indent=2))
|
|
3379
4935
|
|
|
3380
4936
|
|
|
4937
|
+
# --- plain-text styling -----------------------------------------------------
|
|
4938
|
+
# SGR attributes only: the terminal theme supplies the hues, so the output
|
|
4939
|
+
# stays readable on light and dark themes alike.
|
|
4940
|
+
SGR_RESET = "\x1b[0m"
|
|
4941
|
+
SGR_BOLD = "\x1b[1m"
|
|
4942
|
+
SGR_DIM = "\x1b[2m"
|
|
4943
|
+
SGR_GREEN = "\x1b[32m"
|
|
4944
|
+
SGR_YELLOW = "\x1b[33m"
|
|
4945
|
+
|
|
4946
|
+
|
|
4947
|
+
class Style:
|
|
4948
|
+
"""Minimal SGR wrapper; every method is a no-op when disabled."""
|
|
4949
|
+
|
|
4950
|
+
def __init__(self, enabled: bool):
|
|
4951
|
+
self.enabled = enabled
|
|
4952
|
+
|
|
4953
|
+
def _wrap(self, text: str, code: str) -> str:
|
|
4954
|
+
return f"{code}{text}{SGR_RESET}" if self.enabled else text
|
|
4955
|
+
|
|
4956
|
+
def bold(self, text: str) -> str:
|
|
4957
|
+
return self._wrap(text, SGR_BOLD)
|
|
4958
|
+
|
|
4959
|
+
def dim(self, text: str) -> str:
|
|
4960
|
+
return self._wrap(text, SGR_DIM)
|
|
4961
|
+
|
|
4962
|
+
def green(self, text: str) -> str:
|
|
4963
|
+
return self._wrap(text, SGR_GREEN)
|
|
4964
|
+
|
|
4965
|
+
def yellow(self, text: str) -> str:
|
|
4966
|
+
return self._wrap(text, SGR_YELLOW)
|
|
4967
|
+
|
|
4968
|
+
|
|
4969
|
+
def color_enabled(mode: str, stream) -> bool:
|
|
4970
|
+
"""Resolve --color for one stream. Explicit "on" beats NO_COLOR."""
|
|
4971
|
+
if mode == "off":
|
|
4972
|
+
return False
|
|
4973
|
+
if mode == "on":
|
|
4974
|
+
return True
|
|
4975
|
+
if os.environ.get("NO_COLOR"):
|
|
4976
|
+
return False
|
|
4977
|
+
return bool(getattr(stream, "isatty", lambda: False)())
|
|
4978
|
+
|
|
4979
|
+
|
|
3381
4980
|
# ---------------------------------------------------------------------------
|
|
3382
4981
|
# CLI
|
|
3383
4982
|
# ---------------------------------------------------------------------------
|
|
@@ -3395,99 +4994,145 @@ def parse_args(argv=None):
|
|
|
3395
4994
|
action="version",
|
|
3396
4995
|
version=f"%(prog)s {VERSION}",
|
|
3397
4996
|
)
|
|
3398
|
-
parser.
|
|
4997
|
+
output = parser.add_argument_group("output")
|
|
4998
|
+
ranking = parser.add_argument_group("ranking")
|
|
4999
|
+
filters = parser.add_argument_group("filters")
|
|
5000
|
+
sources = parser.add_argument_group("data sources & cache")
|
|
5001
|
+
filters.add_argument(
|
|
3399
5002
|
"--min-context",
|
|
3400
5003
|
type=int,
|
|
3401
5004
|
default=1_000_000,
|
|
3402
5005
|
help="hard minimum context window in tokens (default: 1000000)",
|
|
3403
5006
|
)
|
|
3404
|
-
|
|
5007
|
+
ranking.add_argument(
|
|
3405
5008
|
"--priority",
|
|
3406
5009
|
choices=sorted(PRIORITY_WEIGHTS),
|
|
3407
5010
|
default="balanced",
|
|
3408
5011
|
help="ranking emphasis (default: balanced)",
|
|
3409
5012
|
)
|
|
3410
|
-
|
|
5013
|
+
ranking.add_argument(
|
|
3411
5014
|
"--top", type=int, default=5, help="how many models to list (default: 5)"
|
|
3412
5015
|
)
|
|
3413
|
-
|
|
5016
|
+
output.add_argument(
|
|
3414
5017
|
"--best",
|
|
3415
5018
|
action="store_true",
|
|
3416
5019
|
help="print only the #1 model as an opencode id (openrouter/<model>), for scripting",
|
|
3417
5020
|
)
|
|
3418
|
-
|
|
5021
|
+
output.add_argument(
|
|
3419
5022
|
"--json", action="store_true", help="print the ranked candidates as JSON"
|
|
3420
5023
|
)
|
|
3421
|
-
|
|
5024
|
+
output.add_argument(
|
|
3422
5025
|
"--catalog",
|
|
3423
5026
|
action="store_true",
|
|
3424
5027
|
help="print the full model catalog (ranked candidates and filtered-out models with reasons) as one JSON document; ignores --top and --priority, cannot be combined with --best or --json",
|
|
3425
5028
|
)
|
|
3426
|
-
|
|
5029
|
+
output.add_argument(
|
|
5030
|
+
"--detail",
|
|
5031
|
+
choices=("simple", "expert"),
|
|
5032
|
+
default="expert",
|
|
5033
|
+
help="table density (default: expert; simple hides PREC, TIER, CTX and "
|
|
5034
|
+
"SCORE, mimicking the web page's simple view)",
|
|
5035
|
+
)
|
|
5036
|
+
output.add_argument(
|
|
5037
|
+
"--off-peak",
|
|
5038
|
+
action="store_true",
|
|
5039
|
+
help="also list scheduled off-peak deals, ranked by discount size",
|
|
5040
|
+
)
|
|
5041
|
+
output.add_argument(
|
|
5042
|
+
"--bare",
|
|
5043
|
+
action="store_true",
|
|
5044
|
+
help="print just the table(s), no summary or legend",
|
|
5045
|
+
)
|
|
5046
|
+
output.add_argument(
|
|
5047
|
+
"--no-depths",
|
|
5048
|
+
action="store_true",
|
|
5049
|
+
help=(
|
|
5050
|
+
"hide the per-model depth detail below the table (the DEPTH and "
|
|
5051
|
+
"QUAL columns stay)"
|
|
5052
|
+
),
|
|
5053
|
+
)
|
|
5054
|
+
output.add_argument(
|
|
5055
|
+
"--color",
|
|
5056
|
+
choices=("auto", "on", "off"),
|
|
5057
|
+
default="auto",
|
|
5058
|
+
help="colorize the table output (default: auto; honors NO_COLOR, and on "
|
|
5059
|
+
"forces color when piping through less -R)",
|
|
5060
|
+
)
|
|
5061
|
+
ranking.add_argument(
|
|
3427
5062
|
"--input-share",
|
|
3428
5063
|
type=float,
|
|
3429
5064
|
default=0.75,
|
|
3430
5065
|
help="input-token share used for the blended price, 0-1 (default: 0.75)",
|
|
3431
5066
|
)
|
|
3432
|
-
|
|
5067
|
+
ranking.add_argument(
|
|
3433
5068
|
"--recency-half-life",
|
|
3434
5069
|
type=float,
|
|
3435
5070
|
default=120.0,
|
|
3436
5071
|
help="age decay half-life in days (default: 120)",
|
|
3437
5072
|
)
|
|
3438
|
-
|
|
5073
|
+
ranking.add_argument(
|
|
3439
5074
|
"--max-age-days",
|
|
3440
5075
|
type=float,
|
|
3441
5076
|
default=0.0,
|
|
3442
5077
|
help="drop models listed more than this many days ago (0 = off)",
|
|
3443
5078
|
)
|
|
3444
|
-
|
|
5079
|
+
ranking.add_argument(
|
|
3445
5080
|
"--quality-ref",
|
|
3446
5081
|
type=float,
|
|
3447
5082
|
default=70.0,
|
|
3448
5083
|
help="AA intelligence index that counts as full quality score (default: 70)",
|
|
3449
5084
|
)
|
|
3450
|
-
|
|
5085
|
+
ranking.add_argument(
|
|
5086
|
+
"--reasoning-effort",
|
|
5087
|
+
choices=REASONING_EFFORT_CHOICES,
|
|
5088
|
+
default="default",
|
|
5089
|
+
help=(
|
|
5090
|
+
"reasoning effort used for quality and filtering: a specific "
|
|
5091
|
+
"effort drops models that cannot run it (default: each model's "
|
|
5092
|
+
"effective default, no filter)"
|
|
5093
|
+
),
|
|
5094
|
+
)
|
|
5095
|
+
sources.add_argument(
|
|
3451
5096
|
"--aa-api-key",
|
|
3452
5097
|
default=None,
|
|
3453
5098
|
help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
|
|
3454
5099
|
)
|
|
3455
|
-
|
|
5100
|
+
sources.add_argument(
|
|
3456
5101
|
"--openrouter-api-key",
|
|
3457
5102
|
default=None,
|
|
3458
5103
|
help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
|
|
3459
5104
|
)
|
|
3460
|
-
|
|
5105
|
+
filters.add_argument(
|
|
3461
5106
|
"--no-require-tools",
|
|
3462
5107
|
action="store_true",
|
|
3463
5108
|
help="do not require tool-calling support (coding agents want it)",
|
|
3464
5109
|
)
|
|
3465
|
-
|
|
5110
|
+
filters.add_argument(
|
|
3466
5111
|
"--exclude-free",
|
|
3467
5112
|
action="store_true",
|
|
3468
5113
|
help="drop zero-cost listings such as rate-limited :free variants",
|
|
3469
5114
|
)
|
|
3470
|
-
|
|
5115
|
+
filters.add_argument(
|
|
3471
5116
|
"--include-batch",
|
|
3472
5117
|
action="store_true",
|
|
3473
5118
|
help="keep ':batch' variants (asynchronous completion; cheaper but unsuitable for interactive agents)",
|
|
3474
5119
|
)
|
|
3475
|
-
|
|
5120
|
+
filters.add_argument(
|
|
3476
5121
|
"--discount",
|
|
3477
5122
|
action="store_true",
|
|
3478
5123
|
help="only list models with an active discount",
|
|
3479
5124
|
)
|
|
3480
|
-
|
|
5125
|
+
filters.add_argument(
|
|
3481
5126
|
"--no-zdr",
|
|
3482
5127
|
action="store_true",
|
|
3483
5128
|
help="consider all models, not just zero-data-retention (ZDR) ones",
|
|
3484
5129
|
)
|
|
3485
|
-
|
|
5130
|
+
sources.add_argument(
|
|
3486
5131
|
"--no-cache",
|
|
3487
5132
|
action="store_true",
|
|
3488
5133
|
help="bypass the local response cache and refetch",
|
|
3489
5134
|
)
|
|
3490
|
-
|
|
5135
|
+
sources.add_argument(
|
|
3491
5136
|
"--cache-ttl",
|
|
3492
5137
|
type=int,
|
|
3493
5138
|
default=6 * 3600,
|
|
@@ -3520,7 +5165,13 @@ def main(argv=None) -> int:
|
|
|
3520
5165
|
|
|
3521
5166
|
|
|
3522
5167
|
def run(args, models, or_cached) -> int:
|
|
3523
|
-
|
|
5168
|
+
(
|
|
5169
|
+
discounts,
|
|
5170
|
+
zdr_ids,
|
|
5171
|
+
aa_by_id,
|
|
5172
|
+
frontend_cache_hits,
|
|
5173
|
+
frontend_models,
|
|
5174
|
+
) = fetch_openrouter_frontend(args)
|
|
3524
5175
|
if not args.no_zdr and not zdr_ids:
|
|
3525
5176
|
warn(
|
|
3526
5177
|
"ZDR data unavailable; refusing to rank possibly non-ZDR models "
|
|
@@ -3536,25 +5187,43 @@ def run(args, models, or_cached) -> int:
|
|
|
3536
5187
|
if args.aa_api_key or os.environ.get("AA_API_KEY"):
|
|
3537
5188
|
aa_entries, aa_source, aa_cached, aa_index_version = fetch_aa_entries(args)
|
|
3538
5189
|
catalog_quality = None
|
|
5190
|
+
catalog_depths = None
|
|
3539
5191
|
catalog_generated_at = None
|
|
5192
|
+
# Keyed runs never hydrate schedules from the published catalog.
|
|
5193
|
+
schedule_entries = None
|
|
3540
5194
|
else:
|
|
3541
5195
|
aa_entries, aa_source, aa_cached, aa_index_version = [], None, False, None
|
|
3542
|
-
# None (transport failure) and
|
|
5196
|
+
# None (transport failure) and empty maps (fetched-but-unusable
|
|
3543
5197
|
# artifact) both degrade to no catalog tier -- resolve_quality and
|
|
3544
5198
|
# the footer treat them identically as "catalog unavailable".
|
|
3545
5199
|
catalog_doc = fetch_quality_catalog(args)
|
|
3546
5200
|
catalog_quality = (catalog_doc or {}).get("by_id") or None
|
|
5201
|
+
catalog_depths = (catalog_doc or {}).get("depths_by_id") or None
|
|
5202
|
+
# Spec 6.5: the published schedules array is the unkeyed Deals
|
|
5203
|
+
# source of truth -- schedule-only models never appear in models.
|
|
5204
|
+
schedule_entries = (catalog_doc or {}).get("schedules_by_id") or None
|
|
3547
5205
|
# The catalog meta only means something when hydration actually
|
|
3548
5206
|
# produced rows: an unusable artifact contributes no meta, so the
|
|
3549
5207
|
# sources block and the mode never credit a catalog that yielded
|
|
3550
5208
|
# nothing.
|
|
3551
5209
|
catalog_generated_at = (
|
|
3552
|
-
(catalog_doc or {}).get("generated_at")
|
|
5210
|
+
(catalog_doc or {}).get("generated_at")
|
|
5211
|
+
if (catalog_quality or catalog_depths)
|
|
5212
|
+
else None
|
|
3553
5213
|
)
|
|
3554
|
-
exact, fuzzy = build_aa_lookup(aa_entries)
|
|
5214
|
+
exact, fuzzy = build_aa_lookup(dedupe_aa_entries_by_key(aa_entries))
|
|
5215
|
+
depth_lookup = build_aa_depth_lookup(aa_entries)
|
|
3555
5216
|
|
|
3556
5217
|
filtered = []
|
|
3557
|
-
|
|
5218
|
+
reasoning_by_id = build_reasoning_metadata(models, frontend_models)
|
|
5219
|
+
candidates, dropped = build_candidates(
|
|
5220
|
+
models,
|
|
5221
|
+
args,
|
|
5222
|
+
discounts,
|
|
5223
|
+
zdr_ids,
|
|
5224
|
+
filtered,
|
|
5225
|
+
reasoning_by_id=reasoning_by_id,
|
|
5226
|
+
)
|
|
3558
5227
|
# The provider-aware stage: every surviving candidate is repriced at its
|
|
3559
5228
|
# baseline endpoint (or dropped fail-closed) before any scoring runs.
|
|
3560
5229
|
candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
|
|
@@ -3576,7 +5245,15 @@ def run(args, models, or_cached) -> int:
|
|
|
3576
5245
|
return 2
|
|
3577
5246
|
|
|
3578
5247
|
quality_by_id, quality_source_by_id = resolve_quality(
|
|
3579
|
-
candidates,
|
|
5248
|
+
candidates,
|
|
5249
|
+
aa_by_id,
|
|
5250
|
+
exact,
|
|
5251
|
+
fuzzy,
|
|
5252
|
+
aa_source,
|
|
5253
|
+
catalog_quality,
|
|
5254
|
+
depth_lookup=depth_lookup,
|
|
5255
|
+
catalog_depths=catalog_depths,
|
|
5256
|
+
requested_effort=args.reasoning_effort,
|
|
3580
5257
|
)
|
|
3581
5258
|
weights = compute_scores(candidates, args, quality_by_id)
|
|
3582
5259
|
|
|
@@ -3621,6 +5298,11 @@ def run(args, models, or_cached) -> int:
|
|
|
3621
5298
|
aa_index_version,
|
|
3622
5299
|
catalog_generated_at,
|
|
3623
5300
|
catalog_quality,
|
|
5301
|
+
depth_lookup=depth_lookup,
|
|
5302
|
+
catalog_depths=catalog_depths,
|
|
5303
|
+
reasoning_by_id=reasoning_by_id,
|
|
5304
|
+
requested_effort=args.reasoning_effort,
|
|
5305
|
+
schedule_entries=schedule_entries,
|
|
3624
5306
|
)
|
|
3625
5307
|
)
|
|
3626
5308
|
return 0
|
|
@@ -3629,45 +5311,73 @@ def run(args, models, or_cached) -> int:
|
|
|
3629
5311
|
top = candidates[:limit]
|
|
3630
5312
|
|
|
3631
5313
|
if args.best:
|
|
3632
|
-
|
|
5314
|
+
print_best(top, args)
|
|
3633
5315
|
elif args.json:
|
|
3634
5316
|
print_json(top)
|
|
3635
5317
|
else:
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
5318
|
+
style = Style(color_enabled(args.color, sys.stdout))
|
|
5319
|
+
err_style = Style(color_enabled(args.color, sys.stderr))
|
|
5320
|
+
if not args.bare:
|
|
5321
|
+
source_note = []
|
|
5322
|
+
if or_cached:
|
|
5323
|
+
source_note.append("catalog cached")
|
|
5324
|
+
if "discounts" in frontend_cache_hits:
|
|
5325
|
+
source_note.append("discounts cached")
|
|
5326
|
+
if "zdr" in frontend_cache_hits:
|
|
5327
|
+
source_note.append("ZDR cached")
|
|
5328
|
+
if "aa" in frontend_cache_hits:
|
|
5329
|
+
source_note.append("AA benchmarks cached")
|
|
5330
|
+
if aa_cached and aa_source:
|
|
5331
|
+
source_note.append("quality cached")
|
|
5332
|
+
if source_note:
|
|
5333
|
+
print(
|
|
5334
|
+
f"note: {', '.join(source_note)} (use --no-cache to refresh)",
|
|
5335
|
+
file=sys.stderr,
|
|
5336
|
+
)
|
|
5337
|
+
print(
|
|
5338
|
+
f"{len(candidates)} of {len(models)} listed models ranked",
|
|
5339
|
+
file=sys.stderr,
|
|
3640
5340
|
)
|
|
3641
|
-
|
|
5341
|
+
if dropped:
|
|
5342
|
+
bits = ", ".join(
|
|
5343
|
+
f"{v} {k}"
|
|
5344
|
+
for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
|
|
5345
|
+
)
|
|
5346
|
+
print(err_style.dim(f"(dropped: {bits})"), file=sys.stderr)
|
|
5347
|
+
print(file=sys.stderr)
|
|
3642
5348
|
# The footer lists per-source counts, each source's own as_of and
|
|
3643
5349
|
# the AA index version when known; unmatched candidates and drift
|
|
3644
|
-
# get trailing
|
|
3645
|
-
# fallback
|
|
5350
|
+
# get trailing clauses. No record resolved -> print_table's
|
|
5351
|
+
# quality-blind fallback line.
|
|
3646
5352
|
unmatched = len(candidates) - len(quality_by_id)
|
|
3647
|
-
quality_note = quality_footer_note(
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
if "aa" in frontend_cache_hits:
|
|
3658
|
-
source_note.append("AA benchmarks cached")
|
|
3659
|
-
if aa_cached and aa_source:
|
|
3660
|
-
source_note.append("quality cached")
|
|
3661
|
-
if source_note:
|
|
3662
|
-
print(
|
|
3663
|
-
f"note: {', '.join(source_note)} (use --no-cache to refresh)",
|
|
3664
|
-
file=sys.stderr,
|
|
3665
|
-
)
|
|
3666
|
-
print(
|
|
3667
|
-
f"pool: {len(models)} listed -> {len(candidates)} candidates{drop_note}",
|
|
3668
|
-
file=sys.stderr,
|
|
5353
|
+
quality_note = quality_footer_note(quality_by_id, aa_index_version, unmatched)
|
|
5354
|
+
print_table(
|
|
5355
|
+
top,
|
|
5356
|
+
weights,
|
|
5357
|
+
quality_note,
|
|
5358
|
+
detail=args.detail,
|
|
5359
|
+
style=style,
|
|
5360
|
+
bare=args.bare,
|
|
5361
|
+
input_share=args.input_share,
|
|
5362
|
+
depth_detail=(None if args.bare or args.no_depths else print_depth_detail),
|
|
3669
5363
|
)
|
|
3670
|
-
|
|
5364
|
+
if args.off_peak:
|
|
5365
|
+
catalog_w = catalog_weights(candidates, quality_by_id)
|
|
5366
|
+
# Use the exact unrounded weighted score (not overall_scores,
|
|
5367
|
+
# whose rounded components can disagree by 0.001) so a candidate
|
|
5368
|
+
# in both tables always prints the same SCORE under balanced.
|
|
5369
|
+
balanced_w = catalog_w["balanced"]
|
|
5370
|
+
balanced = {
|
|
5371
|
+
cand["id"]: weighted_score(cand, balanced_w) for cand in candidates
|
|
5372
|
+
}
|
|
5373
|
+
schedules = build_schedules(
|
|
5374
|
+
models, args, quality_by_id, aa_by_id, balanced, catalog_quality
|
|
5375
|
+
)
|
|
5376
|
+
# build_schedules resolves quality for any listed model; the web
|
|
5377
|
+
# page's deals table shows that resolved value even for models
|
|
5378
|
+
# outside the ranked pool, so keep it here too. Only SCORE is
|
|
5379
|
+
# blanked for non-candidates (balanced is None for them).
|
|
5380
|
+
print_deals(off_peak_deals(schedules), style=style, bare=args.bare)
|
|
3671
5381
|
return 0
|
|
3672
5382
|
|
|
3673
5383
|
|