model-compare-opencode-plugin 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,6 +37,9 @@ Examples:
37
37
  model_compare.py --best print only "#1 opencode id" (openrouter/<model>)
38
38
  model_compare.py --priority price --top 3 three cheapest-sensible picks
39
39
  model_compare.py --discount only currently-discounted models
40
+ model_compare.py --off-peak also list scheduled off-peak deals
41
+ model_compare.py --detail simple fewer columns, friendlier labels
42
+ model_compare.py --bare just the table(s): no summary or legend
40
43
  model_compare.py --catalog full catalog (ranked + filtered) as one JSON document
41
44
  MODEL=$(model_compare.py --best) shell integration (opencode --model $MODEL)
42
45
  """
@@ -68,10 +71,27 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
68
71
  # (legacy endpoints 410 after 2026-11-04; see
69
72
  # https://artificialanalysis.ai/data-api/migrate-v2-data).
70
73
  AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
71
- # Cache key for the AA API entries envelope. v3 adds the per-fetch `as_of`
72
- # timestamp on every entry and the envelope's intelligence_index_version;
73
- # payloads under the pre-v3 keys predate those fields and must never be read.
74
- AA_ENTRIES_CACHE = "aa-intelligence-v3"
74
+ # Cache key for the AA API entries envelope. v4 adds the per-effort fields
75
+ # (id, slug, base, effort, creator, cost_per_task) parsed from each row's
76
+ # name/slug; v3 payloads predate them and must never be read.
77
+ AA_ENTRIES_CACHE = "aa-intelligence-v4"
78
+ # Reasoning-effort labels AA encodes in a row's name parenthetical or slug
79
+ # suffix. AA_EFFORT_ORDER is the display order (deepest first);
80
+ # REASONING_GATEWAY_EFFORTS is what OpenRouter accepts as a discrete effort.
81
+ AA_EFFORT_ORDER = (
82
+ "max",
83
+ "xhigh",
84
+ "high",
85
+ "medium",
86
+ "low",
87
+ "minimal",
88
+ "none",
89
+ "reasoning",
90
+ )
91
+ REASONING_GATEWAY_EFFORTS = ("max", "xhigh", "high", "medium", "low", "minimal", "none")
92
+ # --reasoning-effort values: "default" keeps each model's effective default
93
+ # and does not filter; the rest select and filter (spec §7).
94
+ REASONING_EFFORT_CHOICES = ("default",) + tuple(reversed(REASONING_GATEWAY_EFFORTS))
75
95
  # The repo's own published catalog.json -- built by keyed CI runs, hydrated
76
96
  # by unkeyed local runs (design: quality source reconciliation, source 2).
77
97
  # The github.io host 301s to plain http://model-compare.dev/catalog.json, an
@@ -84,7 +104,7 @@ QUALITY_CATALOG_CACHE = "published-catalog"
84
104
  # A published catalog generated more than this many days ago is stale:
85
105
  # warn but use -- the records' as_of makes the staleness visible downstream.
86
106
  QUALITY_CATALOG_MAX_AGE_DAYS = 14
87
- VERSION = "0.5.0"
107
+ VERSION = "0.6.0"
88
108
  USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
89
109
 
90
110
  # opencode expects provider-qualified model ids: openrouter/<provider/model>.
@@ -103,7 +123,7 @@ PRIORITY_WEIGHTS = {
103
123
  "quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
104
124
  }
105
125
 
106
- CATALOG_SCHEMA_VERSION = 4
126
+ CATALOG_SCHEMA_VERSION = 5
107
127
 
108
128
  # The drop-reason keys build_candidates counts, verbatim -- keep in sync with
109
129
  # its drop() call sites. pool.dropped lists all of them zero-filled so the
@@ -125,6 +145,7 @@ CATALOG_DROP_REASONS = (
125
145
  "tool calling",
126
146
  "expired",
127
147
  "age",
148
+ "reasoning effort",
128
149
  "no_baseline_eligible_endpoint",
129
150
  "endpoint_fetch_failed",
130
151
  )
@@ -1217,110 +1238,229 @@ def fetch_session_cost(args) -> dict | None:
1217
1238
  # ---------------------------------------------------------------------------
1218
1239
 
1219
1240
 
1220
- # Pre-reconciliation published catalogs (schema v3, written before the AA
1221
- # tier was relabeled) record the AA tier as "api"; hydration normalizes it
1222
- # to the final "aa" label. Every other recorded label passes through as-is.
1223
- _QUALITY_SOURCE_NORMALIZATION = {"api": "aa"}
1241
+ # Pre-reconciliation published catalogs (schema v3/v4) are retired: v5 is
1242
+ # the only shape whose `quality` means "the index at the requested effort".
1243
+ CATALOG_QUALITY_SCHEMA_VERSION = 5
1224
1244
 
1225
1245
 
1226
- def catalog_quality_by_id(doc) -> dict:
1227
- """Pure parser: a published catalog document -> {id: quality record}.
1246
+ def _catalog_index(value) -> float | None:
1247
+ """A finite in-range catalog index, or None for poisoned/absent input."""
1248
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
1249
+ return None
1250
+ try:
1251
+ index = float(value) # huge ints raise OverflowError
1252
+ except OverflowError:
1253
+ return None
1254
+ if not math.isfinite(index) or not 0.0 <= index <= 100.0:
1255
+ return None
1256
+ return index
1257
+
1258
+
1259
+ def _parse_catalog_v5(doc) -> tuple[dict, dict]:
1260
+ """Pure parser: a published catalog v5 document -> (by_id, depths_by_id).
1228
1261
 
1229
- No fetch, no cache, no args object -- the fetcher owns transport and
1230
- freshness; this owns shape. Records are {"index", "source", "as_of"}
1231
- and feed resolve_quality's catalog_quality parameter on unkeyed runs;
1232
- the values were produced by a keyed CI run, so hydration is an exact
1233
- catalog-id join with no re-matching.
1262
+ by_id maps id -> {"index", "source", "as_of", "effort"} for the model's
1263
+ main quality (the index at the keyed run's requested effort). The
1264
+ depth map maps id -> {"default_effort", "depths": {effort: record}} for
1265
+ the model's full measured depth list; resolve_quality re-selects from
1266
+ it at the unkeyed run's live effective default.
1234
1267
 
1235
1268
  Gates (design: failure modes):
1236
- - schema_version must be an int >= 3; anything older warns
1237
- "published catalog schema vX, expected >= 3; skipping hydration"
1238
- and yields {}. Accepting v3 is the deploy-day skew case: the live
1239
- artifact predates the v4 per-row quality_as_of.
1269
+ - schema_version must be an int == 5. v3/v4 predate the depth contract
1270
+ (their `quality` means max effort), so they warn
1271
+ "published catalog schema vX, expected 5; skipping hydration" and
1272
+ yield empty maps. Mixing semantics is never allowed.
1240
1273
  - Any shape surprise -- a non-dict document, a non-int schema_version,
1241
- a non-list models, a row without a usable id, or a matched row whose
1242
- quality is not a finite number or whose quality_match is not a
1243
- non-empty string -- warns and yields {} for the WHOLE artifact.
1244
- Never a partial hydration, never a raise: the catalog is
1245
- machine-generated, so one malformed row means this is not the
1246
- artifact we think it is.
1274
+ a non-list models, a row without a usable id, a main quality that is
1275
+ not finite/in-range, or malformed depth data -- warns and yields
1276
+ empty maps for the WHOLE artifact. Never a partial hydration, never
1277
+ a raise: the catalog is machine-generated, so one malformed row
1278
+ means this is not the artifact we think it is.
1247
1279
  - Rows with quality None and quality_match None are legitimate
1248
- unmatched models; they are skipped silently (nothing to hydrate).
1280
+ unmatched models; they are skipped in by_id but their depth block is
1281
+ still parsed (a model can have measured depths without a main score).
1249
1282
 
1250
- Normalization: v3 rows map quality_match "api" -> "aa" and date every
1251
- row with the catalog's generated_at -- the live v3 artifact carries
1252
- quality_as_of: null, which must be ignored, not passed through.
1253
- v4+ rows keep their recorded quality_as_of (None when absent) and
1254
- source label as-is: that label is the row-level truth inside the
1255
- published artifact (a keyed CI run emits "aa"/"openrouter", never
1256
- "catalog"). resolve_quality's catalog tier owns the run-level label:
1257
- it relabels every hydrated record to the chain label "catalog".
1283
+ Per-depth records use the published shape {effort, index,
1284
+ cost_per_task_usd, source, as_of} -- the quality record's own shape
1285
+ (`index`, never `quality_index`).
1258
1286
  """
1259
1287
  if not isinstance(doc, dict):
1260
1288
  warn("published catalog returned an unexpected response shape")
1261
- return {}
1289
+ return {}, {}
1262
1290
  schema_version = doc.get("schema_version")
1263
1291
  if type(schema_version) is not int:
1264
1292
  warn("published catalog returned an unexpected response shape")
1265
- return {}
1266
- if schema_version < 3:
1293
+ return {}, {}
1294
+ if schema_version != CATALOG_QUALITY_SCHEMA_VERSION:
1267
1295
  warn(
1268
- f"published catalog schema v{schema_version}, expected >= 3; "
1269
- "skipping hydration"
1296
+ f"published catalog schema v{schema_version}, expected "
1297
+ f"{CATALOG_QUALITY_SCHEMA_VERSION}; skipping hydration"
1270
1298
  )
1271
- return {}
1299
+ return {}, {}
1272
1300
  models = doc.get("models")
1273
1301
  if not isinstance(models, list):
1274
1302
  warn("published catalog returned an unexpected response shape")
1275
- return {}
1276
- generated_at = doc.get("generated_at")
1277
- v3_as_of = generated_at if isinstance(generated_at, str) and generated_at else None
1303
+ return {}, {}
1278
1304
  by_id: dict = {}
1305
+ depths_by_id: dict = {}
1279
1306
  for entry in models:
1280
1307
  if not isinstance(entry, dict):
1281
1308
  warn("published catalog returned an unexpected response shape")
1282
- return {}
1309
+ return {}, {}
1283
1310
  model_id = entry.get("id")
1284
1311
  if not (isinstance(model_id, str) and model_id):
1285
1312
  warn("published catalog returned an unexpected response shape")
1286
- return {}
1313
+ return {}, {}
1287
1314
  quality = entry.get("quality")
1288
1315
  match = entry.get("quality_match")
1289
- if match is None and quality is None:
1290
- continue # legitimate unmatched row: nothing to hydrate
1291
- if not (isinstance(match, str) and match):
1292
- warn("published catalog returned an unexpected response shape")
1293
- return {}
1294
- if not isinstance(quality, (int, float)) or isinstance(quality, bool):
1295
- warn("published catalog returned an unexpected response shape")
1296
- return {}
1297
- try:
1298
- index = float(quality) # huge ints raise OverflowError
1299
- except OverflowError:
1316
+ if not (match is None and quality is None):
1317
+ if not (isinstance(match, str) and match):
1318
+ warn("published catalog returned an unexpected response shape")
1319
+ return {}, {}
1320
+ index = _catalog_index(quality)
1321
+ if index is None:
1322
+ warn("published catalog returned an unexpected response shape")
1323
+ return {}, {}
1324
+ effort = entry.get("quality_effort")
1325
+ if effort is not None and not (isinstance(effort, str) and effort):
1326
+ warn("published catalog returned an unexpected response shape")
1327
+ return {}, {}
1328
+ recorded = entry.get("quality_as_of")
1329
+ by_id[model_id] = {
1330
+ "index": index,
1331
+ "source": match,
1332
+ "as_of": recorded if isinstance(recorded, str) and recorded else None,
1333
+ "effort": effort,
1334
+ }
1335
+ reasoning = entry.get("reasoning")
1336
+ if reasoning is None:
1337
+ continue
1338
+ if not isinstance(reasoning, dict):
1300
1339
  warn("published catalog returned an unexpected response shape")
1301
- return {}
1302
- # Non-finite indexes would serialize as NaN and poison quality.
1303
- if not math.isfinite(index):
1340
+ return {}, {}
1341
+ default_effort = reasoning.get("default_effort")
1342
+ if default_effort is not None and not isinstance(default_effort, str):
1304
1343
  warn("published catalog returned an unexpected response shape")
1305
- return {}
1306
- # The AA indexes run 0..100 (the range the catalog validator
1307
- # enforces on quality); an out-of-range value is poisoned data and
1308
- # degrades the whole artifact. The surface is unkeyed runs: only
1309
- # they hydrate from the published catalog.
1310
- if not 0.0 <= index <= 100.0:
1344
+ return {}, {}
1345
+ raw_depths = reasoning.get("depths")
1346
+ if raw_depths is None:
1347
+ raw_depths = []
1348
+ if not isinstance(raw_depths, list):
1311
1349
  warn("published catalog returned an unexpected response shape")
1312
- return {}
1313
- if schema_version == 3:
1314
- as_of = v3_as_of
1315
- else:
1316
- recorded = entry.get("quality_as_of")
1317
- as_of = recorded if isinstance(recorded, str) and recorded else None
1318
- by_id[model_id] = {
1319
- "index": index,
1320
- "source": _QUALITY_SOURCE_NORMALIZATION.get(match, match),
1321
- "as_of": as_of,
1350
+ return {}, {}
1351
+ depths: dict = {}
1352
+ for depth in raw_depths:
1353
+ if not isinstance(depth, dict):
1354
+ warn("published catalog returned an unexpected response shape")
1355
+ return {}, {}
1356
+ effort = depth.get("effort")
1357
+ if not (isinstance(effort, str) and effort):
1358
+ warn("published catalog returned an unexpected response shape")
1359
+ return {}, {}
1360
+ index = _catalog_index(depth.get("index"))
1361
+ if index is None:
1362
+ warn("published catalog returned an unexpected response shape")
1363
+ return {}, {}
1364
+ cost = depth.get("cost_per_task_usd")
1365
+ if cost is not None:
1366
+ cost = _finite_cost(cost)
1367
+ if cost is None:
1368
+ warn("published catalog returned an unexpected response shape")
1369
+ return {}, {}
1370
+ source = depth.get("source")
1371
+ if not (isinstance(source, str) and source):
1372
+ warn("published catalog returned an unexpected response shape")
1373
+ return {}, {}
1374
+ as_of = depth.get("as_of")
1375
+ if as_of is not None and not isinstance(as_of, str):
1376
+ warn("published catalog returned an unexpected response shape")
1377
+ return {}, {}
1378
+ if effort in depths:
1379
+ # Duplicate efforts are ambiguous: fail closed, never guess.
1380
+ warn("published catalog returned an unexpected response shape")
1381
+ return {}, {}
1382
+ depths[effort] = {
1383
+ "effort": effort,
1384
+ "index": index,
1385
+ "cost_per_task_usd": cost,
1386
+ "source": source,
1387
+ "as_of": as_of if as_of else None,
1388
+ }
1389
+ depths_by_id[model_id] = {
1390
+ "default_effort": default_effort if default_effort else None,
1391
+ "depths": depths,
1392
+ }
1393
+ return by_id, depths_by_id
1394
+
1395
+
1396
+ def catalog_quality_by_id(doc) -> dict:
1397
+ """The v5 main-quality map {id: {"index", "source", "as_of", "effort"}}.
1398
+
1399
+ Pure parser; see _parse_catalog_v5 for the gates and the depth map.
1400
+ """
1401
+ return _parse_catalog_v5(doc)[0]
1402
+
1403
+
1404
+ def _catalog_schedule_entries(doc) -> dict:
1405
+ """Pure parser: published catalog v5 schedules -> id -> recorded provenance.
1406
+
1407
+ Spec 6.5: unkeyed runs rebuild the Deals table from the published
1408
+ catalog's schedules array, because schedule-only models (sub-1M context,
1409
+ non-ZDR) never appear in the models array. Each value is
1410
+ {"quality", "quality_effort", "quality_as_of", "quality_source"},
1411
+ copied verbatim so the rebuild publishes the keyed run's recorded
1412
+ provenance unchanged.
1413
+
1414
+ A non-list schedules block yields an empty map. Individual malformed
1415
+ rows (not an object, no usable id, a non-finite/out-of-range quality, or
1416
+ a wrong-typed provenance field) are skipped rather than guessed; the
1417
+ model then falls through the existing tiers.
1418
+
1419
+ Schema gate: only a v5 catalog contributes schedule entries. v4/v3
1420
+ schedules predate ``quality_effort``, so hydrating them would mix the
1421
+ old max-effort semantics into a default-effort run (spec 9.1).
1422
+ """
1423
+ if not isinstance(doc, dict):
1424
+ return {}
1425
+ schema_version = doc.get("schema_version")
1426
+ if (
1427
+ type(schema_version) is not int
1428
+ or schema_version != CATALOG_QUALITY_SCHEMA_VERSION
1429
+ ):
1430
+ return {}
1431
+ schedules = doc.get("schedules")
1432
+ if not isinstance(schedules, list):
1433
+ return {}
1434
+ entries = {}
1435
+ for entry in schedules:
1436
+ if not isinstance(entry, dict):
1437
+ continue
1438
+ model_id = entry.get("id")
1439
+ if not (isinstance(model_id, str) and model_id):
1440
+ continue
1441
+ quality = entry.get("quality")
1442
+ if quality is not None:
1443
+ quality = _catalog_index(quality)
1444
+ if quality is None:
1445
+ continue
1446
+ quality_effort = entry.get("quality_effort")
1447
+ if quality_effort is not None and not (
1448
+ isinstance(quality_effort, str) and quality_effort
1449
+ ):
1450
+ continue
1451
+ quality_as_of = entry.get("quality_as_of")
1452
+ if quality_as_of is not None and not isinstance(quality_as_of, str):
1453
+ continue
1454
+ quality_source = entry.get("quality_source")
1455
+ if quality_source is not None and not isinstance(quality_source, str):
1456
+ continue
1457
+ entries[model_id] = {
1458
+ "quality": quality,
1459
+ "quality_effort": quality_effort,
1460
+ "quality_as_of": quality_as_of,
1461
+ "quality_source": quality_source,
1322
1462
  }
1323
- return by_id
1463
+ return entries
1324
1464
 
1325
1465
 
1326
1466
  def fetch_quality_catalog(args) -> dict | None:
@@ -1330,47 +1470,60 @@ def fetch_quality_catalog(args) -> dict | None:
1330
1470
  catalog artifact, built by keyed CI runs and hydrated by unkeyed local
1331
1471
  runs. Never called on keyed runs -- a CI/site build hydrating from the
1332
1472
  published catalog would consume its own output (self-reference loop);
1333
- Task 4 owns that wiring.
1473
+ run() owns that wiring.
1334
1474
 
1335
1475
  Returns None only on TRANSPORT failure (after fetch_json's retries);
1336
- otherwise the envelope {"by_id": {id: record}, "generated_at":
1337
- str|None, "schema_version": int}. An empty by_id means
1476
+ otherwise the envelope {"by_id": {id: record}, "depths_by_id":
1477
+ {id: {"default_effort", "depths": {effort: record}}}, "generated_at":
1478
+ str|None, "schedules_by_id": {id: recorded schedule provenance}}. An
1479
+ empty by_id AND empty depths_by_id AND empty schedules_by_id means
1338
1480
  fetched-but-unusable (schema gate fail, malformed shape) -- Task 4's
1339
- footer treats None and empty by_id identically ("catalog unavailable").
1481
+ footer treats None and empty maps identically ("catalog unavailable").
1340
1482
 
1341
1483
  The envelope is cached under QUALITY_CATALOG_CACHE with the standard
1342
- TTL, and only a usable (non-empty by_id) result is ever written: a
1343
- degraded payload must self-heal on the next run. --no-cache neither
1344
- reads nor writes. Staleness: generated_at older than
1345
- QUALITY_CATALOG_MAX_AGE_DAYS warns but uses -- as_of makes the
1346
- staleness visible downstream (the check runs only when rows actually
1347
- hydrate -- warning about the age of a catalog we are not using is
1348
- noise); an unusable generated_at warns that
1349
- staleness cannot be checked and still uses the rows (their as_of then
1350
- shows None rather than a wrong date).
1484
+ TTL. A usable result (non-empty by_id, depths_by_id or schedules_by_id)
1485
+ is written; so is a v4 catalog's deliberately empty envelope -- that is
1486
+ the stable deploy-skew state, and caching it avoids a refetch loop.
1487
+ Older/unknown schemas and malformed v5 payloads stay uncached so a
1488
+ degraded payload self-heals on the next run. A cached envelope from
1489
+ before the v5 depth contract lacks depths_by_id, and one from before
1490
+ schedule hydration lacks schedules_by_id; both are refetched rather
1491
+ than serving depth/schedule-blind quality. --no-cache neither reads nor
1492
+ writes. Staleness: generated_at older than QUALITY_CATALOG_MAX_AGE_DAYS
1493
+ warns but uses -- as_of makes the staleness visible downstream (the
1494
+ check runs only when rows actually hydrate -- warning about the age of
1495
+ a catalog we are not using is noise); an unusable generated_at warns
1496
+ that staleness cannot be checked and still uses the rows (their as_of
1497
+ then shows None rather than a wrong date).
1351
1498
  """
1352
1499
  if not args.no_cache:
1353
1500
  cached = load_cache(QUALITY_CATALOG_CACHE, args.cache_ttl)
1354
- if isinstance(cached, dict) and isinstance(cached.get("by_id"), dict):
1501
+ if (
1502
+ isinstance(cached, dict)
1503
+ and isinstance(cached.get("by_id"), dict)
1504
+ and isinstance(cached.get("depths_by_id"), dict)
1505
+ and isinstance(cached.get("schedules_by_id"), dict)
1506
+ ):
1355
1507
  return cached
1356
1508
  try:
1357
1509
  doc = fetch_json(QUALITY_CATALOG_URL, timeout=30, retries=3)
1358
1510
  except Exception as exc:
1359
1511
  warn(f"published catalog request failed ({exc})")
1360
1512
  return None
1361
- by_id = catalog_quality_by_id(doc)
1513
+ by_id, depths_by_id = _parse_catalog_v5(doc)
1514
+ schedules_by_id = _catalog_schedule_entries(doc)
1362
1515
  generated_at = doc.get("generated_at") if isinstance(doc, dict) else None
1363
1516
  if not isinstance(generated_at, str):
1364
1517
  generated_at = None
1365
- schema_version = doc.get("schema_version") if isinstance(doc, dict) else 0
1366
- if type(schema_version) is not int:
1367
- schema_version = 0
1518
+ schema_version = doc.get("schema_version") if isinstance(doc, dict) else None
1368
1519
  result = {
1369
1520
  "by_id": by_id,
1521
+ "depths_by_id": depths_by_id,
1370
1522
  "generated_at": generated_at,
1371
- "schema_version": schema_version,
1523
+ "schedules_by_id": schedules_by_id,
1372
1524
  }
1373
- if by_id:
1525
+ usable = bool(by_id or depths_by_id or schedules_by_id)
1526
+ if usable:
1374
1527
  generated = parse_iso_datetime(generated_at)
1375
1528
  if generated is None:
1376
1529
  warn(
@@ -1384,6 +1537,11 @@ def fetch_quality_catalog(args) -> dict | None:
1384
1537
  f"published catalog is stale (generated {generated_at}, "
1385
1538
  f"{age_days:.0f} days old); using it anyway"
1386
1539
  )
1540
+ # A v4 catalog is the stable deploy-skew state: its envelope is
1541
+ # deliberately empty (no hydration), but caching it prevents a
1542
+ # refetch-every-run loop. Older/unknown schemas and malformed v5
1543
+ # payloads still self-heal by staying uncached.
1544
+ if usable or (type(schema_version) is int and schema_version == 4):
1387
1545
  if not args.no_cache:
1388
1546
  save_cache(QUALITY_CATALOG_CACHE, result)
1389
1547
  return result
@@ -1411,23 +1569,31 @@ FRONTEND_DISCOUNTS_CACHE = "openrouter-frontend-discounts"
1411
1569
  # v2: AA records are keyed by exact catalog slug and carry as_of (#17);
1412
1570
  # v1 cached the conflated date-stripped shape, so the old key is never read.
1413
1571
  FRONTEND_AA_CACHE = "openrouter-frontend-aa-v2"
1572
+ # The frontend listing projected to the reasoning fields (slug, permaslug,
1573
+ # supports_reasoning, reasoning_config). Cached separately so a fully cached
1574
+ # run keeps the reasoning fallback instead of degrading it to unknown.
1575
+ FRONTEND_REASONING_CACHE = "openrouter-frontend-reasoning-v1"
1414
1576
 
1415
1577
 
1416
1578
  def fetch_openrouter_frontend(args):
1417
- """Derive discounts, ZDR ids and OR-published AA benchmarks.
1418
-
1419
- Two loads: the base frontend URL serves discounts and benchmarks in one
1420
- response; the per-endpoint ZDR list serves ZDR and is skipped entirely
1421
- under --no-zdr. Each payload caches independently under its own key and is
1422
- cached only when non-empty, so a degraded payload self-heals on the
1423
- next run. A base-URL outage never blocks the ZDR fetch, and vice
1424
- versa. Returns (discounts, zdr_ids, aa_by_id, cache_hits) where
1579
+ """Derive discounts, ZDR ids, OR-published AA benchmarks and reasoning.
1580
+
1581
+ Two loads: the base frontend URL serves discounts, benchmarks and the
1582
+ per-model reasoning config in one response; the per-endpoint ZDR list
1583
+ serves ZDR and is skipped entirely under --no-zdr. Each payload caches
1584
+ independently under its own key and is cached only when non-empty, so a
1585
+ degraded payload self-heals on the next run. A base-URL outage never
1586
+ blocks the ZDR fetch, and vice versa. Returns
1587
+ (discounts, zdr_ids, aa_by_id, cache_hits, frontend_models) where
1425
1588
  cache_hits names the payloads served from cache ("discounts", "zdr",
1426
- "aa").
1589
+ "aa"), and frontend_models is the frontend listing projected to the
1590
+ fields build_reasoning_metadata needs (slug, permaslug,
1591
+ supports_reasoning, reasoning_config).
1427
1592
  """
1428
1593
  discounts = {}
1429
1594
  aa_by_id = {}
1430
1595
  zdr_ids = set()
1596
+ frontend_models = []
1431
1597
  cache_hits = set()
1432
1598
 
1433
1599
  aa_from_cache = False
@@ -1441,6 +1607,9 @@ def fetch_openrouter_frontend(args):
1441
1607
  aa_by_id = cached
1442
1608
  aa_from_cache = True
1443
1609
  cache_hits.add("aa")
1610
+ cached = load_cache(FRONTEND_REASONING_CACHE, args.cache_ttl)
1611
+ if cached:
1612
+ frontend_models = cached
1444
1613
 
1445
1614
  if not discounts or not aa_from_cache:
1446
1615
  try:
@@ -1490,21 +1659,31 @@ def fetch_openrouter_frontend(args):
1490
1659
  "no AA benchmark entries found; treating OpenRouter "
1491
1660
  "benchmarks as unavailable"
1492
1661
  )
1662
+ fresh_reasoning = _frontend_reasoning_entries(entries)
1663
+ if fresh_reasoning:
1664
+ frontend_models = fresh_reasoning
1665
+ save_cache(FRONTEND_REASONING_CACHE, frontend_models)
1493
1666
 
1494
1667
  if args.no_zdr:
1495
- return discounts, zdr_ids, aa_by_id, cache_hits
1668
+ return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
1496
1669
  if not args.no_cache:
1497
1670
  # v3: the id set now comes from the per-endpoint ZDR list; v1/v2
1498
1671
  # sets were derived from the models/find response and must never
1499
1672
  # be read.
1500
1673
  cached = load_cache("openrouter-zdr-v3", args.cache_ttl)
1501
1674
  if cached:
1502
- return discounts, set(cached), aa_by_id, cache_hits | {"zdr"}
1675
+ return (
1676
+ discounts,
1677
+ set(cached),
1678
+ aa_by_id,
1679
+ cache_hits | {"zdr"},
1680
+ frontend_models,
1681
+ )
1503
1682
  try:
1504
1683
  payload = fetch_json(OPENROUTER_ZDR_URL, timeout=30)
1505
1684
  except Exception as exc:
1506
1685
  warn(f"could not fetch ZDR data: {exc}")
1507
- return discounts, zdr_ids, aa_by_id, cache_hits
1686
+ return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
1508
1687
  # One entry per ZDR endpoint, keyed by the exact model id (variants
1509
1688
  # such as :free are listed individually). An id is ZDR iff it has at
1510
1689
  # least one ZDR endpoint. The models/find ?zdr=true filter cannot be
@@ -1530,9 +1709,9 @@ def fetch_openrouter_frontend(args):
1530
1709
  ids.add(model_id)
1531
1710
  if not ids:
1532
1711
  warn("no ZDR entries found; treating ZDR data as unavailable")
1533
- return discounts, zdr_ids, aa_by_id, cache_hits
1712
+ return discounts, zdr_ids, aa_by_id, cache_hits, frontend_models
1534
1713
  save_cache("openrouter-zdr-v3", sorted(ids))
1535
- return discounts, ids, aa_by_id, cache_hits
1714
+ return discounts, ids, aa_by_id, cache_hits, frontend_models
1536
1715
 
1537
1716
 
1538
1717
  def parse_price(value) -> float | None:
@@ -1959,18 +2138,146 @@ def parse_iso_datetime(value):
1959
2138
  # Artificial Analysis intelligence index
1960
2139
  # ---------------------------------------------------------------------------
1961
2140
 
2141
+ _AA_DISCRETE_EFFORTS = frozenset(("max", "xhigh", "high", "medium", "low", "minimal"))
2142
+ _AA_TRAILING_PAREN_RE = re.compile(r"\s*\(([^()]*)\)\s*$")
2143
+ _AA_SLUG_EFFORT_RE = re.compile(
2144
+ r"-(max|xhigh|high|medium|low|minimal|non-reasoning|reasoning)$", re.IGNORECASE
2145
+ )
2146
+ _AA_SLUG_EFFORT_LABELS = {
2147
+ "max": "max",
2148
+ "xhigh": "xhigh",
2149
+ "high": "high",
2150
+ "medium": "medium",
2151
+ "low": "low",
2152
+ "minimal": "minimal",
2153
+ "non-reasoning": "none",
2154
+ "reasoning": "reasoning",
2155
+ }
2156
+ # Shapes (normalized parenthetical text, or the no-label case) already
2157
+ # warned about. Spec §6.1: one warning per unknown row shape, not per row.
2158
+ _AA_WARNED_UNKNOWN_SHAPES: set[str] = set()
2159
+
2160
+
2161
+ def _classify_aa_effort_tokens(raw: str) -> tuple[str | None, bool]:
2162
+ """Classify one name parenthetical's comma-separated tokens.
2163
+
2164
+ "Default Fallback" is a qualifier, not a label. A parenthetical that
2165
+ names two different states -- two discrete efforts, or non-reasoning
2166
+ plus an effort -- is ambiguous: no effort, never guessed.
2167
+ """
2168
+ labels = set()
2169
+ for token in (raw or "").split(","):
2170
+ token = token.strip().lower()
2171
+ if not token or token == "default fallback":
2172
+ continue
2173
+ if token in _AA_DISCRETE_EFFORTS:
2174
+ labels.add(token)
2175
+ elif token == "non-reasoning":
2176
+ labels.add("none")
2177
+ elif token == "reasoning":
2178
+ labels.add("reasoning")
2179
+ if len(labels) > 1:
2180
+ return None, True
2181
+ if labels:
2182
+ return labels.pop(), False
2183
+ return None, False
2184
+
2185
+
2186
+ def _warn_unknown_aa_shape(parenthetical: str | None) -> None:
2187
+ """Warn once per distinct unrecognized effort shape (spec §6.1)."""
2188
+ if parenthetical is not None:
2189
+ detail = f"parenthetical {parenthetical!r}"
2190
+ shape = "parenthetical:" + norm_key(parenthetical)
2191
+ else:
2192
+ detail = "no effort label"
2193
+ shape = "no-effort-label"
2194
+ if shape in _AA_WARNED_UNKNOWN_SHAPES:
2195
+ return
2196
+ _AA_WARNED_UNKNOWN_SHAPES.add(shape)
2197
+ warn(f"AA row has an unrecognized effort shape ({detail}); row is display-only")
2198
+
2199
+
2200
+ def parse_aa_effort(name: str, slug: str) -> dict:
2201
+ """Split an AA row's name/slug into its base model and effort label.
2202
+
2203
+ The name's trailing parenthetical is authoritative; when it names no
2204
+ effort (and is not ambiguous) the slug suffix is the fallback. Returns
2205
+ base_name/base_slug with the label stripped, effort (None when unknown
2206
+ or ambiguous), and ambiguous=True for contradictory labels such as
2207
+ "(Non-reasoning, High)" or "(High, Max)". Ambiguous and unknown rows
2208
+ are display-only: callers must never guess an effort for them.
2209
+ """
2210
+ name = name if isinstance(name, str) else ""
2211
+ slug = slug if isinstance(slug, str) else ""
2212
+
2213
+ name_match = _AA_TRAILING_PAREN_RE.search(name)
2214
+ if name_match:
2215
+ base_name = name[: name_match.start()].strip()
2216
+ effort, ambiguous = _classify_aa_effort_tokens(name_match.group(1))
2217
+ else:
2218
+ base_name = name.strip()
2219
+ effort, ambiguous = None, False
2220
+
2221
+ slug_match = _AA_SLUG_EFFORT_RE.search(slug)
2222
+ if slug_match:
2223
+ base_slug = slug[: slug_match.start()]
2224
+ slug_effort = _AA_SLUG_EFFORT_LABELS[slug_match.group(1).lower()]
2225
+ else:
2226
+ base_slug = slug
2227
+ slug_effort = None
2228
+
2229
+ if effort is None and not ambiguous:
2230
+ effort = slug_effort
2231
+
2232
+ if effort is None and not ambiguous:
2233
+ _warn_unknown_aa_shape(name_match.group(1) if name_match is not None else None)
2234
+
2235
+ return {
2236
+ "base_name": base_name,
2237
+ "base_slug": base_slug,
2238
+ "effort": effort,
2239
+ "ambiguous": ambiguous,
2240
+ }
2241
+
2242
+
2243
+ def _finite_cost(value) -> float | None:
2244
+ """A finite float cost, or None for absent/non-numeric/non-finite input."""
2245
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
2246
+ return None
2247
+ try:
2248
+ value = float(value) # huge ints raise OverflowError
2249
+ except OverflowError:
2250
+ return None
2251
+ return value if math.isfinite(value) else None
2252
+
2253
+
2254
+ def _aa_cost_per_task(item: dict) -> float | None:
2255
+ """AA's measured cost per benchmark task, or None when not reported."""
2256
+ block = item.get("artificial_analysis_intelligence_index_cost")
2257
+ if not isinstance(block, dict):
2258
+ return None
2259
+ cost_per_task = block.get("cost_per_task")
2260
+ if not isinstance(cost_per_task, dict):
2261
+ return None
2262
+ return _finite_cost(cost_per_task.get("total_cost"))
2263
+
1962
2264
 
1963
2265
  def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
1964
2266
  """Intelligence entries from AA's V2 language models endpoint.
1965
2267
 
1966
2268
  Documented envelope: {tier, intelligence_index_version, pagination,
1967
2269
  data[]} with snake_case fields; the index lives at
1968
- data[].evaluations.artificial_analysis_intelligence_index. Items
1969
- without a usable slug/name or without a finite numeric index are
1970
- skipped (nulls mean "not measured"). Pagination follows the documented
1971
- has_more/total_pages contract with a 25-page hard cap; a failure past
1972
- page 1 is best-effort (warn, keep earlier pages), a page-1 failure
1973
- propagates.
2270
+ data[].evaluations.artificial_analysis_intelligence_index and the
2271
+ measured benchmark cost at
2272
+ data[].artificial_analysis_intelligence_index_cost.cost_per_task.total_cost.
2273
+ AA returns one row per measured reasoning-effort variant; the effort is
2274
+ parsed from the row's name/slug by parse_aa_effort. Items without a
2275
+ usable slug/name or without a finite numeric index are skipped (nulls
2276
+ mean "not measured"). Rows are deduped by their AA `id` (fallback
2277
+ (slug, name)), never by slug alone: variants often share a slug.
2278
+ Pagination follows the documented has_more/total_pages contract with a
2279
+ 25-page hard cap; a failure past page 1 is best-effort (warn, keep
2280
+ earlier pages), a page-1 failure propagates.
1974
2281
 
1975
2282
  Returns (entries, version): version is the envelope's
1976
2283
  intelligence_index_version (number-or-null) -- provenance for the whole
@@ -2030,18 +2337,36 @@ def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
2030
2337
  continue
2031
2338
  if not math.isfinite(index):
2032
2339
  continue
2033
- entries.setdefault(
2034
- key,
2035
- {
2036
- "key": key,
2037
- "name": name if isinstance(name, str) and name else key,
2038
- "index": index,
2039
- # One fetch timestamp for the whole fetch: every entry
2040
- # from this run shares it, so the cache payload and any
2041
- # downstream record can date the data exactly.
2042
- "as_of": as_of,
2043
- },
2044
- )
2340
+ entry_id = item.get("id")
2341
+ entry_id = entry_id if isinstance(entry_id, str) and entry_id else None
2342
+ slug = slug if isinstance(slug, str) and slug else None
2343
+ raw_name = name if isinstance(name, str) and name else ""
2344
+ parsed = parse_aa_effort(raw_name, slug or "")
2345
+ name = raw_name or key
2346
+ dedupe_key = entry_id if entry_id else (slug, name)
2347
+ if dedupe_key in entries:
2348
+ continue
2349
+ creator = None
2350
+ raw_creator = item.get("model_creator")
2351
+ if isinstance(raw_creator, dict):
2352
+ raw_creator_name = raw_creator.get("name")
2353
+ if isinstance(raw_creator_name, str) and raw_creator_name:
2354
+ creator = raw_creator_name
2355
+ entries[dedupe_key] = {
2356
+ "key": key,
2357
+ "name": name,
2358
+ "index": index,
2359
+ # One fetch timestamp for the whole fetch: every entry
2360
+ # from this run shares it, so the cache payload and any
2361
+ # downstream record can date the data exactly.
2362
+ "as_of": as_of,
2363
+ "id": entry_id,
2364
+ "slug": slug,
2365
+ "base": parsed["base_slug"] or parsed["base_name"],
2366
+ "effort": parsed["effort"],
2367
+ "creator": creator,
2368
+ "cost_per_task": _aa_cost_per_task(item),
2369
+ }
2045
2370
  pag = payload.get("pagination") if isinstance(payload, dict) else None
2046
2371
  if not isinstance(pag, dict):
2047
2372
  break # undocumented shape: single-page degrade
@@ -2070,10 +2395,10 @@ def fetch_aa_entries(args):
2070
2395
  ones.
2071
2396
  """
2072
2397
  if not args.no_cache:
2073
- # v3: payloads cached under earlier keys predate the as_of stamp and
2074
- # the intelligence_index_version capture and must never be read.
2075
- # Only AA API entries are read; build_catalog rejects any other
2076
- # source as unknown.
2398
+ # v4: payloads cached under earlier keys predate the per-effort
2399
+ # fields (id, base, effort, creator, cost_per_task) and must never
2400
+ # be read. Only AA API entries are read; build_catalog rejects any
2401
+ # other source as unknown.
2077
2402
  cached = load_cache(AA_ENTRIES_CACHE, args.cache_ttl)
2078
2403
  if isinstance(cached, dict) and cached.get("source") == "AA API v2":
2079
2404
  return (
@@ -2105,6 +2430,26 @@ def fetch_aa_entries(args):
2105
2430
  # ---------------------------------------------------------------------------
2106
2431
 
2107
2432
 
2433
+ def dedupe_aa_entries_by_key(entries):
2434
+ """Restore the legacy lookup's one-entry-per-key contract.
2435
+
2436
+ The depth path keeps every effort variant (aa_api_entries dedupes by AA
2437
+ id), but build_aa_lookup expects the old slug-keyed selection: for each
2438
+ `key`, the first occurrence in list order wins. This is the seam between
2439
+ the per-effort ingestion and the unchanged legacy matcher.
2440
+ """
2441
+ unique = []
2442
+ seen = set()
2443
+ for entry in entries:
2444
+ key = entry.get("key")
2445
+ if key:
2446
+ if key in seen:
2447
+ continue
2448
+ seen.add(key)
2449
+ unique.append(entry)
2450
+ return unique
2451
+
2452
+
2108
2453
  def build_aa_lookup(entries):
2109
2454
  """Index the AA entries into exact and fuzzy lookup maps.
2110
2455
 
@@ -2145,6 +2490,167 @@ def build_aa_lookup(entries):
2145
2490
  return exact, fuzzy
2146
2491
 
2147
2492
 
2493
+ def build_aa_depth_lookup(entries):
2494
+ """Index AA entries as base key -> effort -> depth record.
2495
+
2496
+ Each record is normalized for downstream publication:
2497
+ {"effort", "index", "cost_per_task_usd", "source": "aa", "as_of"}.
2498
+ Rows whose effort is absent or ambiguous are display-only and skipped.
2499
+ Duplicate (base key, effort) records from different AA rows are stored
2500
+ as None (ambiguous: matched as absent), never guessed. Base keys come
2501
+ from the name and slug with the effort label stripped, both
2502
+ unqualified and creator-qualified ("<creator_norm>:<base_key>"), where
2503
+ creator_norm is normalize_provider(model_creator.name).
2504
+ """
2505
+ lookup = {}
2506
+ seen = set()
2507
+ for entry in entries:
2508
+ if not isinstance(entry, dict):
2509
+ continue
2510
+ entry_id = entry.get("id")
2511
+ dedupe_key = (
2512
+ entry_id
2513
+ if isinstance(entry_id, str) and entry_id
2514
+ else (entry.get("slug"), entry.get("name"))
2515
+ )
2516
+ if dedupe_key in seen:
2517
+ continue
2518
+ parsed = parse_aa_effort(entry.get("name") or "", entry.get("slug") or "")
2519
+ effort = parsed["effort"]
2520
+ if not effort or parsed["ambiguous"]:
2521
+ continue
2522
+ raw_index = entry.get("index")
2523
+ if not isinstance(raw_index, (int, float)) or isinstance(raw_index, bool):
2524
+ continue
2525
+ index = float(raw_index)
2526
+ if not math.isfinite(index):
2527
+ continue
2528
+ seen.add(dedupe_key)
2529
+ record = {
2530
+ "effort": effort,
2531
+ "index": index,
2532
+ "cost_per_task_usd": _finite_cost(entry.get("cost_per_task")),
2533
+ "source": "aa",
2534
+ "as_of": (
2535
+ entry.get("as_of") if isinstance(entry.get("as_of"), str) else None
2536
+ ),
2537
+ }
2538
+ base_keys = []
2539
+ for base in (parsed["base_name"], parsed["base_slug"]):
2540
+ normalized = norm_key(base)
2541
+ if normalized and normalized not in base_keys:
2542
+ base_keys.append(normalized)
2543
+ creator = entry.get("creator")
2544
+ if not isinstance(creator, str) or not creator:
2545
+ raw_creator = entry.get("model_creator")
2546
+ if isinstance(raw_creator, dict) and isinstance(
2547
+ raw_creator.get("name"), str
2548
+ ):
2549
+ creator = raw_creator["name"]
2550
+ creator_norm = normalize_provider(creator) if isinstance(creator, str) else ""
2551
+ keys = list(base_keys)
2552
+ if creator_norm:
2553
+ keys.extend(f"{creator_norm}:{base_key}" for base_key in base_keys)
2554
+ for base_key in keys:
2555
+ efforts = lookup.setdefault(base_key, {})
2556
+ if effort in efforts:
2557
+ efforts[effort] = None
2558
+ else:
2559
+ efforts[effort] = record
2560
+ return lookup
2561
+
2562
+
2563
+ def _depth_group(model, lookup):
2564
+ """The candidate's matching base group (effort -> record), or None.
2565
+
2566
+ Exact tiers only, in the legacy order: full id, base id, display name,
2567
+ display name sans parentheticals. At each tier the creator-qualified
2568
+ key (the OpenRouter author prefix normalized with normalize_provider)
2569
+ is tried before the unqualified key. The first tier that resolves a
2570
+ base group wins; a tier whose group lacks the requested effort yields
2571
+ nothing rather than falling through to a lower tier. Ambiguous
2572
+ (base, effort) records are stored as None by build_aa_depth_lookup and
2573
+ match as absent; exact-key lookup means one tier cannot otherwise
2574
+ resolve to multiple base groups. The fuzzy pass is never used here --
2575
+ it is the #17 conflation bug.
2576
+ """
2577
+ if not isinstance(lookup, dict):
2578
+ return None
2579
+ model_id = model.get("id") or ""
2580
+ name = model.get("name") or ""
2581
+ display = PROVIDER_PREFIX_RE.sub("", name).strip()
2582
+ without_variant = model_id.split(":", 1)[0]
2583
+ _, _, base = without_variant.partition("/")
2584
+ creator = normalize_provider(without_variant.partition("/")[0])
2585
+ for raw in (model_id, base, display, PAREN_RE.sub(" ", display)):
2586
+ normalized = norm_key(raw)
2587
+ if not normalized:
2588
+ continue
2589
+ if creator:
2590
+ qualified = lookup.get(f"{creator}:{normalized}")
2591
+ if qualified is not None:
2592
+ return qualified
2593
+ group = lookup.get(normalized)
2594
+ if group is not None:
2595
+ return group
2596
+ return None
2597
+
2598
+
2599
+ def match_depth_quality(model, lookup, effort):
2600
+ """The depth record at `effort` for the candidate, or None.
2601
+
2602
+ Strict selection: a matched base group with no record at this exact
2603
+ effort yields None. Another effort is never substituted, and the
2604
+ caller must not fall through to a lower tier.
2605
+ """
2606
+ if not effort:
2607
+ return None
2608
+ group = _depth_group(model, lookup)
2609
+ if group is None:
2610
+ return None
2611
+ return group.get(effort)
2612
+
2613
+
2614
+ def _ordered_depth_records(group):
2615
+ """Measured depth records from an effort-keyed group, display-ordered.
2616
+
2617
+ Skips ambiguous/absent entries (a None record) and any record without a
2618
+ finite index. Every record carries the internal shape resolve_quality
2619
+ attaches to cand["reasoning_depths"]:
2620
+ {"effort", "index", "cost_per_task_usd", "source", "as_of"}.
2621
+ """
2622
+ records = []
2623
+ for effort, record in (group or {}).items():
2624
+ if not isinstance(record, dict):
2625
+ continue
2626
+ index = record.get("index")
2627
+ if (
2628
+ not isinstance(index, (int, float))
2629
+ or isinstance(index, bool)
2630
+ or not math.isfinite(index)
2631
+ ):
2632
+ continue
2633
+ records.append(
2634
+ {
2635
+ "effort": record.get("effort") or effort,
2636
+ "index": float(index),
2637
+ "cost_per_task_usd": record.get("cost_per_task_usd"),
2638
+ "source": record.get("source"),
2639
+ "as_of": record.get("as_of"),
2640
+ }
2641
+ )
2642
+
2643
+ def order(record):
2644
+ effort = record["effort"]
2645
+ try:
2646
+ return (AA_EFFORT_ORDER.index(effort), effort)
2647
+ except ValueError:
2648
+ return (len(AA_EFFORT_ORDER), effort or "")
2649
+
2650
+ records.sort(key=order)
2651
+ return records
2652
+
2653
+
2148
2654
  def match_quality(model, exact, fuzzy, allow_fuzzy=False):
2149
2655
  """Match a candidate to an AA entry; exact tiers only unless fuzzy is on.
2150
2656
 
@@ -2193,6 +2699,67 @@ def base_model_id(model_id: str) -> str:
2193
2699
  AA_BENCHMARK_FIELDS = ("intelligence_index", "coding_index", "agentic_index")
2194
2700
 
2195
2701
 
2702
+ def _pair_permaslug_payloads(valid, models):
2703
+ """Resolve permaslug-keyed payloads to exact catalog slugs.
2704
+
2705
+ ``valid`` maps a permaslug (or an undated catalog id) to
2706
+ ``(payload, date)``, where ``date`` is the key's trailing -YYYYMMDD or
2707
+ "" for an undated key. ``models`` carries the catalog's own
2708
+ slug/permaslug pairing. The three passes are the #17-safe pairing
2709
+ documented on build_aa_benchmarks: permaslug pairs first (latest date
2710
+ wins per slug), then undated keys self-map, then an unambiguous dated
2711
+ strip. Returns ``{slug: (payload, date)}``.
2712
+ """
2713
+ permaslug_to_slug = {}
2714
+ for entry in models or []:
2715
+ if not isinstance(entry, dict):
2716
+ continue
2717
+ slug = entry.get("slug") or ""
2718
+ permaslug = entry.get("permaslug") or ""
2719
+ if slug and permaslug:
2720
+ # standard and batch variants repeat an identical pair; the
2721
+ # overwrite is the dedup
2722
+ permaslug_to_slug[permaslug] = slug
2723
+
2724
+ paired = {}
2725
+ claimed = set()
2726
+
2727
+ # Pass 1: permaslug keys pair with their own slug; same-slug
2728
+ # snapshots resolve to the latest date (first wins on an equal date).
2729
+ seen_date = {}
2730
+ for key, (payload, date) in valid.items():
2731
+ slug = permaslug_to_slug.get(key)
2732
+ if slug is None or (slug in seen_date and seen_date[slug] >= date):
2733
+ continue
2734
+ seen_date[slug] = date
2735
+ paired[slug] = (payload, date)
2736
+ claimed.add(slug)
2737
+
2738
+ # Pass 2: undated keys self-map unless pass 1 claimed the id.
2739
+ for key, (payload, date) in valid.items():
2740
+ if date or key in permaslug_to_slug or key in claimed:
2741
+ continue
2742
+ paired[key] = (payload, date)
2743
+ claimed.add(key)
2744
+
2745
+ # Pass 3: unpaired dated keys strip, but only onto an unclaimed id
2746
+ # and only when the strip is unambiguous.
2747
+ strip_candidates = {}
2748
+ for key, (payload, date) in valid.items():
2749
+ if not date or key in permaslug_to_slug:
2750
+ continue
2751
+ stripped = strip_permaslug_date(key)
2752
+ strip_candidates.setdefault(stripped, []).append((date, payload))
2753
+ for bare, candidates in strip_candidates.items():
2754
+ if bare in claimed or len(candidates) > 1:
2755
+ continue
2756
+ date, payload = candidates[0]
2757
+ paired[bare] = (payload, date)
2758
+ claimed.add(bare)
2759
+
2760
+ return paired
2761
+
2762
+
2196
2763
  def build_aa_benchmarks(benchmarks, models=None) -> dict:
2197
2764
  """Map exact catalog slug -> OR-published AA trio from data.benchmarks.
2198
2765
 
@@ -2244,131 +2811,421 @@ def build_aa_benchmarks(benchmarks, models=None) -> dict:
2244
2811
  # the key was dated; the date is then the final 8 characters.
2245
2812
  valid[key] = (trio, "" if bare == key else key[-8:])
2246
2813
 
2247
- permaslug_to_slug = {} # the catalog's own slug/permaslug pairing
2248
- for entry in models or []:
2249
- if not isinstance(entry, dict):
2250
- continue
2251
- slug = entry.get("slug") or ""
2252
- permaslug = entry.get("permaslug") or ""
2253
- if slug and permaslug:
2254
- # standard and batch variants repeat an identical pair; the
2255
- # overwrite is the dedup
2256
- permaslug_to_slug[permaslug] = slug
2814
+ paired = _pair_permaslug_payloads(valid, models)
2815
+ return {
2816
+ slug: {**trio, "as_of": date or None} for slug, (trio, date) in paired.items()
2817
+ }
2257
2818
 
2258
- aa_by_id = {}
2259
- claimed = set()
2260
2819
 
2261
- def record(trio, date):
2262
- return {**trio, "as_of": date or None}
2820
+ FRONTEND_REASONING_FIELDS = (
2821
+ "slug",
2822
+ "permaslug",
2823
+ "supports_reasoning",
2824
+ "reasoning_config",
2825
+ )
2263
2826
 
2264
- # Pass 1: permaslug keys pair with their own slug; same-slug
2265
- # snapshots resolve to the latest date (first wins on an equal date).
2266
- seen_date = {}
2267
- for key, (trio, date) in valid.items():
2268
- slug = permaslug_to_slug.get(key)
2269
- if slug is None or (slug in seen_date and seen_date[slug] >= date):
2270
- continue
2271
- seen_date[slug] = date
2272
- aa_by_id[slug] = record(trio, date)
2273
- claimed.add(slug)
2274
2827
 
2275
- # Pass 2: undated keys self-map unless pass 1 claimed the id.
2276
- for key, (trio, date) in valid.items():
2277
- if date or key in permaslug_to_slug or key in claimed:
2828
+ def _frontend_reasoning_entries(entries):
2829
+ """Project frontend model entries to the fields reasoning metadata needs."""
2830
+ projected = []
2831
+ for entry in entries or []:
2832
+ if not isinstance(entry, dict):
2278
2833
  continue
2279
- aa_by_id[key] = record(trio, date)
2280
- claimed.add(key)
2281
-
2282
- # Pass 3: unpaired dated keys strip, but only onto an unclaimed id
2283
- # and only when the strip is unambiguous.
2284
- strip_candidates = {}
2285
- for key, (trio, date) in valid.items():
2286
- if not date or key in permaslug_to_slug:
2834
+ slug = entry.get("slug") or ""
2835
+ if slug.startswith("~"):
2287
2836
  continue
2288
- stripped = strip_permaslug_date(key)
2289
- strip_candidates.setdefault(stripped, []).append((date, trio))
2290
- for bare, candidates in strip_candidates.items():
2291
- if bare in claimed or len(candidates) > 1:
2837
+ if not (slug or entry.get("permaslug")):
2292
2838
  continue
2293
- date, trio = candidates[0]
2294
- aa_by_id[bare] = record(trio, date)
2295
- claimed.add(bare)
2839
+ projected.append(
2840
+ {field: entry.get(field) for field in FRONTEND_REASONING_FIELDS}
2841
+ )
2842
+ return projected
2296
2843
 
2297
- return aa_by_id
2298
2844
 
2845
+ def _effort_list(value):
2846
+ """Expand a supported-efforts field; null means all gateway efforts."""
2847
+ if value is None:
2848
+ return list(REASONING_GATEWAY_EFFORTS)
2849
+ if not isinstance(value, (list, tuple)):
2850
+ return []
2851
+ return [effort for effort in value if isinstance(effort, str)]
2852
+
2853
+
2854
+ def _reasoning_field(primary, config, primary_key, frontend_key):
2855
+ """Field precedence: models API field, then frontend config, then unknown."""
2856
+ if isinstance(primary, dict) and primary_key in primary:
2857
+ return primary[primary_key]
2858
+ if isinstance(config, dict) and frontend_key in config:
2859
+ return config[frontend_key]
2860
+ return None
2299
2861
 
2300
- def resolve_quality(
2301
- candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality=None
2302
- ):
2303
- """Per-candidate quality records: AA API primary, catalog next, OR last.
2304
2862
 
2305
- Resolution order per candidate (inverted from the pre-reconciliation
2306
- behavior where OpenRouter's benchmarks won):
2307
- 1. AA API entries -- match_quality's exact tiers (fuzzy stays off in
2308
- production: it is the proven variant-conflation bug).
2309
- 2. catalog_quality -- records hydrated from the published catalog on
2310
- unkeyed runs. Every winner is labeled "catalog": the run-level
2311
- provenance is the source chain (read from our published catalog),
2312
- overriding the row's own "aa"/"openrouter" label -- that label is
2313
- the keyed CI run's row-level truth inside the artifact, and no
2314
- real catalog row ever carries "catalog". as_of stays the row's
2315
- own stamp.
2316
- 3. OpenRouter's benchmark trio -- per-slug, matched via
2317
- base_model_id, so a :variant suffix inherits its base's trio.
2318
- 4. No match -- the candidate stays quality-blind downstream.
2319
-
2320
- Every winner is a record {"index", "source", "as_of", "drift",
2321
- "or_index", "or_as_of"}; compute_scores and build_schedules read
2322
- record["index"], so cand["quality"] stays the float index they already
2323
- expect. Drift is computed only when the winner is NOT OpenRouter: the
2324
- OR trio for the same base model acts as the check value, and a
2325
- disagreement of >= 3.0 points flags the record. When the winner IS
2326
- OpenRouter (or no OR value exists) the drift pair is None -- there is
2327
- no second source to disagree with.
2863
+ def _optional_bool(value, field):
2864
+ """A published boolean field: bool, or null when the source is unknown.
2865
+
2866
+ Spec 9.1 publishes ``supports_max_tokens`` as bool or null. The old
2867
+ bool() coercion conflated an omitted/unknown field with false. Explicit
2868
+ booleans pass through; common string forms are coerced; anything else
2869
+ warns and degrades to null (fail closed, never guess).
2328
2870
  """
2329
- quality_by_id = {}
2330
- source_by_id = {}
2331
- for cand in candidates:
2332
- record = None
2333
- # Tier 1: AA API exact match. match_quality returns the winning
2334
- # entry's (index, as_of) pair.
2335
- match = match_quality({"id": cand["id"], "name": cand["name"]}, exact, fuzzy)
2336
- if match is not None:
2337
- index, as_of = match
2338
- record = {"index": index, "source": "aa", "as_of": as_of}
2339
- # Tier 2: catalog record (unkeyed runs hydrate from the published
2340
- # catalog). The run-level provenance label is the source chain:
2341
- # every hydrated record is tagged "catalog" -- read from our
2342
- # published catalog -- overriding the row's own label, which is
2343
- # the keyed CI run's row-level truth inside the artifact ("aa"/
2344
- # "openrouter"; no real catalog row ever carries "catalog").
2345
- # as_of stays the row's own stamp (v4 rows carry the CI run's
2346
- # recorded quality_as_of; v3 rows the catalog's generated_at --
2347
- # catalog_quality_by_id normalizes that).
2348
- if record is None and catalog_quality is not None:
2349
- catalog_record = catalog_quality.get(cand["id"])
2350
- if catalog_record is not None:
2351
- record = {
2352
- "index": catalog_record["index"],
2353
- "source": "catalog",
2354
- "as_of": catalog_record["as_of"],
2355
- }
2356
- # Tier 3: OpenRouter's per-slug benchmark trio (last resort).
2357
- if record is None:
2358
- bench = aa_by_id.get(base_model_id(cand["id"]))
2359
- intelligence = bench.get("intelligence_index") if bench else None
2360
- if intelligence is not None:
2361
- record = {
2362
- "index": intelligence,
2363
- "source": "openrouter",
2364
- "as_of": bench.get("as_of"),
2365
- }
2366
- if record is None:
2367
- continue
2871
+ if value is None:
2872
+ return None
2873
+ if isinstance(value, bool):
2874
+ return value
2875
+ if isinstance(value, str):
2876
+ lowered = value.strip().lower()
2877
+ if lowered in ("true", "1", "yes"):
2878
+ return True
2879
+ if lowered in ("false", "0", "no"):
2880
+ return False
2881
+ warn(f"unexpected reasoning.{field} value {value!r}; treating it as unknown")
2882
+ return None
2368
2883
 
2369
- # Drift: only meaningful when the winner is not OpenRouter itself.
2370
- bench = aa_by_id.get(base_model_id(cand["id"])) or {}
2371
- or_intelligence = bench.get("intelligence_index")
2884
+
2885
+ def _optional_effort(value):
2886
+ """A published effort label: a non-empty string, or null when unknown.
2887
+
2888
+ Malformed external metadata must degrade to unknown rather than poison
2889
+ the published catalog (the catalog validator rejects a non-string
2890
+ default_effort, which would fail the whole artifact).
2891
+ """
2892
+ if value is None:
2893
+ return None
2894
+ if isinstance(value, str) and value.strip():
2895
+ return value
2896
+ warn(f"unexpected reasoning.default_effort value {value!r}; treating it as unknown")
2897
+ return None
2898
+
2899
+
2900
+ def _supported_efforts(primary, config):
2901
+ """The expanded effort list; an omitted field stays empty.
2902
+
2903
+ The models API object is authoritative for effort selection: when it is
2904
+ present but omits supported_efforts, the model has no discrete efforts
2905
+ (single-state), even if the frontend's null "all efforts" marker would
2906
+ expand. The frontend list is only consulted when the primary object is
2907
+ absent entirely.
2908
+ """
2909
+ if isinstance(primary, dict):
2910
+ if "supported_efforts" in primary:
2911
+ return _effort_list(primary["supported_efforts"])
2912
+ return []
2913
+ if isinstance(config, dict) and "supported_reasoning_efforts" in config:
2914
+ return _effort_list(config["supported_reasoning_efforts"])
2915
+ return []
2916
+
2917
+
2918
+ def effective_default_effort(reasoning: dict | None) -> str | None:
2919
+ """The effort a request with no ``reasoning`` field actually runs at (§5.2).
2920
+
2921
+ Single-state/token-budget models (no discrete efforts after expansion)
2922
+ resolve to ``reasoning`` or ``none`` by ``default_enabled``; that branch
2923
+ runs before the discrete-effort rules. Non-reasoning models are ``none``;
2924
+ missing metadata is unknown (None).
2925
+ """
2926
+ if not isinstance(reasoning, dict):
2927
+ return None
2928
+ if reasoning.get("is_reasoning") is False:
2929
+ return "none"
2930
+ if reasoning.get("is_reasoning") is not True:
2931
+ return None
2932
+ efforts = reasoning.get("supported_efforts") or []
2933
+ mandatory = reasoning.get("mandatory")
2934
+ default_enabled = reasoning.get("default_enabled")
2935
+ if mandatory is True and default_enabled is False:
2936
+ warn(
2937
+ "contradictory reasoning metadata: mandatory=true with "
2938
+ "default_enabled=false; treating the effective default as unknown"
2939
+ )
2940
+ return None
2941
+ if not efforts:
2942
+ # Single-state or token-budget model: reasoning is one state.
2943
+ return "reasoning" if default_enabled is not False else "none"
2944
+ if default_enabled is False:
2945
+ return "none"
2946
+ default_effort = reasoning.get("default_effort")
2947
+ if default_effort == "none":
2948
+ return "none"
2949
+ if isinstance(default_effort, str) and default_effort:
2950
+ return default_effort
2951
+ return None
2952
+
2953
+
2954
+ def supports_effort(reasoning: dict | None, effort: str) -> bool:
2955
+ """Whether a model can run ``effort`` (the §7 filter rule)."""
2956
+ if not isinstance(reasoning, dict):
2957
+ return False
2958
+ efforts = reasoning.get("supported_efforts") or []
2959
+ if effort in efforts:
2960
+ return True
2961
+ if effort == "none":
2962
+ if reasoning.get("is_reasoning") is False:
2963
+ return True
2964
+ # A non-mandatory single-state model exposes a reasoning toggle.
2965
+ return (
2966
+ not efforts
2967
+ and reasoning.get("is_reasoning") is True
2968
+ and reasoning.get("mandatory") is not True
2969
+ )
2970
+ if effort == "reasoning":
2971
+ return not efforts and reasoning.get("is_reasoning") is True
2972
+ return False
2973
+
2974
+
2975
+ def build_reasoning_metadata(
2976
+ models: list[dict], frontend_models: list[dict] | None = None
2977
+ ) -> dict[str, dict]:
2978
+ """Per-model reasoning metadata, models API first and frontend second.
2979
+
2980
+ Each value carries the raw flags for provenance plus the *effective*
2981
+ default effort (§5.2) and the expanded supported-effort list. Frontend
2982
+ entries are keyed by slug/permaslug, so they attach to a candidate id
2983
+ through the same three-pass pairing build_aa_benchmarks uses.
2984
+ """
2985
+ models = [
2986
+ model for model in models or [] if isinstance(model, dict) and model.get("id")
2987
+ ]
2988
+ frontend_models = _frontend_reasoning_entries(frontend_models)
2989
+
2990
+ valid = {}
2991
+ for entry in frontend_models:
2992
+ slug = entry.get("slug") or ""
2993
+ permaslug = entry.get("permaslug") or ""
2994
+ key = permaslug or slug
2995
+ if not key:
2996
+ continue
2997
+ bare = strip_permaslug_date(key)
2998
+ valid[key] = (entry, "" if bare == key else key[-8:])
2999
+ if slug and slug != key:
3000
+ # The undated slug is the exact-id self-map (pass 2). Keep it
3001
+ # even when a dated permaslug is present, so a listing whose
3002
+ # permaslug is unpaired still resolves by its own id.
3003
+ valid[slug] = (entry, "")
3004
+
3005
+ catalog_pairs = [
3006
+ {"slug": model["id"], "permaslug": model.get("canonical_slug") or ""}
3007
+ for model in models
3008
+ ]
3009
+ frontend_by_id = {
3010
+ candidate_id: entry
3011
+ for candidate_id, (entry, _date) in _pair_permaslug_payloads(
3012
+ valid, catalog_pairs
3013
+ ).items()
3014
+ }
3015
+ # Exact-slug self-map. The permaslug pairing is authoritative when it
3016
+ # resolves a listing, but a shared canonical slug (base and :batch) is
3017
+ # claimed by only one listing, so the other still needs its own slug.
3018
+ frontend_by_slug = {
3019
+ entry["slug"]: entry for entry in frontend_models if entry.get("slug")
3020
+ }
3021
+
3022
+ metadata = {}
3023
+ for model in models:
3024
+ model_id = model["id"]
3025
+ primary = model.get("reasoning")
3026
+ if not isinstance(primary, dict):
3027
+ primary = None
3028
+ entry = frontend_by_id.get(model_id) or frontend_by_slug.get(model_id)
3029
+ config = None
3030
+ supports_reasoning = None
3031
+ if isinstance(entry, dict):
3032
+ config = entry.get("reasoning_config")
3033
+ if not isinstance(config, dict):
3034
+ config = None
3035
+ supports_reasoning = entry.get("supports_reasoning")
3036
+
3037
+ if primary is not None:
3038
+ is_reasoning = True
3039
+ elif isinstance(supports_reasoning, bool):
3040
+ is_reasoning = supports_reasoning
3041
+ else:
3042
+ is_reasoning = None
3043
+
3044
+ reasoning = {
3045
+ "default_effort": _optional_effort(
3046
+ _reasoning_field(
3047
+ primary, config, "default_effort", "default_reasoning_effort"
3048
+ )
3049
+ ),
3050
+ "supported_efforts": _supported_efforts(primary, config),
3051
+ "default_enabled": _optional_bool(
3052
+ _reasoning_field(
3053
+ primary, config, "default_enabled", "default_reasoning_enabled"
3054
+ ),
3055
+ "default_enabled",
3056
+ ),
3057
+ "mandatory": _optional_bool(
3058
+ _reasoning_field(
3059
+ primary, config, "mandatory", "is_mandatory_reasoning"
3060
+ ),
3061
+ "mandatory",
3062
+ ),
3063
+ "supports_max_tokens": _optional_bool(
3064
+ _reasoning_field(
3065
+ primary,
3066
+ config,
3067
+ "supports_max_tokens",
3068
+ "supports_reasoning_max_tokens",
3069
+ ),
3070
+ "supports_max_tokens",
3071
+ ),
3072
+ "is_reasoning": is_reasoning,
3073
+ }
3074
+ reasoning["default_effort"] = effective_default_effort(reasoning)
3075
+ metadata[model_id] = reasoning
3076
+ return metadata
3077
+
3078
+
3079
+ def resolve_quality(
3080
+ candidates,
3081
+ aa_by_id,
3082
+ exact,
3083
+ fuzzy,
3084
+ aa_source,
3085
+ catalog_quality=None,
3086
+ *,
3087
+ depth_lookup=None,
3088
+ catalog_depths=None,
3089
+ requested_effort="default",
3090
+ ):
3091
+ """Per-candidate quality records: AA API primary, catalog next, OR last.
3092
+
3093
+ Depth-aware mode (either new input is not None; run() always passes the
3094
+ AA depth lookup) selects the record at the requested effort -- the
3095
+ candidate's effective default when requested_effort is "default", else
3096
+ the explicit --reasoning-effort value. Resolution order per candidate:
3097
+ 1. AA depth lookup -- match_depth_quality's exact tiers. source "aa",
3098
+ effort from the record.
3099
+ 2. catalog_depths -- the published catalog's per-model depth map,
3100
+ re-selected at the LIVE effective default (which may differ from
3101
+ the keyed run's recorded default). source "catalog", the record's
3102
+ own as_of.
3103
+ 3. OpenRouter's benchmark trio -- only for a model known to be
3104
+ non-reasoning (is_reasoning False) at effort "none". The trio has
3105
+ no effort label, so a reasoning model never uses it as its main
3106
+ score; unknown reasoning is not non-reasoning.
3107
+ 4. No record -- the candidate stays quality-blind.
3108
+
3109
+ Strict selection: a matched base group with no record at the requested
3110
+ effort yields nothing at that tier. Another effort is never
3111
+ substituted, and a reasoning model never falls through to the OR trio.
3112
+
3113
+ The legacy path (both new inputs None) keeps the pre-depth behavior:
3114
+ match_quality's exact tiers, then the hydrated catalog_quality by id,
3115
+ then the OR trio for any model.
3116
+
3117
+ Every depth-mode record is {"index", "source", "as_of", "effort",
3118
+ "drift", "or_index", "or_as_of"}; legacy records omit "effort".
3119
+ Drift is computed only when the winner is not OpenRouter and only
3120
+ like-for-like: the old |index - or_index| >= 3.0 rule applies at effort
3121
+ "none"; any other effort keeps the OR reference but sets drift None.
3122
+ cand["reasoning_depths"] carries the full measured depth list (AA or
3123
+ catalog group, or the single OR "none" record). The list is single-source:
3124
+ it comes from whichever tier supplied the main record, and sources are
3125
+ never merged.
3126
+ """
3127
+ quality_by_id = {}
3128
+ source_by_id = {}
3129
+ depth_mode = depth_lookup is not None or catalog_depths is not None
3130
+ for cand in candidates:
3131
+ record = None
3132
+ if depth_mode:
3133
+ effort = (
3134
+ requested_effort
3135
+ if requested_effort != "default"
3136
+ else cand.get("default_effort")
3137
+ )
3138
+ measured = []
3139
+ group = _depth_group(cand, depth_lookup) if depth_lookup else None
3140
+ if group is not None:
3141
+ measured = _ordered_depth_records(group)
3142
+ depth_record = group.get(effort) if effort else None
3143
+ if isinstance(depth_record, dict):
3144
+ record = {
3145
+ "index": depth_record["index"],
3146
+ "source": "aa",
3147
+ "as_of": depth_record.get("as_of"),
3148
+ "effort": depth_record.get("effort") or effort,
3149
+ }
3150
+ if record is None and catalog_depths is not None:
3151
+ catalog_entry = catalog_depths.get(cand["id"])
3152
+ if isinstance(catalog_entry, dict):
3153
+ catalog_group = catalog_entry.get("depths")
3154
+ if isinstance(catalog_group, dict):
3155
+ measured = _ordered_depth_records(catalog_group)
3156
+ depth_record = catalog_group.get(effort) if effort else None
3157
+ if isinstance(depth_record, dict):
3158
+ record = {
3159
+ "index": depth_record["index"],
3160
+ "source": "catalog",
3161
+ "as_of": depth_record.get("as_of"),
3162
+ "effort": depth_record.get("effort") or effort,
3163
+ }
3164
+ reasoning = cand.get("reasoning")
3165
+ is_non_reasoning = (
3166
+ isinstance(reasoning, dict) and reasoning.get("is_reasoning") is False
3167
+ )
3168
+ if record is None and is_non_reasoning and effort == "none":
3169
+ bench = aa_by_id.get(base_model_id(cand["id"]))
3170
+ intelligence = bench.get("intelligence_index") if bench else None
3171
+ if intelligence is not None:
3172
+ record = {
3173
+ "index": intelligence,
3174
+ "source": "openrouter",
3175
+ "as_of": bench.get("as_of"),
3176
+ "effort": "none",
3177
+ }
3178
+ measured = [
3179
+ {
3180
+ "effort": "none",
3181
+ "index": intelligence,
3182
+ "cost_per_task_usd": None,
3183
+ "source": "openrouter",
3184
+ "as_of": bench.get("as_of"),
3185
+ }
3186
+ ]
3187
+ cand["reasoning_depths"] = measured
3188
+ else:
3189
+ # Tier 1: AA API exact match. match_quality returns the winning
3190
+ # entry's (index, as_of) pair.
3191
+ match = match_quality(
3192
+ {"id": cand["id"], "name": cand["name"]}, exact, fuzzy
3193
+ )
3194
+ if match is not None:
3195
+ index, as_of = match
3196
+ record = {"index": index, "source": "aa", "as_of": as_of}
3197
+ # Tier 2: catalog record (unkeyed runs hydrate from the published
3198
+ # catalog). The run-level provenance label is the source chain:
3199
+ # every hydrated record is tagged "catalog" -- read from our
3200
+ # published catalog -- overriding the row's own label, which is
3201
+ # the keyed CI run's row-level truth inside the artifact ("aa"/
3202
+ # "openrouter"; no real catalog row ever carries "catalog").
3203
+ # as_of stays the row's own stamp.
3204
+ if record is None and catalog_quality is not None:
3205
+ catalog_record = catalog_quality.get(cand["id"])
3206
+ if catalog_record is not None:
3207
+ record = {
3208
+ "index": catalog_record["index"],
3209
+ "source": "catalog",
3210
+ "as_of": catalog_record["as_of"],
3211
+ }
3212
+ # Tier 3: OpenRouter's per-slug benchmark trio (last resort).
3213
+ if record is None:
3214
+ bench = aa_by_id.get(base_model_id(cand["id"]))
3215
+ intelligence = bench.get("intelligence_index") if bench else None
3216
+ if intelligence is not None:
3217
+ record = {
3218
+ "index": intelligence,
3219
+ "source": "openrouter",
3220
+ "as_of": bench.get("as_of"),
3221
+ }
3222
+ if record is None:
3223
+ continue
3224
+
3225
+ # Drift: only meaningful when the winner is not OpenRouter itself.
3226
+ bench = aa_by_id.get(base_model_id(cand["id"])) or {}
3227
+ or_intelligence = bench.get("intelligence_index")
3228
+ reasoning = cand.get("reasoning") or {}
2372
3229
  if (
2373
3230
  record["source"] in ("aa", "catalog")
2374
3231
  and isinstance(or_intelligence, (int, float))
@@ -2377,7 +3234,18 @@ def resolve_quality(
2377
3234
  ):
2378
3235
  record["or_index"] = or_intelligence
2379
3236
  record["or_as_of"] = bench.get("as_of")
2380
- record["drift"] = abs(record["index"] - or_intelligence) >= 3.0
3237
+ # The comparable case is a non-reasoning MODEL at effort "none",
3238
+ # not merely an effort label: a reasoning model whose effective
3239
+ # default is "none" still has an OR snapshot measured at its
3240
+ # reasoning state, so a difference may be a depth difference.
3241
+ comparable = (not depth_mode) or (
3242
+ reasoning.get("is_reasoning") is False
3243
+ and record.get("effort") == "none"
3244
+ )
3245
+ if comparable:
3246
+ record["drift"] = abs(record["index"] - or_intelligence) >= 3.0
3247
+ else:
3248
+ record["drift"] = None
2381
3249
  else:
2382
3250
  record["or_index"] = None
2383
3251
  record["or_as_of"] = None
@@ -2410,7 +3278,18 @@ def _per_1m(price_per_token: float) -> float:
2410
3278
 
2411
3279
 
2412
3280
  def build_schedules(
2413
- models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality=None
3281
+ models,
3282
+ args,
3283
+ quality_by_id,
3284
+ aa_by_id,
3285
+ balanced_by_id,
3286
+ catalog_quality=None,
3287
+ *,
3288
+ depth_lookup=None,
3289
+ catalog_depths=None,
3290
+ reasoning_by_id=None,
3291
+ requested_effort="default",
3292
+ schedule_entries=None,
2414
3293
  ):
2415
3294
  """Catalog-wide off-peak index built from the RAW model list (spec 5.4).
2416
3295
 
@@ -2420,12 +3299,39 @@ def build_schedules(
2420
3299
  (~ids) and malformed ids are excluded; :batch variants are included as
2421
3300
  distinct deals. Sorted by id.
2422
3301
 
2423
- Quality falls back through the same chain resolve_quality walks, minus
2424
- its AA tier (the AA lookup only ever matched candidates): candidate
2425
- records, then the hydrated catalog (unkeyed runs), then OpenRouter's
2426
- benchmark trio. Every schedule row carries the winner's provenance
2427
- (quality_source/quality_as_of) so the site's QUAL tooltip can render.
3302
+ Depth-aware mode (run() always supplies the AA depth lookup; an unkeyed
3303
+ run supplies the catalog depth map and the published schedule entries)
3304
+ follows spec 6.5:
3305
+ 0. ``schedule_entries`` (unkeyed runs): the published catalog's v5
3306
+ schedules array is the source of truth for Deals. Whenever an
3307
+ entry exists with a non-null quality AND a non-empty
3308
+ ``quality_effort``, its quality and all three provenance fields
3309
+ are copied VERBATIM (no run-level relabeling); requested_effort is
3310
+ ignored because there is no depth list to re-select from. An entry
3311
+ with no effort label is legacy/unknown (spec 6.5): its quality is
3312
+ null and the lower tiers are skipped. A null-quality or absent
3313
+ entry falls through to the tiers below.
3314
+ 1. AA depth lookup at the effective default effort (or the explicit
3315
+ ``requested_effort``) -- keyed runs only.
3316
+ 2. Catalog depth map at the entry's recorded default effort -- unkeyed
3317
+ runs use that provenance as-is, never re-selecting at the live
3318
+ default.
3319
+ 3. Catalog main-quality map, only when the record carries an effort
3320
+ (a record without one is legacy/unknown: quality stays null).
3321
+ 4. OpenRouter's benchmark trio, only for a model known to be
3322
+ non-reasoning (``is_reasoning`` False) at effort "none". A
3323
+ reasoning model never falls through to the trio: it has no effort
3324
+ label, so using it would publish a max-effort snapshot as the
3325
+ default.
3326
+ Strict selection at tiers 1-2: a matched group with no record at the
3327
+ selected effort yields nothing; another effort is never substituted.
3328
+
3329
+ The legacy path (both new inputs None) keeps the pre-depth fallback
3330
+ chain: candidate record, hydrated catalog, then the OR trio. Every row
3331
+ carries ``quality_effort`` (string or null) alongside the winning
3332
+ record's provenance (quality_source/quality_as_of).
2428
3333
  """
3334
+ depth_mode = depth_lookup is not None or catalog_depths is not None
2429
3335
  entries = []
2430
3336
  for model in models:
2431
3337
  model_id = model.get("id") or ""
@@ -2452,35 +3358,107 @@ def build_schedules(
2452
3358
  # quality_by_id values are records {"index", "source", ...}; the
2453
3359
  # schedule row keeps the bare float index verbatim (never a dict)
2454
3360
  # and echoes the winning record's provenance for the QUAL tooltip.
2455
- # Fallback chain mirrors resolve_quality's minus its AA tier:
2456
- # candidate record, hydrated catalog record (unkeyed runs; every
2457
- # hydrated record is relabeled with the run-level chain label
2458
- # "catalog", exactly like resolve_quality's catalog tier), then
2459
- # OpenRouter's benchmark trio (row label "openrouter").
2460
- record = quality_by_id.get(model_id)
2461
- if isinstance(record, dict):
2462
- quality = record.get("index")
2463
- quality_source = record.get("source")
2464
- quality_as_of = record.get("as_of")
3361
+ quality = None
3362
+ quality_source = None
3363
+ quality_as_of = None
3364
+ quality_effort = None
3365
+ schedule_entry = (schedule_entries or {}).get(model_id)
3366
+ if (
3367
+ isinstance(schedule_entry, dict)
3368
+ and schedule_entry.get("quality") is not None
3369
+ ):
3370
+ effort_label = schedule_entry.get("quality_effort")
3371
+ if isinstance(effort_label, str) and effort_label:
3372
+ # Unkeyed source of truth: recorded provenance verbatim.
3373
+ quality = schedule_entry.get("quality")
3374
+ quality_source = schedule_entry.get("quality_source")
3375
+ quality_as_of = schedule_entry.get("quality_as_of")
3376
+ quality_effort = effort_label
3377
+ # else: a recorded entry with no effort label is legacy/unknown
3378
+ # (spec 6.5): its quality is null, and the lower tiers are
3379
+ # skipped rather than re-deriving a max-effort OR snapshot.
3380
+ elif depth_mode:
3381
+ reasoning = (reasoning_by_id or {}).get(model_id)
3382
+ if not isinstance(reasoning, dict):
3383
+ reasoning = None
3384
+ effort = (
3385
+ requested_effort
3386
+ if requested_effort != "default"
3387
+ else (reasoning or {}).get("default_effort")
3388
+ )
3389
+ # Tier 1: AA depth lookup (keyed runs).
3390
+ group = _depth_group(model, depth_lookup) if depth_lookup else None
3391
+ record = group.get(effort) if group is not None and effort else None
3392
+ if isinstance(record, dict):
3393
+ quality = record.get("index")
3394
+ quality_source = "aa"
3395
+ quality_as_of = record.get("as_of")
3396
+ quality_effort = record.get("effort") or effort
3397
+ # Tier 2: catalog depth map (unkeyed runs), recorded default
3398
+ # effort as-is -- the keyed run's provenance, not a live
3399
+ # re-selection.
3400
+ if quality is None and catalog_depths is not None:
3401
+ catalog_entry = catalog_depths.get(model_id)
3402
+ if isinstance(catalog_entry, dict):
3403
+ recorded_default = catalog_entry.get("default_effort")
3404
+ group = catalog_entry.get("depths")
3405
+ record = (
3406
+ group.get(recorded_default)
3407
+ if isinstance(group, dict) and recorded_default
3408
+ else None
3409
+ )
3410
+ if isinstance(record, dict):
3411
+ quality = record.get("index")
3412
+ quality_source = "catalog"
3413
+ quality_as_of = record.get("as_of")
3414
+ quality_effort = record.get("effort") or recorded_default
3415
+ # Tier 3: catalog main-quality map (unkeyed runs). A record with
3416
+ # no quality_effort is legacy/unknown: null, never a guess.
3417
+ if quality is None and catalog_quality is not None:
3418
+ catalog_record = catalog_quality.get(model_id)
3419
+ if isinstance(catalog_record, dict) and catalog_record.get("effort"):
3420
+ quality = catalog_record.get("index")
3421
+ quality_source = "catalog"
3422
+ quality_as_of = catalog_record.get("as_of")
3423
+ quality_effort = catalog_record.get("effort")
3424
+ # Tier 4: OR trio only for known non-reasoning models.
3425
+ if (
3426
+ quality is None
3427
+ and reasoning is not None
3428
+ and reasoning.get("is_reasoning") is False
3429
+ ):
3430
+ benchmark = aa_by_id.get(base_model_id(model_id)) or {}
3431
+ quality = benchmark.get("intelligence_index")
3432
+ if quality is not None:
3433
+ quality_source = "openrouter"
3434
+ quality_as_of = benchmark.get("as_of")
3435
+ quality_effort = "none"
2465
3436
  else:
2466
- quality = record
2467
- quality_source = None
2468
- quality_as_of = None
2469
- if quality is None and catalog_quality is not None:
2470
- catalog_record = catalog_quality.get(model_id)
2471
- if catalog_record is not None:
2472
- quality = catalog_record["index"]
2473
- quality_source = "catalog"
2474
- quality_as_of = catalog_record["as_of"]
2475
- if quality is None:
2476
- benchmark = aa_by_id.get(base_model_id(model_id)) or {}
2477
- quality = benchmark.get("intelligence_index")
2478
- if quality is not None:
2479
- quality_source = "openrouter"
2480
- quality_as_of = benchmark.get("as_of")
3437
+ # Legacy path: candidate record, hydrated catalog, OR trio.
3438
+ record = quality_by_id.get(model_id)
3439
+ if isinstance(record, dict):
3440
+ quality = record.get("index")
3441
+ quality_source = record.get("source")
3442
+ quality_as_of = record.get("as_of")
3443
+ quality_effort = record.get("effort")
2481
3444
  else:
2482
- quality_source = None
2483
- quality_as_of = None
3445
+ quality = record
3446
+ if quality is None and catalog_quality is not None:
3447
+ catalog_record = catalog_quality.get(model_id)
3448
+ if catalog_record is not None:
3449
+ quality = catalog_record["index"]
3450
+ quality_source = "catalog"
3451
+ quality_as_of = catalog_record["as_of"]
3452
+ quality_effort = catalog_record.get("effort")
3453
+ if quality is None:
3454
+ benchmark = aa_by_id.get(base_model_id(model_id)) or {}
3455
+ quality = benchmark.get("intelligence_index")
3456
+ if quality is not None:
3457
+ quality_source = "openrouter"
3458
+ quality_as_of = benchmark.get("as_of")
3459
+ else:
3460
+ quality_source = None
3461
+ quality_as_of = None
2484
3462
  name = model.get("name") or model_id
2485
3463
  entries.append(
2486
3464
  {
@@ -2492,6 +3470,7 @@ def build_schedules(
2492
3470
  # quality came without any -- e.g. a bare-float test map).
2493
3471
  "quality_source": quality_source,
2494
3472
  "quality_as_of": quality_as_of,
3473
+ "quality_effort": quality_effort,
2495
3474
  "score": balanced_by_id.get(model_id),
2496
3475
  "max_discount": eff["max_discount"],
2497
3476
  "sched_note": fmt_sched_note(eff["max_discount"]),
@@ -2543,14 +3522,54 @@ def catalog_weights(candidates, quality_by_id):
2543
3522
  return weights
2544
3523
 
2545
3524
 
3525
+ def overall_scores(cand, weights=None) -> dict:
3526
+ """Per-priority overall scores from rounded components (catalog parity).
3527
+
3528
+ weights is the per-priority map from catalog_weights; callers that must
3529
+ match the catalog on quality-blind runs pass it, otherwise the base
3530
+ PRIORITY_WEIGHTS apply.
3531
+ """
3532
+ weights = weights or PRIORITY_WEIGHTS
3533
+ scores = {
3534
+ "price": round(cand["price_score"], 4),
3535
+ "quality": round(cand["quality_score"], 4),
3536
+ "context": round(cand["context_score"], 4),
3537
+ "age": round(cand["age_score"], 4),
3538
+ }
3539
+ return {
3540
+ priority: round(
3541
+ weights[priority].get("quality", 0.0) * scores["quality"]
3542
+ + weights[priority].get("price", 0.0) * scores["price"]
3543
+ + weights[priority].get("context", 0.0) * scores["context"]
3544
+ + weights[priority].get("age", 0.0) * scores["age"],
3545
+ 4,
3546
+ )
3547
+ for priority in PRIORITY_WEIGHTS
3548
+ }
3549
+
3550
+
3551
+ def off_peak_deals(schedules, limit=10):
3552
+ rows = [row for row in schedules if f"{row['max_discount']:.0%}" != "0%"]
3553
+ rows.sort(key=lambda row: (-row["max_discount"], row["id"]))
3554
+ return rows[:limit]
3555
+
3556
+
2546
3557
  # ---------------------------------------------------------------------------
2547
3558
  # Filtering and scoring
2548
3559
  # ---------------------------------------------------------------------------
2549
3560
 
2550
3561
 
2551
- def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
3562
+ def build_candidates(
3563
+ models,
3564
+ args,
3565
+ discounts,
3566
+ zdr_ids,
3567
+ filtered_out=None,
3568
+ reasoning_by_id: dict[str, dict] | None = None,
3569
+ ):
2552
3570
  now = time.time()
2553
3571
  require_tools = not args.no_require_tools
3572
+ requested_effort = getattr(args, "reasoning_effort", "default")
2554
3573
  dropped = {}
2555
3574
  candidates = []
2556
3575
 
@@ -2598,6 +3617,22 @@ def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
2598
3617
  # schedule-only AND mixed models fail closed here.
2599
3618
  drop("schedule", model_id, model.get("name"))
2600
3619
  continue
3620
+ # The effort filter runs before the ZDR/discount gates: it is the
3621
+ # caller's explicit request, so it is the most useful drop reason
3622
+ # when several gates would reject the model.
3623
+ reasoning = (reasoning_by_id or {}).get(model_id)
3624
+ if not isinstance(reasoning, dict):
3625
+ reasoning = None
3626
+ # A model can always run at its own effective default (§7 equality
3627
+ # carve-out), even when that effort is absent from supported_efforts
3628
+ # -- e.g. default_enabled false resolves to "none" while the list
3629
+ # omits it.
3630
+ if requested_effort != "default" and not (
3631
+ supports_effort(reasoning, requested_effort)
3632
+ or requested_effort == (reasoning or {}).get("default_effort")
3633
+ ):
3634
+ drop("reasoning effort", model_id, model.get("name"))
3635
+ continue
2601
3636
  price_in, price_out = eff["price_in"], eff["price_out"]
2602
3637
  if args.exclude_free and price_in == 0 and price_out == 0:
2603
3638
  drop("free", model_id, model.get("name"))
@@ -2666,6 +3701,11 @@ def build_candidates(models, args, discounts, zdr_ids, filtered_out=None):
2666
3701
  "tool_calling": tool_calling,
2667
3702
  "zdr": None if args.no_zdr else model_id in (zdr_ids or set()),
2668
3703
  "expired": expired,
3704
+ "reasoning": reasoning,
3705
+ "default_effort": (reasoning or {}).get("default_effort"),
3706
+ "supported_efforts": list(
3707
+ (reasoning or {}).get("supported_efforts") or []
3708
+ ),
2669
3709
  }
2670
3710
  )
2671
3711
 
@@ -2809,6 +3849,42 @@ def fmt_discount(value) -> str:
2809
3849
  return f"{value:.0%}" if has_discount(value) else "--"
2810
3850
 
2811
3851
 
3852
+ def _display_default_effort(cand):
3853
+ """The effort a default run uses, for the DEPTH cell and the depth block.
3854
+
3855
+ Production candidates carry it on both ``cand["default_effort"]`` and the
3856
+ reasoning block; partial rows (tests, legacy paths) may carry only one.
3857
+ """
3858
+ return cand.get("default_effort") or (cand.get("reasoning") or {}).get(
3859
+ "default_effort"
3860
+ )
3861
+
3862
+
3863
+ def _published_reasoning_block(cand):
3864
+ """The six published reasoning keys plus the measured depth records.
3865
+
3866
+ ``is_reasoning`` is an internal flag: neither catalog v5 nor spec §8.4
3867
+ publishes it, so it never appears here.
3868
+ """
3869
+ reasoning = cand.get("reasoning") or {}
3870
+ return {
3871
+ "default_effort": reasoning.get("default_effort"),
3872
+ "supported_efforts": reasoning.get("supported_efforts") or [],
3873
+ "default_enabled": reasoning.get("default_enabled"),
3874
+ "mandatory": reasoning.get("mandatory"),
3875
+ "supports_max_tokens": reasoning.get("supports_max_tokens"),
3876
+ "depths": cand.get("reasoning_depths") or [],
3877
+ }
3878
+
3879
+
3880
+ def fmt_depth(cand) -> str:
3881
+ """DEPTH cell: the effort QUAL was measured at, ``*`` on the default."""
3882
+ effort = (cand.get("quality_record") or {}).get("effort")
3883
+ if not effort:
3884
+ return "-"
3885
+ return f"{effort}*" if effort == _display_default_effort(cand) else effort
3886
+
3887
+
2812
3888
  def fmt_prec(cand) -> str:
2813
3889
  """PREC cell: the precision the ranked price is quoted at.
2814
3890
 
@@ -2901,20 +3977,24 @@ def _later_as_of(value, current) -> bool:
2901
3977
  return new > old
2902
3978
 
2903
3979
 
3980
+ QUALITY_SOURCE_LABEL = "Quality source: "
3981
+
3982
+
2904
3983
  def quality_footer_note(quality_by_id, aa_index_version, unmatched):
2905
3984
  """Compose the footer's quality note from the resolved records.
2906
3985
 
2907
- Lists per-source counts, each source's own as_of and the AA index
2908
- version when known, e.g. "AA index v4.3 - live AA (58, checked 13:15Z)
2909
- | our catalog (12, as of 2026-10-23 13:15Z) | OpenRouter snapshot (2,
2910
- as of 2026-10-17)". A trailing "unmatched score 0" names candidates
2911
- with no quality record, and a drift footnote appears when any resolved
2912
- record disagrees with the OpenRouter check value. Returns None when no
2913
- record resolved -- run() falls back to the quality-blind note.
3986
+ Returns {"sources": [...], "clauses": [...], "warn": bool}. Each source
3987
+ names its model count, its own as_of and the AA index version when
3988
+ known, e.g. "AA index v4.3 - live AA (58 models, checked 13:15Z)".
3989
+ Clauses carry the unmatched-candidates note and the drift footnote when
3990
+ any resolved record disagrees with the OpenRouter check value; the drift
3991
+ count covers only comparable (non-reasoning) rows and says so. Returns
3992
+ None when no record resolved -- run() falls back to the quality-blind
3993
+ note.
2914
3994
  """
2915
3995
  labels = {
2916
3996
  "aa": "live AA",
2917
- "catalog": "our catalog",
3997
+ "catalog": "our published catalog",
2918
3998
  "openrouter": "OpenRouter snapshot",
2919
3999
  }
2920
4000
  stamp_words = {"aa": "checked", "catalog": "as of", "openrouter": "as of"}
@@ -2935,29 +4015,61 @@ def quality_footer_note(quality_by_id, aa_index_version, unmatched):
2935
4015
  count = counts.get(source, 0)
2936
4016
  if not count:
2937
4017
  continue
4018
+ plural = "" if count == 1 else "s"
2938
4019
  stamp = _footer_as_of(source, latest.get(source))
2939
4020
  if stamp:
2940
- blocks.append(f"{labels[source]} ({count}, {stamp_words[source]} {stamp})")
4021
+ blocks.append(
4022
+ f"{labels[source]} ({count} model{plural}, {stamp_words[source]} {stamp})"
4023
+ )
2941
4024
  else:
2942
- blocks.append(f"{labels[source]} ({count})")
4025
+ blocks.append(f"{labels[source]} ({count} model{plural})")
2943
4026
  if not blocks:
2944
4027
  return None
2945
- note = "quality: "
2946
- if aa_index_version is not None:
2947
- note += f"AA index v{aa_index_version} - "
2948
- note += " | ".join(blocks)
4028
+ if aa_index_version is not None and counts.get("aa"):
4029
+ # Only the AA block gets the index-version prefix; a run whose first
4030
+ # (or only) source is the OpenRouter snapshot must not be mislabelled
4031
+ # as an AA index version.
4032
+ blocks[0] = f"AA index v{aa_index_version} - {blocks[0]}"
4033
+ clauses = []
2949
4034
  if unmatched:
2950
- note += "; unmatched score 0"
4035
+ plural = "" if unmatched == 1 else "s"
4036
+ clauses.append(
4037
+ f"{unmatched} model{plural} unmatched (no quality data; quality score 0)"
4038
+ )
2951
4039
  drift = sum(1 for record in quality_by_id.values() if record.get("drift"))
2952
4040
  if drift:
2953
- note += f"; drift vs OpenRouter snapshot ({drift})"
2954
- return note
4041
+ rows = "row" if drift == 1 else "rows"
4042
+ clauses.append(
4043
+ f"drift vs OpenRouter snapshot: {drift} comparable (non-reasoning) {rows}"
4044
+ )
4045
+ return {"sources": blocks, "clauses": clauses, "warn": bool(unmatched)}
4046
+
4047
+
4048
+ def quality_source_lines(note) -> list[str]:
4049
+ """Render the footer as one source per line, 16-space continuation."""
4050
+ if not note or not note.get("sources"):
4051
+ return [QUALITY_SOURCE_LABEL + "unavailable; ranked on price/context/age"]
4052
+ indent = " " * len(QUALITY_SOURCE_LABEL)
4053
+ sources = note["sources"]
4054
+ clauses = note.get("clauses") or []
4055
+ lines = []
4056
+ for index, source in enumerate(sources):
4057
+ prefix = QUALITY_SOURCE_LABEL if index == 0 else indent
4058
+ if index < len(sources) - 1:
4059
+ lines.append(prefix + source + " |")
4060
+ elif clauses:
4061
+ lines.append(prefix + source + ";")
4062
+ else:
4063
+ lines.append(prefix + source)
4064
+ lines.extend(indent + clause for clause in clauses)
4065
+ return lines
2955
4066
 
2956
4067
 
2957
- def print_table(top, total_candidates, weights, quality_note):
2958
- headers = [
4068
+ TABLE_COLUMNS = {
4069
+ "expert": (
2959
4070
  "RANK",
2960
4071
  "MODEL",
4072
+ "DEPTH",
2961
4073
  "QUAL",
2962
4074
  "$IN/M",
2963
4075
  "$OUT/M",
@@ -2967,72 +4079,448 @@ def print_table(top, total_candidates, weights, quality_note):
2967
4079
  "CTX",
2968
4080
  "AGE",
2969
4081
  "SCORE",
4082
+ ),
4083
+ "simple": ("RANK", "MODEL", "QUALITY", "IN $/M", "OUT $/M", "DISCOUNT", "AGE"),
4084
+ }
4085
+
4086
+ LEGENDS = {
4087
+ "expert": (
4088
+ (
4089
+ "QUALITY",
4090
+ ['Artificial Analysis intelligence index; "-" = unknown (scores 0)'],
4091
+ ),
4092
+ (
4093
+ "DEPTH",
4094
+ [
4095
+ "reasoning effort the QUAL score was measured at; * = OpenRouter's",
4096
+ "default; - = no known effort; --no-depths hides the depth detail",
4097
+ ],
4098
+ ),
4099
+ (
4100
+ "$IN/$OUT",
4101
+ [
4102
+ "USD per 1M tokens at the cheapest eligible endpoint, blended",
4103
+ "{in_pct} input / {out_pct} output (--input-share); differs from the",
4104
+ "openrouter.ai default-endpoint price by design. A higher rate",
4105
+ "may apply once the prompt outgrows a model's short-context tier.",
4106
+ ],
4107
+ ),
4108
+ ("DISC", ["active discount"]),
4109
+ (
4110
+ "PREC",
4111
+ [
4112
+ 'quoted precision ("!" = default routing favors cheaper',
4113
+ "sub-default endpoints)",
4114
+ ],
4115
+ ),
4116
+ (
4117
+ "TIER",
4118
+ [
4119
+ "how prices vary: >Nk = pricier above that prompt size; -N% =",
4120
+ "scheduled off-peak discount (price shown: the standard rate);",
4121
+ "SCHED = schedule with a discount under 1% (--off-peak lists them)",
4122
+ ],
4123
+ ),
4124
+ ("CTX", ["context window; AGE = days since listed on OpenRouter"]),
4125
+ ("SCORE", ["weighted blend of the four criteria above"]),
4126
+ ),
4127
+ "simple": (
4128
+ ("RANK", ["position in this ranking"]),
4129
+ ("MODEL", ["provider/model id as listed; --best prefixes openrouter/"]),
4130
+ (
4131
+ "QUALITY",
4132
+ [
4133
+ 'Artificial Analysis intelligence index; "-" = unknown (scores 0)',
4134
+ "the suffix after · is the reasoning effort the score was measured at",
4135
+ ],
4136
+ ),
4137
+ (
4138
+ "IN/OUT $/M",
4139
+ [
4140
+ "USD per 1M tokens at the cheapest eligible endpoint, blended",
4141
+ "{in_pct} input / {out_pct} output (--input-share)",
4142
+ ],
4143
+ ),
4144
+ ("DISCOUNT", ["active discount, if any"]),
4145
+ ("AGE", ["days since listed on OpenRouter"]),
4146
+ ),
4147
+ }
4148
+
4149
+
4150
+ def fmt_quality(value) -> str:
4151
+ return "-" if value is None else f"{value:.1f}"
4152
+
4153
+
4154
+ _CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]")
4155
+
4156
+
4157
+ def display_text(value) -> str:
4158
+ """Return untrusted text safe to print: strip C0/C1 control characters.
4159
+
4160
+ Model ids come from the OpenRouter API. An id carrying ESC (SGR/CSI
4161
+ injection) or a newline (forged output lines) must never reach the
4162
+ terminal raw. Normal ids pass through unchanged.
4163
+ """
4164
+ return _CONTROL_CHARS_RE.sub("", str(value))
4165
+
4166
+
4167
+ def table_cells(cand, rank):
4168
+ effort = (cand.get("quality_record") or {}).get("effort")
4169
+ quality_cell = fmt_quality(cand["quality"])
4170
+ if effort and cand["quality"] is not None:
4171
+ # Simple view has no DEPTH column; the effort rides on QUALITY.
4172
+ quality_cell += f" · {effort}"
4173
+ return {
4174
+ "RANK": str(rank),
4175
+ "MODEL": display_text(cand["id"]),
4176
+ "DEPTH": fmt_depth(cand),
4177
+ "QUAL": fmt_quality(cand["quality"]),
4178
+ "QUALITY": quality_cell,
4179
+ "$IN/M": fmt_price(cand["price_in"]),
4180
+ "IN $/M": fmt_price(cand["price_in"]),
4181
+ "$OUT/M": fmt_price(cand["price_out"]),
4182
+ "OUT $/M": fmt_price(cand["price_out"]),
4183
+ "DISC": fmt_discount(cand["discount"]),
4184
+ "DISCOUNT": fmt_discount(cand["discount"]),
4185
+ "PREC": fmt_prec(cand),
4186
+ "TIER": cand["tier_note"] or cand["sched_note"] or "--",
4187
+ "CTX": fmt_context(cand["context"]),
4188
+ "AGE": fmt_age(cand["age_days"]),
4189
+ "SCORE": f"{cand['score']:.3f}",
4190
+ }
4191
+
4192
+
4193
+ def format_weights(weights) -> str:
4194
+ """Render weights as integer percentages that sum to exactly 100.
4195
+
4196
+ Naive per-item ``:.0%`` rounding can print 101% (or 99%) because each
4197
+ share rounds independently. Largest-remainder rounding floors every
4198
+ share, then hands the leftover points to the largest fractional parts
4199
+ (deterministic name tie-break) so the line is always self-consistent.
4200
+ """
4201
+ items = sorted(weights.items(), reverse=True)
4202
+ total = sum(value for _, value in items)
4203
+ if total <= 0:
4204
+ return ", ".join(f"{name} 0%" for name, _ in items)
4205
+ exact = [(name, value / total * 100) for name, value in items]
4206
+ floors = [
4207
+ (name, math.floor(share), share - math.floor(share)) for name, share in exact
2970
4208
  ]
2971
- rows = []
4209
+ remainder = 100 - sum(floor for _, floor, _ in floors)
4210
+ order = sorted(range(len(floors)), key=lambda i: (-floors[i][2], floors[i][0]))
4211
+ bumped = set(order[:remainder])
4212
+ return ", ".join(
4213
+ f"{name} {floor + (1 if index in bumped else 0)}%"
4214
+ for index, (name, floor, _) in enumerate(floors)
4215
+ )
4216
+
4217
+
4218
+ def print_table(
4219
+ top,
4220
+ weights,
4221
+ quality_note,
4222
+ detail="expert",
4223
+ style=None,
4224
+ bare=False,
4225
+ input_share=0.75,
4226
+ depth_detail=None,
4227
+ ):
4228
+ style = style or Style(False)
4229
+ columns = TABLE_COLUMNS[detail]
4230
+ rows = [table_cells(cand, rank) for rank, cand in enumerate(top, 1)]
4231
+ widths = {col: max([len(col)] + [len(row[col]) for row in rows]) for col in columns}
4232
+
4233
+ def pad(col, value):
4234
+ if col == "MODEL":
4235
+ return value.ljust(widths[col])
4236
+ return value.rjust(widths[col])
4237
+
4238
+ header = []
4239
+ for col in columns:
4240
+ text = col.ljust(widths[col])
4241
+ header.append(text if col == "SCORE" else style.bold(text))
4242
+ print(" ".join(header))
4243
+ print(style.dim(" ".join("-" * widths[col] for col in columns)))
4244
+ for rank, row in enumerate(rows, 1):
4245
+ cells = []
4246
+ for col in columns:
4247
+ text = pad(col, row[col])
4248
+ if col == "RANK":
4249
+ text = style.bold(text)
4250
+ if rank == 1:
4251
+ text = style.green(text)
4252
+ elif rank == 1:
4253
+ text = style.green(text)
4254
+ elif col in ("QUAL", "QUALITY") and row[col] == "-":
4255
+ text = style.dim(text)
4256
+ cells.append(text)
4257
+ print(" ".join(cells))
4258
+ if bare:
4259
+ return
4260
+ print()
4261
+ print("Weights: " + format_weights(weights))
4262
+ warn = not quality_note or quality_note.get("warn")
4263
+ for line in quality_source_lines(quality_note):
4264
+ print(style.yellow(line) if warn else line)
4265
+ if depth_detail is not None:
4266
+ print()
4267
+ depth_detail(top)
4268
+ print()
4269
+ legend = LEGENDS[detail]
4270
+ width = max(len(term) for term, _ in legend)
4271
+ in_pct = f"{input_share:.0%}"
4272
+ out_pct = f"{1 - input_share:.0%}"
4273
+ for term, lines in legend:
4274
+ lines = [
4275
+ line.replace("{in_pct}", in_pct).replace("{out_pct}", out_pct)
4276
+ for line in lines
4277
+ ]
4278
+ print(style.bold(term.ljust(width)) + " " + lines[0])
4279
+ for extra in lines[1:]:
4280
+ print(" " * (width + 2) + extra)
4281
+
4282
+
4283
+ DEALS_COLUMNS = (
4284
+ "MODEL",
4285
+ "OFF-PEAK",
4286
+ "SCHEDULE",
4287
+ "QUAL",
4288
+ "$IN/M",
4289
+ "$OUT/M",
4290
+ "CTX",
4291
+ "SCORE",
4292
+ )
4293
+
4294
+ DEALS_LEGEND = (
4295
+ ("OFF-PEAK", "largest scheduled discount vs the standard rate"),
4296
+ ("SCHEDULE", "when the discounted rate applies (UTC)"),
4297
+ )
4298
+
4299
+
4300
+ def print_deals(rows, style=None, bare=False):
4301
+ style = style or Style(False)
4302
+ if not rows:
4303
+ if not bare:
4304
+ print()
4305
+ print("No scheduled off-peak discounts in the current data.")
4306
+ return
4307
+ cells = [
4308
+ {
4309
+ "MODEL": display_text(row["id"]),
4310
+ "OFF-PEAK": row["sched_note"],
4311
+ "SCHEDULE": row["sched_detail"] or "--",
4312
+ "QUAL": fmt_quality(row["quality"]),
4313
+ "$IN/M": fmt_price(row["offpeak"]["input_per_1m"]),
4314
+ "$OUT/M": fmt_price(row["offpeak"]["output_per_1m"]),
4315
+ "CTX": fmt_context(row["context"]),
4316
+ "SCORE": "-" if row["score"] is None else f"{row['score']:.3f}",
4317
+ }
4318
+ for row in rows
4319
+ ]
4320
+ widths = {
4321
+ col: max([len(col)] + [len(cell[col]) for cell in cells])
4322
+ for col in DEALS_COLUMNS
4323
+ }
4324
+
4325
+ def pad(col, value):
4326
+ if col in ("MODEL", "SCHEDULE"):
4327
+ return value.ljust(widths[col])
4328
+ return value.rjust(widths[col])
4329
+
4330
+ if not bare:
4331
+ print()
4332
+ print(
4333
+ style.bold(
4334
+ "Off-peak deals (cheapest off-peak window's prices; "
4335
+ "the standard rate applies at all other times)"
4336
+ )
4337
+ )
4338
+ print()
4339
+ header = []
4340
+ for col in DEALS_COLUMNS:
4341
+ # Every deals header is left-aligned (matching the main table and
4342
+ # the frozen mockup); only the data cells keep numeric right-align.
4343
+ text = col.ljust(widths[col])
4344
+ header.append(text if col == "SCORE" else style.bold(text))
4345
+ print(" ".join(header))
4346
+ print(style.dim(" ".join("-" * widths[col] for col in DEALS_COLUMNS)))
4347
+ for cell in cells:
4348
+ out = []
4349
+ for col in DEALS_COLUMNS:
4350
+ text = pad(col, cell[col])
4351
+ if col == "OFF-PEAK":
4352
+ text = style.bold(text)
4353
+ elif col == "QUAL" and cell[col] == "-":
4354
+ text = style.dim(text)
4355
+ out.append(text)
4356
+ print(" ".join(out))
4357
+ if bare:
4358
+ return
4359
+ print()
4360
+ width = max(len(term) for term, _ in DEALS_LEGEND)
4361
+ for term, text in DEALS_LEGEND:
4362
+ print(style.bold(term.ljust(width)) + " " + text)
4363
+
4364
+
4365
+ def print_depth_detail(top):
4366
+ """Per-model AA depth detail: measured scores and selectable variants.
4367
+
4368
+ One block per model in the given (rank) order, listing the union of
4369
+ OpenRouter-supported and AA-measured efforts, deepest first. Models with
4370
+ no measured depths get a single ``no AA depth data`` line. Display only:
4371
+ AA $/task never enters the weighted score.
4372
+ """
4373
+ if not top:
4374
+ return
4375
+
4376
+ headers = ["effort", "AA idx", "AA $/task", "vs default", "select"]
4377
+
4378
+ def finite(value):
4379
+ if (
4380
+ isinstance(value, (int, float))
4381
+ and not isinstance(value, bool)
4382
+ and math.isfinite(value)
4383
+ ):
4384
+ return value
4385
+ return None
4386
+
4387
+ def order(effort):
4388
+ try:
4389
+ return (AA_EFFORT_ORDER.index(effort), effort)
4390
+ except ValueError:
4391
+ return (len(AA_EFFORT_ORDER), effort or "")
4392
+
4393
+ blocks = []
2972
4394
  for rank, cand in enumerate(top, 1):
2973
- rows.append(
2974
- [
2975
- str(rank),
2976
- cand["id"],
2977
- "-" if cand["quality"] is None else f"{cand['quality']:.1f}",
2978
- fmt_price(cand["price_in"]),
2979
- fmt_price(cand["price_out"]),
2980
- fmt_discount(cand["discount"]),
2981
- fmt_prec(cand),
2982
- cand["tier_note"] or cand["sched_note"] or "--",
2983
- fmt_context(cand["context"]),
2984
- fmt_age(cand["age_days"]),
2985
- f"{cand['score']:.3f}",
2986
- ]
4395
+ reasoning = cand.get("reasoning") or {}
4396
+ supported = list(reasoning.get("supported_efforts") or [])
4397
+ records = [
4398
+ record
4399
+ for record in (cand.get("reasoning_depths") or [])
4400
+ if isinstance(record, dict) and record.get("effort")
4401
+ ]
4402
+ measured = {record["effort"]: record for record in records}
4403
+ default = _display_default_effort(cand)
4404
+ is_non_reasoning = reasoning.get("is_reasoning") is False
4405
+ # A non-mandatory single-state model exposes only the "none" toggle.
4406
+ single_state_none = (
4407
+ reasoning.get("is_reasoning") is True
4408
+ and reasoning.get("mandatory") is not True
4409
+ and not supported
2987
4410
  )
2988
- widths = [len(h) for h in headers]
2989
- for row in rows:
2990
- for i, cell in enumerate(row):
2991
- widths[i] = max(widths[i], len(cell))
2992
-
2993
- def emit(row):
2994
- if row[1] == "MODEL":
2995
- line = " ".join(h.ljust(w) for h, w in zip(row, widths))
2996
- print(line)
2997
- print(" ".join("-" * w for w in widths))
2998
- else:
2999
- cells = [row[0].rjust(widths[0]), row[1].ljust(widths[1])]
3000
- cells += [c.rjust(w) for c, w in zip(row[2:], widths[2:])]
3001
- print(" ".join(cells))
3002
-
3003
- emit(headers)
3004
- for row in rows:
3005
- emit(row)
3006
- weight_bits = ", ".join(
3007
- f"{k} {v:.0%}" for k, v in sorted(weights.items(), reverse=True)
3008
- )
4411
+ default_cost = (
4412
+ finite((measured.get(default) or {}).get("cost_per_task_usd"))
4413
+ if default
4414
+ else None
4415
+ )
4416
+ efforts = sorted(set(supported) | set(measured), key=order)
4417
+ rows = []
4418
+ for effort in efforts:
4419
+ record = measured.get(effort) or {}
4420
+ index = finite(record.get("index"))
4421
+ cost = finite(record.get("cost_per_task_usd"))
4422
+ if effort == "reasoning":
4423
+ # Single-state reasoning row: the bare id is the only form.
4424
+ select = "-"
4425
+ elif effort == "none" and is_non_reasoning:
4426
+ # A non-reasoning model's bare id is the only form.
4427
+ select = "-"
4428
+ elif effort in supported:
4429
+ select = f"#{effort}"
4430
+ elif effort == "none" and single_state_none:
4431
+ select = "#none"
4432
+ else:
4433
+ select = "- not selectable"
4434
+ ratio = "-"
4435
+ if cost and default_cost:
4436
+ value = cost / default_cost
4437
+ ratio = f"{value:.1f}x" if value >= 1 else f"{value:.2f}x"
4438
+ rows.append(
4439
+ [
4440
+ f"{effort}*" if effort == default else effort,
4441
+ "-" if index is None else f"{index:.1f}",
4442
+ "-" if cost is None else f"{cost:.2f}",
4443
+ ratio,
4444
+ select,
4445
+ ]
4446
+ )
4447
+ has_measured = any(
4448
+ finite(record.get("index")) is not None for record in measured.values()
4449
+ )
4450
+ blocks.append(
4451
+ {
4452
+ "rank": rank,
4453
+ "cand": cand,
4454
+ "default": default,
4455
+ "rows": rows,
4456
+ "has_records": bool(records),
4457
+ "missing_default": (
4458
+ bool(default)
4459
+ and default in efforts
4460
+ and finite((measured.get(default) or {}).get("index")) is None
4461
+ and has_measured
4462
+ ),
4463
+ }
4464
+ )
4465
+
4466
+ widths = [len(header) for header in headers]
4467
+ for block in blocks:
4468
+ for row in block["rows"]:
4469
+ for i, cell in enumerate(row):
4470
+ widths[i] = max(widths[i], len(cell))
4471
+
4472
+ def emit(cells):
4473
+ padded = [cell.ljust(width) for cell, width in zip(cells[:-1], widths[:-1])]
4474
+ print(" " + " ".join(padded + [cells[-1]]))
4475
+
4476
+ print("depth detail (AA index / AA $ per task; OR $/M is constant across depths)")
4477
+ for position, block in enumerate(blocks):
4478
+ if position:
4479
+ print()
4480
+ cand = block["cand"]
4481
+ print(
4482
+ f"#{block['rank']} {cand['id']} "
4483
+ f"OR ${cand['price_in']:.2f} in / ${cand['price_out']:.2f} out per 1M "
4484
+ f"default: {block['default'] or '-'}"
4485
+ )
4486
+ if not block["has_records"]:
4487
+ print(" no AA depth data")
4488
+ continue
4489
+ if block["missing_default"]:
4490
+ print(f" AA has no {block['default']} record for this model.")
4491
+ emit(headers)
4492
+ for row in block["rows"]:
4493
+ emit(row)
4494
+
3009
4495
  print()
3010
4496
  print(
3011
- f"pool: {total_candidates} candidates | weights: {weight_bits} | {quality_note}"
3012
- )
3013
- print(
3014
- "quality: Artificial Analysis intelligence index (- = unknown); prices in USD per 1M tokens; "
3015
- "DISC = active discount; AGE = time since listed on OpenRouter"
3016
- )
3017
- print(
3018
- "Prices shown are what a long session pays: several models bill at a "
3019
- "higher rate once the prompt outgrows their cheap short-context tier."
3020
- )
3021
- print(
3022
- "TIER marks how prices vary: a token threshold is the prompt size "
3023
- "above which the higher rate applies; a percentage is a scheduled "
3024
- "off-peak discount (price shown: the standard rate); SCHED marks a "
3025
- "schedule whose discount is under 1%. --json lists each model's "
3026
- "schedule."
3027
- )
3028
- print(
3029
- "PREC = precision the ranked price is quoted at (maker's default when "
3030
- "disclosed, best available otherwise; ! = default routing favors "
3031
- "cheaper sub-default endpoints); prices are the cheapest live endpoint "
3032
- "at that precision."
4497
+ "* = OpenRouter default; append #effort to the model id in OpenCode; "
4498
+ "- in select = not selectable; AA $/task is AA's measured benchmark "
4499
+ "cost at AA's reference price, not the OR per-token price; it never "
4500
+ "enters the weighted score."
3033
4501
  )
3034
4502
 
3035
4503
 
4504
+ def print_best(top, args):
4505
+ """Print the top candidate as an OpenCode model id.
4506
+
4507
+ A non-default ``--reasoning-effort`` appends ``#<effort>`` only when the
4508
+ model exposes that variant; otherwise the bare id selects the default.
4509
+ """
4510
+ cand = top[0]
4511
+ model_id = opencode_model_id(cand["id"])
4512
+ effort = args.reasoning_effort
4513
+ reasoning = cand.get("reasoning")
4514
+ if (
4515
+ effort != "default"
4516
+ and isinstance(reasoning, dict)
4517
+ and reasoning.get("is_reasoning") is not False
4518
+ and supports_effort(reasoning, effort)
4519
+ ):
4520
+ model_id += f"#{effort}"
4521
+ print(display_text(model_id))
4522
+
4523
+
3036
4524
  def print_json(top):
3037
4525
  payload = [
3038
4526
  {
@@ -3050,6 +4538,18 @@ def print_json(top):
3050
4538
  "quality_drift": (cand.get("quality_record") or {}).get("drift"),
3051
4539
  "quality_or_index": (cand.get("quality_record") or {}).get("or_index"),
3052
4540
  "quality_or_as_of": (cand.get("quality_record") or {}).get("or_as_of"),
4541
+ # The effort the quality record was measured at. Legacy records
4542
+ # (pre-depth paths) carry no "effort" key, so this degrades to
4543
+ # None exactly like a missing record.
4544
+ "quality_effort": (cand.get("quality_record") or {}).get("effort"),
4545
+ # The model's effective default effort, duplicated at top level:
4546
+ # data.json rows must keep it after the nested reasoning block is
4547
+ # dropped downstream. Same resolver as the DEPTH cell.
4548
+ "default_effort": _display_default_effort(cand),
4549
+ # The six published reasoning fields plus the measured depth
4550
+ # records; the internal is_reasoning flag never leaves the
4551
+ # process (catalog v5 and spec §8.4 publish only these keys).
4552
+ "reasoning": _published_reasoning_block(cand),
3053
4553
  "input_usd_per_m": round(cand["price_in"], 6),
3054
4554
  "output_usd_per_m": round(cand["price_out"], 6),
3055
4555
  "blended_usd_per_m": round(cand["blended"], 6),
@@ -3123,6 +4623,12 @@ def build_catalog(
3123
4623
  index_version=None,
3124
4624
  catalog_generated_at=None,
3125
4625
  catalog_quality=None,
4626
+ *,
4627
+ depth_lookup=None,
4628
+ catalog_depths=None,
4629
+ reasoning_by_id=None,
4630
+ requested_effort="default",
4631
+ schedule_entries=None,
3126
4632
  ):
3127
4633
  """Assemble the full-catalog document (see README "Catalog output").
3128
4634
 
@@ -3141,6 +4647,8 @@ def build_catalog(
3141
4647
  catalog_quality is the hydrated record map (or None): candidates read
3142
4648
  it through resolve_quality, and build_schedules falls back to it for
3143
4649
  non-candidates.
4650
+ The keyword-only depth inputs feed build_schedules (spec 6.5); run()
4651
+ always supplies them, direct callers may omit them for the legacy path.
3144
4652
  """
3145
4653
  now = datetime.now(timezone.utc)
3146
4654
  weights = catalog_weights(candidates, quality_by_id)
@@ -3167,6 +4675,14 @@ def build_catalog(
3167
4675
  )
3168
4676
  if unknown:
3169
4677
  raise ValueError(f"unknown quality source: {unknown[0]!r}")
4678
+ # Depth counts are candidate-scoped and separate from the disjoint label
4679
+ # buckets above (spec 9.1): schedule entries carry their own provenance
4680
+ # and never enter these counts or the pool invariants. matched_default
4681
+ # is the number of main quality records under strict selection -- exactly
4682
+ # the label buckets' sum, kept explicit for the publish gate.
4683
+ matched_default = matched_aa + matched_catalog + matched_openrouter
4684
+ depth_rows = sum(len(cand.get("reasoning_depths") or []) for cand in candidates)
4685
+ matched_depth = sum(1 for cand in candidates if cand.get("reasoning_depths"))
3170
4686
  # mode is the run's quality chain, not a match outcome: "aa" when the
3171
4687
  # AA API returned usable entries, "catalog" when unkeyed runs hydrated
3172
4688
  # the published catalog, "none" when neither tier produced data.
@@ -3199,23 +4715,31 @@ def build_catalog(
3199
4715
  "context": round(cand["context_score"], 4),
3200
4716
  "age": round(cand["age_score"], 4),
3201
4717
  }
3202
- overall = {}
3203
- for priority in PRIORITY_WEIGHTS:
3204
- w = weights[priority]
3205
- overall[priority] = round(
3206
- w.get("quality", 0.0) * scores["quality"]
3207
- + w.get("price", 0.0) * scores["price"]
3208
- + w.get("context", 0.0) * scores["context"]
3209
- + w.get("age", 0.0) * scores["age"],
3210
- 4,
3211
- )
4718
+ overall = overall_scores(cand, weights)
3212
4719
  bench = aa_by_id.get(base_model_id(cand["id"])) or {}
3213
4720
  # The quality record map is already a parameter; the winning
3214
4721
  # record's as_of and drift verdict become entry fields verbatim.
3215
4722
  # Unmatched candidates degrade to null -- same shape as the v3
3216
- # additive blocks below.
3217
- record = quality_by_id.get(cand["id"])
3218
- record = record if isinstance(record, dict) else None
4723
+ # additive blocks below. The candidate's own record wins (it is the
4724
+ # same object compute_scores attached), with the map as the fallback
4725
+ # for direct callers that skip compute_scores.
4726
+ record = cand.get("quality_record")
4727
+ if not isinstance(record, dict):
4728
+ record = quality_by_id.get(cand["id"])
4729
+ record = record if isinstance(record, dict) else None
4730
+ reasoning = cand.get("reasoning")
4731
+ reasoning = reasoning if isinstance(reasoning, dict) else {}
4732
+ depths = [
4733
+ {
4734
+ "effort": depth.get("effort"),
4735
+ "index": depth.get("index"),
4736
+ "cost_per_task_usd": depth.get("cost_per_task_usd"),
4737
+ "source": depth.get("source"),
4738
+ "as_of": depth.get("as_of"),
4739
+ }
4740
+ for depth in (cand.get("reasoning_depths") or [])
4741
+ if isinstance(depth, dict)
4742
+ ]
3219
4743
  entries.append(
3220
4744
  {
3221
4745
  "id": cand["id"],
@@ -3286,8 +4810,22 @@ def build_catalog(
3286
4810
  # Provenance (schema v4), verbatim from the record: the
3287
4811
  # winning source's own as_of and the drift verdict against
3288
4812
  # OpenRouter's check value.
3289
- "quality_as_of": record["as_of"] if record else None,
3290
- "quality_drift": record["drift"] if record else None,
4813
+ "quality_as_of": record.get("as_of") if record else None,
4814
+ "quality_drift": record.get("drift") if record else None,
4815
+ # Schema v5 depth contract: the effort the main quality was
4816
+ # measured at, and the model's reasoning metadata plus every
4817
+ # measured depth record (record shape: effort/index/
4818
+ # cost_per_task_usd/source/as_of -- `index`, never
4819
+ # `quality_index`).
4820
+ "quality_effort": record.get("effort") if record else None,
4821
+ "reasoning": {
4822
+ "default_effort": reasoning.get("default_effort"),
4823
+ "supported_efforts": reasoning.get("supported_efforts") or [],
4824
+ "default_enabled": reasoning.get("default_enabled"),
4825
+ "mandatory": reasoning.get("mandatory"),
4826
+ "supports_max_tokens": reasoning.get("supports_max_tokens"),
4827
+ "depths": depths,
4828
+ },
3291
4829
  "scores": {**scores, "overall": overall},
3292
4830
  # Additive provider-aware blocks (schema v3), verbatim from
3293
4831
  # the candidate. Candidates that never went through the
@@ -3312,7 +4850,17 @@ def build_catalog(
3312
4850
 
3313
4851
  return {
3314
4852
  "schedules": build_schedules(
3315
- models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality
4853
+ models,
4854
+ args,
4855
+ quality_by_id,
4856
+ aa_by_id,
4857
+ balanced_by_id,
4858
+ catalog_quality,
4859
+ depth_lookup=depth_lookup,
4860
+ catalog_depths=catalog_depths,
4861
+ reasoning_by_id=reasoning_by_id,
4862
+ requested_effort=requested_effort,
4863
+ schedule_entries=schedule_entries,
3316
4864
  ),
3317
4865
  "schema_version": CATALOG_SCHEMA_VERSION,
3318
4866
  "tool": "model-compare",
@@ -3327,6 +4875,9 @@ def build_catalog(
3327
4875
  "require_tools": not args.no_require_tools,
3328
4876
  "exclude_free": args.exclude_free,
3329
4877
  "include_batch": args.include_batch,
4878
+ # The requested effort documents the semantic of every model's
4879
+ # quality value (default = each model's effective default).
4880
+ "reasoning_effort": getattr(args, "reasoning_effort", "default"),
3330
4881
  "weights": weights,
3331
4882
  },
3332
4883
  "sources": {
@@ -3336,6 +4887,11 @@ def build_catalog(
3336
4887
  "matched_aa": matched_aa,
3337
4888
  "matched_catalog": matched_catalog,
3338
4889
  "matched_openrouter": matched_openrouter,
4890
+ # Depth coverage (schema v5), candidate-scoped like the
4891
+ # label buckets above. Schedule entries never contribute.
4892
+ "depth_rows": depth_rows,
4893
+ "matched_depth": matched_depth,
4894
+ "matched_default": matched_default,
3339
4895
  # Provenance meta (schema v4): the published catalog's stamp
3340
4896
  # when this run hydrated from it, and the AA envelope's
3341
4897
  # intelligence index version when the AA API ran. Both null
@@ -3378,6 +4934,49 @@ def print_catalog(document):
3378
4934
  print(json.dumps(document, indent=2))
3379
4935
 
3380
4936
 
4937
+ # --- plain-text styling -----------------------------------------------------
4938
+ # SGR attributes only: the terminal theme supplies the hues, so the output
4939
+ # stays readable on light and dark themes alike.
4940
+ SGR_RESET = "\x1b[0m"
4941
+ SGR_BOLD = "\x1b[1m"
4942
+ SGR_DIM = "\x1b[2m"
4943
+ SGR_GREEN = "\x1b[32m"
4944
+ SGR_YELLOW = "\x1b[33m"
4945
+
4946
+
4947
+ class Style:
4948
+ """Minimal SGR wrapper; every method is a no-op when disabled."""
4949
+
4950
+ def __init__(self, enabled: bool):
4951
+ self.enabled = enabled
4952
+
4953
+ def _wrap(self, text: str, code: str) -> str:
4954
+ return f"{code}{text}{SGR_RESET}" if self.enabled else text
4955
+
4956
+ def bold(self, text: str) -> str:
4957
+ return self._wrap(text, SGR_BOLD)
4958
+
4959
+ def dim(self, text: str) -> str:
4960
+ return self._wrap(text, SGR_DIM)
4961
+
4962
+ def green(self, text: str) -> str:
4963
+ return self._wrap(text, SGR_GREEN)
4964
+
4965
+ def yellow(self, text: str) -> str:
4966
+ return self._wrap(text, SGR_YELLOW)
4967
+
4968
+
4969
+ def color_enabled(mode: str, stream) -> bool:
4970
+ """Resolve --color for one stream. Explicit "on" beats NO_COLOR."""
4971
+ if mode == "off":
4972
+ return False
4973
+ if mode == "on":
4974
+ return True
4975
+ if os.environ.get("NO_COLOR"):
4976
+ return False
4977
+ return bool(getattr(stream, "isatty", lambda: False)())
4978
+
4979
+
3381
4980
  # ---------------------------------------------------------------------------
3382
4981
  # CLI
3383
4982
  # ---------------------------------------------------------------------------
@@ -3395,99 +4994,145 @@ def parse_args(argv=None):
3395
4994
  action="version",
3396
4995
  version=f"%(prog)s {VERSION}",
3397
4996
  )
3398
- parser.add_argument(
4997
+ output = parser.add_argument_group("output")
4998
+ ranking = parser.add_argument_group("ranking")
4999
+ filters = parser.add_argument_group("filters")
5000
+ sources = parser.add_argument_group("data sources & cache")
5001
+ filters.add_argument(
3399
5002
  "--min-context",
3400
5003
  type=int,
3401
5004
  default=1_000_000,
3402
5005
  help="hard minimum context window in tokens (default: 1000000)",
3403
5006
  )
3404
- parser.add_argument(
5007
+ ranking.add_argument(
3405
5008
  "--priority",
3406
5009
  choices=sorted(PRIORITY_WEIGHTS),
3407
5010
  default="balanced",
3408
5011
  help="ranking emphasis (default: balanced)",
3409
5012
  )
3410
- parser.add_argument(
5013
+ ranking.add_argument(
3411
5014
  "--top", type=int, default=5, help="how many models to list (default: 5)"
3412
5015
  )
3413
- parser.add_argument(
5016
+ output.add_argument(
3414
5017
  "--best",
3415
5018
  action="store_true",
3416
5019
  help="print only the #1 model as an opencode id (openrouter/<model>), for scripting",
3417
5020
  )
3418
- parser.add_argument(
5021
+ output.add_argument(
3419
5022
  "--json", action="store_true", help="print the ranked candidates as JSON"
3420
5023
  )
3421
- parser.add_argument(
5024
+ output.add_argument(
3422
5025
  "--catalog",
3423
5026
  action="store_true",
3424
5027
  help="print the full model catalog (ranked candidates and filtered-out models with reasons) as one JSON document; ignores --top and --priority, cannot be combined with --best or --json",
3425
5028
  )
3426
- parser.add_argument(
5029
+ output.add_argument(
5030
+ "--detail",
5031
+ choices=("simple", "expert"),
5032
+ default="expert",
5033
+ help="table density (default: expert; simple hides PREC, TIER, CTX and "
5034
+ "SCORE, mimicking the web page's simple view)",
5035
+ )
5036
+ output.add_argument(
5037
+ "--off-peak",
5038
+ action="store_true",
5039
+ help="also list scheduled off-peak deals, ranked by discount size",
5040
+ )
5041
+ output.add_argument(
5042
+ "--bare",
5043
+ action="store_true",
5044
+ help="print just the table(s), no summary or legend",
5045
+ )
5046
+ output.add_argument(
5047
+ "--no-depths",
5048
+ action="store_true",
5049
+ help=(
5050
+ "hide the per-model depth detail below the table (the DEPTH and "
5051
+ "QUAL columns stay)"
5052
+ ),
5053
+ )
5054
+ output.add_argument(
5055
+ "--color",
5056
+ choices=("auto", "on", "off"),
5057
+ default="auto",
5058
+ help="colorize the table output (default: auto; honors NO_COLOR, and on "
5059
+ "forces color when piping through less -R)",
5060
+ )
5061
+ ranking.add_argument(
3427
5062
  "--input-share",
3428
5063
  type=float,
3429
5064
  default=0.75,
3430
5065
  help="input-token share used for the blended price, 0-1 (default: 0.75)",
3431
5066
  )
3432
- parser.add_argument(
5067
+ ranking.add_argument(
3433
5068
  "--recency-half-life",
3434
5069
  type=float,
3435
5070
  default=120.0,
3436
5071
  help="age decay half-life in days (default: 120)",
3437
5072
  )
3438
- parser.add_argument(
5073
+ ranking.add_argument(
3439
5074
  "--max-age-days",
3440
5075
  type=float,
3441
5076
  default=0.0,
3442
5077
  help="drop models listed more than this many days ago (0 = off)",
3443
5078
  )
3444
- parser.add_argument(
5079
+ ranking.add_argument(
3445
5080
  "--quality-ref",
3446
5081
  type=float,
3447
5082
  default=70.0,
3448
5083
  help="AA intelligence index that counts as full quality score (default: 70)",
3449
5084
  )
3450
- parser.add_argument(
5085
+ ranking.add_argument(
5086
+ "--reasoning-effort",
5087
+ choices=REASONING_EFFORT_CHOICES,
5088
+ default="default",
5089
+ help=(
5090
+ "reasoning effort used for quality and filtering: a specific "
5091
+ "effort drops models that cannot run it (default: each model's "
5092
+ "effective default, no filter)"
5093
+ ),
5094
+ )
5095
+ sources.add_argument(
3451
5096
  "--aa-api-key",
3452
5097
  default=None,
3453
5098
  help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
3454
5099
  )
3455
- parser.add_argument(
5100
+ sources.add_argument(
3456
5101
  "--openrouter-api-key",
3457
5102
  default=None,
3458
5103
  help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
3459
5104
  )
3460
- parser.add_argument(
5105
+ filters.add_argument(
3461
5106
  "--no-require-tools",
3462
5107
  action="store_true",
3463
5108
  help="do not require tool-calling support (coding agents want it)",
3464
5109
  )
3465
- parser.add_argument(
5110
+ filters.add_argument(
3466
5111
  "--exclude-free",
3467
5112
  action="store_true",
3468
5113
  help="drop zero-cost listings such as rate-limited :free variants",
3469
5114
  )
3470
- parser.add_argument(
5115
+ filters.add_argument(
3471
5116
  "--include-batch",
3472
5117
  action="store_true",
3473
5118
  help="keep ':batch' variants (asynchronous completion; cheaper but unsuitable for interactive agents)",
3474
5119
  )
3475
- parser.add_argument(
5120
+ filters.add_argument(
3476
5121
  "--discount",
3477
5122
  action="store_true",
3478
5123
  help="only list models with an active discount",
3479
5124
  )
3480
- parser.add_argument(
5125
+ filters.add_argument(
3481
5126
  "--no-zdr",
3482
5127
  action="store_true",
3483
5128
  help="consider all models, not just zero-data-retention (ZDR) ones",
3484
5129
  )
3485
- parser.add_argument(
5130
+ sources.add_argument(
3486
5131
  "--no-cache",
3487
5132
  action="store_true",
3488
5133
  help="bypass the local response cache and refetch",
3489
5134
  )
3490
- parser.add_argument(
5135
+ sources.add_argument(
3491
5136
  "--cache-ttl",
3492
5137
  type=int,
3493
5138
  default=6 * 3600,
@@ -3520,7 +5165,13 @@ def main(argv=None) -> int:
3520
5165
 
3521
5166
 
3522
5167
  def run(args, models, or_cached) -> int:
3523
- discounts, zdr_ids, aa_by_id, frontend_cache_hits = fetch_openrouter_frontend(args)
5168
+ (
5169
+ discounts,
5170
+ zdr_ids,
5171
+ aa_by_id,
5172
+ frontend_cache_hits,
5173
+ frontend_models,
5174
+ ) = fetch_openrouter_frontend(args)
3524
5175
  if not args.no_zdr and not zdr_ids:
3525
5176
  warn(
3526
5177
  "ZDR data unavailable; refusing to rank possibly non-ZDR models "
@@ -3536,25 +5187,43 @@ def run(args, models, or_cached) -> int:
3536
5187
  if args.aa_api_key or os.environ.get("AA_API_KEY"):
3537
5188
  aa_entries, aa_source, aa_cached, aa_index_version = fetch_aa_entries(args)
3538
5189
  catalog_quality = None
5190
+ catalog_depths = None
3539
5191
  catalog_generated_at = None
5192
+ # Keyed runs never hydrate schedules from the published catalog.
5193
+ schedule_entries = None
3540
5194
  else:
3541
5195
  aa_entries, aa_source, aa_cached, aa_index_version = [], None, False, None
3542
- # None (transport failure) and an empty by_id (fetched-but-unusable
5196
+ # None (transport failure) and empty maps (fetched-but-unusable
3543
5197
  # artifact) both degrade to no catalog tier -- resolve_quality and
3544
5198
  # the footer treat them identically as "catalog unavailable".
3545
5199
  catalog_doc = fetch_quality_catalog(args)
3546
5200
  catalog_quality = (catalog_doc or {}).get("by_id") or None
5201
+ catalog_depths = (catalog_doc or {}).get("depths_by_id") or None
5202
+ # Spec 6.5: the published schedules array is the unkeyed Deals
5203
+ # source of truth -- schedule-only models never appear in models.
5204
+ schedule_entries = (catalog_doc or {}).get("schedules_by_id") or None
3547
5205
  # The catalog meta only means something when hydration actually
3548
5206
  # produced rows: an unusable artifact contributes no meta, so the
3549
5207
  # sources block and the mode never credit a catalog that yielded
3550
5208
  # nothing.
3551
5209
  catalog_generated_at = (
3552
- (catalog_doc or {}).get("generated_at") if catalog_quality else None
5210
+ (catalog_doc or {}).get("generated_at")
5211
+ if (catalog_quality or catalog_depths)
5212
+ else None
3553
5213
  )
3554
- exact, fuzzy = build_aa_lookup(aa_entries)
5214
+ exact, fuzzy = build_aa_lookup(dedupe_aa_entries_by_key(aa_entries))
5215
+ depth_lookup = build_aa_depth_lookup(aa_entries)
3555
5216
 
3556
5217
  filtered = []
3557
- candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
5218
+ reasoning_by_id = build_reasoning_metadata(models, frontend_models)
5219
+ candidates, dropped = build_candidates(
5220
+ models,
5221
+ args,
5222
+ discounts,
5223
+ zdr_ids,
5224
+ filtered,
5225
+ reasoning_by_id=reasoning_by_id,
5226
+ )
3558
5227
  # The provider-aware stage: every surviving candidate is repriced at its
3559
5228
  # baseline endpoint (or dropped fail-closed) before any scoring runs.
3560
5229
  candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
@@ -3576,7 +5245,15 @@ def run(args, models, or_cached) -> int:
3576
5245
  return 2
3577
5246
 
3578
5247
  quality_by_id, quality_source_by_id = resolve_quality(
3579
- candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality
5248
+ candidates,
5249
+ aa_by_id,
5250
+ exact,
5251
+ fuzzy,
5252
+ aa_source,
5253
+ catalog_quality,
5254
+ depth_lookup=depth_lookup,
5255
+ catalog_depths=catalog_depths,
5256
+ requested_effort=args.reasoning_effort,
3580
5257
  )
3581
5258
  weights = compute_scores(candidates, args, quality_by_id)
3582
5259
 
@@ -3621,6 +5298,11 @@ def run(args, models, or_cached) -> int:
3621
5298
  aa_index_version,
3622
5299
  catalog_generated_at,
3623
5300
  catalog_quality,
5301
+ depth_lookup=depth_lookup,
5302
+ catalog_depths=catalog_depths,
5303
+ reasoning_by_id=reasoning_by_id,
5304
+ requested_effort=args.reasoning_effort,
5305
+ schedule_entries=schedule_entries,
3624
5306
  )
3625
5307
  )
3626
5308
  return 0
@@ -3629,45 +5311,73 @@ def run(args, models, or_cached) -> int:
3629
5311
  top = candidates[:limit]
3630
5312
 
3631
5313
  if args.best:
3632
- print(opencode_model_id(top[0]["id"]))
5314
+ print_best(top, args)
3633
5315
  elif args.json:
3634
5316
  print_json(top)
3635
5317
  else:
3636
- drop_note = ""
3637
- if dropped:
3638
- bits = ", ".join(
3639
- f"{v} {k}" for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
5318
+ style = Style(color_enabled(args.color, sys.stdout))
5319
+ err_style = Style(color_enabled(args.color, sys.stderr))
5320
+ if not args.bare:
5321
+ source_note = []
5322
+ if or_cached:
5323
+ source_note.append("catalog cached")
5324
+ if "discounts" in frontend_cache_hits:
5325
+ source_note.append("discounts cached")
5326
+ if "zdr" in frontend_cache_hits:
5327
+ source_note.append("ZDR cached")
5328
+ if "aa" in frontend_cache_hits:
5329
+ source_note.append("AA benchmarks cached")
5330
+ if aa_cached and aa_source:
5331
+ source_note.append("quality cached")
5332
+ if source_note:
5333
+ print(
5334
+ f"note: {', '.join(source_note)} (use --no-cache to refresh)",
5335
+ file=sys.stderr,
5336
+ )
5337
+ print(
5338
+ f"{len(candidates)} of {len(models)} listed models ranked",
5339
+ file=sys.stderr,
3640
5340
  )
3641
- drop_note = f" (dropped: {bits})"
5341
+ if dropped:
5342
+ bits = ", ".join(
5343
+ f"{v} {k}"
5344
+ for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
5345
+ )
5346
+ print(err_style.dim(f"(dropped: {bits})"), file=sys.stderr)
5347
+ print(file=sys.stderr)
3642
5348
  # The footer lists per-source counts, each source's own as_of and
3643
5349
  # the AA index version when known; unmatched candidates and drift
3644
- # get trailing footnotes. No record resolved -> the quality-blind
3645
- # fallback note.
5350
+ # get trailing clauses. No record resolved -> print_table's
5351
+ # quality-blind fallback line.
3646
5352
  unmatched = len(candidates) - len(quality_by_id)
3647
- quality_note = quality_footer_note(
3648
- quality_by_id, aa_index_version, unmatched
3649
- ) or ("quality data unavailable, ranked on price/context/age")
3650
- source_note = []
3651
- if or_cached:
3652
- source_note.append("catalog cached")
3653
- if "discounts" in frontend_cache_hits:
3654
- source_note.append("discounts cached")
3655
- if "zdr" in frontend_cache_hits:
3656
- source_note.append("ZDR cached")
3657
- if "aa" in frontend_cache_hits:
3658
- source_note.append("AA benchmarks cached")
3659
- if aa_cached and aa_source:
3660
- source_note.append("quality cached")
3661
- if source_note:
3662
- print(
3663
- f"note: {', '.join(source_note)} (use --no-cache to refresh)",
3664
- file=sys.stderr,
3665
- )
3666
- print(
3667
- f"pool: {len(models)} listed -> {len(candidates)} candidates{drop_note}",
3668
- file=sys.stderr,
5353
+ quality_note = quality_footer_note(quality_by_id, aa_index_version, unmatched)
5354
+ print_table(
5355
+ top,
5356
+ weights,
5357
+ quality_note,
5358
+ detail=args.detail,
5359
+ style=style,
5360
+ bare=args.bare,
5361
+ input_share=args.input_share,
5362
+ depth_detail=(None if args.bare or args.no_depths else print_depth_detail),
3669
5363
  )
3670
- print_table(top, len(candidates), weights, quality_note)
5364
+ if args.off_peak:
5365
+ catalog_w = catalog_weights(candidates, quality_by_id)
5366
+ # Use the exact unrounded weighted score (not overall_scores,
5367
+ # whose rounded components can disagree by 0.001) so a candidate
5368
+ # in both tables always prints the same SCORE under balanced.
5369
+ balanced_w = catalog_w["balanced"]
5370
+ balanced = {
5371
+ cand["id"]: weighted_score(cand, balanced_w) for cand in candidates
5372
+ }
5373
+ schedules = build_schedules(
5374
+ models, args, quality_by_id, aa_by_id, balanced, catalog_quality
5375
+ )
5376
+ # build_schedules resolves quality for any listed model; the web
5377
+ # page's deals table shows that resolved value even for models
5378
+ # outside the ranked pool, so keep it here too. Only SCORE is
5379
+ # blanked for non-candidates (balanced is None for them).
5380
+ print_deals(off_peak_deals(schedules), style=style, bare=args.bare)
3671
5381
  return 0
3672
5382
 
3673
5383