model-compare-opencode-plugin 0.1.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,6 +54,7 @@ import time
54
54
  import urllib.error
55
55
  import urllib.parse
56
56
  import urllib.request
57
+ from concurrent.futures import ThreadPoolExecutor
57
58
  from datetime import datetime, timezone
58
59
 
59
60
  OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
@@ -67,7 +68,23 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
67
68
  # (legacy endpoints 410 after 2026-11-04; see
68
69
  # https://artificialanalysis.ai/data-api/migrate-v2-data).
69
70
  AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
70
- VERSION = "0.3.1"
71
+ # Cache key for the AA API entries envelope. v3 adds the per-fetch `as_of`
72
+ # timestamp on every entry and the envelope's intelligence_index_version;
73
+ # payloads under the pre-v3 keys predate those fields and must never be read.
74
+ AA_ENTRIES_CACHE = "aa-intelligence-v3"
75
+ # The repo's own published catalog.json -- built by keyed CI runs, hydrated
76
+ # by unkeyed local runs (design: quality source reconciliation, source 2).
77
+ # The github.io host 301s to plain http://model-compare.dev/catalog.json, an
78
+ # unauthenticated downgrade on ranking-critical data; the https direct URL is
79
+ # verified 200 application/json. Do not point this back at github.io.
80
+ QUALITY_CATALOG_URL = "https://model-compare.dev/catalog.json"
81
+ # Cache key for the hydration envelope {"by_id", "generated_at",
82
+ # "schema_version"}. Only usable (non-empty by_id) results are ever written.
83
+ QUALITY_CATALOG_CACHE = "published-catalog"
84
+ # A published catalog generated more than this many days ago is stale:
85
+ # warn but use -- the records' as_of makes the staleness visible downstream.
86
+ QUALITY_CATALOG_MAX_AGE_DAYS = 14
87
+ VERSION = "0.5.0"
71
88
  USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
72
89
 
73
90
  # opencode expects provider-qualified model ids: openrouter/<provider/model>.
@@ -86,11 +103,15 @@ PRIORITY_WEIGHTS = {
86
103
  "quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
87
104
  }
88
105
 
89
- CATALOG_SCHEMA_VERSION = 2
106
+ CATALOG_SCHEMA_VERSION = 4
90
107
 
91
108
  # The drop-reason keys build_candidates counts, verbatim -- keep in sync with
92
109
  # its drop() call sites. pool.dropped lists all of them zero-filled so the
93
- # contract is self-describing.
110
+ # contract is self-describing. The last two come from the endpoint stage
111
+ # (apply_endpoint_pricing's drop sites), not build_candidates: snake_case
112
+ # while the older reasons are human phrases. The mixed naming convention is
113
+ # a deliberate, documented extension -- machine-readable reasons for the
114
+ # endpoint-stage drops the publish drop-rate gate keys on.
94
115
  CATALOG_DROP_REASONS = (
95
116
  "malformed id",
96
117
  "context",
@@ -104,11 +125,74 @@ CATALOG_DROP_REASONS = (
104
125
  "tool calling",
105
126
  "expired",
106
127
  "age",
128
+ "no_baseline_eligible_endpoint",
129
+ "endpoint_fetch_failed",
130
+ )
131
+
132
+ # The only reasons the endpoint stage itself produces (the last two
133
+ # CATALOG_DROP_REASONS entries) -- the empty-pool warning keys on them;
134
+ # web/publish.py keeps its own copy of this tuple by design.
135
+ ENDPOINT_DROP_REASONS = (
136
+ "no_baseline_eligible_endpoint",
137
+ "endpoint_fetch_failed",
107
138
  )
108
139
 
109
140
  PAREN_RE = re.compile(r"\([^)]*\)")
110
141
  PROVIDER_PREFIX_RE = re.compile(r"^[^:\s]{1,30}:\s*")
111
142
 
143
+ # Quantization precision ladder: label -> rank, highest fidelity first. The
144
+ # ladder ordering in the design doc is authoritative: fp6 > int8 is strict,
145
+ # and formats within one fidelity tier share the same rank. "unknown" never
146
+ # enters the ladder -- it only routes through the first-party eligibility
147
+ # carve-out, never through rank comparison.
148
+ PRECISION_LADDER: dict[str, int] = {
149
+ "fp32": 4,
150
+ "bf16": 3,
151
+ "fp16": 3,
152
+ "fp8": 2,
153
+ "mxfp8": 2,
154
+ "fp6": 1,
155
+ "int8": 0,
156
+ "fp4": -1,
157
+ "mxfp4": -1,
158
+ "nvfp4": -1,
159
+ "int4": -2,
160
+ }
161
+
162
+ # Every quantization value an endpoint may report: the ladder keys plus the
163
+ # "unknown" sentinel. Whether a recorded value is recognized at all is
164
+ # membership here, not ladder rank.
165
+ QUANTIZATION_ENUM: frozenset[str] = frozenset(PRECISION_LADDER) | {"unknown"}
166
+
167
+ # Official partner hosts that serve closed labs' models: an endpoint on one
168
+ # of these providers counts as first-party for the eligibility carve-out
169
+ # even though it is not the maker itself. Entries are compared after
170
+ # normalize_provider (casefold + strip non-alphanumerics), so the slugs here
171
+ # are display forms -- "aws-bedrock" and "AWS Bedrock" both hit. Extend only
172
+ # with live-data evidence (e.g. an endpoint tag matching <lab>-official).
173
+ FIRST_PARTY_HOSTS = frozenset(
174
+ {"bedrock", "azure", "vertex", "azure-ai", "google-vertex", "aws-bedrock"}
175
+ )
176
+
177
+ # Explicit first-party alias pairs that normalization alone cannot bridge
178
+ # (design §4.2): makers that self-name differently than their OpenRouter
179
+ # author slug. Key = the exact live provider_name string, value = the
180
+ # author slug the pair is first-party FOR; both sides are compared
181
+ # normalized (case/punctuation variants match), and the alias only fires
182
+ # when the model's author equals the value. Verified against live data
183
+ # 2026-10-20 (task-13-fix-report.md section 8b; endpoints fetched fresh,
184
+ # every endpoint quantization "unknown", status 0):
185
+ # ("Amazon Bedrock", "amazon") -- amazon/nova-2-lite-v1, 2 endpoints
186
+ # ("Google AI Studio", "google") -- google/gemini-2.5-pro-preview, 3 endpoints
187
+ # ("Mistral", "mistralai") -- mistralai/mistral-large-4-0, 3 endpoints
188
+ # Add entries only after the same live verification -- never pin an
189
+ # unverified string.
190
+ FIRST_PARTY_ALIASES: dict[str, str] = {
191
+ "Amazon Bedrock": "amazon",
192
+ "Google AI Studio": "google",
193
+ "Mistral": "mistralai",
194
+ }
195
+
112
196
 
113
197
  def warn(msg: str) -> None:
114
198
  print(f"warning: {msg}", file=sys.stderr)
@@ -118,6 +202,44 @@ def norm_key(s: str) -> str:
118
202
  return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
119
203
 
120
204
 
205
+ # ---------------------------------------------------------------------------
206
+ # Provider normalization + first-party detection
207
+ # ---------------------------------------------------------------------------
208
+
209
+
210
+ def normalize_provider(name: str) -> str:
211
+ """Casefold a provider name and strip ALL non-alphanumeric characters.
212
+
213
+ "Z.AI", "z-ai" and "z ai" all normalize to "zai", so a live-recorded
214
+ maker name matches its author slug regardless of punctuation or case.
215
+ None or empty input normalizes to "".
216
+ """
217
+ return re.sub(r"[^a-z0-9]+", "", (name or "").casefold())
218
+
219
+
220
+ def is_first_party(provider_name: str, author: str) -> bool:
221
+ """True iff the provider is the model's own maker or a partner host.
222
+
223
+ Criterion 1: the normalized provider name equals the normalized author
224
+ slug. Criterion 2: it matches a normalized FIRST_PARTY_HOSTS entry, or
225
+ a FIRST_PARTY_ALIASES key whose value equals the normalized author slug
226
+ (the §4.2 alias pairs: a maker self-naming differently than its author
227
+ slug is first-party only for that author). Everything else is
228
+ third-party.
229
+ """
230
+ provider = normalize_provider(provider_name)
231
+ if provider == normalize_provider(author):
232
+ return True
233
+ if provider in {normalize_provider(host) for host in FIRST_PARTY_HOSTS}:
234
+ return True
235
+ author_slug = normalize_provider(author)
236
+ return any(
237
+ provider == normalize_provider(alias_provider)
238
+ and author_slug == normalize_provider(alias_author)
239
+ for alias_provider, alias_author in FIRST_PARTY_ALIASES.items()
240
+ )
241
+
242
+
121
243
  # ---------------------------------------------------------------------------
122
244
  # HTTP + cache
123
245
  # ---------------------------------------------------------------------------
@@ -224,6 +346,1049 @@ def save_cache(name: str, payload) -> None:
224
346
  warn(f"could not write cache: {exc}")
225
347
 
226
348
 
349
+ # ---------------------------------------------------------------------------
350
+ # OpenRouter endpoints (per-model provider list)
351
+ # ---------------------------------------------------------------------------
352
+
353
+
354
+ OPENROUTER_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{id}/endpoints"
355
+
356
+
357
+ def load_cache_expired(name: str) -> dict | None:
358
+ """Return the cached payload regardless of age; None when absent/corrupt.
359
+
360
+ load_cache drops payloads past their TTL; this variant serves them anyway
361
+ so a dropped model's last known endpoint annotations can still be shown
362
+ (marked stale by the caller). Never used for ranking.
363
+ """
364
+ path = cache_path(name)
365
+ if not path:
366
+ return None
367
+ try:
368
+ with open(path) as fh:
369
+ blob = json.load(fh)
370
+ return blob["payload"]
371
+ except (OSError, ValueError, KeyError, TypeError):
372
+ return None
373
+
374
+
375
+ def parse_endpoints(payload) -> list[dict]:
376
+ """Keep the verified per-endpoint fields of an endpoints response.
377
+
378
+ The live GET /api/v1/models/{id}/endpoints response nests the list one
379
+ level below the usual envelope: `data` is the model object (id, name,
380
+ architecture, ...) and the per-provider entries live in
381
+ `data["endpoints"]`. `pricing` keeps prompt/completion (parsed through
382
+ parse_price) and the raw `overrides` list -- the tiered-pricing
383
+ machinery parses the entries itself. Fields beyond the verified
384
+ per-entry shape are dropped, and the parse keeps `status` as-is (the
385
+ caller filters).
386
+ """
387
+ # Task 13 lesson: the first unit fixtures fed the models/find shape
388
+ # ({"data": [entry, ...]}) -- a list directly under "data" -- so the
389
+ # suite stayed green while every live parse returned [] and the whole
390
+ # pool died as no_baseline_eligible_endpoint. Pin the live envelope:
391
+ # the list is read ONLY from data["endpoints"], and every other shape
392
+ # parses to [] (no compat branch for the wrong fixture shape).
393
+ data = payload.get("data") if isinstance(payload, dict) else None
394
+ entries = data.get("endpoints") if isinstance(data, dict) else None
395
+ endpoints = []
396
+ for entry in entries or []:
397
+ if not isinstance(entry, dict):
398
+ continue
399
+ raw = entry.get("pricing")
400
+ raw_pricing = raw if isinstance(raw, dict) else {}
401
+ endpoints.append(
402
+ {
403
+ "provider_name": entry.get("provider_name"),
404
+ "tag": entry.get("tag"),
405
+ "quantization": entry.get("quantization"),
406
+ "pricing": {
407
+ "prompt": parse_price(raw_pricing.get("prompt")),
408
+ "completion": parse_price(raw_pricing.get("completion")),
409
+ "overrides": raw_pricing.get("overrides"),
410
+ },
411
+ "context_length": entry.get("context_length"),
412
+ "status": entry.get("status"),
413
+ "service_tier": entry.get("service_tier"),
414
+ }
415
+ )
416
+ return endpoints
417
+
418
+
419
+ def fetch_model_endpoints(model_id: str, args) -> list[dict]:
420
+ """Fetch + parse one model's endpoint list, cached per model.
421
+
422
+ The cache name flattens the model id ("/" -> "_") to keep the cache
423
+ files flat. Fetches retry 3 times (4 attempts total) via fetch_json,
424
+ which raises at once on 4xx and after the retries on 5xx/network
425
+ errors -- the caller owns the fail-closed policy.
426
+ """
427
+ cache_name = f"endpoints:{model_id.replace('/', '_')}"
428
+ if not args.no_cache:
429
+ cached = load_cache(cache_name, args.cache_ttl)
430
+ if cached:
431
+ return cached
432
+ payload = fetch_json(
433
+ OPENROUTER_ENDPOINTS_URL.format(id=model_id), timeout=30, retries=3
434
+ )
435
+ endpoints = parse_endpoints(payload)
436
+ save_cache(cache_name, endpoints)
437
+ return endpoints
438
+
439
+
440
+ def is_variant_endpoint(endpoint) -> bool:
441
+ """True iff the endpoint is a :free/:batch variant or entirely free.
442
+
443
+ Variant iff its slug or tag carries a `:free`/`:batch` marker OR both
444
+ input and output prices are zero. `:batch` is cheaper-but-restricted,
445
+ not zero-priced -- only the marker test catches it. Missing or
446
+ unparseable prices are not zero.
447
+ """
448
+ for field in ("slug", "tag"):
449
+ marker = endpoint.get(field)
450
+ if isinstance(marker, str) and (":free" in marker or ":batch" in marker):
451
+ return True
452
+ pricing = endpoint.get("pricing")
453
+ prices = pricing if isinstance(pricing, dict) else {}
454
+ return (
455
+ parse_price(prices.get("prompt")) == 0.0
456
+ and parse_price(prices.get("completion")) == 0.0
457
+ )
458
+
459
+
460
+ def endpoint_schedule_ok(endpoint, args) -> bool:
461
+ """False iff the endpoint's pricing.overrides fails the v2 schedule rule.
462
+
463
+ Reuses effective_pricing in full: the gate fires on malformed windows,
464
+ exactly-one-of utc_start/utc_end pairs, bad utc_days, combined
465
+ token+time entries and missing inheritance -- not just non-tiling
466
+ windows. An endpoint without overrides has no schedule to fail.
467
+ """
468
+ result = effective_pricing(
469
+ endpoint.get("pricing"), args.min_context, args.input_share
470
+ )
471
+ return not result["schedule_error"]
472
+
473
+
474
+ # Schedule gate for args-less callers: effective_pricing's schedule_error
475
+ # depends only on the SHAPE of pricing.overrides (malformed windows, bad
476
+ # utc_days, ...), never on min_context/input_share -- those only affect the
477
+ # resolved prices. derive_default_precision has no args parameter (pinned
478
+ # interface), so it gates through this fixed stand-in; any args would give
479
+ # the same schedule_error verdict.
480
+ _SCHEDULE_GATE_ARGS = argparse.Namespace(min_context=0, input_share=0.75)
481
+
482
+
483
+ def _baseline_universe(endpoints: list[dict], args=None) -> list[dict]:
484
+ """The shared derivation universe (design section 4.1).
485
+
486
+ Live (status == 0), not a variant endpoint (:free/:batch marker or
487
+ zero-priced), and passing the schedule gate. Schedule-broken endpoints
488
+ must not pin the default that governs healthy endpoints, same rationale
489
+ as the variant exclusion. args defaults to the structural stand-in
490
+ (see _SCHEDULE_GATE_ARGS); callers with real args pass them.
491
+ """
492
+ if args is None:
493
+ args = _SCHEDULE_GATE_ARGS
494
+ return [
495
+ endpoint
496
+ for endpoint in endpoints
497
+ if endpoint.get("status") == 0
498
+ and not is_variant_endpoint(endpoint)
499
+ and endpoint_schedule_ok(endpoint, args)
500
+ ]
501
+
502
+
503
+ def derive_default_precision(endpoints: list[dict], author: str) -> str | None:
504
+ """Derive a model's default precision from its endpoint list (section 4.1).
505
+
506
+ Universe: live non-variant schedule-passing endpoints (_baseline_universe).
507
+ Primary rule: the highest-ranked KNOWN quantization among the maker's
508
+ endpoints -- matched by normalized author-slug equality only, so partner
509
+ hosts never pin the maker's default (section 4.2 criterion 2 does not
510
+ apply here). Fallback: when the maker has no known quantization, the
511
+ highest-ranked known quantization across the FULL universe (partners
512
+ included -- best-available). None when no endpoint in the universe has a
513
+ known quantization (the all-unknown closed-model case): eligibility then
514
+ relies solely on the first-party carve-out.
515
+
516
+ Known means a PRECISION_LADDER key: "unknown" and unparseable/absent
517
+ values never enter the ladder. An endpoint whose provider_name is
518
+ absent/empty (or normalizes to empty) is skipped for maker matching
519
+ (never first-party -- fail-closed), but still feeds the fallback
520
+ universe.
521
+ """
522
+ universe = _baseline_universe(endpoints, _SCHEDULE_GATE_ARGS)
523
+ maker = normalize_provider(author)
524
+ maker_best = None
525
+ fallback_best = None
526
+ for endpoint in universe:
527
+ quantization = endpoint.get("quantization")
528
+ if not isinstance(quantization, str) or quantization not in PRECISION_LADDER:
529
+ continue # "unknown", null, unrecognized: no rank, no pinning
530
+ rank = PRECISION_LADDER[quantization]
531
+ if fallback_best is None or rank > fallback_best[0]:
532
+ fallback_best = (rank, quantization)
533
+ # Maker matching only: a provider_name that normalizes to nothing
534
+ # must never match an (equally empty) author slug.
535
+ provider_slug = (
536
+ normalize_provider(endpoint.get("provider_name"))
537
+ if isinstance(endpoint.get("provider_name"), str)
538
+ else ""
539
+ )
540
+ if (
541
+ provider_slug
542
+ and provider_slug == maker
543
+ and (maker_best is None or rank > maker_best[0])
544
+ ):
545
+ maker_best = (rank, quantization)
546
+ return (maker_best or fallback_best or (None, None))[1]
547
+
548
+
549
+ def baseline_eligible(endpoint, default_precision, author, args) -> bool:
550
+ """Baseline-eligibility rule (design section 4.3, fail-closed).
551
+
552
+ True iff the endpoint is live (status == 0), not a variant, passes the
553
+ schedule gate, and then either:
554
+
555
+ * its quantization is a known ladder label with rank >=
556
+ rank(default_precision) -- provider-blind (default_routing considers
557
+ every endpoint OpenRouter may send traffic to), or
558
+ * its quantization is "unknown" or null AND the endpoint is first-party
559
+ (the D3 carve-out: the maker or a partner host).
560
+
561
+ When default_precision is None (all-unknown model) the rank comparison is
562
+ meaningless -- only the carve-out applies. A provider_name that is
563
+ absent/empty (or normalizes to empty) is never first-party, so such an
564
+ endpoint can only qualify through the rank branch. An unrecognized
565
+ quantization string (neither a ladder label nor "unknown"/null) is a
566
+ schema anomaly: no rank comparison and no carve-out -- fail-closed
567
+ (fleet-wide new-label drift is what the publish drop-rate gate exists
568
+ to catch).
569
+ """
570
+ if (
571
+ endpoint.get("status") != 0
572
+ or is_variant_endpoint(endpoint)
573
+ or not endpoint_schedule_ok(endpoint, args)
574
+ ):
575
+ return False
576
+ quantization = endpoint.get("quantization")
577
+ if isinstance(quantization, str) and quantization in PRECISION_LADDER:
578
+ # Known quantization: the rank branch, provider-blind.
579
+ if default_precision is None:
580
+ return False
581
+ return PRECISION_LADDER[quantization] >= PRECISION_LADDER[default_precision]
582
+ # Unverifiable: the carve-out applies to "unknown" and null only --
583
+ # an unrecognized string fails closed. A provider_name that normalizes
584
+ # to nothing is never first-party (fail-closed; the literal
585
+ # is_first_party("", "") iff is guarded out here).
586
+ provider_name = endpoint.get("provider_name")
587
+ provider_slug = (
588
+ normalize_provider(provider_name) if isinstance(provider_name, str) else ""
589
+ )
590
+ return (
591
+ (quantization is None or quantization == "unknown")
592
+ and bool(provider_slug)
593
+ and is_first_party(provider_name, author)
594
+ )
595
+
596
+
597
+ def pick_baseline(
598
+ endpoints: list[dict], eligible_flags: list[bool], args
599
+ ) -> dict | None:
600
+ """Pick the baseline endpoint among the eligible ones (design section 4.5).
601
+
602
+ Cheapest blended per-token price wins, computed from effective_pricing
603
+ with the args' min_context and input_share (the shared --input-share
604
+ convention). Ties broken by (normalized provider name, tag), then --
605
+ still tied -- first in list order (deterministic; Python's sort is
606
+ stable and the endpoints list order is the API's). Returns None when no
607
+ eligible endpoint has a usable price (the caller owns the drop).
608
+ """
609
+ candidates = []
610
+ for endpoint, eligible in zip(endpoints, eligible_flags):
611
+ if not eligible:
612
+ continue
613
+ eff = effective_pricing(
614
+ endpoint.get("pricing"), args.min_context, args.input_share
615
+ )
616
+ price_in, price_out = eff["price_in"], eff["price_out"]
617
+ if (
618
+ price_in is None
619
+ or price_out is None
620
+ or not math.isfinite(price_in)
621
+ or not math.isfinite(price_out)
622
+ ):
623
+ continue # unpriceable: no executable blended price
624
+ blended = args.input_share * price_in + (1 - args.input_share) * price_out
625
+ provider_name = endpoint.get("provider_name")
626
+ candidates.append(
627
+ (
628
+ blended,
629
+ normalize_provider(provider_name)
630
+ if isinstance(provider_name, str)
631
+ else "",
632
+ endpoint.get("tag") or "",
633
+ endpoint,
634
+ )
635
+ )
636
+ if not candidates:
637
+ return None
638
+ return min(candidates, key=lambda candidate: candidate[:3])[3]
639
+
640
+
641
+ def _blended_price(endpoint, args) -> float | None:
642
+ """Blended per-token price of one endpoint (pick_baseline's price rule).
643
+
644
+ effective_pricing with the args' min_context and input_share, then
645
+ input_share * price_in + (1 - input_share) * price_out. None when either
646
+ resolved price is missing or non-finite -- no executable blended price,
647
+ the same skip pick_baseline applies. A zero result is returned as-is:
648
+ zero-priced endpoints are variants (already excluded upstream), so the
649
+ callers apply their own zero guards defensively.
650
+ """
651
+ eff = effective_pricing(endpoint.get("pricing"), args.min_context, args.input_share)
652
+ price_in, price_out = eff["price_in"], eff["price_out"]
653
+ if (
654
+ price_in is None
655
+ or price_out is None
656
+ or not math.isfinite(price_in)
657
+ or not math.isfinite(price_out)
658
+ ):
659
+ return None
660
+ blended = args.input_share * price_in + (1 - args.input_share) * price_out
661
+ return blended if math.isfinite(blended) else None
662
+
663
+
664
+ def _flag_list(endpoints: list[dict], eligible_flags: list[bool]):
665
+ """Zip endpoints with their parallel eligibility flags, hardened.
666
+
667
+ eligible_flags is a parallel list to endpoints (the baseline_eligible
668
+ verdict per endpoint, same order -- Task 4 contract). A length mismatch
669
+ must raise, never truncate silently (Task 4 review ruling).
670
+ """
671
+ if len(endpoints) != len(eligible_flags):
672
+ raise ValueError(
673
+ "eligible_flags must be a parallel list to endpoints: "
674
+ f"{len(endpoints)} endpoints, {len(eligible_flags)} flags"
675
+ )
676
+ return zip(endpoints, eligible_flags, strict=True)
677
+
678
+
679
+ def NON_ELIGIBLE_UNIVERSE(endpoints: list[dict], eligible_flags: list[bool]) -> list:
680
+ """The shared annotations universe (design section 4.6).
681
+
682
+ Live (status == 0), non-variant endpoints that are NOT baseline-eligible:
683
+ sub-default known quantization, third-party unknown/null quantization, or
684
+ schedule-gate-failed. All annotations derive from this one universe.
685
+ Eligibility comes from the parallel eligible_flags list (the
686
+ baseline_eligible verdicts); variants and non-live endpoints are excluded
687
+ even though they too are not eligible -- unlike the steering `ignore`
688
+ list, which includes live variants (section 4.7).
689
+
690
+ Raises ValueError when eligible_flags does not match endpoints in length.
691
+ """
692
+ return [
693
+ endpoint
694
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags)
695
+ if endpoint.get("status") == 0
696
+ and not is_variant_endpoint(endpoint)
697
+ and not eligible
698
+ ]
699
+
700
+
701
+ def build_annotations(
702
+ endpoints: list[dict],
703
+ eligible_flags: list[bool],
704
+ baseline,
705
+ catalog_context,
706
+ args,
707
+ ) -> dict:
708
+ """Per-model endpoint annotations (design section 4.6).
709
+
710
+ All four annotations derive from one shared universe -- the live
711
+ non-variant endpoints that are not baseline-eligible
712
+ (NON_ELIGIBLE_UNIVERSE):
713
+
714
+ * price_spread: cheapest non-eligible blended price divided by the
715
+ cheapest eligible blended price, both blended with the args'
716
+ min_context and input_share (the shared --input-share convention).
717
+ None when the universe is empty -- generalized to None when either
718
+ side has no priceable member at all -- and None when either cheapest
719
+ price is 0 (zero-priced endpoints are variants by definition, so the
720
+ zero half of the guard is defensive only). Unpriceable endpoints are
721
+ skipped when looking for a cheapest price.
722
+ * cheapest_quantized: {provider, quantization, price} of the cheapest
723
+ member of the quantization-caused subset of the universe -- endpoints
724
+ non-eligible due to quantization only (sub-default known quantization
725
+ or third-party unknown/null), NOT schedule-gate-failed ones, which are
726
+ excluded even when cheaper. Unpriceable subset members are skipped
727
+ (defensively also non-positive ones, matching the repo-wide treatment
728
+ of negative prices as malformed); None when the subset has no
729
+ priceable member. Price ties keep the first in list order
730
+ (deterministic).
731
+ * risk_flag: True iff the sum of 1/blended**2 over the universe strictly
732
+ exceeds the sum over the baseline-eligible endpoints (exact tie ->
733
+ False). Any endpoint whose blended price is 0/None (defensively also
734
+ negative) is skipped from BOTH sums -- variants are already excluded,
735
+ so this only guards malformed data, and the flag stays deterministic.
736
+ * baseline_context_below_catalog: True iff the baseline endpoint's
737
+ context_length is smaller than the catalog context (the models-API
738
+ value). A None baseline -- or missing context on either side -- yields
739
+ False: the caveat marks a positive finding, never missing data.
740
+
741
+ Raises ValueError when eligible_flags does not match endpoints in length.
742
+ """
743
+ universe = NON_ELIGIBLE_UNIVERSE(endpoints, eligible_flags)
744
+
745
+ universe_prices = []
746
+ for endpoint in universe:
747
+ price = _blended_price(endpoint, args)
748
+ if price is not None:
749
+ universe_prices.append(price)
750
+ eligible_prices = []
751
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
752
+ if not eligible:
753
+ continue
754
+ price = _blended_price(endpoint, args)
755
+ if price is not None:
756
+ eligible_prices.append(price)
757
+ cheapest_non_eligible = min(universe_prices) if universe_prices else None
758
+ cheapest_eligible = min(eligible_prices) if eligible_prices else None
759
+ price_spread = (
760
+ cheapest_non_eligible / cheapest_eligible
761
+ if cheapest_non_eligible is not None
762
+ and cheapest_eligible is not None
763
+ and cheapest_non_eligible > 0
764
+ and cheapest_eligible > 0
765
+ else None
766
+ )
767
+
768
+ quantized_best = None
769
+ for endpoint in universe:
770
+ if not endpoint_schedule_ok(endpoint, args):
771
+ continue # schedule-gate-failed: not a quantization-caused exclusion
772
+ price = _blended_price(endpoint, args)
773
+ if price is None or price <= 0:
774
+ continue # unpriceable/non-positive: no comparable price
775
+ if quantized_best is None or price < quantized_best[0]:
776
+ quantized_best = (price, endpoint) # ties keep the first in list order
777
+ if quantized_best is None:
778
+ cheapest_quantized = None
779
+ else:
780
+ price, endpoint = quantized_best
781
+ cheapest_quantized = {
782
+ "provider": endpoint.get("provider_name"),
783
+ "quantization": endpoint.get("quantization"),
784
+ "price": price,
785
+ }
786
+
787
+ non_eligible_weight = 0.0
788
+ for endpoint in universe:
789
+ price = _blended_price(endpoint, args)
790
+ if price is None or price <= 0:
791
+ continue # skipped from BOTH sums, deterministically
792
+ non_eligible_weight += 1.0 / (price * price)
793
+ eligible_weight = 0.0
794
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
795
+ if not eligible:
796
+ continue
797
+ price = _blended_price(endpoint, args)
798
+ if price is None or price <= 0:
799
+ continue
800
+ eligible_weight += 1.0 / (price * price)
801
+ risk_flag = non_eligible_weight > eligible_weight
802
+
803
+ baseline_context = baseline.get("context_length") if baseline is not None else None
804
+ baseline_context_below_catalog = (
805
+ baseline_context is not None
806
+ and catalog_context is not None
807
+ and baseline_context < catalog_context
808
+ )
809
+
810
+ return {
811
+ "price_spread": price_spread,
812
+ "cheapest_quantized": cheapest_quantized,
813
+ "risk_flag": risk_flag,
814
+ "baseline_context_below_catalog": baseline_context_below_catalog,
815
+ }
816
+
817
+
818
+ def build_steering(
819
+ model_id: str,
820
+ endpoints: list[dict],
821
+ eligible_flags: list[bool],
822
+ default_precision: str | None,
823
+ args,
824
+ ) -> dict:
825
+ """OpenRouter provider steering object (design section 4.7).
826
+
827
+ {"model": model_id, "provider": {"order": [...], "ignore": [...],
828
+ "quantizations": [...], "allow_fallbacks": True}}.
829
+
830
+ * order: tags of the baseline-eligible endpoints, cheapest blended first
831
+ (blended price asc, ties broken by (normalized provider name, tag) --
832
+ the same comparator as pick_baseline, stable so the final tie is list
833
+ order). Eligible endpoints whose blended price is unpriceable cannot
834
+ be price-ranked; they are appended after the ranked ones (still
835
+ (provider, tag)-ordered) so every eligible endpoint keeps a place --
836
+ ranked last, never silently left to default priority.
837
+ * ignore: tags of EVERY live endpoint that is not baseline-eligible --
838
+ sub-default quantization, third-party unknown/null, schedule-gate-
839
+ failed AND variants (variants are live but not eligible; the universal
840
+ quantifier includes them). Non-live (status != 0) endpoints are never
841
+ listed. List order (the API's).
842
+ * quantizations: sorted KNOWN quantizations (PRECISION_LADDER labels)
843
+ with rank >= rank(default_precision) among live non-variant endpoints,
844
+ deduplicated -- no schedule or eligibility filter. OMITTED from the
845
+ provider dict entirely when default_precision is None -- never an
846
+ empty list (OpenRouter semantics would filter out every endpoint).
847
+ * allow_fallbacks: True.
848
+
849
+ Signature deviation from the plan (controller ruling at Task 5 spawn):
850
+ args is a required parameter -- the order ranking is real pricing math
851
+ (effective_pricing via min_context/input_share), and reusing the
852
+ shape-only _SCHEDULE_GATE_ARGS stand-in for pricing was explicitly
853
+ ruled out in the Task 4 review.
854
+
855
+ Raises ValueError when eligible_flags does not match endpoints in length.
856
+ """
857
+ ignore = []
858
+ ranked = [] # (blended, normalized provider name, tag) -- pick_baseline order
859
+ unranked = [] # (normalized provider name, tag) -- eligible but unpriceable
860
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
861
+ if endpoint.get("status") == 0 and not eligible:
862
+ ignore.append(endpoint.get("tag") or "")
863
+ if not eligible:
864
+ continue
865
+ provider_name = endpoint.get("provider_name")
866
+ provider_key = (
867
+ normalize_provider(provider_name) if isinstance(provider_name, str) else ""
868
+ )
869
+ tag = endpoint.get("tag") or ""
870
+ price = _blended_price(endpoint, args)
871
+ if price is None:
872
+ unranked.append((provider_key, tag))
873
+ else:
874
+ ranked.append((price, provider_key, tag))
875
+ ranked.sort() # stable: the final tie is list order, as in pick_baseline
876
+ order = [tag for _, _, tag in ranked] + [tag for _, tag in sorted(unranked)]
877
+
878
+ provider = {"order": order, "ignore": ignore}
879
+ if default_precision is not None:
880
+ default_rank = PRECISION_LADDER[default_precision]
881
+ quants = set()
882
+ for endpoint in endpoints:
883
+ if endpoint.get("status") != 0 or is_variant_endpoint(endpoint):
884
+ continue
885
+ quantization = endpoint.get("quantization")
886
+ if (
887
+ not isinstance(quantization, str)
888
+ or quantization not in PRECISION_LADDER
889
+ ):
890
+ continue # "unknown"/null/unrecognized: not a known quantization
891
+ if PRECISION_LADDER[quantization] >= default_rank:
892
+ quants.add(quantization)
893
+ provider["quantizations"] = sorted(quants)
894
+ provider["allow_fallbacks"] = True
895
+ return {"model": model_id, "provider": provider}
896
+
897
+
898
+ def _stale_filtered_annotations(model_id, catalog_context, args):
899
+ """Annotations for a dropped model's filtered entry, or None.
900
+
901
+ Built from the model's last cached endpoint list (load_cache_expired --
902
+ TTL ignored), never from a fresh fetch: the model is out of the ranking,
903
+ so the block is marked "stale": True for display in the filtered section
904
+ only (design section 4.9). Same builders as the live stage: eligibility
905
+ and baseline are re-derived from the cached list, so the shape matches a
906
+ ranked candidate's annotations. A model whose cache holds no endpoint
907
+ list at all yields None (the entry carries the drop reason alone).
908
+ """
909
+ cached = load_cache_expired(f"endpoints:{model_id.replace('/', '_')}")
910
+ if not isinstance(cached, list):
911
+ return None
912
+ author = model_id.partition("/")[0]
913
+ try:
914
+ default_precision = derive_default_precision(cached, author)
915
+ eligible_flags = [
916
+ baseline_eligible(endpoint, default_precision, author, args)
917
+ for endpoint in cached
918
+ ]
919
+ baseline = pick_baseline(cached, eligible_flags, args)
920
+ annotations = build_annotations(
921
+ cached, eligible_flags, baseline, catalog_context, args
922
+ )
923
+ except Exception:
924
+ # Display-only block for an already-dropped model: never fail the run.
925
+ return None
926
+ annotations["stale"] = True
927
+ return annotations
928
+
929
+
930
+ def _endpoint_stage_drop(extra_dropped, filtered, cand, reason, args):
931
+ """Count one endpoint-stage drop and append its filtered entry.
932
+
933
+ Same validated entry shape as build_candidates' drop() plus the optional
934
+ stale annotations block (absent cache -> reasons only).
935
+ """
936
+ extra_dropped[reason] = extra_dropped.get(reason, 0) + 1
937
+ entry = {"id": cand["id"], "name": cand["name"] or cand["id"], "reasons": [reason]}
938
+ annotations = _stale_filtered_annotations(cand["id"], cand.get("context"), args)
939
+ if annotations is not None:
940
+ entry["annotations"] = annotations
941
+ filtered.append(entry)
942
+
943
+
944
+ def apply_endpoint_pricing(candidates, args, filtered):
945
+ """Reprice every candidate at its baseline endpoint (design section 4.9).
946
+
947
+ The CLI stage between build_candidates and resolve_quality. Endpoint
948
+ fetches run concurrently (ThreadPoolExecutor, 8 workers) through
949
+ fetch_model_endpoints' per-model cache, but the results are applied PER
950
+ CANDIDATE ID in the original candidates order -- never in completion
951
+ order -- so the stage is deterministic.
952
+
953
+ Per candidate: endpoints = fetch. On fetch failure after fetch_json's
954
+ internal retries the model is dropped "endpoint_fetch_failed" and removed
955
+ from the returned list. With endpoints, default_precision is derived,
956
+ every endpoint scored for baseline eligibility and the baseline picked;
957
+ no usable baseline drops the model "no_baseline_eligible_endpoint"
958
+ (the stale-annotations filtered-entry rule covers both drop reasons).
959
+ Otherwise the candidate is mutated in place: the pricing block
960
+ (price_in/price_out/blended/base_*/tier/schedule/notes) is replaced with
961
+ the baseline endpoint's effective_pricing outputs scaled per-1M exactly
962
+ like build_candidates, and the additive keys "endpoints", "default_precision",
963
+ "annotations" (catalog context = the candidate's existing "context"),
964
+ "advertised_pricing" (the models-API scalars the candidate carried before
965
+ the mutation, source-tagged) and "steering" are attached.
966
+
967
+ A baseline whose prices are missing/non-finite counts as unusable
968
+ (pick_baseline's blended-price skip is authoritative for the top-level
969
+ effective prices). The same bar applies to the base prices the mutation
970
+ would publish: parse_price returns float | None, so a None base price
971
+ (tier-over-None-base endpoint) is reachable -- the isinstance(..., float)
972
+ half of the guard is load-bearing for None, not a redundant type check.
973
+ Returns (surviving candidates in the original order, dropped counts for
974
+ the two new reasons).
975
+ """
976
+ extra_dropped: dict[str, int] = {}
977
+ if not candidates:
978
+ return candidates, extra_dropped
979
+ endpoints_by_id: dict[str, list] = {}
980
+ failed_ids: set[str] = set()
981
+ with ThreadPoolExecutor(max_workers=8) as pool:
982
+ futures = {}
983
+ for cand in candidates:
984
+ # Duplicate ids fetch once and share the result.
985
+ futures.setdefault(
986
+ cand["id"], pool.submit(fetch_model_endpoints, cand["id"], args)
987
+ )
988
+ for cand_id, future in futures.items():
989
+ try:
990
+ endpoints_by_id[cand_id] = future.result()
991
+ except Exception:
992
+ # fetch_json already retried 3 times; the caller owns the
993
+ # fail-closed policy.
994
+ failed_ids.add(cand_id)
995
+ survivors = []
996
+ for cand in candidates:
997
+ cand_id = cand["id"]
998
+ if cand_id in failed_ids:
999
+ _endpoint_stage_drop(
1000
+ extra_dropped, filtered, cand, "endpoint_fetch_failed", args
1001
+ )
1002
+ continue
1003
+ endpoints = endpoints_by_id[cand_id]
1004
+ author = cand_id.partition("/")[0]
1005
+ default_precision = derive_default_precision(endpoints, author)
1006
+ eligible_flags = [
1007
+ baseline_eligible(endpoint, default_precision, author, args)
1008
+ for endpoint in endpoints
1009
+ ]
1010
+ baseline = pick_baseline(endpoints, eligible_flags, args)
1011
+ eff = (
1012
+ effective_pricing(
1013
+ baseline.get("pricing"), args.min_context, args.input_share
1014
+ )
1015
+ if baseline is not None
1016
+ else None
1017
+ )
1018
+ if baseline is None or not all(
1019
+ isinstance(eff[key], float) and math.isfinite(eff[key])
1020
+ for key in ("price_in", "price_out", "base_price_in", "base_price_out")
1021
+ ):
1022
+ _endpoint_stage_drop(
1023
+ extra_dropped, filtered, cand, "no_baseline_eligible_endpoint", args
1024
+ )
1025
+ continue
1026
+ # The advertised models-API scalars, captured before the mutation.
1027
+ advertised_pricing = {
1028
+ "input_per_1m": cand["price_in"],
1029
+ "output_per_1m": cand["price_out"],
1030
+ "blended_per_1m": cand["blended"],
1031
+ "source": "models-api",
1032
+ }
1033
+ # Replace the pricing block with the baseline endpoint's outputs,
1034
+ # scaled per-1M exactly like build_candidates.
1035
+ cand["price_in"] = eff["price_in"] * 1_000_000.0
1036
+ cand["price_out"] = eff["price_out"] * 1_000_000.0
1037
+ cand["blended"] = (
1038
+ args.input_share * cand["price_in"]
1039
+ + (1.0 - args.input_share) * cand["price_out"]
1040
+ )
1041
+ cand["base_price_in"] = eff["base_price_in"] * 1_000_000.0
1042
+ cand["base_price_out"] = eff["base_price_out"] * 1_000_000.0
1043
+ cand["tier_prompt_tokens"] = eff["tier_prompt_tokens"]
1044
+ cand["tiers"] = eff["tiers"]
1045
+ cand["schedule"] = eff["schedule"]
1046
+ cand["max_discount"] = eff["max_discount"]
1047
+ cand["tier_note"] = fmt_tier_note(eff["tier_prompt_tokens"])
1048
+ cand["sched_note"] = fmt_sched_note(eff["max_discount"])
1049
+ cand["sched_detail"] = fmt_sched_detail(eff["schedule"], eff["peak_blended"])
1050
+ cand["endpoints"] = endpoints
1051
+ cand["default_precision"] = default_precision
1052
+ cand["annotations"] = build_annotations(
1053
+ endpoints, eligible_flags, baseline, cand["context"], args
1054
+ )
1055
+ cand["advertised_pricing"] = advertised_pricing
1056
+ cand["steering"] = build_steering(
1057
+ cand_id, endpoints, eligible_flags, default_precision, args
1058
+ )
1059
+ survivors.append(cand)
1060
+ return survivors, extra_dropped
1061
+
1062
+
1063
+ # ---------------------------------------------------------------------------
1064
+ # OpenRouter session-cost dataset (per-task cost, outside the score)
1065
+ # ---------------------------------------------------------------------------
1066
+
1067
+
1068
+ SESSION_COST_URL = "https://openrouter.ai/api/v1/datasets/session-cost"
1069
+ SESSION_COST_HARNESS_SLUG = "hermes-agent"
1070
+ # The turn range the catalog/CLI surfaces (Task 8).
1071
+ SESSION_COST_DISPLAY_TURN_RANGE = "10-49-turns"
1072
+ # The dataset's four turn ranges (brainstorm section 2.6) -- the catalog
1073
+ # contract's closed set (web/build_site_data.py pins the same tuple).
1074
+ # Cells naming anything else are dropped: an unknown range is dataset
1075
+ # drift, and the block must never fail the publish (design section 4.8).
1076
+ SESSION_COST_TURN_RANGES = ("1-turn", "2-9-turns", "10-49-turns", "50-plus-turns")
1077
+ SESSION_COST_ATTRIBUTION = "Data © OpenRouter, CC BY 4.0"
1078
+
1079
+
1080
+ def strip_permaslug_date(permaslug: str) -> str:
1081
+ """Permaslug without its trailing -YYYYMMDD (undated ids map to themselves)."""
1082
+ match = re.match(r"^(.*)-(\d{8})$", permaslug)
1083
+ return match.group(1) if match else permaslug
1084
+
1085
+
1086
+ def fetch_session_cost(args) -> dict | None:
1087
+ """Per-task session-cost medians from OpenRouter's dataset, or None.
1088
+
1089
+ A separate published dimension (design section 4.8): the median USD
1090
+ cost of real sessions through the pinned published harness -- never
1091
+ part of the weighted score. One bulk paginated fetch per invocation
1092
+ (?app_slug=<harness>&limit=500&offset=k) regardless of the model list;
1093
+ pagination stops on a page shorter than the limit or once the next
1094
+ offset would exceed the API's 5000 cap. Cells key on dated permaslugs
1095
+ (z-ai/glm-5.3-flash-20260826), stripped to the bare catalog id with
1096
+ strip_permaslug_date; when several strip to the same id, the latest
1097
+ date wins each turn-range cell (same rule as build_aa_benchmarks).
1098
+
1099
+ Returns None (graceful degradation, design section 4.9) when no key is
1100
+ configured (the --openrouter-api-key flag, else OPENROUTER_API_KEY),
1101
+ when the fetch fails after fetch_json's retries, or when the response
1102
+ shape is unexpected -- a malformed payload must never crash the run.
1103
+ The merged result is cached under "session-cost" with the standard TTL.
1104
+ """
1105
+ api_key = args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
1106
+ if not api_key:
1107
+ return None
1108
+ if not args.no_cache:
1109
+ cached = load_cache("session-cost", args.cache_ttl)
1110
+ if isinstance(cached, dict) and isinstance(cached.get("medians"), dict):
1111
+ return cached
1112
+ cells = []
1113
+ # The API caps limit at 500 and offset at 5000 (11 full pages max).
1114
+ offset = 0
1115
+ while offset <= 5000:
1116
+ try:
1117
+ payload = fetch_json(
1118
+ f"{SESSION_COST_URL}?app_slug={SESSION_COST_HARNESS_SLUG}"
1119
+ f"&limit=500&offset={offset}",
1120
+ headers={"Authorization": f"Bearer {api_key}"},
1121
+ timeout=30,
1122
+ retries=3,
1123
+ )
1124
+ except Exception as exc:
1125
+ warn(f"session-cost dataset request failed ({exc})")
1126
+ return None
1127
+ data = payload.get("data") if isinstance(payload, dict) else None
1128
+ if not isinstance(data, list):
1129
+ warn("session-cost dataset returned an unexpected response shape")
1130
+ return None
1131
+ cells.extend(cell for cell in data if isinstance(cell, dict))
1132
+ if len(data) < 500:
1133
+ break
1134
+ offset += 500
1135
+ medians: dict[str, dict[str, float]] = {}
1136
+ seen_date: dict[tuple[str, str], str] = {}
1137
+
1138
+ def accept(bare, date, turn_range, median):
1139
+ key = (bare, turn_range)
1140
+ if key in seen_date and seen_date[key] >= date:
1141
+ return
1142
+ seen_date[key] = date
1143
+ medians.setdefault(bare, {})[turn_range] = median
1144
+
1145
+ for cell in cells:
1146
+ permaslug = cell.get("model_permaslug")
1147
+ turn_range = cell.get("turn_range")
1148
+ raw = cell.get("median_session_cost_usd")
1149
+ if not (
1150
+ isinstance(permaslug, str)
1151
+ and permaslug
1152
+ and isinstance(turn_range, str)
1153
+ and turn_range
1154
+ ):
1155
+ continue
1156
+ # Closed contract set: a cell naming a new turn range is dropped,
1157
+ # never published (the catalog validator rejects unknown ranges,
1158
+ # and session cost must never fail the publish).
1159
+ if turn_range not in SESSION_COST_TURN_RANGES:
1160
+ continue
1161
+ # Non-finite medians would serialize as NaN and poison the catalog.
1162
+ if not isinstance(raw, (int, float)) or isinstance(raw, bool):
1163
+ continue
1164
+ try:
1165
+ median = float(raw) # huge ints raise OverflowError
1166
+ except OverflowError:
1167
+ continue
1168
+ if not math.isfinite(median):
1169
+ continue
1170
+ match = re.match(r"^(.*)-(\d{8})$", permaslug)
1171
+ accept(
1172
+ strip_permaslug_date(permaslug),
1173
+ match.group(2) if match else "",
1174
+ turn_range,
1175
+ median,
1176
+ )
1177
+ if not medians:
1178
+ warn("session-cost dataset returned no usable cells")
1179
+ return None
1180
+ meta = payload.get("meta") if isinstance(payload, dict) else None
1181
+ if not isinstance(meta, dict):
1182
+ warn("session-cost dataset returned an unexpected response shape")
1183
+ return None
1184
+ as_of = meta.get("as_of")
1185
+ window_days = meta.get("window_days")
1186
+ window_end_date = meta.get("window_end_date")
1187
+ # The catalog validator demands non-empty as_of/window_end_date strings
1188
+ # and a positive integer window_days; anything else is dataset drift.
1189
+ # Degrade to None (block omitted, "not available this refresh") rather
1190
+ # than emit a block the validator would reject -- session cost must
1191
+ # never fail the publish (design section 4.8).
1192
+ if not (
1193
+ isinstance(as_of, str)
1194
+ and as_of
1195
+ and isinstance(window_end_date, str)
1196
+ and window_end_date
1197
+ and isinstance(window_days, int)
1198
+ and not isinstance(window_days, bool)
1199
+ and window_days > 0
1200
+ ):
1201
+ warn("session-cost dataset returned malformed meta; omitting the block")
1202
+ return None
1203
+ result = {
1204
+ "attribution": SESSION_COST_ATTRIBUTION,
1205
+ "app_slug": SESSION_COST_HARNESS_SLUG,
1206
+ "as_of": as_of,
1207
+ "window_days": window_days,
1208
+ "window_end_date": window_end_date,
1209
+ "medians": medians,
1210
+ }
1211
+ save_cache("session-cost", result)
1212
+ return result
1213
+
1214
+
1215
+ # ---------------------------------------------------------------------------
1216
+ # Published catalog quality (hydration for unkeyed runs)
1217
+ # ---------------------------------------------------------------------------
1218
+
1219
+
1220
+ # Pre-reconciliation published catalogs (schema v3, written before the AA
1221
+ # tier was relabeled) record the AA tier as "api"; hydration normalizes it
1222
+ # to the final "aa" label. Every other recorded label passes through as-is.
1223
+ _QUALITY_SOURCE_NORMALIZATION = {"api": "aa"}
1224
+
1225
+
1226
+ def catalog_quality_by_id(doc) -> dict:
1227
+ """Pure parser: a published catalog document -> {id: quality record}.
1228
+
1229
+ No fetch, no cache, no args object -- the fetcher owns transport and
1230
+ freshness; this owns shape. Records are {"index", "source", "as_of"}
1231
+ and feed resolve_quality's catalog_quality parameter on unkeyed runs;
1232
+ the values were produced by a keyed CI run, so hydration is an exact
1233
+ catalog-id join with no re-matching.
1234
+
1235
+ Gates (design: failure modes):
1236
+ - schema_version must be an int >= 3; anything older warns
1237
+ "published catalog schema vX, expected >= 3; skipping hydration"
1238
+ and yields {}. Accepting v3 is the deploy-day skew case: the live
1239
+ artifact predates the v4 per-row quality_as_of.
1240
+ - Any shape surprise -- a non-dict document, a non-int schema_version,
1241
+ a non-list models, a row without a usable id, or a matched row whose
1242
+ quality is not a finite number or whose quality_match is not a
1243
+ non-empty string -- warns and yields {} for the WHOLE artifact.
1244
+ Never a partial hydration, never a raise: the catalog is
1245
+ machine-generated, so one malformed row means this is not the
1246
+ artifact we think it is.
1247
+ - Rows with quality None and quality_match None are legitimate
1248
+ unmatched models; they are skipped silently (nothing to hydrate).
1249
+
1250
+ Normalization: v3 rows map quality_match "api" -> "aa" and date every
1251
+ row with the catalog's generated_at -- the live v3 artifact carries
1252
+ quality_as_of: null, which must be ignored, not passed through.
1253
+ v4+ rows keep their recorded quality_as_of (None when absent) and
1254
+ source label as-is: that label is the row-level truth inside the
1255
+ published artifact (a keyed CI run emits "aa"/"openrouter", never
1256
+ "catalog"). resolve_quality's catalog tier owns the run-level label:
1257
+ it relabels every hydrated record to the chain label "catalog".
1258
+ """
1259
+ if not isinstance(doc, dict):
1260
+ warn("published catalog returned an unexpected response shape")
1261
+ return {}
1262
+ schema_version = doc.get("schema_version")
1263
+ if type(schema_version) is not int:
1264
+ warn("published catalog returned an unexpected response shape")
1265
+ return {}
1266
+ if schema_version < 3:
1267
+ warn(
1268
+ f"published catalog schema v{schema_version}, expected >= 3; "
1269
+ "skipping hydration"
1270
+ )
1271
+ return {}
1272
+ models = doc.get("models")
1273
+ if not isinstance(models, list):
1274
+ warn("published catalog returned an unexpected response shape")
1275
+ return {}
1276
+ generated_at = doc.get("generated_at")
1277
+ v3_as_of = generated_at if isinstance(generated_at, str) and generated_at else None
1278
+ by_id: dict = {}
1279
+ for entry in models:
1280
+ if not isinstance(entry, dict):
1281
+ warn("published catalog returned an unexpected response shape")
1282
+ return {}
1283
+ model_id = entry.get("id")
1284
+ if not (isinstance(model_id, str) and model_id):
1285
+ warn("published catalog returned an unexpected response shape")
1286
+ return {}
1287
+ quality = entry.get("quality")
1288
+ match = entry.get("quality_match")
1289
+ if match is None and quality is None:
1290
+ continue # legitimate unmatched row: nothing to hydrate
1291
+ if not (isinstance(match, str) and match):
1292
+ warn("published catalog returned an unexpected response shape")
1293
+ return {}
1294
+ if not isinstance(quality, (int, float)) or isinstance(quality, bool):
1295
+ warn("published catalog returned an unexpected response shape")
1296
+ return {}
1297
+ try:
1298
+ index = float(quality) # huge ints raise OverflowError
1299
+ except OverflowError:
1300
+ warn("published catalog returned an unexpected response shape")
1301
+ return {}
1302
+ # Non-finite indexes would serialize as NaN and poison quality.
1303
+ if not math.isfinite(index):
1304
+ warn("published catalog returned an unexpected response shape")
1305
+ return {}
1306
+ # The AA indexes run 0..100 (the range the catalog validator
1307
+ # enforces on quality); an out-of-range value is poisoned data and
1308
+ # degrades the whole artifact. The surface is unkeyed runs: only
1309
+ # they hydrate from the published catalog.
1310
+ if not 0.0 <= index <= 100.0:
1311
+ warn("published catalog returned an unexpected response shape")
1312
+ return {}
1313
+ if schema_version == 3:
1314
+ as_of = v3_as_of
1315
+ else:
1316
+ recorded = entry.get("quality_as_of")
1317
+ as_of = recorded if isinstance(recorded, str) and recorded else None
1318
+ by_id[model_id] = {
1319
+ "index": index,
1320
+ "source": _QUALITY_SOURCE_NORMALIZATION.get(match, match),
1321
+ "as_of": as_of,
1322
+ }
1323
+ return by_id
1324
+
1325
+
1326
+ def fetch_quality_catalog(args) -> dict | None:
1327
+ """Quality records from the published catalog.json, or None.
1328
+
1329
+ The third quality source (design: source 2): the repo's own published
1330
+ catalog artifact, built by keyed CI runs and hydrated by unkeyed local
1331
+ runs. Never called on keyed runs -- a CI/site build hydrating from the
1332
+ published catalog would consume its own output (self-reference loop);
1333
+ Task 4 owns that wiring.
1334
+
1335
+ Returns None only on TRANSPORT failure (after fetch_json's retries);
1336
+ otherwise the envelope {"by_id": {id: record}, "generated_at":
1337
+ str|None, "schema_version": int}. An empty by_id means
1338
+ fetched-but-unusable (schema gate fail, malformed shape) -- Task 4's
1339
+ footer treats None and empty by_id identically ("catalog unavailable").
1340
+
1341
+ The envelope is cached under QUALITY_CATALOG_CACHE with the standard
1342
+ TTL, and only a usable (non-empty by_id) result is ever written: a
1343
+ degraded payload must self-heal on the next run. --no-cache neither
1344
+ reads nor writes. Staleness: generated_at older than
1345
+ QUALITY_CATALOG_MAX_AGE_DAYS warns but uses -- as_of makes the
1346
+ staleness visible downstream (the check runs only when rows actually
1347
+ hydrate -- warning about the age of a catalog we are not using is
1348
+ noise); an unusable generated_at warns that
1349
+ staleness cannot be checked and still uses the rows (their as_of then
1350
+ shows None rather than a wrong date).
1351
+ """
1352
+ if not args.no_cache:
1353
+ cached = load_cache(QUALITY_CATALOG_CACHE, args.cache_ttl)
1354
+ if isinstance(cached, dict) and isinstance(cached.get("by_id"), dict):
1355
+ return cached
1356
+ try:
1357
+ doc = fetch_json(QUALITY_CATALOG_URL, timeout=30, retries=3)
1358
+ except Exception as exc:
1359
+ warn(f"published catalog request failed ({exc})")
1360
+ return None
1361
+ by_id = catalog_quality_by_id(doc)
1362
+ generated_at = doc.get("generated_at") if isinstance(doc, dict) else None
1363
+ if not isinstance(generated_at, str):
1364
+ generated_at = None
1365
+ schema_version = doc.get("schema_version") if isinstance(doc, dict) else 0
1366
+ if type(schema_version) is not int:
1367
+ schema_version = 0
1368
+ result = {
1369
+ "by_id": by_id,
1370
+ "generated_at": generated_at,
1371
+ "schema_version": schema_version,
1372
+ }
1373
+ if by_id:
1374
+ generated = parse_iso_datetime(generated_at)
1375
+ if generated is None:
1376
+ warn(
1377
+ "published catalog generated_at is missing or unparseable; "
1378
+ "cannot check staleness"
1379
+ )
1380
+ else:
1381
+ age_days = (datetime.now(timezone.utc) - generated).total_seconds() / 86400
1382
+ if age_days > QUALITY_CATALOG_MAX_AGE_DAYS:
1383
+ warn(
1384
+ f"published catalog is stale (generated {generated_at}, "
1385
+ f"{age_days:.0f} days old); using it anyway"
1386
+ )
1387
+ if not args.no_cache:
1388
+ save_cache(QUALITY_CATALOG_CACHE, result)
1389
+ return result
1390
+
1391
+
227
1392
  # ---------------------------------------------------------------------------
228
1393
  # OpenRouter catalog
229
1394
  # ---------------------------------------------------------------------------
@@ -243,7 +1408,9 @@ def fetch_openrouter_models(args):
243
1408
 
244
1409
 
245
1410
  FRONTEND_DISCOUNTS_CACHE = "openrouter-frontend-discounts"
246
- FRONTEND_AA_CACHE = "openrouter-frontend-aa"
1411
+ # v2: AA records are keyed by exact catalog slug and carry as_of (#17);
1412
+ # v1 cached the conflated date-stripped shape, so the old key is never read.
1413
+ FRONTEND_AA_CACHE = "openrouter-frontend-aa-v2"
247
1414
 
248
1415
 
249
1416
  def fetch_openrouter_frontend(args):
@@ -301,7 +1468,8 @@ def fetch_openrouter_frontend(args):
301
1468
  key = slug if variant in ("", "standard") else f"{slug}:{variant}"
302
1469
  fresh_discounts.setdefault(key, float(raw))
303
1470
  fresh_aa = build_aa_benchmarks(
304
- data.get("benchmarks") if isinstance(data, dict) else None
1471
+ data.get("benchmarks") if isinstance(data, dict) else None,
1472
+ entries,
305
1473
  )
306
1474
  if fresh_discounts:
307
1475
  discounts = fresh_discounts
@@ -792,7 +1960,7 @@ def parse_iso_datetime(value):
792
1960
  # ---------------------------------------------------------------------------
793
1961
 
794
1962
 
795
- def aa_api_entries(api_key: str) -> list:
1963
+ def aa_api_entries(api_key: str) -> tuple[list, float | int | None]:
796
1964
  """Intelligence entries from AA's V2 language models endpoint.
797
1965
 
798
1966
  Documented envelope: {tier, intelligence_index_version, pagination,
@@ -803,8 +1971,16 @@ def aa_api_entries(api_key: str) -> list:
803
1971
  has_more/total_pages contract with a 25-page hard cap; a failure past
804
1972
  page 1 is best-effort (warn, keep earlier pages), a page-1 failure
805
1973
  propagates.
1974
+
1975
+ Returns (entries, version): version is the envelope's
1976
+ intelligence_index_version (number-or-null) -- provenance for the whole
1977
+ fetch. Every surviving entry carries `as_of`, one ISO fetch timestamp
1978
+ shared by the whole fetch, so the cache and any downstream record can
1979
+ date the data even after the envelope's version is lost.
806
1980
  """
807
1981
  entries = {}
1982
+ version = None
1983
+ as_of = datetime.now(timezone.utc).isoformat()
808
1984
  page = 1
809
1985
  while page <= 25: # hard cap: halt after processing page 25
810
1986
  try:
@@ -819,6 +1995,15 @@ def aa_api_entries(api_key: str) -> list:
819
1995
  warn(f"AA API page {page} failed ({exc}); using entries collected so far")
820
1996
  break
821
1997
  data = payload.get("data") if isinstance(payload, dict) else None
1998
+ # The envelope's version is one provenance fact for the whole
1999
+ # fetch; the last page's value wins (they are identical in practice).
2000
+ raw_version = (
2001
+ payload.get("intelligence_index_version")
2002
+ if isinstance(payload, dict)
2003
+ else None
2004
+ )
2005
+ if isinstance(raw_version, (int, float)) and not isinstance(raw_version, bool):
2006
+ version = raw_version
822
2007
  for item in data if isinstance(data, list) else []:
823
2008
  if not isinstance(item, dict):
824
2009
  continue
@@ -851,6 +2036,10 @@ def aa_api_entries(api_key: str) -> list:
851
2036
  "key": key,
852
2037
  "name": name if isinstance(name, str) and name else key,
853
2038
  "index": index,
2039
+ # One fetch timestamp for the whole fetch: every entry
2040
+ # from this run shares it, so the cache payload and any
2041
+ # downstream record can date the data exactly.
2042
+ "as_of": as_of,
854
2043
  },
855
2044
  )
856
2045
  pag = payload.get("pagination") if isinstance(payload, dict) else None
@@ -862,37 +2051,53 @@ def aa_api_entries(api_key: str) -> list:
862
2051
  if isinstance(total_pages, int) and page >= total_pages:
863
2052
  break
864
2053
  page += 1
865
- return list(entries.values())
2054
+ return list(entries.values()), version
866
2055
 
867
2056
 
868
2057
  def fetch_aa_entries(args):
869
2058
  """AA intelligence entries from the keyed AA API v2, or nothing.
870
2059
 
871
- Returns (entries, source, cached). With a key, an API failure or an
872
- empty result warns and yields ([], None, False). Without a key this is
873
- silent and yields ([], None, False): OpenRouter's benchmarks are the
874
- primary source and the catalog records the absence (sources.aa).
2060
+ Returns (entries, source, cached, index_version). With a key, an API
2061
+ failure or an empty result warns and yields ([], None, False, None).
2062
+ Without a key this is silent and yields ([], None, False, None):
2063
+ OpenRouter's benchmarks cover what they cover and the catalog records
2064
+ the absence (sources.aa).
2065
+
2066
+ `index_version` is the envelope's intelligence_index_version (number
2067
+ or null) -- provenance for the whole fetch. Entries carry `as_of`, one
2068
+ ISO fetch timestamp for the whole fetch; a cache hit reuses the
2069
+ stamped payload verbatim so cached rows stay dated exactly like fresh
2070
+ ones.
875
2071
  """
876
2072
  if not args.no_cache:
877
- # v2: entries cached under the old key predate the V2 endpoint
878
- # migration and must never be read. Only AA API entries are read;
879
- # build_catalog rejects any other source as unknown.
880
- cached = load_cache("aa-intelligence-v2", args.cache_ttl)
2073
+ # v3: payloads cached under earlier keys predate the as_of stamp and
2074
+ # the intelligence_index_version capture and must never be read.
2075
+ # Only AA API entries are read; build_catalog rejects any other
2076
+ # source as unknown.
2077
+ cached = load_cache(AA_ENTRIES_CACHE, args.cache_ttl)
881
2078
  if isinstance(cached, dict) and cached.get("source") == "AA API v2":
882
- return cached.get("entries", []), cached.get("source"), True
2079
+ return (
2080
+ cached.get("entries", []),
2081
+ cached.get("source"),
2082
+ True,
2083
+ cached.get("version"),
2084
+ )
883
2085
  api_key = args.aa_api_key or os.environ.get("AA_API_KEY")
884
2086
  if not api_key:
885
- return [], None, False
2087
+ return [], None, False, None
886
2088
  try:
887
- entries = aa_api_entries(api_key)
2089
+ entries, version = aa_api_entries(api_key)
888
2090
  except Exception as exc:
889
2091
  warn(f"AA API request failed ({exc})")
890
- return [], None, False
2092
+ return [], None, False, None
891
2093
  if not entries:
892
2094
  warn("AA API returned no intelligence scores")
893
- return [], None, False
894
- save_cache("aa-intelligence-v2", {"entries": entries, "source": "AA API v2"})
895
- return entries, "AA API v2", False
2095
+ return [], None, False, None
2096
+ save_cache(
2097
+ AA_ENTRIES_CACHE,
2098
+ {"entries": entries, "source": "AA API v2", "version": version},
2099
+ )
2100
+ return entries, "AA API v2", False, version
896
2101
 
897
2102
 
898
2103
  # ---------------------------------------------------------------------------
@@ -901,6 +2106,13 @@ def fetch_aa_entries(args):
901
2106
 
902
2107
 
903
2108
  def build_aa_lookup(entries):
2109
+ """Index the AA entries into exact and fuzzy lookup maps.
2110
+
2111
+ Values are (index, as_of) pairs, not bare floats: the as_of travels
2112
+ with the match so resolve_quality can date every AA-sourced record
2113
+ without re-scanning the raw entry list. Entries without a usable key
2114
+ or a finite numeric index are skipped.
2115
+ """
904
2116
  exact = {}
905
2117
  fuzzy = []
906
2118
  for entry in entries:
@@ -913,11 +2125,12 @@ def build_aa_lookup(entries):
913
2125
  # Non-finite indexes would serialize as NaN and poison quality.
914
2126
  if not math.isfinite(index):
915
2127
  continue
2128
+ as_of = entry.get("as_of") if isinstance(entry.get("as_of"), str) else None
916
2129
 
917
2130
  def put(tier, raw):
918
2131
  normalized = norm_key(raw)
919
2132
  if normalized:
920
- exact.setdefault((tier, normalized), index)
2133
+ exact.setdefault((tier, normalized), (index, as_of))
921
2134
 
922
2135
  if "/" in key:
923
2136
  put(0, key)
@@ -928,11 +2141,17 @@ def build_aa_lookup(entries):
928
2141
  put(3, PAREN_RE.sub(" ", name))
929
2142
  tokens = set(norm_key(key).split()) | set(norm_key(name).split())
930
2143
  if tokens:
931
- fuzzy.append((tokens, index))
2144
+ fuzzy.append((tokens, (index, as_of)))
932
2145
  return exact, fuzzy
933
2146
 
934
2147
 
935
2148
  def match_quality(model, exact, fuzzy, allow_fuzzy=False):
2149
+ """Match a candidate to an AA entry; exact tiers only unless fuzzy is on.
2150
+
2151
+ Returns the winning entry's (index, as_of) pair, or None when no tier
2152
+ matches. Values come straight out of build_aa_lookup's maps, so the
2153
+ as_of is the entry's own fetch stamp, never a re-lookup.
2154
+ """
936
2155
  model_id = model["id"]
937
2156
  name = model.get("name") or ""
938
2157
  display = PROVIDER_PREFIX_RE.sub("", name).strip()
@@ -955,14 +2174,14 @@ def match_quality(model, exact, fuzzy, allow_fuzzy=False):
955
2174
  own = set(norm_key(base).split()) | set(norm_key(display).split())
956
2175
  if own:
957
2176
  best = 0.0
958
- best_index = None
959
- for tokens, index in fuzzy:
2177
+ best_match = None
2178
+ for tokens, match in fuzzy:
960
2179
  overlap = len(own & tokens) / len(own | tokens)
961
2180
  if overlap > best:
962
2181
  best = overlap
963
- best_index = index
964
- if best >= 0.5 and best_index is not None:
965
- return best_index
2182
+ best_match = match
2183
+ if best >= 0.5 and best_match is not None:
2184
+ return best_match
966
2185
  return None
967
2186
 
968
2187
 
@@ -974,24 +2193,37 @@ def base_model_id(model_id: str) -> str:
974
2193
  AA_BENCHMARK_FIELDS = ("intelligence_index", "coding_index", "agentic_index")
975
2194
 
976
2195
 
977
- def build_aa_benchmarks(benchmarks) -> dict:
978
- """Map bare OpenRouter id -> OR-published AA trio from data.benchmarks.
979
-
980
- Keys are dated permaslugs (z-ai/glm-5.3-flash-20260826): the trailing
981
- -YYYYMMDD is stripped and undated keys map to themselves; when two keys
982
- strip to the same id the latest date wins. Values failing the
983
- finite-number guard are treated as absent; entries with at least one
984
- valid field are kept.
2196
+ def build_aa_benchmarks(benchmarks, models=None) -> dict:
2197
+ """Map exact catalog slug -> OR-published AA trio from data.benchmarks.
2198
+
2199
+ Benchmark keys are dated permaslugs (deepseek/deepseek-v4.1-flash-
2200
+ 20261017) whose date-stripped form can collide across listings of one
2201
+ model family: the snapshot listing deepseek/deepseek-v4.1-flash-1017
2202
+ and the base listing deepseek/deepseek-v4.1-flash both strip to
2203
+ deepseek/deepseek-v4.1-flash, so blind stripping with latest-date-wins
2204
+ made the base listing borrow the snapshot's score (#17). Keys resolve
2205
+ through three explicit passes over the surviving entries, and every
2206
+ accepted id is claimed by the pass that resolved it, so later passes
2207
+ never overwrite an earlier one:
2208
+
2209
+ 1. Primary: a key equal to a models entry's permaslug maps to that
2210
+ entry's slug -- the catalog's own pairing, no guessing. Several
2211
+ dated permaslugs may legitimately pair with one slug (snapshot
2212
+ variants of one listing); among those the latest date wins.
2213
+ 2. Undated: a key without a -YYYYMMDD suffix maps to itself, unless
2214
+ pass 1 claimed the id.
2215
+ 3. Dated fallback: a dated key the catalog does not pair strips via
2216
+ strip_permaslug_date, accepted only when the stripped id is
2217
+ unclaimed and no other pass-3 key strips to the same id -- an
2218
+ ambiguous strip is the #17 conflation shape, so every candidate
2219
+ for that id drops rather than guess.
2220
+
2221
+ Entries whose values fail the finite-number guard are treated as
2222
+ absent before the passes run; entries with at least one valid field
2223
+ are kept. Records carry "as_of": the YYYYMMDD of the permaslug the
2224
+ trio came from, or None for undated keys.
985
2225
  """
986
- aa_by_id = {}
987
- seen_date = {}
988
-
989
- def accept(bare, date, trio):
990
- if bare in seen_date and seen_date[bare] >= date:
991
- return
992
- seen_date[bare] = date
993
- aa_by_id[bare] = trio
994
-
2226
+ valid = {} # benchmark key -> (trio, permaslug date or "")
995
2227
  for key, node in (benchmarks or {}).items():
996
2228
  aa = node.get("aa") if isinstance(node, dict) else None
997
2229
  if not isinstance(aa, dict):
@@ -1007,36 +2239,152 @@ def build_aa_benchmarks(benchmarks) -> dict:
1007
2239
  trio[field] = float(value)
1008
2240
  if not trio:
1009
2241
  continue
1010
- match = re.match(r"^(.*)-(\d{8})$", key)
1011
- if match:
1012
- accept(match.group(1), match.group(2), trio)
1013
- else:
1014
- accept(key, "", trio)
1015
- return aa_by_id
2242
+ bare = strip_permaslug_date(key)
2243
+ # strip_permaslug_date removed a trailing -YYYYMMDD exactly when
2244
+ # the key was dated; the date is then the final 8 characters.
2245
+ valid[key] = (trio, "" if bare == key else key[-8:])
1016
2246
 
2247
+ permaslug_to_slug = {} # the catalog's own slug/permaslug pairing
2248
+ for entry in models or []:
2249
+ if not isinstance(entry, dict):
2250
+ continue
2251
+ slug = entry.get("slug") or ""
2252
+ permaslug = entry.get("permaslug") or ""
2253
+ if slug and permaslug:
2254
+ # standard and batch variants repeat an identical pair; the
2255
+ # overwrite is the dedup
2256
+ permaslug_to_slug[permaslug] = slug
2257
+
2258
+ aa_by_id = {}
2259
+ claimed = set()
1017
2260
 
1018
- def resolve_quality(candidates, aa_by_id, exact, fuzzy, aa_source):
1019
- """Per-candidate quality: OR benchmarks first, AA exact fallback, else None.
2261
+ def record(trio, date):
2262
+ return {**trio, "as_of": date or None}
1020
2263
 
1021
- OR's per-slug values cannot mispair; the AA fallback is exact-tier only
1022
- (match_quality's fuzzy tier is the proven variant-conflation bug and
1023
- stays off in production). Source values match the catalog contract:
1024
- "openrouter", "api".
2264
+ # Pass 1: permaslug keys pair with their own slug; same-slug
2265
+ # snapshots resolve to the latest date (first wins on an equal date).
2266
+ seen_date = {}
2267
+ for key, (trio, date) in valid.items():
2268
+ slug = permaslug_to_slug.get(key)
2269
+ if slug is None or (slug in seen_date and seen_date[slug] >= date):
2270
+ continue
2271
+ seen_date[slug] = date
2272
+ aa_by_id[slug] = record(trio, date)
2273
+ claimed.add(slug)
2274
+
2275
+ # Pass 2: undated keys self-map unless pass 1 claimed the id.
2276
+ for key, (trio, date) in valid.items():
2277
+ if date or key in permaslug_to_slug or key in claimed:
2278
+ continue
2279
+ aa_by_id[key] = record(trio, date)
2280
+ claimed.add(key)
2281
+
2282
+ # Pass 3: unpaired dated keys strip, but only onto an unclaimed id
2283
+ # and only when the strip is unambiguous.
2284
+ strip_candidates = {}
2285
+ for key, (trio, date) in valid.items():
2286
+ if not date or key in permaslug_to_slug:
2287
+ continue
2288
+ stripped = strip_permaslug_date(key)
2289
+ strip_candidates.setdefault(stripped, []).append((date, trio))
2290
+ for bare, candidates in strip_candidates.items():
2291
+ if bare in claimed or len(candidates) > 1:
2292
+ continue
2293
+ date, trio = candidates[0]
2294
+ aa_by_id[bare] = record(trio, date)
2295
+ claimed.add(bare)
2296
+
2297
+ return aa_by_id
2298
+
2299
+
2300
+ def resolve_quality(
2301
+ candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality=None
2302
+ ):
2303
+ """Per-candidate quality records: AA API primary, catalog next, OR last.
2304
+
2305
+ Resolution order per candidate (inverted from the pre-reconciliation
2306
+ behavior where OpenRouter's benchmarks won):
2307
+ 1. AA API entries -- match_quality's exact tiers (fuzzy stays off in
2308
+ production: it is the proven variant-conflation bug).
2309
+ 2. catalog_quality -- records hydrated from the published catalog on
2310
+ unkeyed runs. Every winner is labeled "catalog": the run-level
2311
+ provenance is the source chain (read from our published catalog),
2312
+ overriding the row's own "aa"/"openrouter" label -- that label is
2313
+ the keyed CI run's row-level truth inside the artifact, and no
2314
+ real catalog row ever carries "catalog". as_of stays the row's
2315
+ own stamp.
2316
+ 3. OpenRouter's benchmark trio -- per-slug, matched via
2317
+ base_model_id, so a :variant suffix inherits its base's trio.
2318
+ 4. No match -- the candidate stays quality-blind downstream.
2319
+
2320
+ Every winner is a record {"index", "source", "as_of", "drift",
2321
+ "or_index", "or_as_of"}; compute_scores and build_schedules read
2322
+ record["index"], so cand["quality"] stays the float index they already
2323
+ expect. Drift is computed only when the winner is NOT OpenRouter: the
2324
+ OR trio for the same base model acts as the check value, and a
2325
+ disagreement of >= 3.0 points flags the record. When the winner IS
2326
+ OpenRouter (or no OR value exists) the drift pair is None -- there is
2327
+ no second source to disagree with.
1025
2328
  """
1026
- fallback_source = {"AA API v2": "api"}.get(aa_source)
1027
2329
  quality_by_id = {}
1028
2330
  source_by_id = {}
1029
2331
  for cand in candidates:
1030
- bench = aa_by_id.get(base_model_id(cand["id"]))
1031
- intelligence = bench.get("intelligence_index") if bench else None
1032
- if intelligence is not None:
1033
- quality_by_id[cand["id"]] = intelligence
1034
- source_by_id[cand["id"]] = "openrouter"
2332
+ record = None
2333
+ # Tier 1: AA API exact match. match_quality returns the winning
2334
+ # entry's (index, as_of) pair.
2335
+ match = match_quality({"id": cand["id"], "name": cand["name"]}, exact, fuzzy)
2336
+ if match is not None:
2337
+ index, as_of = match
2338
+ record = {"index": index, "source": "aa", "as_of": as_of}
2339
+ # Tier 2: catalog record (unkeyed runs hydrate from the published
2340
+ # catalog). The run-level provenance label is the source chain:
2341
+ # every hydrated record is tagged "catalog" -- read from our
2342
+ # published catalog -- overriding the row's own label, which is
2343
+ # the keyed CI run's row-level truth inside the artifact ("aa"/
2344
+ # "openrouter"; no real catalog row ever carries "catalog").
2345
+ # as_of stays the row's own stamp (v4 rows carry the CI run's
2346
+ # recorded quality_as_of; v3 rows the catalog's generated_at --
2347
+ # catalog_quality_by_id normalizes that).
2348
+ if record is None and catalog_quality is not None:
2349
+ catalog_record = catalog_quality.get(cand["id"])
2350
+ if catalog_record is not None:
2351
+ record = {
2352
+ "index": catalog_record["index"],
2353
+ "source": "catalog",
2354
+ "as_of": catalog_record["as_of"],
2355
+ }
2356
+ # Tier 3: OpenRouter's per-slug benchmark trio (last resort).
2357
+ if record is None:
2358
+ bench = aa_by_id.get(base_model_id(cand["id"]))
2359
+ intelligence = bench.get("intelligence_index") if bench else None
2360
+ if intelligence is not None:
2361
+ record = {
2362
+ "index": intelligence,
2363
+ "source": "openrouter",
2364
+ "as_of": bench.get("as_of"),
2365
+ }
2366
+ if record is None:
1035
2367
  continue
1036
- index = match_quality({"id": cand["id"], "name": cand["name"]}, exact, fuzzy)
1037
- if index is not None:
1038
- quality_by_id[cand["id"]] = index
1039
- source_by_id[cand["id"]] = fallback_source
2368
+
2369
+ # Drift: only meaningful when the winner is not OpenRouter itself.
2370
+ bench = aa_by_id.get(base_model_id(cand["id"])) or {}
2371
+ or_intelligence = bench.get("intelligence_index")
2372
+ if (
2373
+ record["source"] in ("aa", "catalog")
2374
+ and isinstance(or_intelligence, (int, float))
2375
+ and not isinstance(or_intelligence, bool)
2376
+ and math.isfinite(or_intelligence)
2377
+ ):
2378
+ record["or_index"] = or_intelligence
2379
+ record["or_as_of"] = bench.get("as_of")
2380
+ record["drift"] = abs(record["index"] - or_intelligence) >= 3.0
2381
+ else:
2382
+ record["or_index"] = None
2383
+ record["or_as_of"] = None
2384
+ record["drift"] = None
2385
+
2386
+ quality_by_id[cand["id"]] = record
2387
+ source_by_id[cand["id"]] = record["source"]
1040
2388
  return quality_by_id, source_by_id
1041
2389
 
1042
2390
 
@@ -1061,7 +2409,9 @@ def _per_1m(price_per_token: float) -> float:
1061
2409
  return round(price_per_token * 1_000_000.0, 6)
1062
2410
 
1063
2411
 
1064
- def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
2412
+ def build_schedules(
2413
+ models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality=None
2414
+ ):
1065
2415
  """Catalog-wide off-peak index built from the RAW model list (spec 5.4).
1066
2416
 
1067
2417
  Deliberately unfiltered by the candidate pool: sub-floor-context and
@@ -1069,6 +2419,12 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
1069
2419
  schedule models mostly sit below the 1M context floor). Router aliases
1070
2420
  (~ids) and malformed ids are excluded; :batch variants are included as
1071
2421
  distinct deals. Sorted by id.
2422
+
2423
+ Quality falls back through the same chain resolve_quality walks, minus
2424
+ its AA tier (the AA lookup only ever matched candidates): candidate
2425
+ records, then the hydrated catalog (unkeyed runs), then OpenRouter's
2426
+ benchmark trio. Every schedule row carries the winner's provenance
2427
+ (quality_source/quality_as_of) so the site's QUAL tooltip can render.
1072
2428
  """
1073
2429
  entries = []
1074
2430
  for model in models:
@@ -1093,10 +2449,38 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
1093
2449
  continue
1094
2450
  peak = max(windows, key=lambda window: window["blended"])
1095
2451
  offpeak = min(windows, key=lambda window: window["blended"])
1096
- quality = quality_by_id.get(model_id)
2452
+ # quality_by_id values are records {"index", "source", ...}; the
2453
+ # schedule row keeps the bare float index verbatim (never a dict)
2454
+ # and echoes the winning record's provenance for the QUAL tooltip.
2455
+ # Fallback chain mirrors resolve_quality's minus its AA tier:
2456
+ # candidate record, hydrated catalog record (unkeyed runs; every
2457
+ # hydrated record is relabeled with the run-level chain label
2458
+ # "catalog", exactly like resolve_quality's catalog tier), then
2459
+ # OpenRouter's benchmark trio (row label "openrouter").
2460
+ record = quality_by_id.get(model_id)
2461
+ if isinstance(record, dict):
2462
+ quality = record.get("index")
2463
+ quality_source = record.get("source")
2464
+ quality_as_of = record.get("as_of")
2465
+ else:
2466
+ quality = record
2467
+ quality_source = None
2468
+ quality_as_of = None
2469
+ if quality is None and catalog_quality is not None:
2470
+ catalog_record = catalog_quality.get(model_id)
2471
+ if catalog_record is not None:
2472
+ quality = catalog_record["index"]
2473
+ quality_source = "catalog"
2474
+ quality_as_of = catalog_record["as_of"]
1097
2475
  if quality is None:
1098
2476
  benchmark = aa_by_id.get(base_model_id(model_id)) or {}
1099
2477
  quality = benchmark.get("intelligence_index")
2478
+ if quality is not None:
2479
+ quality_source = "openrouter"
2480
+ quality_as_of = benchmark.get("as_of")
2481
+ else:
2482
+ quality_source = None
2483
+ quality_as_of = None
1100
2484
  name = model.get("name") or model_id
1101
2485
  entries.append(
1102
2486
  {
@@ -1104,6 +2488,10 @@ def build_schedules(models, args, quality_by_id, aa_by_id, balanced_by_id):
1104
2488
  "name": PROVIDER_PREFIX_RE.sub("", name).strip() or name,
1105
2489
  "context": coerce_int(model.get("context_length"), 0),
1106
2490
  "quality": quality,
2491
+ # Provenance for the site's QUAL tooltip (null when the
2492
+ # quality came without any -- e.g. a bare-float test map).
2493
+ "quality_source": quality_source,
2494
+ "quality_as_of": quality_as_of,
1107
2495
  "score": balanced_by_id.get(model_id),
1108
2496
  "max_discount": eff["max_discount"],
1109
2497
  "sched_note": fmt_sched_note(eff["max_discount"]),
@@ -1313,11 +2701,19 @@ def compute_scores(candidates, args, quality_by_id):
1313
2701
  0.0, min(1.0, 1.0 - (math.log10(price + 0.01) - low) / price_span)
1314
2702
  )
1315
2703
  quality = quality_by_id.get(cand["id"])
1316
- cand["quality"] = quality
1317
- if quality is None:
2704
+ # quality_by_id values are records {"index", "source", ...}; the
2705
+ # candidate keeps the bare float index (that is what ranking_key,
2706
+ # build_catalog's quality field and print_json's quality_index all
2707
+ # expect) and the full record travels alongside for provenance.
2708
+ record = quality if isinstance(quality, dict) else None
2709
+ cand["quality"] = record["index"] if record else quality
2710
+ cand["quality_record"] = record
2711
+ if cand["quality"] is None:
1318
2712
  cand["quality_score"] = 0.0
1319
2713
  else:
1320
- cand["quality_score"] = max(0.0, min(1.0, quality / args.quality_ref))
2714
+ cand["quality_score"] = max(
2715
+ 0.0, min(1.0, cand["quality"] / args.quality_ref)
2716
+ )
1321
2717
  context = cand["context"]
1322
2718
  cand["context_score"] = (
1323
2719
  max(0.0, min(1.0, math.log(context / floor_ctx) / ctx_span))
@@ -1413,6 +2809,151 @@ def fmt_discount(value) -> str:
1413
2809
  return f"{value:.0%}" if has_discount(value) else "--"
1414
2810
 
1415
2811
 
2812
+ def fmt_prec(cand) -> str:
2813
+ """PREC cell: the precision the ranked price is quoted at.
2814
+
2815
+ The maker's default precision when disclosed, the best available
2816
+ otherwise, "n/a" when none could be derived; "!" flags default routing
2817
+ that favors cheaper sub-default endpoints (annotations.risk_flag).
2818
+ Candidates that never went through the endpoint stage degrade to "n/a".
2819
+ """
2820
+ cell = cand.get("default_precision") or "n/a"
2821
+ if (cand.get("annotations") or {}).get("risk_flag"):
2822
+ cell += "!"
2823
+ return cell
2824
+
2825
+
2826
+ def fmt_quant_note(annotations: dict) -> str | None:
2827
+ """One-line note on the cheapest sub-default endpoint (PREC tooltip).
2828
+
2829
+ The exact string the site shows (Task 11 renders it verbatim), so the
2830
+ format is pinned by test. None when there is no non-eligible live
2831
+ endpoint -- nothing cheaper to route to. cheapest_quantized.price is
2832
+ stored per-token, the way build_annotations leaves it; the note shows
2833
+ USD per 1M like every other price the CLI prints. A missing spread
2834
+ (defensive) drops the parenthetical. A null quantization (third-party
2835
+ null == "unknown" for eligibility) and a null provider_name on
2836
+ malformed data render the design's "unknown" sentinel, never Python's
2837
+ None repr.
2838
+ """
2839
+ quant = (annotations or {}).get("cheapest_quantized")
2840
+ if not quant:
2841
+ return None
2842
+ quantization = quant["quantization"] or "unknown"
2843
+ provider = quant["provider"] or "unknown"
2844
+ note = (
2845
+ f"cheapest non-baseline: {quantization} @ {provider}, "
2846
+ f"${fmt_price(quant['price'] * 1_000_000)}/M"
2847
+ )
2848
+ spread = (annotations or {}).get("price_spread")
2849
+ if spread is not None:
2850
+ note += f" in ({spread:.2f}x baseline)"
2851
+ return note
2852
+
2853
+
2854
+ def _footer_as_of(source, value):
2855
+ """Format a record's as_of for its footer block (None when unusable).
2856
+
2857
+ Two stamp shapes circulate. A YYYYMMDD date (OpenRouter's benchmark
2858
+ date, and catalog rows hydrated from OR rows) renders as a plain date
2859
+ ("as of 2026-10-17") for ANY source -- a date-only stamp never
2860
+ fabricates a time of day. ISO stamps: live AA records carry one fetch
2861
+ timestamp for the whole fetch (checked today), so their block shows
2862
+ the time of day only; other rows carry the full stamp ("as of
2863
+ 2026-10-23 13:15Z").
2864
+ """
2865
+ if not value:
2866
+ return None
2867
+ text = str(value)
2868
+ if len(text) == 8 and text.isdigit():
2869
+ return f"{text[:4]}-{text[4:6]}-{text[6:]}"
2870
+ parsed = parse_iso_datetime(value)
2871
+ if parsed is None:
2872
+ return None
2873
+ if source == "aa":
2874
+ return parsed.strftime("%H:%MZ")
2875
+ return parsed.strftime("%Y-%m-%d %H:%MZ")
2876
+
2877
+
2878
+ def _as_of_instant(value):
2879
+ """Parse an as_of stamp to a comparable UTC instant (None when unusable).
2880
+
2881
+ The pipeline carries ISO stamps and the YYYYMMDD benchmark date; a
2882
+ date-only stamp means "that day" and compares as midnight UTC.
2883
+ """
2884
+ text = str(value)
2885
+ if len(text) == 8 and text.isdigit():
2886
+ return parse_iso_datetime(f"{text[:4]}-{text[4:6]}-{text[6:]}")
2887
+ return parse_iso_datetime(value)
2888
+
2889
+
2890
+ def _later_as_of(value, current) -> bool:
2891
+ """True when the value stamp names a later instant than current.
2892
+
2893
+ Parsed instants, never raw strings: a YYYYMMDD stamp must not
2894
+ lexicographically beat a fresher ISO stamp ("20261017" sorts above
2895
+ every "2026-..." string). Ties keep the earlier pick; an unparseable
2896
+ stamp never displaces anything.
2897
+ """
2898
+ new, old = _as_of_instant(value), _as_of_instant(current)
2899
+ if new is None or old is None:
2900
+ return new is not None
2901
+ return new > old
2902
+
2903
+
2904
+ def quality_footer_note(quality_by_id, aa_index_version, unmatched):
2905
+ """Compose the footer's quality note from the resolved records.
2906
+
2907
+ Lists per-source counts, each source's own as_of and the AA index
2908
+ version when known, e.g. "AA index v4.3 - live AA (58, checked 13:15Z)
2909
+ | our catalog (12, as of 2026-10-23 13:15Z) | OpenRouter snapshot (2,
2910
+ as of 2026-10-17)". A trailing "unmatched score 0" names candidates
2911
+ with no quality record, and a drift footnote appears when any resolved
2912
+ record disagrees with the OpenRouter check value. Returns None when no
2913
+ record resolved -- run() falls back to the quality-blind note.
2914
+ """
2915
+ labels = {
2916
+ "aa": "live AA",
2917
+ "catalog": "our catalog",
2918
+ "openrouter": "OpenRouter snapshot",
2919
+ }
2920
+ stamp_words = {"aa": "checked", "catalog": "as of", "openrouter": "as of"}
2921
+ counts = {}
2922
+ latest = {}
2923
+ for record in quality_by_id.values():
2924
+ source = record.get("source")
2925
+ if source not in labels:
2926
+ continue
2927
+ counts[source] = counts.get(source, 0) + 1
2928
+ value = record.get("as_of")
2929
+ if value:
2930
+ current = latest.get(source)
2931
+ if current is None or _later_as_of(value, current):
2932
+ latest[source] = value
2933
+ blocks = []
2934
+ for source in ("aa", "catalog", "openrouter"):
2935
+ count = counts.get(source, 0)
2936
+ if not count:
2937
+ continue
2938
+ stamp = _footer_as_of(source, latest.get(source))
2939
+ if stamp:
2940
+ blocks.append(f"{labels[source]} ({count}, {stamp_words[source]} {stamp})")
2941
+ else:
2942
+ blocks.append(f"{labels[source]} ({count})")
2943
+ if not blocks:
2944
+ return None
2945
+ note = "quality: "
2946
+ if aa_index_version is not None:
2947
+ note += f"AA index v{aa_index_version} - "
2948
+ note += " | ".join(blocks)
2949
+ if unmatched:
2950
+ note += "; unmatched score 0"
2951
+ drift = sum(1 for record in quality_by_id.values() if record.get("drift"))
2952
+ if drift:
2953
+ note += f"; drift vs OpenRouter snapshot ({drift})"
2954
+ return note
2955
+
2956
+
1416
2957
  def print_table(top, total_candidates, weights, quality_note):
1417
2958
  headers = [
1418
2959
  "RANK",
@@ -1421,6 +2962,7 @@ def print_table(top, total_candidates, weights, quality_note):
1421
2962
  "$IN/M",
1422
2963
  "$OUT/M",
1423
2964
  "DISC",
2965
+ "PREC",
1424
2966
  "TIER",
1425
2967
  "CTX",
1426
2968
  "AGE",
@@ -1436,6 +2978,7 @@ def print_table(top, total_candidates, weights, quality_note):
1436
2978
  fmt_price(cand["price_in"]),
1437
2979
  fmt_price(cand["price_out"]),
1438
2980
  fmt_discount(cand["discount"]),
2981
+ fmt_prec(cand),
1439
2982
  cand["tier_note"] or cand["sched_note"] or "--",
1440
2983
  fmt_context(cand["context"]),
1441
2984
  fmt_age(cand["age_days"]),
@@ -1482,6 +3025,12 @@ def print_table(top, total_candidates, weights, quality_note):
1482
3025
  "schedule whose discount is under 1%. --json lists each model's "
1483
3026
  "schedule."
1484
3027
  )
3028
+ print(
3029
+ "PREC = precision the ranked price is quoted at (maker's default when "
3030
+ "disclosed, best available otherwise; ! = default routing favors "
3031
+ "cheaper sub-default endpoints); prices are the cheapest live endpoint "
3032
+ "at that precision."
3033
+ )
1485
3034
 
1486
3035
 
1487
3036
  def print_json(top):
@@ -1492,6 +3041,15 @@ def print_json(top):
1492
3041
  "name": cand["name"],
1493
3042
  "score": round(cand["score"], 4),
1494
3043
  "quality_index": cand["quality"],
3044
+ # Provenance block, verbatim from the quality record resolved
3045
+ # in compute_scores: the winning tier's source and as_of plus
3046
+ # the drift check against OpenRouter's benchmarks. Rows without
3047
+ # a record (pre-endpoint test paths) degrade to None.
3048
+ "quality_source": (cand.get("quality_record") or {}).get("source"),
3049
+ "quality_as_of": (cand.get("quality_record") or {}).get("as_of"),
3050
+ "quality_drift": (cand.get("quality_record") or {}).get("drift"),
3051
+ "quality_or_index": (cand.get("quality_record") or {}).get("or_index"),
3052
+ "quality_or_as_of": (cand.get("quality_record") or {}).get("or_as_of"),
1495
3053
  "input_usd_per_m": round(cand["price_in"], 6),
1496
3054
  "output_usd_per_m": round(cand["price_out"], 6),
1497
3055
  "blended_usd_per_m": round(cand["blended"], 6),
@@ -1518,6 +3076,31 @@ def print_json(top):
1518
3076
  "age_days": round(cand["age_days"], 1)
1519
3077
  if cand["age_days"] is not None
1520
3078
  else None,
3079
+ # Additive keys (Task 9's ROW_KEYS extension pins this exact
3080
+ # key set and order). Risk/spread come from the annotations
3081
+ # block; session_cost_usd is run()'s per-candidate slice of the
3082
+ # Task 7 envelope, shown for the display turn range (run()
3083
+ # attaches it before any output path). Candidates that never
3084
+ # went through either stage (historical test paths) degrade to
3085
+ # null, as does a model the dataset covers only in other turn
3086
+ # ranges.
3087
+ "default_precision": cand.get("default_precision"),
3088
+ "risk_flag": (cand.get("annotations") or {}).get("risk_flag"),
3089
+ "price_spread": (cand.get("annotations") or {}).get("price_spread"),
3090
+ "session_cost_usd": (cand.get("session_cost") or {}).get(
3091
+ SESSION_COST_DISPLAY_TURN_RANGE
3092
+ ),
3093
+ # The cheapest-sub-default note, exactly as the CLI prints it
3094
+ # (Task 11 shows it verbatim). Emit-first: Task 9's ROW_KEYS
3095
+ # pins this key last; pre-endpoint candidates degrade to None.
3096
+ "quant_note": fmt_quant_note(cand.get("annotations")),
3097
+ # One extra row key AFTER quant_note -- steering is NOT in
3098
+ # ROW_KEYS. build_site_data's row check is missing-key-only, so
3099
+ # extras pass through (tier_note precedent) and the site
3100
+ # ignores it. The OpenRouter provider routing object (design
3101
+ # section 4.7) the plugin contract reads; pre-endpoint
3102
+ # candidates degrade to None.
3103
+ "steering": cand.get("steering"),
1521
3104
  }
1522
3105
  for cand in top
1523
3106
  ]
@@ -1535,6 +3118,11 @@ def build_catalog(
1535
3118
  aa_source,
1536
3119
  aa_by_id,
1537
3120
  quality_source_by_id,
3121
+ session_cost=None,
3122
+ session_cost_mode="skipped",
3123
+ index_version=None,
3124
+ catalog_generated_at=None,
3125
+ catalog_quality=None,
1538
3126
  ):
1539
3127
  """Assemble the full-catalog document (see README "Catalog output").
1540
3128
 
@@ -1542,21 +3130,56 @@ def build_catalog(
1542
3130
  component scores) and never mutates them. Deterministic apart from
1543
3131
  generated_at: models sort by (-overall.balanced, id), filtered by id,
1544
3132
  and age uses date precision so two runs in the same UTC day match.
3133
+
3134
+ session_cost is the Task 7 fetch envelope (verbatim, medians included)
3135
+ or None; session_cost_mode is run()'s resolution of the key-presence
3136
+ fact ("ok" / "unavailable" / "skipped") -- build_catalog stays pure and
3137
+ never fetches or inspects the environment itself. index_version is the
3138
+ AA envelope's intelligence_index_version (number or null) and
3139
+ catalog_generated_at the published catalog's generated_at when this
3140
+ run hydrated from it (else None); both pass verbatim into sources.aa.
3141
+ catalog_quality is the hydrated record map (or None): candidates read
3142
+ it through resolve_quality, and build_schedules falls back to it for
3143
+ non-candidates.
1545
3144
  """
1546
3145
  now = datetime.now(timezone.utc)
1547
3146
  weights = catalog_weights(candidates, quality_by_id)
1548
- aa_modes = {"AA API v2": "api"}
1549
- if aa_source is not None and aa_source not in aa_modes:
1550
- # Never claim mode "none" for a source we do not know: the document
1551
- # would contradict itself (mode none with matched quality scores).
3147
+ if aa_source is not None and aa_source != "AA API v2":
3148
+ # Never publish a document whose AA source we cannot name: the
3149
+ # consumers' mode enum has no slot for it.
1552
3150
  raise ValueError(f"unknown AA source: {aa_source!r}")
3151
+ # The three counts are disjoint label buckets: resolve_quality's strict
3152
+ # rank gives every candidate at most one record, so their sum is exactly
3153
+ # the number of entries with a non-null quality_match -- the invariant
3154
+ # the site validator enforces. A label outside the buckets would break
3155
+ # it silently, so it fails loudly here.
3156
+ matched_aa = sum(1 for s in quality_source_by_id.values() if s == "aa")
3157
+ matched_catalog = sum(1 for s in quality_source_by_id.values() if s == "catalog")
1553
3158
  matched_openrouter = sum(
1554
3159
  1 for s in quality_source_by_id.values() if s == "openrouter"
1555
3160
  )
1556
- # fallback is what the AA API yielded on its own; mode
1557
- # hides it behind "openrouter" as soon as one OpenRouter match exists.
1558
- aa_fallback = aa_modes.get(aa_source, "none")
1559
- aa_mode = "openrouter" if matched_openrouter else aa_fallback
3161
+ unknown = sorted(
3162
+ {
3163
+ s
3164
+ for s in quality_source_by_id.values()
3165
+ if s not in ("aa", "catalog", "openrouter")
3166
+ }
3167
+ )
3168
+ if unknown:
3169
+ raise ValueError(f"unknown quality source: {unknown[0]!r}")
3170
+ # mode is the run's quality chain, not a match outcome: "aa" when the
3171
+ # AA API returned usable entries, "catalog" when unkeyed runs hydrated
3172
+ # the published catalog, "none" when neither tier produced data.
3173
+ # OpenRouter's benchmarks are the chain's shared last resort, never a
3174
+ # mode of their own -- matched_openrouter carries their count. The
3175
+ # catalog branch also fires on the counts alone: hydrated records are
3176
+ # proof of a usable catalog even when its generated_at was unusable.
3177
+ if aa_source is not None:
3178
+ aa_mode = "aa"
3179
+ elif catalog_generated_at is not None or matched_catalog or matched_aa:
3180
+ aa_mode = "catalog"
3181
+ else:
3182
+ aa_mode = "none"
1560
3183
 
1561
3184
  entries = []
1562
3185
  for cand in candidates:
@@ -1587,6 +3210,12 @@ def build_catalog(
1587
3210
  4,
1588
3211
  )
1589
3212
  bench = aa_by_id.get(base_model_id(cand["id"])) or {}
3213
+ # The quality record map is already a parameter; the winning
3214
+ # record's as_of and drift verdict become entry fields verbatim.
3215
+ # Unmatched candidates degrade to null -- same shape as the v3
3216
+ # additive blocks below.
3217
+ record = quality_by_id.get(cand["id"])
3218
+ record = record if isinstance(record, dict) else None
1590
3219
  entries.append(
1591
3220
  {
1592
3221
  "id": cand["id"],
@@ -1654,7 +3283,25 @@ def build_catalog(
1654
3283
  },
1655
3284
  "quality": cand["quality"],
1656
3285
  "quality_match": quality_source_by_id.get(cand["id"]),
3286
+ # Provenance (schema v4), verbatim from the record: the
3287
+ # winning source's own as_of and the drift verdict against
3288
+ # OpenRouter's check value.
3289
+ "quality_as_of": record["as_of"] if record else None,
3290
+ "quality_drift": record["drift"] if record else None,
1657
3291
  "scores": {**scores, "overall": overall},
3292
+ # Additive provider-aware blocks (schema v3), verbatim from
3293
+ # the candidate. Candidates that never went through the
3294
+ # endpoint stage (historical test paths) degrade to null --
3295
+ # .get() with null defaults for these NEW keys only; the
3296
+ # pre-existing keys above stay direct-indexed.
3297
+ "endpoints": cand.get("endpoints"),
3298
+ "default_precision": cand.get("default_precision"),
3299
+ "advertised_pricing": cand.get("advertised_pricing"),
3300
+ "annotations": cand.get("annotations"),
3301
+ # The OpenRouter provider routing object (design section
3302
+ # 4.7) the plugin contract reads -- same null-default
3303
+ # degradation shape as the blocks above.
3304
+ "steering": cand.get("steering"),
1658
3305
  }
1659
3306
  )
1660
3307
  entries.sort(key=lambda e: (-e["scores"]["overall"]["balanced"], e["id"]))
@@ -1665,7 +3312,7 @@ def build_catalog(
1665
3312
 
1666
3313
  return {
1667
3314
  "schedules": build_schedules(
1668
- models, args, quality_by_id, aa_by_id, balanced_by_id
3315
+ models, args, quality_by_id, aa_by_id, balanced_by_id, catalog_quality
1669
3316
  ),
1670
3317
  "schema_version": CATALOG_SCHEMA_VERSION,
1671
3318
  "tool": "model-compare",
@@ -1686,13 +3333,31 @@ def build_catalog(
1686
3333
  "openrouter": "ok",
1687
3334
  "aa": {
1688
3335
  "mode": aa_mode,
1689
- "fallback": aa_fallback,
1690
- "matched": len(quality_by_id),
3336
+ "matched_aa": matched_aa,
3337
+ "matched_catalog": matched_catalog,
1691
3338
  "matched_openrouter": matched_openrouter,
3339
+ # Provenance meta (schema v4): the published catalog's stamp
3340
+ # when this run hydrated from it, and the AA envelope's
3341
+ # intelligence index version when the AA API ran. Both null
3342
+ # when their source was not consulted.
3343
+ "catalog_generated_at": catalog_generated_at,
3344
+ "intelligence_index_version": index_version,
1692
3345
  },
1693
3346
  "zdr": "skipped" if args.no_zdr else "ok",
1694
3347
  "discounts": "ok" if discounts else "unavailable",
3348
+ # The endpoint stage always ran before build_catalog is reached;
3349
+ # its failures surface as pool.dropped counts
3350
+ # (endpoint_fetch_failed / no_baseline_eligible_endpoint), never
3351
+ # as source state -- so this is deterministic "ok".
3352
+ "endpoints": "ok",
3353
+ # "ok" (envelope fetched) / "unavailable" (key configured, fetch
3354
+ # failed) / "skipped" (no key) -- resolved in run().
3355
+ "session_cost": session_cost_mode,
1695
3356
  },
3357
+ # Per-task session-cost medians (Task 7), verbatim from the fetch
3358
+ # envelope. Omitted entirely when unavailable: consumers branch on
3359
+ # key presence, never on a null shape.
3360
+ **({"session_cost": session_cost} if session_cost is not None else {}),
1696
3361
  "pool": {
1697
3362
  "listed": len(models),
1698
3363
  "candidates": len(candidates),
@@ -1787,6 +3452,11 @@ def parse_args(argv=None):
1787
3452
  default=None,
1788
3453
  help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
1789
3454
  )
3455
+ parser.add_argument(
3456
+ "--openrouter-api-key",
3457
+ default=None,
3458
+ help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
3459
+ )
1790
3460
  parser.add_argument(
1791
3461
  "--no-require-tools",
1792
3462
  action="store_true",
@@ -1857,23 +3527,82 @@ def run(args, models, or_cached) -> int:
1857
3527
  "(--no-zdr to override)"
1858
3528
  )
1859
3529
  return 1
1860
- aa_entries, aa_source, aa_cached = fetch_aa_entries(args)
3530
+ # Mode-scoped quality chains: a keyed run consults the AA API only; an
3531
+ # unkeyed run hydrates from the published catalog only. The catalog
3532
+ # fetcher must NEVER run on keyed builds -- a CI/site build hydrating
3533
+ # from the published catalog would consume its own output (a
3534
+ # self-reference loop). Both chains end in OpenRouter's benchmarks as
3535
+ # the last resort, then nothing.
3536
+ if args.aa_api_key or os.environ.get("AA_API_KEY"):
3537
+ aa_entries, aa_source, aa_cached, aa_index_version = fetch_aa_entries(args)
3538
+ catalog_quality = None
3539
+ catalog_generated_at = None
3540
+ else:
3541
+ aa_entries, aa_source, aa_cached, aa_index_version = [], None, False, None
3542
+ # None (transport failure) and an empty by_id (fetched-but-unusable
3543
+ # artifact) both degrade to no catalog tier -- resolve_quality and
3544
+ # the footer treat them identically as "catalog unavailable".
3545
+ catalog_doc = fetch_quality_catalog(args)
3546
+ catalog_quality = (catalog_doc or {}).get("by_id") or None
3547
+ # The catalog meta only means something when hydration actually
3548
+ # produced rows: an unusable artifact contributes no meta, so the
3549
+ # sources block and the mode never credit a catalog that yielded
3550
+ # nothing.
3551
+ catalog_generated_at = (
3552
+ (catalog_doc or {}).get("generated_at") if catalog_quality else None
3553
+ )
1861
3554
  exact, fuzzy = build_aa_lookup(aa_entries)
1862
3555
 
1863
3556
  filtered = []
1864
3557
  candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
3558
+ # The provider-aware stage: every surviving candidate is repriced at its
3559
+ # baseline endpoint (or dropped fail-closed) before any scoring runs.
3560
+ candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
3561
+ dropped.update(extra_dropped)
1865
3562
  if not candidates:
1866
- warn(
1867
- f"no models satisfy the filters (min-context={args.min_context}, "
1868
- f"tools={'off' if args.no_require_tools else 'required'}); relax them and retry"
1869
- )
3563
+ # When the endpoint stage wiped the pool, naming the filter knobs is
3564
+ # misleading (they passed; the drops came after) -- warn with the
3565
+ # endpoint-stage drop counts instead (both reasons, zero-filled).
3566
+ counts = {r: extra_dropped.get(r, 0) for r in ENDPOINT_DROP_REASONS}
3567
+ if any(counts.values()):
3568
+ bits = ", ".join(f"{k}={v}" for k, v in counts.items())
3569
+ warn(f"no models after the endpoint stage: {bits}")
3570
+ else:
3571
+ warn(
3572
+ f"no models satisfy the filters (min-context={args.min_context}, "
3573
+ f"tools={'off' if args.no_require_tools else 'required'}); "
3574
+ "relax them and retry"
3575
+ )
1870
3576
  return 2
1871
3577
 
1872
3578
  quality_by_id, quality_source_by_id = resolve_quality(
1873
- candidates, aa_by_id, exact, fuzzy, aa_source
3579
+ candidates, aa_by_id, exact, fuzzy, aa_source, catalog_quality
1874
3580
  )
1875
3581
  weights = compute_scores(candidates, args, quality_by_id)
1876
3582
 
3583
+ # Per-task session-cost medians (Task 7): one fetch per invocation, but
3584
+ # only on the output paths that surface it (--catalog block, --json
3585
+ # rows; publish runs --json per priority, so the cache makes the repeat
3586
+ # cost free). A keyed --best/plain run skips the paginated fetch
3587
+ # entirely -- the medians never reach those outputs. Keyless,
3588
+ # fetch_session_cost returns None before any network or cache access.
3589
+ if args.catalog or args.json:
3590
+ session_cost = fetch_session_cost(args)
3591
+ # The key-presence fact distinguishes "skipped" from "unavailable";
3592
+ # same expression fetch_session_cost uses for its own guard.
3593
+ session_cost_mode = (
3594
+ "skipped"
3595
+ if not (args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY"))
3596
+ else ("ok" if session_cost else "unavailable")
3597
+ )
3598
+ else:
3599
+ session_cost = None
3600
+ session_cost_mode = "skipped"
3601
+ # run() owns the per-candidate attachment (build_catalog stays pure);
3602
+ # print_json reads it for the session_cost_usd display cell.
3603
+ for cand in candidates:
3604
+ cand["session_cost"] = (session_cost or {}).get("medians", {}).get(cand["id"])
3605
+
1877
3606
  if args.catalog:
1878
3607
  print_catalog(
1879
3608
  build_catalog(
@@ -1887,6 +3616,11 @@ def run(args, models, or_cached) -> int:
1887
3616
  aa_source,
1888
3617
  aa_by_id,
1889
3618
  quality_source_by_id,
3619
+ session_cost,
3620
+ session_cost_mode,
3621
+ aa_index_version,
3622
+ catalog_generated_at,
3623
+ catalog_quality,
1890
3624
  )
1891
3625
  )
1892
3626
  return 0
@@ -1905,23 +3639,14 @@ def run(args, models, or_cached) -> int:
1905
3639
  f"{v} {k}" for k, v in sorted(dropped.items(), key=lambda kv: -kv[1])
1906
3640
  )
1907
3641
  drop_note = f" (dropped: {bits})"
1908
- matched_or = sum(1 for s in quality_source_by_id.values() if s == "openrouter")
1909
- matched_aa = len(quality_by_id) - matched_or
3642
+ # The footer lists per-source counts, each source's own as_of and
3643
+ # the AA index version when known; unmatched candidates and drift
3644
+ # get trailing footnotes. No record resolved -> the quality-blind
3645
+ # fallback note.
1910
3646
  unmatched = len(candidates) - len(quality_by_id)
1911
- bits = []
1912
- if matched_or:
1913
- bits.append(f"OpenRouter benchmarks ({matched_or})")
1914
- if matched_aa:
1915
- bits.append(f"{aa_source} exact ({matched_aa})")
1916
- if bits:
1917
- quality_note = (
1918
- f"quality via {' + '.join(bits)}: matched "
1919
- f"{len(quality_by_id)}/{len(candidates)} candidates"
1920
- )
1921
- if unmatched:
1922
- quality_note += "; unmatched candidates score 0 on quality"
1923
- else:
1924
- quality_note = "quality data unavailable, ranked on price/context/age"
3647
+ quality_note = quality_footer_note(
3648
+ quality_by_id, aa_index_version, unmatched
3649
+ ) or ("quality data unavailable, ranked on price/context/age")
1925
3650
  source_note = []
1926
3651
  if or_cached:
1927
3652
  source_note.append("catalog cached")