model-compare-opencode-plugin 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,6 +54,7 @@ import time
54
54
  import urllib.error
55
55
  import urllib.parse
56
56
  import urllib.request
57
+ from concurrent.futures import ThreadPoolExecutor
57
58
  from datetime import datetime, timezone
58
59
 
59
60
  OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
@@ -67,7 +68,7 @@ OPENROUTER_ZDR_URL = "https://openrouter.ai/api/v1/endpoints/zdr"
67
68
  # (legacy endpoints 410 after 2026-11-04; see
68
69
  # https://artificialanalysis.ai/data-api/migrate-v2-data).
69
70
  AA_API_URL = "https://artificialanalysis.ai/api/v2/language/models/free"
70
- VERSION = "0.3.1"
71
+ VERSION = "0.4.0"
71
72
  USER_AGENT = f"model-compare/{VERSION} (https://github.com/canonical/model-compare)"
72
73
 
73
74
  # opencode expects provider-qualified model ids: openrouter/<provider/model>.
@@ -86,11 +87,15 @@ PRIORITY_WEIGHTS = {
86
87
  "quality": {"quality": 0.60, "price": 0.20, "context": 0.10, "age": 0.10},
87
88
  }
88
89
 
89
- CATALOG_SCHEMA_VERSION = 2
90
+ CATALOG_SCHEMA_VERSION = 3
90
91
 
91
92
  # The drop-reason keys build_candidates counts, verbatim -- keep in sync with
92
93
  # its drop() call sites. pool.dropped lists all of them zero-filled so the
93
- # contract is self-describing.
94
+ # contract is self-describing. The last two come from the endpoint stage
95
+ # (apply_endpoint_pricing's drop sites), not build_candidates: snake_case
96
+ # while the older reasons are human phrases. The mixed naming convention is
97
+ # a deliberate, documented extension -- machine-readable reasons for the
98
+ # endpoint-stage drops the publish drop-rate gate keys on.
94
99
  CATALOG_DROP_REASONS = (
95
100
  "malformed id",
96
101
  "context",
@@ -104,11 +109,74 @@ CATALOG_DROP_REASONS = (
104
109
  "tool calling",
105
110
  "expired",
106
111
  "age",
112
+ "no_baseline_eligible_endpoint",
113
+ "endpoint_fetch_failed",
114
+ )
115
+
116
+ # The only reasons the endpoint stage itself produces (the last two
117
+ # CATALOG_DROP_REASONS entries) -- the empty-pool warning keys on them;
118
+ # web/publish.py keeps its own copy of this tuple by design.
119
+ ENDPOINT_DROP_REASONS = (
120
+ "no_baseline_eligible_endpoint",
121
+ "endpoint_fetch_failed",
107
122
  )
108
123
 
109
124
  PAREN_RE = re.compile(r"\([^)]*\)")
110
125
  PROVIDER_PREFIX_RE = re.compile(r"^[^:\s]{1,30}:\s*")
111
126
 
127
+ # Quantization precision ladder: label -> rank, highest fidelity first. The
128
+ # ladder ordering in the design doc is authoritative: fp6 > int8 is strict,
129
+ # and formats within one fidelity tier share the same rank. "unknown" never
130
+ # enters the ladder -- it only routes through the first-party eligibility
131
+ # carve-out, never through rank comparison.
132
+ PRECISION_LADDER: dict[str, int] = {
133
+ "fp32": 4,
134
+ "bf16": 3,
135
+ "fp16": 3,
136
+ "fp8": 2,
137
+ "mxfp8": 2,
138
+ "fp6": 1,
139
+ "int8": 0,
140
+ "fp4": -1,
141
+ "mxfp4": -1,
142
+ "nvfp4": -1,
143
+ "int4": -2,
144
+ }
145
+
146
+ # Every quantization value an endpoint may report: the ladder keys plus the
147
+ # "unknown" sentinel. Whether a recorded value is recognized at all is
148
+ # membership here, not ladder rank.
149
+ QUANTIZATION_ENUM: frozenset[str] = frozenset(PRECISION_LADDER) | {"unknown"}
150
+
151
+ # Official partner hosts that serve closed labs' models: an endpoint on one
152
+ # of these providers counts as first-party for the eligibility carve-out
153
+ # even though it is not the maker itself. Entries are compared after
154
+ # normalize_provider (casefold + strip non-alphanumerics), so the slugs here
155
+ # are display forms -- "aws-bedrock" and "AWS Bedrock" both hit. Extend only
156
+ # with live-data evidence (e.g. an endpoint tag matching <lab>-official).
157
+ FIRST_PARTY_HOSTS = frozenset(
158
+ {"bedrock", "azure", "vertex", "azure-ai", "google-vertex", "aws-bedrock"}
159
+ )
160
+
161
+ # Explicit first-party alias pairs that normalization alone cannot bridge
162
+ # (design §4.2): makers that self-name differently than their OpenRouter
163
+ # author slug. Key = the exact live provider_name string, value = the
164
+ # author slug the pair is first-party FOR; both sides are compared
165
+ # normalized (case/punctuation variants match), and the alias only fires
166
+ # when the model's author equals the value. Verified against live data
167
+ # 2026-10-20 (task-13-fix-report.md section 8b; endpoints fetched fresh,
168
+ # every endpoint quantization "unknown", status 0):
169
+ # ("Amazon Bedrock", "amazon") -- amazon/nova-2-lite-v1, 2 endpoints
170
+ # ("Google AI Studio", "google") -- google/gemini-2.5-pro-preview, 3 endpoints
171
+ # ("Mistral", "mistralai") -- mistralai/mistral-large-4-0, 3 endpoints
172
+ # Add entries only after the same live verification -- never pin an
173
+ # unverified string.
174
+ FIRST_PARTY_ALIASES: dict[str, str] = {
175
+ "Amazon Bedrock": "amazon",
176
+ "Google AI Studio": "google",
177
+ "Mistral": "mistralai",
178
+ }
179
+
112
180
 
113
181
  def warn(msg: str) -> None:
114
182
  print(f"warning: {msg}", file=sys.stderr)
@@ -118,6 +186,44 @@ def norm_key(s: str) -> str:
118
186
  return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
119
187
 
120
188
 
189
+ # ---------------------------------------------------------------------------
190
+ # Provider normalization + first-party detection
191
+ # ---------------------------------------------------------------------------
192
+
193
+
194
+ def normalize_provider(name: str) -> str:
195
+ """Casefold a provider name and strip ALL non-alphanumeric characters.
196
+
197
+ "Z.AI", "z-ai" and "z ai" all normalize to "zai", so a live-recorded
198
+ maker name matches its author slug regardless of punctuation or case.
199
+ None or empty input normalizes to "".
200
+ """
201
+ return re.sub(r"[^a-z0-9]+", "", (name or "").casefold())
202
+
203
+
204
+ def is_first_party(provider_name: str, author: str) -> bool:
205
+ """True iff the provider is the model's own maker or a partner host.
206
+
207
+ Criterion 1: the normalized provider name equals the normalized author
208
+ slug. Criterion 2: it matches a normalized FIRST_PARTY_HOSTS entry, or
209
+ a FIRST_PARTY_ALIASES key whose value equals the normalized author slug
210
+ (the §4.2 alias pairs: a maker self-naming differently than its author
211
+ slug is first-party only for that author). Everything else is
212
+ third-party.
213
+ """
214
+ provider = normalize_provider(provider_name)
215
+ if provider == normalize_provider(author):
216
+ return True
217
+ if provider in {normalize_provider(host) for host in FIRST_PARTY_HOSTS}:
218
+ return True
219
+ author_slug = normalize_provider(author)
220
+ return any(
221
+ provider == normalize_provider(alias_provider)
222
+ and author_slug == normalize_provider(alias_author)
223
+ for alias_provider, alias_author in FIRST_PARTY_ALIASES.items()
224
+ )
225
+
226
+
121
227
  # ---------------------------------------------------------------------------
122
228
  # HTTP + cache
123
229
  # ---------------------------------------------------------------------------
@@ -224,6 +330,872 @@ def save_cache(name: str, payload) -> None:
224
330
  warn(f"could not write cache: {exc}")
225
331
 
226
332
 
333
+ # ---------------------------------------------------------------------------
334
+ # OpenRouter endpoints (per-model provider list)
335
+ # ---------------------------------------------------------------------------
336
+
337
+
338
+ OPENROUTER_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{id}/endpoints"
339
+
340
+
341
+ def load_cache_expired(name: str) -> dict | None:
342
+ """Return the cached payload regardless of age; None when absent/corrupt.
343
+
344
+ load_cache drops payloads past their TTL; this variant serves them anyway
345
+ so a dropped model's last known endpoint annotations can still be shown
346
+ (marked stale by the caller). Never used for ranking.
347
+ """
348
+ path = cache_path(name)
349
+ if not path:
350
+ return None
351
+ try:
352
+ with open(path) as fh:
353
+ blob = json.load(fh)
354
+ return blob["payload"]
355
+ except (OSError, ValueError, KeyError, TypeError):
356
+ return None
357
+
358
+
359
+ def parse_endpoints(payload) -> list[dict]:
360
+ """Keep the verified per-endpoint fields of an endpoints response.
361
+
362
+ The live GET /api/v1/models/{id}/endpoints response nests the list one
363
+ level below the usual envelope: `data` is the model object (id, name,
364
+ architecture, ...) and the per-provider entries live in
365
+ `data["endpoints"]`. `pricing` keeps prompt/completion (parsed through
366
+ parse_price) and the raw `overrides` list -- the tiered-pricing
367
+ machinery parses the entries itself. Fields beyond the verified
368
+ per-entry shape are dropped, and the parse keeps `status` as-is (the
369
+ caller filters).
370
+ """
371
+ # Task 13 lesson: the first unit fixtures fed the models/find shape
372
+ # ({"data": [entry, ...]}) -- a list directly under "data" -- so the
373
+ # suite stayed green while every live parse returned [] and the whole
374
+ # pool died as no_baseline_eligible_endpoint. Pin the live envelope:
375
+ # the list is read ONLY from data["endpoints"], and every other shape
376
+ # parses to [] (no compat branch for the wrong fixture shape).
377
+ data = payload.get("data") if isinstance(payload, dict) else None
378
+ entries = data.get("endpoints") if isinstance(data, dict) else None
379
+ endpoints = []
380
+ for entry in entries or []:
381
+ if not isinstance(entry, dict):
382
+ continue
383
+ raw = entry.get("pricing")
384
+ raw_pricing = raw if isinstance(raw, dict) else {}
385
+ endpoints.append(
386
+ {
387
+ "provider_name": entry.get("provider_name"),
388
+ "tag": entry.get("tag"),
389
+ "quantization": entry.get("quantization"),
390
+ "pricing": {
391
+ "prompt": parse_price(raw_pricing.get("prompt")),
392
+ "completion": parse_price(raw_pricing.get("completion")),
393
+ "overrides": raw_pricing.get("overrides"),
394
+ },
395
+ "context_length": entry.get("context_length"),
396
+ "status": entry.get("status"),
397
+ "service_tier": entry.get("service_tier"),
398
+ }
399
+ )
400
+ return endpoints
401
+
402
+
403
+ def fetch_model_endpoints(model_id: str, args) -> list[dict]:
404
+ """Fetch + parse one model's endpoint list, cached per model.
405
+
406
+ The cache name flattens the model id ("/" -> "_") to keep the cache
407
+ files flat. Fetches retry 3 times (4 attempts total) via fetch_json,
408
+ which raises at once on 4xx and after the retries on 5xx/network
409
+ errors -- the caller owns the fail-closed policy.
410
+ """
411
+ cache_name = f"endpoints:{model_id.replace('/', '_')}"
412
+ if not args.no_cache:
413
+ cached = load_cache(cache_name, args.cache_ttl)
414
+ if cached:
415
+ return cached
416
+ payload = fetch_json(
417
+ OPENROUTER_ENDPOINTS_URL.format(id=model_id), timeout=30, retries=3
418
+ )
419
+ endpoints = parse_endpoints(payload)
420
+ save_cache(cache_name, endpoints)
421
+ return endpoints
422
+
423
+
424
+ def is_variant_endpoint(endpoint) -> bool:
425
+ """True iff the endpoint is a :free/:batch variant or entirely free.
426
+
427
+ Variant iff its slug or tag carries a `:free`/`:batch` marker OR both
428
+ input and output prices are zero. `:batch` is cheaper-but-restricted,
429
+ not zero-priced -- only the marker test catches it. Missing or
430
+ unparseable prices are not zero.
431
+ """
432
+ for field in ("slug", "tag"):
433
+ marker = endpoint.get(field)
434
+ if isinstance(marker, str) and (":free" in marker or ":batch" in marker):
435
+ return True
436
+ pricing = endpoint.get("pricing")
437
+ prices = pricing if isinstance(pricing, dict) else {}
438
+ return (
439
+ parse_price(prices.get("prompt")) == 0.0
440
+ and parse_price(prices.get("completion")) == 0.0
441
+ )
442
+
443
+
444
+ def endpoint_schedule_ok(endpoint, args) -> bool:
445
+ """False iff the endpoint's pricing.overrides fails the v2 schedule rule.
446
+
447
+ Reuses effective_pricing in full: the gate fires on malformed windows,
448
+ exactly-one-of utc_start/utc_end pairs, bad utc_days, combined
449
+ token+time entries and missing inheritance -- not just non-tiling
450
+ windows. An endpoint without overrides has no schedule to fail.
451
+ """
452
+ result = effective_pricing(
453
+ endpoint.get("pricing"), args.min_context, args.input_share
454
+ )
455
+ return not result["schedule_error"]
456
+
457
+
458
+ # Schedule gate for args-less callers: effective_pricing's schedule_error
459
+ # depends only on the SHAPE of pricing.overrides (malformed windows, bad
460
+ # utc_days, ...), never on min_context/input_share -- those only affect the
461
+ # resolved prices. derive_default_precision has no args parameter (pinned
462
+ # interface), so it gates through this fixed stand-in; any args would give
463
+ # the same schedule_error verdict.
464
+ _SCHEDULE_GATE_ARGS = argparse.Namespace(min_context=0, input_share=0.75)
465
+
466
+
467
+ def _baseline_universe(endpoints: list[dict], args=None) -> list[dict]:
468
+ """The shared derivation universe (design section 4.1).
469
+
470
+ Live (status == 0), not a variant endpoint (:free/:batch marker or
471
+ zero-priced), and passing the schedule gate. Schedule-broken endpoints
472
+ must not pin the default that governs healthy endpoints, same rationale
473
+ as the variant exclusion. args defaults to the structural stand-in
474
+ (see _SCHEDULE_GATE_ARGS); callers with real args pass them.
475
+ """
476
+ if args is None:
477
+ args = _SCHEDULE_GATE_ARGS
478
+ return [
479
+ endpoint
480
+ for endpoint in endpoints
481
+ if endpoint.get("status") == 0
482
+ and not is_variant_endpoint(endpoint)
483
+ and endpoint_schedule_ok(endpoint, args)
484
+ ]
485
+
486
+
487
+ def derive_default_precision(endpoints: list[dict], author: str) -> str | None:
488
+ """Derive a model's default precision from its endpoint list (section 4.1).
489
+
490
+ Universe: live non-variant schedule-passing endpoints (_baseline_universe).
491
+ Primary rule: the highest-ranked KNOWN quantization among the maker's
492
+ endpoints -- matched by normalized author-slug equality only, so partner
493
+ hosts never pin the maker's default (section 4.2 criterion 2 does not
494
+ apply here). Fallback: when the maker has no known quantization, the
495
+ highest-ranked known quantization across the FULL universe (partners
496
+ included -- best-available). None when no endpoint in the universe has a
497
+ known quantization (the all-unknown closed-model case): eligibility then
498
+ relies solely on the first-party carve-out.
499
+
500
+ Known means a PRECISION_LADDER key: "unknown" and unparseable/absent
501
+ values never enter the ladder. An endpoint whose provider_name is
502
+ absent/empty (or normalizes to empty) is skipped for maker matching
503
+ (never first-party -- fail-closed), but still feeds the fallback
504
+ universe.
505
+ """
506
+ universe = _baseline_universe(endpoints, _SCHEDULE_GATE_ARGS)
507
+ maker = normalize_provider(author)
508
+ maker_best = None
509
+ fallback_best = None
510
+ for endpoint in universe:
511
+ quantization = endpoint.get("quantization")
512
+ if not isinstance(quantization, str) or quantization not in PRECISION_LADDER:
513
+ continue # "unknown", null, unrecognized: no rank, no pinning
514
+ rank = PRECISION_LADDER[quantization]
515
+ if fallback_best is None or rank > fallback_best[0]:
516
+ fallback_best = (rank, quantization)
517
+ # Maker matching only: a provider_name that normalizes to nothing
518
+ # must never match an (equally empty) author slug.
519
+ provider_slug = (
520
+ normalize_provider(endpoint.get("provider_name"))
521
+ if isinstance(endpoint.get("provider_name"), str)
522
+ else ""
523
+ )
524
+ if (
525
+ provider_slug
526
+ and provider_slug == maker
527
+ and (maker_best is None or rank > maker_best[0])
528
+ ):
529
+ maker_best = (rank, quantization)
530
+ return (maker_best or fallback_best or (None, None))[1]
531
+
532
+
533
+ def baseline_eligible(endpoint, default_precision, author, args) -> bool:
534
+ """Baseline-eligibility rule (design section 4.3, fail-closed).
535
+
536
+ True iff the endpoint is live (status == 0), not a variant, passes the
537
+ schedule gate, and then either:
538
+
539
+ * its quantization is a known ladder label with rank >=
540
+ rank(default_precision) -- provider-blind (default_routing considers
541
+ every endpoint OpenRouter may send traffic to), or
542
+ * its quantization is "unknown" or null AND the endpoint is first-party
543
+ (the D3 carve-out: the maker or a partner host).
544
+
545
+ When default_precision is None (all-unknown model) the rank comparison is
546
+ meaningless -- only the carve-out applies. A provider_name that is
547
+ absent/empty (or normalizes to empty) is never first-party, so such an
548
+ endpoint can only qualify through the rank branch. An unrecognized
549
+ quantization string (neither a ladder label nor "unknown"/null) is a
550
+ schema anomaly: no rank comparison and no carve-out -- fail-closed
551
+ (fleet-wide new-label drift is what the publish drop-rate gate exists
552
+ to catch).
553
+ """
554
+ if (
555
+ endpoint.get("status") != 0
556
+ or is_variant_endpoint(endpoint)
557
+ or not endpoint_schedule_ok(endpoint, args)
558
+ ):
559
+ return False
560
+ quantization = endpoint.get("quantization")
561
+ if isinstance(quantization, str) and quantization in PRECISION_LADDER:
562
+ # Known quantization: the rank branch, provider-blind.
563
+ if default_precision is None:
564
+ return False
565
+ return PRECISION_LADDER[quantization] >= PRECISION_LADDER[default_precision]
566
+ # Unverifiable: the carve-out applies to "unknown" and null only --
567
+ # an unrecognized string fails closed. A provider_name that normalizes
568
+ # to nothing is never first-party (fail-closed; the literal
569
+ # is_first_party("", "") iff is guarded out here).
570
+ provider_name = endpoint.get("provider_name")
571
+ provider_slug = (
572
+ normalize_provider(provider_name) if isinstance(provider_name, str) else ""
573
+ )
574
+ return (
575
+ (quantization is None or quantization == "unknown")
576
+ and bool(provider_slug)
577
+ and is_first_party(provider_name, author)
578
+ )
579
+
580
+
581
+ def pick_baseline(
582
+ endpoints: list[dict], eligible_flags: list[bool], args
583
+ ) -> dict | None:
584
+ """Pick the baseline endpoint among the eligible ones (design section 4.5).
585
+
586
+ Cheapest blended per-token price wins, computed from effective_pricing
587
+ with the args' min_context and input_share (the shared --input-share
588
+ convention). Ties broken by (normalized provider name, tag), then --
589
+ still tied -- first in list order (deterministic; Python's sort is
590
+ stable and the endpoints list order is the API's). Returns None when no
591
+ eligible endpoint has a usable price (the caller owns the drop).
592
+ """
593
+ candidates = []
594
+ for endpoint, eligible in zip(endpoints, eligible_flags):
595
+ if not eligible:
596
+ continue
597
+ eff = effective_pricing(
598
+ endpoint.get("pricing"), args.min_context, args.input_share
599
+ )
600
+ price_in, price_out = eff["price_in"], eff["price_out"]
601
+ if (
602
+ price_in is None
603
+ or price_out is None
604
+ or not math.isfinite(price_in)
605
+ or not math.isfinite(price_out)
606
+ ):
607
+ continue # unpriceable: no executable blended price
608
+ blended = args.input_share * price_in + (1 - args.input_share) * price_out
609
+ provider_name = endpoint.get("provider_name")
610
+ candidates.append(
611
+ (
612
+ blended,
613
+ normalize_provider(provider_name)
614
+ if isinstance(provider_name, str)
615
+ else "",
616
+ endpoint.get("tag") or "",
617
+ endpoint,
618
+ )
619
+ )
620
+ if not candidates:
621
+ return None
622
+ return min(candidates, key=lambda candidate: candidate[:3])[3]
623
+
624
+
625
+ def _blended_price(endpoint, args) -> float | None:
626
+ """Blended per-token price of one endpoint (pick_baseline's price rule).
627
+
628
+ effective_pricing with the args' min_context and input_share, then
629
+ input_share * price_in + (1 - input_share) * price_out. None when either
630
+ resolved price is missing or non-finite -- no executable blended price,
631
+ the same skip pick_baseline applies. A zero result is returned as-is:
632
+ zero-priced endpoints are variants (already excluded upstream), so the
633
+ callers apply their own zero guards defensively.
634
+ """
635
+ eff = effective_pricing(endpoint.get("pricing"), args.min_context, args.input_share)
636
+ price_in, price_out = eff["price_in"], eff["price_out"]
637
+ if (
638
+ price_in is None
639
+ or price_out is None
640
+ or not math.isfinite(price_in)
641
+ or not math.isfinite(price_out)
642
+ ):
643
+ return None
644
+ blended = args.input_share * price_in + (1 - args.input_share) * price_out
645
+ return blended if math.isfinite(blended) else None
646
+
647
+
648
+ def _flag_list(endpoints: list[dict], eligible_flags: list[bool]):
649
+ """Zip endpoints with their parallel eligibility flags, hardened.
650
+
651
+ eligible_flags is a parallel list to endpoints (the baseline_eligible
652
+ verdict per endpoint, same order -- Task 4 contract). A length mismatch
653
+ must raise, never truncate silently (Task 4 review ruling).
654
+ """
655
+ if len(endpoints) != len(eligible_flags):
656
+ raise ValueError(
657
+ "eligible_flags must be a parallel list to endpoints: "
658
+ f"{len(endpoints)} endpoints, {len(eligible_flags)} flags"
659
+ )
660
+ return zip(endpoints, eligible_flags, strict=True)
661
+
662
+
663
+ def NON_ELIGIBLE_UNIVERSE(endpoints: list[dict], eligible_flags: list[bool]) -> list:
664
+ """The shared annotations universe (design section 4.6).
665
+
666
+ Live (status == 0), non-variant endpoints that are NOT baseline-eligible:
667
+ sub-default known quantization, third-party unknown/null quantization, or
668
+ schedule-gate-failed. All annotations derive from this one universe.
669
+ Eligibility comes from the parallel eligible_flags list (the
670
+ baseline_eligible verdicts); variants and non-live endpoints are excluded
671
+ even though they too are not eligible -- unlike the steering `ignore`
672
+ list, which includes live variants (section 4.7).
673
+
674
+ Raises ValueError when eligible_flags does not match endpoints in length.
675
+ """
676
+ return [
677
+ endpoint
678
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags)
679
+ if endpoint.get("status") == 0
680
+ and not is_variant_endpoint(endpoint)
681
+ and not eligible
682
+ ]
683
+
684
+
685
+ def build_annotations(
686
+ endpoints: list[dict],
687
+ eligible_flags: list[bool],
688
+ baseline,
689
+ catalog_context,
690
+ args,
691
+ ) -> dict:
692
+ """Per-model endpoint annotations (design section 4.6).
693
+
694
+ All four annotations derive from one shared universe -- the live
695
+ non-variant endpoints that are not baseline-eligible
696
+ (NON_ELIGIBLE_UNIVERSE):
697
+
698
+ * price_spread: cheapest non-eligible blended price divided by the
699
+ cheapest eligible blended price, both blended with the args'
700
+ min_context and input_share (the shared --input-share convention).
701
+ None when the universe is empty -- generalized to None when either
702
+ side has no priceable member at all -- and None when either cheapest
703
+ price is 0 (zero-priced endpoints are variants by definition, so the
704
+ zero half of the guard is defensive only). Unpriceable endpoints are
705
+ skipped when looking for a cheapest price.
706
+ * cheapest_quantized: {provider, quantization, price} of the cheapest
707
+ member of the quantization-caused subset of the universe -- endpoints
708
+ non-eligible due to quantization only (sub-default known quantization
709
+ or third-party unknown/null), NOT schedule-gate-failed ones, which are
710
+ excluded even when cheaper. Unpriceable subset members are skipped
711
+ (defensively also non-positive ones, matching the repo-wide treatment
712
+ of negative prices as malformed); None when the subset has no
713
+ priceable member. Price ties keep the first in list order
714
+ (deterministic).
715
+ * risk_flag: True iff the sum of 1/blended**2 over the universe strictly
716
+ exceeds the sum over the baseline-eligible endpoints (exact tie ->
717
+ False). Any endpoint whose blended price is 0/None (defensively also
718
+ negative) is skipped from BOTH sums -- variants are already excluded,
719
+ so this only guards malformed data, and the flag stays deterministic.
720
+ * baseline_context_below_catalog: True iff the baseline endpoint's
721
+ context_length is smaller than the catalog context (the models-API
722
+ value). A None baseline -- or missing context on either side -- yields
723
+ False: the caveat marks a positive finding, never missing data.
724
+
725
+ Raises ValueError when eligible_flags does not match endpoints in length.
726
+ """
727
+ universe = NON_ELIGIBLE_UNIVERSE(endpoints, eligible_flags)
728
+
729
+ universe_prices = []
730
+ for endpoint in universe:
731
+ price = _blended_price(endpoint, args)
732
+ if price is not None:
733
+ universe_prices.append(price)
734
+ eligible_prices = []
735
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
736
+ if not eligible:
737
+ continue
738
+ price = _blended_price(endpoint, args)
739
+ if price is not None:
740
+ eligible_prices.append(price)
741
+ cheapest_non_eligible = min(universe_prices) if universe_prices else None
742
+ cheapest_eligible = min(eligible_prices) if eligible_prices else None
743
+ price_spread = (
744
+ cheapest_non_eligible / cheapest_eligible
745
+ if cheapest_non_eligible is not None
746
+ and cheapest_eligible is not None
747
+ and cheapest_non_eligible > 0
748
+ and cheapest_eligible > 0
749
+ else None
750
+ )
751
+
752
+ quantized_best = None
753
+ for endpoint in universe:
754
+ if not endpoint_schedule_ok(endpoint, args):
755
+ continue # schedule-gate-failed: not a quantization-caused exclusion
756
+ price = _blended_price(endpoint, args)
757
+ if price is None or price <= 0:
758
+ continue # unpriceable/non-positive: no comparable price
759
+ if quantized_best is None or price < quantized_best[0]:
760
+ quantized_best = (price, endpoint) # ties keep the first in list order
761
+ if quantized_best is None:
762
+ cheapest_quantized = None
763
+ else:
764
+ price, endpoint = quantized_best
765
+ cheapest_quantized = {
766
+ "provider": endpoint.get("provider_name"),
767
+ "quantization": endpoint.get("quantization"),
768
+ "price": price,
769
+ }
770
+
771
+ non_eligible_weight = 0.0
772
+ for endpoint in universe:
773
+ price = _blended_price(endpoint, args)
774
+ if price is None or price <= 0:
775
+ continue # skipped from BOTH sums, deterministically
776
+ non_eligible_weight += 1.0 / (price * price)
777
+ eligible_weight = 0.0
778
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
779
+ if not eligible:
780
+ continue
781
+ price = _blended_price(endpoint, args)
782
+ if price is None or price <= 0:
783
+ continue
784
+ eligible_weight += 1.0 / (price * price)
785
+ risk_flag = non_eligible_weight > eligible_weight
786
+
787
+ baseline_context = baseline.get("context_length") if baseline is not None else None
788
+ baseline_context_below_catalog = (
789
+ baseline_context is not None
790
+ and catalog_context is not None
791
+ and baseline_context < catalog_context
792
+ )
793
+
794
+ return {
795
+ "price_spread": price_spread,
796
+ "cheapest_quantized": cheapest_quantized,
797
+ "risk_flag": risk_flag,
798
+ "baseline_context_below_catalog": baseline_context_below_catalog,
799
+ }
800
+
801
+
802
+ def build_steering(
803
+ model_id: str,
804
+ endpoints: list[dict],
805
+ eligible_flags: list[bool],
806
+ default_precision: str | None,
807
+ args,
808
+ ) -> dict:
809
+ """OpenRouter provider steering object (design section 4.7).
810
+
811
+ {"model": model_id, "provider": {"order": [...], "ignore": [...],
812
+ "quantizations": [...], "allow_fallbacks": True}}.
813
+
814
+ * order: tags of the baseline-eligible endpoints, cheapest blended first
815
+ (blended price asc, ties broken by (normalized provider name, tag) --
816
+ the same comparator as pick_baseline, stable so the final tie is list
817
+ order). Eligible endpoints whose blended price is unpriceable cannot
818
+ be price-ranked; they are appended after the ranked ones (still
819
+ (provider, tag)-ordered) so every eligible endpoint keeps a place --
820
+ ranked last, never silently left to default priority.
821
+ * ignore: tags of EVERY live endpoint that is not baseline-eligible --
822
+ sub-default quantization, third-party unknown/null, schedule-gate-
823
+ failed AND variants (variants are live but not eligible; the universal
824
+ quantifier includes them). Non-live (status != 0) endpoints are never
825
+ listed. List order (the API's).
826
+ * quantizations: sorted KNOWN quantizations (PRECISION_LADDER labels)
827
+ with rank >= rank(default_precision) among live non-variant endpoints,
828
+ deduplicated -- no schedule or eligibility filter. OMITTED from the
829
+ provider dict entirely when default_precision is None -- never an
830
+ empty list (OpenRouter semantics would filter out every endpoint).
831
+ * allow_fallbacks: True.
832
+
833
+ Signature deviation from the plan (controller ruling at Task 5 spawn):
834
+ args is a required parameter -- the order ranking is real pricing math
835
+ (effective_pricing via min_context/input_share), and reusing the
836
+ shape-only _SCHEDULE_GATE_ARGS stand-in for pricing was explicitly
837
+ ruled out in the Task 4 review.
838
+
839
+ Raises ValueError when eligible_flags does not match endpoints in length.
840
+ """
841
+ ignore = []
842
+ ranked = [] # (blended, normalized provider name, tag) -- pick_baseline order
843
+ unranked = [] # (normalized provider name, tag) -- eligible but unpriceable
844
+ for endpoint, eligible in _flag_list(endpoints, eligible_flags):
845
+ if endpoint.get("status") == 0 and not eligible:
846
+ ignore.append(endpoint.get("tag") or "")
847
+ if not eligible:
848
+ continue
849
+ provider_name = endpoint.get("provider_name")
850
+ provider_key = (
851
+ normalize_provider(provider_name) if isinstance(provider_name, str) else ""
852
+ )
853
+ tag = endpoint.get("tag") or ""
854
+ price = _blended_price(endpoint, args)
855
+ if price is None:
856
+ unranked.append((provider_key, tag))
857
+ else:
858
+ ranked.append((price, provider_key, tag))
859
+ ranked.sort() # stable: the final tie is list order, as in pick_baseline
860
+ order = [tag for _, _, tag in ranked] + [tag for _, tag in sorted(unranked)]
861
+
862
+ provider = {"order": order, "ignore": ignore}
863
+ if default_precision is not None:
864
+ default_rank = PRECISION_LADDER[default_precision]
865
+ quants = set()
866
+ for endpoint in endpoints:
867
+ if endpoint.get("status") != 0 or is_variant_endpoint(endpoint):
868
+ continue
869
+ quantization = endpoint.get("quantization")
870
+ if (
871
+ not isinstance(quantization, str)
872
+ or quantization not in PRECISION_LADDER
873
+ ):
874
+ continue # "unknown"/null/unrecognized: not a known quantization
875
+ if PRECISION_LADDER[quantization] >= default_rank:
876
+ quants.add(quantization)
877
+ provider["quantizations"] = sorted(quants)
878
+ provider["allow_fallbacks"] = True
879
+ return {"model": model_id, "provider": provider}
880
+
881
+
882
+ def _stale_filtered_annotations(model_id, catalog_context, args):
883
+ """Annotations for a dropped model's filtered entry, or None.
884
+
885
+ Built from the model's last cached endpoint list (load_cache_expired --
886
+ TTL ignored), never from a fresh fetch: the model is out of the ranking,
887
+ so the block is marked "stale": True for display in the filtered section
888
+ only (design section 4.9). Same builders as the live stage: eligibility
889
+ and baseline are re-derived from the cached list, so the shape matches a
890
+ ranked candidate's annotations. A model whose cache holds no endpoint
891
+ list at all yields None (the entry carries the drop reason alone).
892
+ """
893
+ cached = load_cache_expired(f"endpoints:{model_id.replace('/', '_')}")
894
+ if not isinstance(cached, list):
895
+ return None
896
+ author = model_id.partition("/")[0]
897
+ try:
898
+ default_precision = derive_default_precision(cached, author)
899
+ eligible_flags = [
900
+ baseline_eligible(endpoint, default_precision, author, args)
901
+ for endpoint in cached
902
+ ]
903
+ baseline = pick_baseline(cached, eligible_flags, args)
904
+ annotations = build_annotations(
905
+ cached, eligible_flags, baseline, catalog_context, args
906
+ )
907
+ except Exception:
908
+ # Display-only block for an already-dropped model: never fail the run.
909
+ return None
910
+ annotations["stale"] = True
911
+ return annotations
912
+
913
+
914
+ def _endpoint_stage_drop(extra_dropped, filtered, cand, reason, args):
915
+ """Count one endpoint-stage drop and append its filtered entry.
916
+
917
+ Same validated entry shape as build_candidates' drop() plus the optional
918
+ stale annotations block (absent cache -> reasons only).
919
+ """
920
+ extra_dropped[reason] = extra_dropped.get(reason, 0) + 1
921
+ entry = {"id": cand["id"], "name": cand["name"] or cand["id"], "reasons": [reason]}
922
+ annotations = _stale_filtered_annotations(cand["id"], cand.get("context"), args)
923
+ if annotations is not None:
924
+ entry["annotations"] = annotations
925
+ filtered.append(entry)
926
+
927
+
928
+ def apply_endpoint_pricing(candidates, args, filtered):
929
+ """Reprice every candidate at its baseline endpoint (design section 4.9).
930
+
931
+ The CLI stage between build_candidates and resolve_quality. Endpoint
932
+ fetches run concurrently (ThreadPoolExecutor, 8 workers) through
933
+ fetch_model_endpoints' per-model cache, but the results are applied PER
934
+ CANDIDATE ID in the original candidates order -- never in completion
935
+ order -- so the stage is deterministic.
936
+
937
+ Per candidate: endpoints = fetch. On fetch failure after fetch_json's
938
+ internal retries the model is dropped "endpoint_fetch_failed" and removed
939
+ from the returned list. With endpoints, default_precision is derived,
940
+ every endpoint scored for baseline eligibility and the baseline picked;
941
+ no usable baseline drops the model "no_baseline_eligible_endpoint"
942
+ (the stale-annotations filtered-entry rule covers both drop reasons).
943
+ Otherwise the candidate is mutated in place: the pricing block
944
+ (price_in/price_out/blended/base_*/tier/schedule/notes) is replaced with
945
+ the baseline endpoint's effective_pricing outputs scaled per-1M exactly
946
+ like build_candidates, and the additive keys "endpoints", "default_precision",
947
+ "annotations" (catalog context = the candidate's existing "context"),
948
+ "advertised_pricing" (the models-API scalars the candidate carried before
949
+ the mutation, source-tagged) and "steering" are attached.
950
+
951
+ A baseline whose prices are missing/non-finite counts as unusable
952
+ (pick_baseline's blended-price skip is authoritative for the top-level
953
+ effective prices). The same bar applies to the base prices the mutation
954
+ would publish: parse_price returns float | None, so a None base price
955
+ (tier-over-None-base endpoint) is reachable -- the isinstance(..., float)
956
+ half of the guard is load-bearing for None, not a redundant type check.
957
+ Returns (surviving candidates in the original order, dropped counts for
958
+ the two new reasons).
959
+ """
960
+ extra_dropped: dict[str, int] = {}
961
+ if not candidates:
962
+ return candidates, extra_dropped
963
+ endpoints_by_id: dict[str, list] = {}
964
+ failed_ids: set[str] = set()
965
+ with ThreadPoolExecutor(max_workers=8) as pool:
966
+ futures = {}
967
+ for cand in candidates:
968
+ # Duplicate ids fetch once and share the result.
969
+ futures.setdefault(
970
+ cand["id"], pool.submit(fetch_model_endpoints, cand["id"], args)
971
+ )
972
+ for cand_id, future in futures.items():
973
+ try:
974
+ endpoints_by_id[cand_id] = future.result()
975
+ except Exception:
976
+ # fetch_json already retried 3 times; the caller owns the
977
+ # fail-closed policy.
978
+ failed_ids.add(cand_id)
979
+ survivors = []
980
+ for cand in candidates:
981
+ cand_id = cand["id"]
982
+ if cand_id in failed_ids:
983
+ _endpoint_stage_drop(
984
+ extra_dropped, filtered, cand, "endpoint_fetch_failed", args
985
+ )
986
+ continue
987
+ endpoints = endpoints_by_id[cand_id]
988
+ author = cand_id.partition("/")[0]
989
+ default_precision = derive_default_precision(endpoints, author)
990
+ eligible_flags = [
991
+ baseline_eligible(endpoint, default_precision, author, args)
992
+ for endpoint in endpoints
993
+ ]
994
+ baseline = pick_baseline(endpoints, eligible_flags, args)
995
+ eff = (
996
+ effective_pricing(
997
+ baseline.get("pricing"), args.min_context, args.input_share
998
+ )
999
+ if baseline is not None
1000
+ else None
1001
+ )
1002
+ if baseline is None or not all(
1003
+ isinstance(eff[key], float) and math.isfinite(eff[key])
1004
+ for key in ("price_in", "price_out", "base_price_in", "base_price_out")
1005
+ ):
1006
+ _endpoint_stage_drop(
1007
+ extra_dropped, filtered, cand, "no_baseline_eligible_endpoint", args
1008
+ )
1009
+ continue
1010
+ # The advertised models-API scalars, captured before the mutation.
1011
+ advertised_pricing = {
1012
+ "input_per_1m": cand["price_in"],
1013
+ "output_per_1m": cand["price_out"],
1014
+ "blended_per_1m": cand["blended"],
1015
+ "source": "models-api",
1016
+ }
1017
+ # Replace the pricing block with the baseline endpoint's outputs,
1018
+ # scaled per-1M exactly like build_candidates.
1019
+ cand["price_in"] = eff["price_in"] * 1_000_000.0
1020
+ cand["price_out"] = eff["price_out"] * 1_000_000.0
1021
+ cand["blended"] = (
1022
+ args.input_share * cand["price_in"]
1023
+ + (1.0 - args.input_share) * cand["price_out"]
1024
+ )
1025
+ cand["base_price_in"] = eff["base_price_in"] * 1_000_000.0
1026
+ cand["base_price_out"] = eff["base_price_out"] * 1_000_000.0
1027
+ cand["tier_prompt_tokens"] = eff["tier_prompt_tokens"]
1028
+ cand["tiers"] = eff["tiers"]
1029
+ cand["schedule"] = eff["schedule"]
1030
+ cand["max_discount"] = eff["max_discount"]
1031
+ cand["tier_note"] = fmt_tier_note(eff["tier_prompt_tokens"])
1032
+ cand["sched_note"] = fmt_sched_note(eff["max_discount"])
1033
+ cand["sched_detail"] = fmt_sched_detail(eff["schedule"], eff["peak_blended"])
1034
+ cand["endpoints"] = endpoints
1035
+ cand["default_precision"] = default_precision
1036
+ cand["annotations"] = build_annotations(
1037
+ endpoints, eligible_flags, baseline, cand["context"], args
1038
+ )
1039
+ cand["advertised_pricing"] = advertised_pricing
1040
+ cand["steering"] = build_steering(
1041
+ cand_id, endpoints, eligible_flags, default_precision, args
1042
+ )
1043
+ survivors.append(cand)
1044
+ return survivors, extra_dropped
1045
+
1046
+
1047
+ # ---------------------------------------------------------------------------
1048
+ # OpenRouter session-cost dataset (per-task cost, outside the score)
1049
+ # ---------------------------------------------------------------------------
1050
+
1051
+
1052
+ SESSION_COST_URL = "https://openrouter.ai/api/v1/datasets/session-cost"
1053
+ SESSION_COST_HARNESS_SLUG = "hermes-agent"
1054
+ # The turn range the catalog/CLI surfaces (Task 8).
1055
+ SESSION_COST_DISPLAY_TURN_RANGE = "10-49-turns"
1056
+ # The dataset's four turn ranges (brainstorm section 2.6) -- the catalog
1057
+ # contract's closed set (web/build_site_data.py pins the same tuple).
1058
+ # Cells naming anything else are dropped: an unknown range is dataset
1059
+ # drift, and the block must never fail the publish (design section 4.8).
1060
+ SESSION_COST_TURN_RANGES = ("1-turn", "2-9-turns", "10-49-turns", "50-plus-turns")
1061
+ SESSION_COST_ATTRIBUTION = "Data © OpenRouter, CC BY 4.0"
1062
+
1063
+
1064
+ def strip_permaslug_date(permaslug: str) -> str:
1065
+ """Permaslug without its trailing -YYYYMMDD (undated ids map to themselves)."""
1066
+ match = re.match(r"^(.*)-(\d{8})$", permaslug)
1067
+ return match.group(1) if match else permaslug
1068
+
1069
+
1070
+ def fetch_session_cost(args) -> dict | None:
1071
+ """Per-task session-cost medians from OpenRouter's dataset, or None.
1072
+
1073
+ A separate published dimension (design section 4.8): the median USD
1074
+ cost of real sessions through the pinned published harness -- never
1075
+ part of the weighted score. One bulk paginated fetch per invocation
1076
+ (?app_slug=<harness>&limit=500&offset=k) regardless of the model list;
1077
+ pagination stops on a page shorter than the limit or once the next
1078
+ offset would exceed the API's 5000 cap. Cells key on dated permaslugs
1079
+ (z-ai/glm-5.3-flash-20260826), stripped to the bare catalog id with
1080
+ strip_permaslug_date; when several strip to the same id, the latest
1081
+ date wins each turn-range cell (same rule as build_aa_benchmarks).
1082
+
1083
+ Returns None (graceful degradation, design section 4.9) when no key is
1084
+ configured (the --openrouter-api-key flag, else OPENROUTER_API_KEY),
1085
+ when the fetch fails after fetch_json's retries, or when the response
1086
+ shape is unexpected -- a malformed payload must never crash the run.
1087
+ The merged result is cached under "session-cost" with the standard TTL.
1088
+ """
1089
+ api_key = args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
1090
+ if not api_key:
1091
+ return None
1092
+ if not args.no_cache:
1093
+ cached = load_cache("session-cost", args.cache_ttl)
1094
+ if isinstance(cached, dict) and isinstance(cached.get("medians"), dict):
1095
+ return cached
1096
+ cells = []
1097
+ # The API caps limit at 500 and offset at 5000 (11 full pages max).
1098
+ offset = 0
1099
+ while offset <= 5000:
1100
+ try:
1101
+ payload = fetch_json(
1102
+ f"{SESSION_COST_URL}?app_slug={SESSION_COST_HARNESS_SLUG}"
1103
+ f"&limit=500&offset={offset}",
1104
+ headers={"Authorization": f"Bearer {api_key}"},
1105
+ timeout=30,
1106
+ retries=3,
1107
+ )
1108
+ except Exception as exc:
1109
+ warn(f"session-cost dataset request failed ({exc})")
1110
+ return None
1111
+ data = payload.get("data") if isinstance(payload, dict) else None
1112
+ if not isinstance(data, list):
1113
+ warn("session-cost dataset returned an unexpected response shape")
1114
+ return None
1115
+ cells.extend(cell for cell in data if isinstance(cell, dict))
1116
+ if len(data) < 500:
1117
+ break
1118
+ offset += 500
1119
+ medians: dict[str, dict[str, float]] = {}
1120
+ seen_date: dict[tuple[str, str], str] = {}
1121
+
1122
+ def accept(bare, date, turn_range, median):
1123
+ key = (bare, turn_range)
1124
+ if key in seen_date and seen_date[key] >= date:
1125
+ return
1126
+ seen_date[key] = date
1127
+ medians.setdefault(bare, {})[turn_range] = median
1128
+
1129
+ for cell in cells:
1130
+ permaslug = cell.get("model_permaslug")
1131
+ turn_range = cell.get("turn_range")
1132
+ raw = cell.get("median_session_cost_usd")
1133
+ if not (
1134
+ isinstance(permaslug, str)
1135
+ and permaslug
1136
+ and isinstance(turn_range, str)
1137
+ and turn_range
1138
+ ):
1139
+ continue
1140
+ # Closed contract set: a cell naming a new turn range is dropped,
1141
+ # never published (the catalog validator rejects unknown ranges,
1142
+ # and session cost must never fail the publish).
1143
+ if turn_range not in SESSION_COST_TURN_RANGES:
1144
+ continue
1145
+ # Non-finite medians would serialize as NaN and poison the catalog.
1146
+ if not isinstance(raw, (int, float)) or isinstance(raw, bool):
1147
+ continue
1148
+ try:
1149
+ median = float(raw) # huge ints raise OverflowError
1150
+ except OverflowError:
1151
+ continue
1152
+ if not math.isfinite(median):
1153
+ continue
1154
+ match = re.match(r"^(.*)-(\d{8})$", permaslug)
1155
+ accept(
1156
+ strip_permaslug_date(permaslug),
1157
+ match.group(2) if match else "",
1158
+ turn_range,
1159
+ median,
1160
+ )
1161
+ if not medians:
1162
+ warn("session-cost dataset returned no usable cells")
1163
+ return None
1164
+ meta = payload.get("meta") if isinstance(payload, dict) else None
1165
+ if not isinstance(meta, dict):
1166
+ warn("session-cost dataset returned an unexpected response shape")
1167
+ return None
1168
+ as_of = meta.get("as_of")
1169
+ window_days = meta.get("window_days")
1170
+ window_end_date = meta.get("window_end_date")
1171
+ # The catalog validator demands non-empty as_of/window_end_date strings
1172
+ # and a positive integer window_days; anything else is dataset drift.
1173
+ # Degrade to None (block omitted, "not available this refresh") rather
1174
+ # than emit a block the validator would reject -- session cost must
1175
+ # never fail the publish (design section 4.8).
1176
+ if not (
1177
+ isinstance(as_of, str)
1178
+ and as_of
1179
+ and isinstance(window_end_date, str)
1180
+ and window_end_date
1181
+ and isinstance(window_days, int)
1182
+ and not isinstance(window_days, bool)
1183
+ and window_days > 0
1184
+ ):
1185
+ warn("session-cost dataset returned malformed meta; omitting the block")
1186
+ return None
1187
+ result = {
1188
+ "attribution": SESSION_COST_ATTRIBUTION,
1189
+ "app_slug": SESSION_COST_HARNESS_SLUG,
1190
+ "as_of": as_of,
1191
+ "window_days": window_days,
1192
+ "window_end_date": window_end_date,
1193
+ "medians": medians,
1194
+ }
1195
+ save_cache("session-cost", result)
1196
+ return result
1197
+
1198
+
227
1199
  # ---------------------------------------------------------------------------
228
1200
  # OpenRouter catalog
229
1201
  # ---------------------------------------------------------------------------
@@ -1413,6 +2385,48 @@ def fmt_discount(value) -> str:
1413
2385
  return f"{value:.0%}" if has_discount(value) else "--"
1414
2386
 
1415
2387
 
2388
+ def fmt_prec(cand) -> str:
2389
+ """PREC cell: the precision the ranked price is quoted at.
2390
+
2391
+ The maker's default precision when disclosed, the best available
2392
+ otherwise, "n/a" when none could be derived; "!" flags default routing
2393
+ that favors cheaper sub-default endpoints (annotations.risk_flag).
2394
+ Candidates that never went through the endpoint stage degrade to "n/a".
2395
+ """
2396
+ cell = cand.get("default_precision") or "n/a"
2397
+ if (cand.get("annotations") or {}).get("risk_flag"):
2398
+ cell += "!"
2399
+ return cell
2400
+
2401
+
2402
+ def fmt_quant_note(annotations: dict) -> str | None:
2403
+ """One-line note on the cheapest sub-default endpoint (PREC tooltip).
2404
+
2405
+ The exact string the site shows (Task 11 renders it verbatim), so the
2406
+ format is pinned by test. None when there is no non-eligible live
2407
+ endpoint -- nothing cheaper to route to. cheapest_quantized.price is
2408
+ stored per-token, the way build_annotations leaves it; the note shows
2409
+ USD per 1M like every other price the CLI prints. A missing spread
2410
+ (defensive) drops the parenthetical. A null quantization (third-party
2411
+ null == "unknown" for eligibility) and a null provider_name on
2412
+ malformed data render the design's "unknown" sentinel, never Python's
2413
+ None repr.
2414
+ """
2415
+ quant = (annotations or {}).get("cheapest_quantized")
2416
+ if not quant:
2417
+ return None
2418
+ quantization = quant["quantization"] or "unknown"
2419
+ provider = quant["provider"] or "unknown"
2420
+ note = (
2421
+ f"cheapest non-baseline: {quantization} @ {provider}, "
2422
+ f"${fmt_price(quant['price'] * 1_000_000)}/M"
2423
+ )
2424
+ spread = (annotations or {}).get("price_spread")
2425
+ if spread is not None:
2426
+ note += f" in ({spread:.2f}x baseline)"
2427
+ return note
2428
+
2429
+
1416
2430
  def print_table(top, total_candidates, weights, quality_note):
1417
2431
  headers = [
1418
2432
  "RANK",
@@ -1421,6 +2435,7 @@ def print_table(top, total_candidates, weights, quality_note):
1421
2435
  "$IN/M",
1422
2436
  "$OUT/M",
1423
2437
  "DISC",
2438
+ "PREC",
1424
2439
  "TIER",
1425
2440
  "CTX",
1426
2441
  "AGE",
@@ -1436,6 +2451,7 @@ def print_table(top, total_candidates, weights, quality_note):
1436
2451
  fmt_price(cand["price_in"]),
1437
2452
  fmt_price(cand["price_out"]),
1438
2453
  fmt_discount(cand["discount"]),
2454
+ fmt_prec(cand),
1439
2455
  cand["tier_note"] or cand["sched_note"] or "--",
1440
2456
  fmt_context(cand["context"]),
1441
2457
  fmt_age(cand["age_days"]),
@@ -1482,6 +2498,12 @@ def print_table(top, total_candidates, weights, quality_note):
1482
2498
  "schedule whose discount is under 1%. --json lists each model's "
1483
2499
  "schedule."
1484
2500
  )
2501
+ print(
2502
+ "PREC = precision the ranked price is quoted at (maker's default when "
2503
+ "disclosed, best available otherwise; ! = default routing favors "
2504
+ "cheaper sub-default endpoints); prices are the cheapest live endpoint "
2505
+ "at that precision."
2506
+ )
1485
2507
 
1486
2508
 
1487
2509
  def print_json(top):
@@ -1518,6 +2540,31 @@ def print_json(top):
1518
2540
  "age_days": round(cand["age_days"], 1)
1519
2541
  if cand["age_days"] is not None
1520
2542
  else None,
2543
+ # Additive keys (Task 9's ROW_KEYS extension pins this exact
2544
+ # key set and order). Risk/spread come from the annotations
2545
+ # block; session_cost_usd is run()'s per-candidate slice of the
2546
+ # Task 7 envelope, shown for the display turn range (run()
2547
+ # attaches it before any output path). Candidates that never
2548
+ # went through either stage (historical test paths) degrade to
2549
+ # null, as does a model the dataset covers only in other turn
2550
+ # ranges.
2551
+ "default_precision": cand.get("default_precision"),
2552
+ "risk_flag": (cand.get("annotations") or {}).get("risk_flag"),
2553
+ "price_spread": (cand.get("annotations") or {}).get("price_spread"),
2554
+ "session_cost_usd": (cand.get("session_cost") or {}).get(
2555
+ SESSION_COST_DISPLAY_TURN_RANGE
2556
+ ),
2557
+ # The cheapest-sub-default note, exactly as the CLI prints it
2558
+ # (Task 11 shows it verbatim). Emit-first: Task 9's ROW_KEYS
2559
+ # pins this key last; pre-endpoint candidates degrade to None.
2560
+ "quant_note": fmt_quant_note(cand.get("annotations")),
2561
+ # One extra row key AFTER quant_note -- steering is NOT in
2562
+ # ROW_KEYS. build_site_data's row check is missing-key-only, so
2563
+ # extras pass through (tier_note precedent) and the site
2564
+ # ignores it. The OpenRouter provider routing object (design
2565
+ # section 4.7) the plugin contract reads; pre-endpoint
2566
+ # candidates degrade to None.
2567
+ "steering": cand.get("steering"),
1521
2568
  }
1522
2569
  for cand in top
1523
2570
  ]
@@ -1535,6 +2582,8 @@ def build_catalog(
1535
2582
  aa_source,
1536
2583
  aa_by_id,
1537
2584
  quality_source_by_id,
2585
+ session_cost=None,
2586
+ session_cost_mode="skipped",
1538
2587
  ):
1539
2588
  """Assemble the full-catalog document (see README "Catalog output").
1540
2589
 
@@ -1542,6 +2591,11 @@ def build_catalog(
1542
2591
  component scores) and never mutates them. Deterministic apart from
1543
2592
  generated_at: models sort by (-overall.balanced, id), filtered by id,
1544
2593
  and age uses date precision so two runs in the same UTC day match.
2594
+
2595
+ session_cost is the Task 7 fetch envelope (verbatim, medians included)
2596
+ or None; session_cost_mode is run()'s resolution of the key-presence
2597
+ fact ("ok" / "unavailable" / "skipped") -- build_catalog stays pure and
2598
+ never fetches or inspects the environment itself.
1545
2599
  """
1546
2600
  now = datetime.now(timezone.utc)
1547
2601
  weights = catalog_weights(candidates, quality_by_id)
@@ -1655,6 +2709,19 @@ def build_catalog(
1655
2709
  "quality": cand["quality"],
1656
2710
  "quality_match": quality_source_by_id.get(cand["id"]),
1657
2711
  "scores": {**scores, "overall": overall},
2712
+ # Additive provider-aware blocks (schema v3), verbatim from
2713
+ # the candidate. Candidates that never went through the
2714
+ # endpoint stage (historical test paths) degrade to null --
2715
+ # .get() with null defaults for these NEW keys only; the
2716
+ # pre-existing keys above stay direct-indexed.
2717
+ "endpoints": cand.get("endpoints"),
2718
+ "default_precision": cand.get("default_precision"),
2719
+ "advertised_pricing": cand.get("advertised_pricing"),
2720
+ "annotations": cand.get("annotations"),
2721
+ # The OpenRouter provider routing object (design section
2722
+ # 4.7) the plugin contract reads -- same null-default
2723
+ # degradation shape as the blocks above.
2724
+ "steering": cand.get("steering"),
1658
2725
  }
1659
2726
  )
1660
2727
  entries.sort(key=lambda e: (-e["scores"]["overall"]["balanced"], e["id"]))
@@ -1692,7 +2759,19 @@ def build_catalog(
1692
2759
  },
1693
2760
  "zdr": "skipped" if args.no_zdr else "ok",
1694
2761
  "discounts": "ok" if discounts else "unavailable",
2762
+ # The endpoint stage always ran before build_catalog is reached;
2763
+ # its failures surface as pool.dropped counts
2764
+ # (endpoint_fetch_failed / no_baseline_eligible_endpoint), never
2765
+ # as source state -- so this is deterministic "ok".
2766
+ "endpoints": "ok",
2767
+ # "ok" (envelope fetched) / "unavailable" (key configured, fetch
2768
+ # failed) / "skipped" (no key) -- resolved in run().
2769
+ "session_cost": session_cost_mode,
1695
2770
  },
2771
+ # Per-task session-cost medians (Task 7), verbatim from the fetch
2772
+ # envelope. Omitted entirely when unavailable: consumers branch on
2773
+ # key presence, never on a null shape.
2774
+ **({"session_cost": session_cost} if session_cost is not None else {}),
1696
2775
  "pool": {
1697
2776
  "listed": len(models),
1698
2777
  "candidates": len(candidates),
@@ -1787,6 +2866,11 @@ def parse_args(argv=None):
1787
2866
  default=None,
1788
2867
  help="Artificial Analysis API key (or set AA_API_KEY; free at artificialanalysis.ai)",
1789
2868
  )
2869
+ parser.add_argument(
2870
+ "--openrouter-api-key",
2871
+ default=None,
2872
+ help="OpenRouter API key (or set OPENROUTER_API_KEY; free at openrouter.ai/keys)",
2873
+ )
1790
2874
  parser.add_argument(
1791
2875
  "--no-require-tools",
1792
2876
  action="store_true",
@@ -1862,11 +2946,24 @@ def run(args, models, or_cached) -> int:
1862
2946
 
1863
2947
  filtered = []
1864
2948
  candidates, dropped = build_candidates(models, args, discounts, zdr_ids, filtered)
2949
+ # The provider-aware stage: every surviving candidate is repriced at its
2950
+ # baseline endpoint (or dropped fail-closed) before any scoring runs.
2951
+ candidates, extra_dropped = apply_endpoint_pricing(candidates, args, filtered)
2952
+ dropped.update(extra_dropped)
1865
2953
  if not candidates:
1866
- warn(
1867
- f"no models satisfy the filters (min-context={args.min_context}, "
1868
- f"tools={'off' if args.no_require_tools else 'required'}); relax them and retry"
1869
- )
2954
+ # When the endpoint stage wiped the pool, naming the filter knobs is
2955
+ # misleading (they passed; the drops came after) -- warn with the
2956
+ # endpoint-stage drop counts instead (both reasons, zero-filled).
2957
+ counts = {r: extra_dropped.get(r, 0) for r in ENDPOINT_DROP_REASONS}
2958
+ if any(counts.values()):
2959
+ bits = ", ".join(f"{k}={v}" for k, v in counts.items())
2960
+ warn(f"no models after the endpoint stage: {bits}")
2961
+ else:
2962
+ warn(
2963
+ f"no models satisfy the filters (min-context={args.min_context}, "
2964
+ f"tools={'off' if args.no_require_tools else 'required'}); "
2965
+ "relax them and retry"
2966
+ )
1870
2967
  return 2
1871
2968
 
1872
2969
  quality_by_id, quality_source_by_id = resolve_quality(
@@ -1874,6 +2971,29 @@ def run(args, models, or_cached) -> int:
1874
2971
  )
1875
2972
  weights = compute_scores(candidates, args, quality_by_id)
1876
2973
 
2974
+ # Per-task session-cost medians (Task 7): one fetch per invocation, but
2975
+ # only on the output paths that surface it (--catalog block, --json
2976
+ # rows; publish runs --json per priority, so the cache makes the repeat
2977
+ # cost free). A keyed --best/plain run skips the paginated fetch
2978
+ # entirely -- the medians never reach those outputs. Keyless,
2979
+ # fetch_session_cost returns None before any network or cache access.
2980
+ if args.catalog or args.json:
2981
+ session_cost = fetch_session_cost(args)
2982
+ # The key-presence fact distinguishes "skipped" from "unavailable";
2983
+ # same expression fetch_session_cost uses for its own guard.
2984
+ session_cost_mode = (
2985
+ "skipped"
2986
+ if not (args.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY"))
2987
+ else ("ok" if session_cost else "unavailable")
2988
+ )
2989
+ else:
2990
+ session_cost = None
2991
+ session_cost_mode = "skipped"
2992
+ # run() owns the per-candidate attachment (build_catalog stays pure);
2993
+ # print_json reads it for the session_cost_usd display cell.
2994
+ for cand in candidates:
2995
+ cand["session_cost"] = (session_cost or {}).get("medians", {}).get(cand["id"])
2996
+
1877
2997
  if args.catalog:
1878
2998
  print_catalog(
1879
2999
  build_catalog(
@@ -1887,6 +3007,8 @@ def run(args, models, or_cached) -> int:
1887
3007
  aa_source,
1888
3008
  aa_by_id,
1889
3009
  quality_source_by_id,
3010
+ session_cost,
3011
+ session_cost_mode,
1890
3012
  )
1891
3013
  )
1892
3014
  return 0