@wendellresearch-ai/benchwarmers 0.3.3 → 0.5.0

package/metrics/discover.py

@@ -1,10 +1,16 @@
-"""One list of models worth adding, from models.dev and FreeLLMAPI, deduped.
+"""One list of models worth adding, from FreeLLMAPI and models.dev, deduped.
 
     python metrics/discover.py           # summary: free models by reach, new this week
 
 The shopping list: Discover's Add (serve.py) puts a model on the bench by one
 of the routes below. A model offered by the same provider in both sources
-appears once, carrying both sources and the better-filled fields.
+appears once, carrying both sources. FreeLLMAPI leads where it lists the
+model (its name, its provider's name, context, tools, limits, its smartest
+and fastest ranks, size, monthly budget, quirks and freellmapi.co page);
+models.dev fills the rest, and alone has the paid models and prices, API base
+URLs (reach, direct routes), modalities, release and knowledge dates.
+Image and audio generation in FreeLLMAPI's list are left out: the bench asks
+for text.
 
 FREE
 ----
@@ -149,7 +155,7 @@ def hub_routes(rows: list[dict]) -> tuple[list[list[dict]], list[str]]:
                 mine.append({"via": "hub", "hub": which, "connection": conn, "endpoint": ep,
                              "label": f"{r['provider_name']} via {label} ({conn} connection)",
                              "note": f"Also serves it to everything else that uses {label}.",
-                             "steps": ["; ".join(said), "The bench: register, a 4-call probe, then the suite"],
+                             "steps": ["; ".join(said), "The bench: register it and onboard it as chosen below"],
                              "on_bench_as": next((a for e, _c, _n, a, s in bench if e == ep and s == served), None)})
         out.append(mine)
     return out, errors
@@ -158,6 +164,8 @@ def hub_routes(rows: list[dict]) -> tuple[list[list[dict]], list[str]]:
 def build() -> list[dict]:
     import endpoints
     md, fl = sources.cached("modelsdev"), sources.cached("freellmapi")
+    pages = set(sources.cached("freellmapi-pages").get("slugs") or [])
+    platform_names = {p.get("id"): p.get("name") for p in fl.get("platforms") or []}
     md_conn = {v: k for k, v in sources.connection_to_modelsdev().items()}
     bench = _on_bench()
     # Open WebUI's connections, as far as the bench can tell without asking it.
@@ -185,31 +193,37 @@ def build() -> list[dict]:
                 "vision": "image" in ((m.get("modalities") or {}).get("input") or []),
                 "released": m.get("release_date"), "knowledge": m.get("knowledge"),
                 "open_weights": m.get("open_weights"), "api": p.get("api"),
-                "openai_like": openai_like, "connection": md_conn.get(prov), "limits": None}
+                "openai_like": openai_like, "connection": md_conn.get(prov), "limits": None,
+                "rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None}
 
     for m in fl.get("models") or []:
+        if (m.get("modality") or "chat") != "chat":
+            continue                                # image or audio generation
         plat = m.get("platform", "")
         prov = sources.FREELLMAPI_TO_MODELSDEV.get(plat, plat)
         key = (prov, sources.norm_model(m.get("modelId", "")))
-        lim = {k: v for k, v in (m.get("limits") or {}).items() if v} or None
+        lead = {"sources": ["freellmapi"], "free": True,          # listed in a free-tier catalog
+                "name": sources.freellmapi_name(m.get("displayName") or "") or m.get("modelId"),
+                "provider_name": platform_names.get(plat) or prov,
+                "limits": {k: v for k, v in (m.get("limits") or {}).items() if v} or None,
+                "rank": m.get("intelligenceRank"), "speed": m.get("speedRank"),
+                "size": m.get("sizeLabel") or None, "budget": m.get("monthlyTokenBudget") or None,
+                "quirks": [{k: q.get(k) for k in ("title", "body", "severity")} for q in m.get("quirks") or []],
+                "page": sources.freellmapi_page(m.get("displayName") or "", pages)}
         row = rows.get(key)
         if row:
-            row["sources"].append("freellmapi")
-            row["free"] = True                      # listed in a free-tier catalog
-            row["limits"] = lim
-            row["context"] = row["context"] or m.get("contextWindow")
-            row["tools"] = row["tools"] if row["tools"] is not None else m.get("supportsTools")
+            row.update(lead, sources=list(dict.fromkeys(["freellmapi", *row["sources"]])),
+                       context=m.get("contextWindow") or row["context"],          # the free tier's, where it says
+                       tools=row["tools"] if m.get("supportsTools") is None else m.get("supportsTools"))
         else:
             rows[key] = {
-                "in_mods": ["text", "image"] if m.get("supportsVision") else ["text"], "out_mods": ["text"],
-                "provider": prov, "provider_name": prov, "model": m.get("modelId"),
-                "name": m.get("displayName") or m.get("modelId"), "sources": ["freellmapi"],
-                "free": True, "cost": None, "context": m.get("contextWindow"), "output": None,
+                **lead, "in_mods": ["text", "image"] if m.get("supportsVision") else ["text"], "out_mods": ["text"],
+                "provider": prov, "model": m.get("modelId"),
+                "cost": None, "context": m.get("contextWindow"), "output": None,
                 "tools": m.get("supportsTools"), "reasoning": None,
                 "vision": m.get("supportsVision"), "released": None, "knowledge": None,
                 "open_weights": None, "api": None,
-                "openai_like": False, "connection": FREELLMAPI_TO_CONNECTION.get(plat),
-                "limits": lim}
+                "openai_like": False, "connection": FREELLMAPI_TO_CONNECTION.get(plat)}
 
     # The same model through ANOTHER provider is fine to add, but worth knowing.
     by_model: dict[str, list[str]] = {}
@@ -245,7 +259,11 @@ def build() -> list[dict]:
         out.append(r)
     _write_seen(seen)
     order = {"connected": 0, "easy to add": 1, "on bench": 2, "other": 3}
-    out.sort(key=lambda r: (not r["free"], not r["new"], order[r["reach"]], r["provider"], r["model"]))
+    # Within each reach, FreeLLMAPI's smartest first; the models it does not rank
+    # after, new ones first. Not new first overall: models.dev adds hundreds a
+    # week, which buried the ranked ones (the badge still marks them).
+    out.sort(key=lambda r: (not r["free"], order[r["reach"]], r["rank"] is None, r["rank"] or 0, not r["new"],
+                            r["provider"], r["model"]))
     return out
 
 
@@ -293,6 +311,38 @@ def _selftest() -> None:
     over = sources.free_in_own_client_only({"opencode": False, "x": True})
     assert over["opencode"] is False and over["x"] is True and over["opencode-go"] is True, over
     assert sources.free_in_own_client_only(True) == sources.free_in_own_client_only({})   # not a map: ignored
+    # freellmapi.co names a page by the display name without its note; only its sitemap's pages exist.
+    assert sources.freellmapi_page("Llama 4 Scout (CF)", {"llama-4-scout"}) == "https://freellmapi.co/model/llama-4-scout"
+    assert sources.freellmapi_page("GLM-4.7 FlashX", {"llama-4-scout"}) is None
+    # FreeLLMAPI leads where it lists a model, models.dev fills the rest; image models out; smartest first.
+    md = {"acme": {"name": "Acme", "api": "https://acme.example/v1", "models": {
+        "acme/big": {"name": "Big", "cost": {"input": 1, "output": 2}, "limit": {"context": 8000},
+                     "modalities": {"input": ["text", "image"], "output": ["text"]}, "release_date": "2026-01-01"},
+        "acme/small": {"name": "Small", "cost": {"input": 0, "output": 0}},
+        "acme/tiny": {"name": "Tiny", "cost": {"input": 0, "output": 0}}}}}
+    fl = {"platforms": [{"id": "acme", "name": "Acme Cloud"}], "models": [
+        {"platform": "acme", "modelId": "acme/big", "displayName": "Big One (Acme)", "intelligenceRank": 3,
+         "speedRank": 9, "sizeLabel": "Large", "monthlyTokenBudget": "~3M", "contextWindow": 4000,
+         "limits": {"rpm": 10, "rpd": None}, "quirks": [{"slug": "q", "title": "T", "body": "B", "severity": "info"}]},
+        {"platform": "acme", "modelId": "acme/painter", "displayName": "Painter", "modality": "image"},
+        {"platform": "acme", "modelId": "acme/tiny", "displayName": "Tiny", "intelligenceRank": 1}]}
+    feeds = {"modelsdev": md, "freellmapi": fl, "freellmapi-pages": {"slugs": ["big-one"]}}
+    import tempfile
+    kept = sources.cached, globals()["_on_bench"], globals()["SEEN"]
+    try:
+        sources.cached = lambda name: feeds.get(name, {})
+        globals().update(_on_bench=lambda: [], SEEN=pathlib.Path(tempfile.mkdtemp()) / "seen.json")
+        rows = [r for r in build() if r["provider"] == "acme"]
+    finally:
+        sources.cached = kept[0]
+        globals().update(_on_bench=kept[1], SEEN=kept[2])
+    big = next(r for r in rows if r["model"] == "acme/big")
+    assert (big["sources"], big["name"], big["provider_name"]) == (["freellmapi", "models.dev"], "Big One", "Acme Cloud"), big
+    assert big["free"] and big["context"] == 4000 and big["limits"] == {"rpm": 10} and big["released"] == "2026-01-01", big
+    assert (big["rank"], big["speed"], big["size"], big["budget"]) == (3, 9, "Large", "~3M"), big
+    assert big["page"] == "https://freellmapi.co/model/big-one" and big["quirks"] == [
+        {"title": "T", "body": "B", "severity": "info"}] and big["vision"], big
+    assert [r["model"] for r in rows] == ["acme/tiny", "acme/big", "acme/small"], [r["model"] for r in rows]
     print("discover selftest ok")
 
 
@@ -306,7 +356,7 @@ def main() -> None:
     by = {}
     for r in free:
         by[r["reach"]] = by.get(r["reach"], 0) + 1
-    print(f"{len(rows)} models ({len(free)} free) from models.dev + FreeLLMAPI;"
+    print(f"{len(rows)} models ({len(free)} free) from FreeLLMAPI + models.dev;"
           f" free by reach: {by}; new this week: {sum(r['new'] for r in rows)}")
     both = sum(1 for r in rows if len(r["sources"]) > 1)
     print(f"{both} listed by both sources (shown once)")

package/metrics/endpoints.py

@@ -154,13 +154,29 @@ def connections() -> dict:
     return data if isinstance(data, dict) else {}
 
 
+_default: tuple = (None, "")
+
+
 def default() -> str:
+    """The default endpoint's name, read again only when endpoints.json
+    changes: snapshot_key asks once per model, hundreds of times a page."""
+    global _default
+    try:
+        st = FILE.stat()
+        key = (str(FILE), st.st_mtime_ns, st.st_size)
+    except OSError:
+        key = None
+    seen = _default
+    if key and seen[0] == key:
+        return seen[1]
     data = load()
     if not data:
         raise RuntimeError(f"no endpoints configured: add one from the dashboard (python serve.py), run"
                            f" scripts/init.py --endpoint NAME BASE_URL, or create {FILE}"
                            ' like {"name": {"base": "https://.../v1", "key": "name", "default": true}}')
-    return next((n for n, e in data.items() if e.get("default")), next(iter(data)))
+    name = next((n for n, e in data.items() if e.get("default")), next(iter(data)))
+    _default = (key, name)
+    return name
 
 
 def kind(name: str, data: dict | None = None) -> str:
@@ -322,13 +338,29 @@ def endpoint_name(reg: dict, data: dict | None = None) -> str | None:
     return next((n for n in same if data[n].get("key") == reg.get("api_key_name")), same[0] if same else None)
 
 
+_parsed: tuple = (None, {})
+
+
 def registrations() -> dict[str, dict]:
     """alias -> its yaml entry as written by yaml_block: model_name, api_base,
-    api_key_name and a `headers` dict. Not a YAML parser; the file is ours."""
+    api_key_name and a `headers` dict. Not a YAML parser; the file is ours.
+
+    Parsed again only when the file changes (a dashboard render asks hundreds
+    of times); every caller gets its own copies to change."""
+    global _parsed
+    try:
+        st = YAML.stat()
+    except OSError:                                # a fresh install has registered nothing yet
+        return {}
+    key = (str(YAML), st.st_mtime_ns, st.st_size)
+    if _parsed[0] != key:
+        _parsed = (key, _parse_registrations())
+    return {a: dict(r, headers=dict(r["headers"])) for a, r in _parsed[1].items()}
+
+
+def _parse_registrations() -> dict[str, dict]:
     out: dict[str, dict] = {}
     current = None
-    if not YAML.exists():                          # a fresh install has registered nothing yet
-        return out
     for line in YAML.read_text(encoding="utf-8").splitlines():
         if (m := re.match(r"\s*-\s*model_id:\s*(.+)", line)):
             current = out.setdefault(m.group(1).strip().strip("\"'"), {"headers": {}})

package/metrics/model_roles.py

@@ -20,7 +20,9 @@ roles:   general  reasoning / do-it-all: eligible for every shape
                   charts), weak at text-only work: admitted by the vision shape
                   alone, and a collection member with this role sits only when
                   the request carries images (gateway.py). A model that reads
-                  images AND does everything else stays general.
+                  images AND does everything else stays general. `bench onboard
+                  --vision` (the picker's Vision model box) onboards one this
+                  way; its suite is the vision tasks alone (suite/run.applies).
          decision typed answers with probabilities through a Decisions API, not
                   chat; no shape admits them (config/decision-models.json)
 

package/metrics/pacing.py

@@ -177,6 +177,28 @@ def _ago(s: float) -> str:
     return "just now" if s < 60 else f"{s / 60:.0f} min ago" if s < 7200 else f"{s / 3600:.0f} h ago"
 
 
+def _since(rs: list[dict], records: list[dict]) -> float:
+    """The oldest ledger line any of these windows or quota records counts."""
+    spans = [AXES[r["axis"]][1] for r in rs]
+    return min([ledger.month_start(time.time()) if s == "month" else time.time() - s for s in spans]
+               + [_ts(q["observed_at"]) for q in records if q.get("observed_at")])
+
+
+def held(alias: str, connection: str, est_tokens: int = 1) -> tuple[float, str] | None:
+    """(seconds until there is room, why) when a call to `alias` would have to
+    wait or be refused now, else None. Read only: no slot is taken. For
+    picking seats, so a model the bench knows will be refused is not seated
+    (gateway.refused_now); `admit` still decides each call."""
+    import quota
+    rs, records = rules(alias, connection), quota.live(connection, alias)
+    if not rs and not records:
+        return None
+    now = time.time()
+    block = _blocked(rs, records, alias, connection, est_tokens, now,
+                     list(ledger.rows(_since(rs, records))), _inflight(now))
+    return None if block is None else (max(block[0] - now, 0.0), block[1])
+
+
 def admit(alias: str, connection: str, est_tokens: int, left: float, sleep=time.sleep) -> tuple[str | None, str]:
     """Wait for room, or refuse. Returns (ticket, refusal): a ticket to pass
     to `release` once the call's ledger line is written (None when no limit
@@ -189,9 +211,7 @@ def admit(alias: str, connection: str, est_tokens: int, left: float, sleep=time.
         records = quota.live(connection, alias)
         if not rs and not records:
             return None, ""
-        spans = [AXES[r["axis"]][1] for r in rs]
-        since = min([ledger.month_start(time.time()) if s == "month" else time.time() - s for s in spans]
-                    + [_ts(q["observed_at"]) for q in records if q.get("observed_at")])
+        since = _since(rs, records)
         with locked.lock(LOCK):
             now = time.time()
             rows = list(ledger.rows(since))