@wendellresearch-ai/benchwarmers 0.4.0 → 0.5.1

package/metrics/scheduler.py

@@ -168,6 +168,7 @@ def _start(name, args, due, running) -> None:
 
 
 _stop = threading.Event()
+RUNNING: dict = {}                                     # job name -> True while it runs (serve.restart reads it)
 
 
 def _loop() -> None:
@@ -180,7 +181,7 @@ def _loop() -> None:
     except TimeoutError:
         return                                          # another dashboard already runs it
     try:
-        running: dict = {}
+        running = RUNNING
         conf = settings()
         _log(f"scheduler started, mode {conf['mode']}, jobs {', '.join(enabled(conf))}")
         while True:

package/metrics/sources.py

@@ -1,6 +1,10 @@
-"""Public model lists the bench reads: models.dev and FreeLLMAPI's free catalog.
+"""Public model lists the bench reads: FreeLLMAPI's free catalog and models.dev.
 
-    python metrics/sources.py            # fetch both now (normally once a day)
+    python metrics/sources.py            # fetch them now (normally once a day)
+
+Discover leads with FreeLLMAPI (easier for a person to follow: a page per
+model, its limits, ranks and quirks) and fills the rest from models.dev, which
+alone has prices, paid models, providers' API base URLs and modalities.
 
 WHY THESE TWO
 -------------
@@ -37,6 +41,9 @@ CACHE = DATA / "runs" / "cache"
 FEEDS = {
     "modelsdev": "https://models.dev/api.json",
     "freellmapi": "https://api.freellmapi.co/v1/latest",
+    # The model pages freellmapi.co has (its sitemap): a page exists only for
+    # what its monthly snapshot lists, so Discover links only to those.
+    "freellmapi-pages": "https://freellmapi.co/sitemap-core.xml",
 }
 MAX_AGE_S = 20 * 3600
 
@@ -90,7 +97,8 @@ def fetch(name: str, force: bool = False) -> dict:
         return json.loads(path.read_text(encoding="utf-8"))
     req = urllib.request.Request(FEEDS[name], headers={"User-Agent": "benchwarmers"})
     with urllib.request.urlopen(req, timeout=90) as r:
-        data = json.load(r)
+        data = (json.load(r) if not FEEDS[name].endswith(".xml") else
+                {"slugs": sorted(set(re.findall(r"/model/([^<\"/]+)<", r.read().decode("utf-8"))))})
     CACHE.mkdir(parents=True, exist_ok=True)
     tmp = path.with_suffix(".part")
     tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
@@ -144,6 +152,20 @@ def freellmapi_entry(served: str, platform: str | None = None) -> dict | None:
     return None
 
 
+def freellmapi_name(display_name: str) -> str:
+    """A FreeLLMAPI display name without its trailing note, "Llama 4 Scout (CF)"
+    -> "Llama 4 Scout": the name freellmapi.co gives the model on every platform."""
+    return re.sub(r"\s*\([^)]*\)\s*$", "", display_name or "").strip()
+
+
+def freellmapi_page(display_name: str, pages: set[str]) -> str | None:
+    """The model's freellmapi.co page, if `pages` (its sitemap) has it. The site
+    names a page by the display name without its note, lowercased, each run of
+    other characters one dash (buildBundles in its app.js, read 2026-10-28)."""
+    slug = "-".join(re.findall(r"[a-z0-9]+", freellmapi_name(display_name).lower()))
+    return f"https://freellmapi.co/model/{slug}" if slug in pages else None
+
+
 def is_free(cost: dict | None) -> bool | None:
     if not cost:
         return None
@@ -156,9 +178,10 @@ def main() -> None:
     for name in FEEDS:
         try:
             data = fetch(name, force=True)
-            size = len(data) if name == "modelsdev" else len(data.get("models", []))
-            print(f"  {name}: {size} {'providers' if name == 'modelsdev' else 'models'}"
-                  f" at {dt.datetime.now():%H:%M}")
+            size, unit = {"modelsdev": (len(data), "providers"),
+                          "freellmapi-pages": (len(data.get("slugs", [])), "model pages")}.get(
+                              name, (len(data.get("models", [])), "models"))
+            print(f"  {name}: {size} {unit} at {dt.datetime.now():%H:%M}")
         except Exception as exc:                       # noqa: BLE001
             print(f"  {name}: could not fetch ({type(exc).__name__}: {exc}); keeping the cached copy")
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/check.py

@@ -39,6 +39,7 @@ CHECKS = [
     ("metrics/bench_collections.py", [], False),
     ("metrics/learn.py", ["--selftest"], False),
     ("metrics/quota.py", ["--selftest"], False),
+    ("metrics/quotaview.py", ["--selftest"], False),
     ("metrics/discover.py", ["--selftest"], False),
     ("metrics/test_decide.py", [], False),
     ("metrics/wrong.py", [], False),

package/scripts/test_hubs.py

@@ -278,16 +278,21 @@ def discover_add() -> None:
         os.environ["BENCHWARMERS_KEY_CPA_ADMIN"] = "mgmt-test"
 
     onboarded = []
-    serve.queue_onboard = lambda items, who="ui": onboarded.append((items[0]["served"], items[0]["endpoint"])) or {
+    serve.queue_onboard = lambda items, who="ui": onboarded.append(
+        (items[0]["served"], items[0]["endpoint"], items[0]["mode"], items[0]["vision"])) or {
         "ok": True, "errors": [], "items": []}
     owui.add = lambda c, m: (m, [f"owui added {m} on {c}"])
     r = serve.discover_add("groq", "vendor/new", endpoint="lite")
-    assert r["ok"] and "/lite/model/new" in writes() and onboarded[-1] == ("vendor/new", "lite"), r
-    r = serve.discover_add("openrouter", "vendor/y:free", endpoint="cpa")
-    assert r["ok"] and onboarded[-1] == ("vendor/y:free", "cpa"), r
+    assert r["ok"] and "/lite/model/new" in writes() and onboarded[-1] == ("vendor/new", "lite", "full", False), r
+    # The confirm's Onboard as and Vision box reach the queue; a bad mode is refused before the hub has it.
+    r = serve.discover_add("openrouter", "vendor/y:free", endpoint="cpa", mode="quick-adhoc", vision=True)
+    assert r["ok"] and onboarded[-1] == ("vendor/y:free", "cpa", "quick-adhoc", True), r
     assert [m for m, p, _b in SEEN if "management" in p and m == "PATCH"] == ["PATCH"]
+    before = len(SEEN)
+    assert "mode must be" in serve.discover_add("openrouter", "vendor/q", endpoint="cpa", mode="slow")["errors"][0]
+    assert len(SEEN) == before, SEEN[before:]
     r = serve.discover_add("groq-api", "vendor/z")                                 # no endpoint: Open WebUI, as before
-    assert r["steps"] == ["owui added vendor/z on groq-api"] and onboarded[-1] == ("vendor/z", "owui"), r
+    assert r["steps"] == ["owui added vendor/z on groq-api"] and onboarded[-1] == ("vendor/z", "owui", "full", False), r
     assert serve.discover_add("groq-api", "vendor/z", endpoint="owui")["ok"]
     assert "not a hub" in serve.discover_add("groq", "v/z", endpoint="plain")["errors"][0]
     changes = [json.loads(line)["field"] for line in serve.CHANGES_PATH.read_text(encoding="utf-8").splitlines()]

package/scripts/test_no_owui.py

@@ -132,6 +132,15 @@ def stranger(fake: str) -> None:
     assert c["ok"] and not c["errors"], c
     assert sorted(x["served"] for x in c["candidates"]) == ["vendor/c", "vendor/d", "vendor/e", "vendor/f"], c
     assert {x["group"] for x in c["candidates"]} == {"openrouter (direct)"} and not any(x["new"] for x in c["candidates"])
+    # Queued or onboarding now: left out until the queue is done with it.
+    kept_items = serve._OQ["items"]
+    serve._OQ["items"] = [{"served": "vendor/c", "endpoint": "openrouter", "state": "running"},
+                          {"served": "vendor/d", "endpoint": "openrouter", "state": "done"}]
+    try:
+        busy = sorted(x["served"] for x in serve.onboard_candidates()["candidates"])
+    finally:
+        serve._OQ["items"] = kept_items
+    assert busy == ["vendor/d", "vendor/e", "vendor/f"], busy
 
     # The served check: the registered models only, not the aggregator's catalogue.
     assert check_deprecations.served() == {"vendor/a:free", "vendor/b:free"}

package/scripts/test_quota.py

@@ -290,6 +290,16 @@ def main() -> int:
     check("rate limit (not sent" in r["error"] and "a calendar month" in r["error"] and "this month" in r["error"],
           "rpmo 3: the fourth call this month is refused", r["error"])
     check(client.chat("t-ok", "hi")["text"] != "", "another connection is not held by that cap")
+    # Seating asks the same question first (gateway.refused_now): held past any
+    # seat's wait, so not seated; a model with room is.
+    sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+    import gateway
+    import pacing
+    h = pacing.held("t-month", "monthprov")
+    check(h is not None and h[0] > gateway.SEAT_TIMEOUT and "this month" in h[1], "held() sees the monthly cap", h)
+    check(pacing.held("t-ok", gateway.route.connection_of("t-ok")) is None, "held() is None with room")
+    out = gateway.refused_now(["t-month", "t-ok"], 10, gateway.SEAT_TIMEOUT)
+    check(list(out) == ["t-month"] and "rate limit" in out["t-month"], "refused_now names only the held model", out)
 
     # The free-tier guard: a model that went paid is held until allowed.
     before = len(seen.get("paid", []))
@@ -322,8 +332,8 @@ def main() -> int:
           and "rpmo" not in ((serve.catalog.overrides().get("t-ok") or {}).get("limits") or {}),
           "a bad rpmo is refused; a model's own limits have none", bad)
     page = serve.quotas_page()
-    check("What providers said is left" in page and "fakeprov" in page and "What the bench sent" in page,
-          "the Quotas page shows the records and the ledger's counts")
+    check("id='quota-data'" in page and "fakeprov" in page and "What the bench sent" in page,
+          "the Quotas page carries its tiles and the ledger's counts")
 
     srv.shutdown()
     print("\nthe ledger, quotas and pacing hold" if not failures else f"\n{failures} FAILED")

package/scripts/test_serve_edit.py

@@ -398,7 +398,7 @@ def main() -> int:
         stopped = {i["served"]: i["state"] for i in serve.onboard_queue()["items"]}
         order.remove("quiet")
         serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m, "vision": s == "quiet"} for s, m in
-                             (("ok", "full"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
+                             (("ok", "full-adhoc"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
         settle()
         batch = {i["served"]: (i["state"], i["alias"], bool(i["note"])) for i in serve.onboard_queue()["items"]}
         cleared = serve.clear_onboard_queue()["items"]
@@ -412,10 +412,55 @@ def main() -> int:
           and batch == {"ok": ("done", "ok-x", False), "quiet": ("no answer", "quiet-x", True),
                         "bad": ("failed", "bad-x", True)}
           and cleared == [] and bad_input == (False, False, False)
-          and modes == {"a": "full", "ok": "full", "quiet": "quick vision", "bad": "quick-adhoc"})
+          and modes == {"a": "full", "ok": "full-adhoc", "quiet": "quick vision", "bad": "quick-adhoc"}
+          and serve.ONBOARD_MODES["full-adhoc"] == ["--ad-hoc"])            # the suite now, then ad hoc
     print(("ok   " if ok else "FAIL ") + "onboarding queue" + ("" if ok else f" -> {stopped} {order} {batch} {modes}"))
     failures += 0 if ok else 1
 
+    # Restart: refused unless serve.py's parent started it (nothing would start it
+    # again); refused while a job runs, unless forced; then the server shuts down.
+    import scheduler
+    kept = os.environ.get("BENCH_SERVED"), serve.SERVER
+    shut = []
+    try:
+        os.environ.pop("BENCH_SERVED", None)
+        alone = serve.restart()
+        os.environ["BENCH_SERVED"] = "1"
+        serve.SERVER = type("Server", (), {"shutdown": lambda self: shut.append(1)})()
+        scheduler.RUNNING["suite"] = True
+        busy, forced = serve.restart(), serve.restart(force=True)
+        time.sleep(0.8)                                  # the shutdown waits half a second
+    finally:
+        scheduler.RUNNING.pop("suite", None)
+        serve.SERVER = kept[1]
+        os.environ.pop("BENCH_SERVED", None) if kept[0] is None else os.environ.update(BENCH_SERVED=kept[0])
+    ok = (not alone["ok"] and busy.get("busy") and not busy["ok"] and "suite" in busy["errors"][0]
+          and forced["ok"] and forced["cut"] and shut == [1])
+    print(("ok   " if ok else "FAIL ") + "restart" + ("" if ok else f" -> {alone} {busy} {forced} {shut}"))
+    failures += 0 if ok else 1
+
+    # Action needed: an alert whose models have all left the bench is not shown,
+    # before the daily check rewrites the file (2026-10-28: a kilo connection
+    # alert stayed after Retire all 4).
+    kept_data, kept_reg = serve.DATA, serve.registered
+    with tempfile.TemporaryDirectory() as tmp:
+        (pathlib.Path(tmp) / "metrics").mkdir()
+        (pathlib.Path(tmp) / "metrics" / "alerts.json").write_text(json.dumps({
+            "connection:held:gone.example": {"kind": "connection", "message": "every model on gone.example is failing",
+                                             "aliases": ["g1", "g2"], "first_seen": "2026-10-28"},
+            "connection:held:here.example": {"kind": "connection", "message": "every model on here.example is failing",
+                                             "aliases": ["h1", "g1"], "first_seen": "2026-10-28"},
+            "retiring:x:old": {"kind": "retiring", "alias": "old", "message": "old retires", "first_seen": "2026-10-28"}}),
+            encoding="utf-8")
+        serve.DATA, serve.registered = pathlib.Path(tmp), lambda: {"h1-served": "h1"}
+        try:
+            shown = serve.problems()
+        finally:
+            serve.DATA, serve.registered = kept_data, kept_reg
+    ok = "here.example" in shown and "gone.example" not in shown and "old retires" not in shown
+    print(("ok   " if ok else "FAIL ") + "action needed drops alerts on retired models")
+    failures += 0 if ok else 1
+
     # Dismiss: one picker key or many in one write, a change-log line each;
     # undo brings back only the ones named; nothing named is refused.
     serve.DISMISSED_PATH = cfg / "dismissed.json"
@@ -433,6 +478,23 @@ def main() -> int:
     print(("ok   " if ok else "FAIL ") + "dismiss in bulk, and undo" + ("" if ok else f" -> {gone} {left} {log} {refused}"))
     failures += 0 if ok else 1
 
+    # The member picker's provider chips: a proxy's vendor, else the bare host.
+    names = {"nvidia-api": "nvidia", "api.unorouter.com": "unorouter", "router.requesty.ai": "requesty",
+             "cli-proxy-api/codex": "codex", "openrouter": "openrouter", "apihub.agnes-ai.com": "agnes-ai"}
+    got = {t: serve.provider_label(t) for t in names}
+    ok = got == names
+    print(("ok   " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
+    failures += 0 if ok else 1
+
+    # Got it clears a notice (NVIDIA changed its figures), never a state still wrong.
+    alerts.raise_alert("test:notice", "limits", "a notice", dismiss=True)
+    alerts.raise_alert("test:state", "limits", "a state")
+    ok = (serve.dismiss_alert("test:notice")["ok"] and not serve.dismiss_alert("test:state")["ok"]
+          and "test:notice" not in alerts.active() and "test:state" in alerts.active())
+    alerts.clear("test:state")
+    print(("ok   " if ok else "FAIL ") + "Got it dismisses a notice, not a state")
+    failures += 0 if ok else 1
+
     # The owner's files are as they were. alerts.json can change under a
     # running suite, so only what this test could do counts: the victim's
     # alerts cleared from the real file.