@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0

package/metrics/route.py

@@ -101,8 +101,9 @@ from aliases import current as current_alias  # noqa: E402
 
 
 def _json(path: pathlib.Path) -> dict:
+    import locked
     try:
-        return json.loads(path.read_text(encoding="utf-8"))
+        return json.loads(locked.read_text(path))      # inside a render, from its snapshot
     except (OSError, ValueError):
         return {}
 
@@ -135,7 +136,14 @@ def full_tag_of(alias: str) -> str:
     name, else "?": a hub fronts many connections, and an untagged model
     behind it (Open WebUI stopped listing it) is not on a connection called
     `openwebui` with every other untagged one.
+
+    Worked out once per alias in a dashboard render, which asked 2,600 times.
     """
+    import locked
+    return locked.once(("full_tag_of", alias), lambda: _full_tag_of(alias))
+
+
+def _full_tag_of(alias: str) -> str:
     from run import _registered_names
     import endpoints
     # Tags are keyed by the served name. A decision model is not in the llm
@@ -273,7 +281,7 @@ def suite_recall() -> dict[str, tuple[int, int, int]]:
 def panel(shape: str | None = None, large: bool = True,
           nominated: list[str] | None = None,
           members: list[str] | None = None, explorers: int = 2,
-          avoid: set[str] | None = None) -> dict:
+          avoid: set[str] | None = None, stand_ins: int = 0) -> dict:
     """`large=True` means the task is a big one — most of ours are.
 
     `members` is a bench-gateway COLLECTION: models the caller always runs. With
@@ -282,7 +290,8 @@ def panel(shape: str | None = None, large: bool = True,
     never with a member or a member's family. `avoid` keeps models out of
     those automatic picks (the gateway passes the ones on a retiring alert, so
     a seat is not spent on a model already known to be failing); a nomination
-    still seats one.
+    still seats one. `stand_ins` more are picked the same way, for members out
+    of quota (the gateway); fewer come back if the pool runs out.
 
     `nominated` names models the operator wants on the panel today, and they
     take EXPLORER seats — never the primary or the second.
@@ -469,6 +478,12 @@ def panel(shape: str | None = None, large: bool = True,
 
     explorer, explorer2 = seats[0], seats[1]
 
+    standing_in = []
+    while len(standing_in) < stand_ins and (pick := stalest()):
+        chosen.add(pick["model"])
+        seated_families.add(pick["family"])
+        standing_in.append(pick)
+
     # RESERVE: the fallback when the panel meets a rate limit.
     #
     # This replaces the hard rule that seats must sit on different
@@ -489,7 +504,7 @@ def panel(shape: str | None = None, large: bool = True,
                       key=lambda r: (r["found"], paid(r)), default=None)
 
     return {"primary": primary, "second": second, "explorer": explorer,
-            "explorer2": explorer2, "reserve": reserve,
+            "explorer2": explorer2, "reserve": reserve, "stand_ins": standing_in,
             "considered": len(rows),
             "untried": sum(1 for r in rows if not r["offloads"]),
             "contenders": len(contenders),

package/metrics/scheduler.py

@@ -6,7 +6,8 @@
 RELEASE-PLAN.md Phase 3. The checks used to exist only as Windows scheduled
 tasks (scripts/schedule.ps1), which macOS, Linux and Docker do not have. serve.py
 starts this in a background thread; the jobs below are the same seven, at the
-same times, with the same arguments (schedule.ps1 explains each choice).
+same times, with the same arguments (schedule.ps1 explains each choice), and
+since 2026-11-08 the decision benchmark, which has no Windows task.
 
 MODE, from config/scheduler.json:
   absent / "off"   nothing runs. A fresh install spends no model calls until
@@ -55,6 +56,11 @@ JOBS = {
     "perf": (["perf/run.py", "-n", "3", "--timeout", "90", "--client", "builtin"], [(0, "07:00")]),
     "capabilities": (["metrics/probe_caps.py", "--gap", "30"], [(6, "07:00")]),
     "suite": (["suite/run.py", "--next"], [(None, t) for t in ("03:00", "09:00", "15:00", "21:00")]),
+    # Not in schedule.ps1 (2026-11-08). One question per model per set, 30 s
+    # apart round-robin: most decision models are OpenRouter `:free`, whose
+    # daily budget the suite and the probe share (metrics/decide.py).
+    "decisions": (["metrics/decide.py", "bench", "--per", "1", "--gap", "30", "--new-errors", "3"],
+                  [(None, "04:30")]),
 }
 # Each mode as the Settings page names and explains it (the docstring above).
 MODES = {"off": ("Off", "Nothing runs. A fresh install spends no model calls until someone opts in."),
@@ -77,6 +83,9 @@ ABOUT = {
                     " per model, spaced out.",
     "suite": "Scores one paid model on the review suite every six hours: up to six tasks it has not"
              " answered yet. These scores are what the router ranks by.",
+    "decisions": "Measures the decision models: one labelled question per model on each set (error classes,"
+                 " collections, verdicts), 30 s apart, stopping a model at its first failure; and asks"
+                 " about up to three new unrecognised error shapes. The gateway's picks ask the best one.",
 }
 
 
@@ -213,6 +222,7 @@ def _selftest() -> None:
     assert next_due(JOBS["perf"][1], mon) == dt.datetime(2026, 10, 19, 7, 0)
     assert next_due(JOBS["perf"][1], mon.replace(hour=8)) == dt.datetime(2026, 10, 26, 7, 0)
     assert next_due(JOBS["capabilities"][1], mon) == dt.datetime(2026, 10, 25, 7, 0)
+    assert next_due(JOBS["decisions"][1], mon) == dt.datetime(2026, 10, 19, 4, 30) and set(ABOUT) == set(JOBS)
     conf = {"mode": "log-only", "jobs": {n: n == "availability" for n in JOBS}}
     state: dict = {}
     global _log

package/metrics/sources.py

@@ -93,17 +93,30 @@ FREELLMAPI_TO_MODELSDEV = {
 def fetch(name: str, force: bool = False) -> dict:
     """A feed, from the cache when it is under a day old."""
     path = CACHE / f"{name}.json"
-    if not force and path.exists() and time.time() - path.stat().st_mtime < MAX_AGE_S:
+    age = age_s(name)
+    if not force and age is not None and age < MAX_AGE_S:
         return json.loads(path.read_text(encoding="utf-8"))
     req = urllib.request.Request(FEEDS[name], headers={"User-Agent": "benchwarmers"})
     with urllib.request.urlopen(req, timeout=90) as r:
         data = (json.load(r) if not FEEDS[name].endswith(".xml") else
                 {"slugs": sorted(set(re.findall(r"/model/([^<\"/]+)<", r.read().decode("utf-8"))))})
+    store(name, data)
+    return data
+
+
+def store(name: str, data: dict) -> None:
+    """Write one cached list whole, never half."""
     CACHE.mkdir(parents=True, exist_ok=True)
+    path = CACHE / f"{name}.json"
     tmp = path.with_suffix(".part")
     tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
     tmp.replace(path)
-    return data
+
+
+def age_s(name: str) -> float | None:
+    """Seconds since a cached list was written; None if it never was."""
+    path = CACHE / f"{name}.json"
+    return time.time() - path.stat().st_mtime if path.exists() else None
 
 
 def cached(name: str) -> dict:

package/metrics/test_decide.py

@@ -1,14 +1,87 @@
-"""python metrics/test_decide.py — error parsing and the question-type guard, no network."""
+"""python metrics/test_decide.py — the decision client and its log, the benchmark's
+sets, measures and pass, who is asked, the error-shape cache and the role
+suggestion. No network: urlopen is a stub. BENCHWARMERS_HOME is a temp folder,
+set before anything is imported."""
 import io
+import json
+import os
+import pathlib
+import shutil
 import sys
+import tempfile
 import urllib.error
 
-sys.path.insert(0, __file__.rsplit("\\", 1)[0].rsplit("/", 1)[0])
+TMP = pathlib.Path(tempfile.mkdtemp(prefix="bench-decide-"))
+for sub in ("config", "metrics", "perf", "runs/gateway", "llm"):
+    (TMP / sub).mkdir(parents=True)
+os.environ.update(BENCHWARMERS_HOME=str(TMP), LLM_USER_PATH=str(TMP / "llm"), BENCHWARMERS_KEY_HUB="sk-test")
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path[:0] = [str(HERE), str(HERE.parent / "suite")]
+
+
+def write(path: str, data) -> None:
+    text = data if isinstance(data, str) else json.dumps(data)
+    (TMP / path).write_text(text, encoding="utf-8")
+
+
+def jsonl(path: str, rows: list[dict]) -> None:
+    write(path, "".join(json.dumps(r) + "\n" for r in rows))
+
+
+write("config/endpoints.json", {"hub": {"base": "http://127.0.0.1:9/api", "key": "hub", "default": True,
+                                        "decisions": "http://127.0.0.1:9/openai/{path}"}})
+# dm2 first: before any result it is the one asked (the first that takes choice, as before),
+# except per request, where its :free budget is spared while dm1 can answer.
+MODELS = {
+    "dm2": {"served": "vendor/d2:free", "via": "hub", "question_types": ["noul", "choice", "score"]},
+    "dm1": {"served": "d1", "via": "hub", "path": "systemone", "question_types": ["noul", "choice", "score"]},
+    "yesno": {"served": "y", "via": "hub", "question_types": ["noul"]},
+    "dear": {"served": "dd", "via": "hub", "question_types": ["noul", "choice"]}}
+write("config/decision-models.json", MODELS)
+write("config/model-roles.json", {"models": {"m1": "general"}, "ad_hoc": ["dear"]})
+write("config/collections.json", {
+    "core": {"members": ["m1"], "description": "General second opinions.", "tags": ["general"], "shape": "general"},
+    "fast": {"members": ["m1"], "description": "", "tags": ["fast"], "shape": "quick"},
+    "vision": {"members": ["m1"], "description": "Photos with questions.", "tags": ["vision"]},
+    "dear": {"members": ["m1"], "description": "Costly.", "ad_hoc": True}})
+BUSY = "Error: Error code: 503 - The model is temporarily unavailable due to high demand. (Trace ID: 3fa2c9d81b)"
+QUOTA = "Error: Error code: 429 - Rate limit exceeded: free-models-per-day. Add 10 credits"
+ODD = "Error: the gremlins ate it"
+jsonl("runs/gateway/calls.jsonl", [
+    {"panel": "p1", "model": "bench/core", "shape": "general", "shape_by": "set on the collection", "system": "",
+     "prompt": "Review this function for bugs.",
+     "seats": [{"call": "p1.member1", "status": "ok", "answer": "It double-frees on line 3.", "error": ""},
+               {"call": "p1.explorer", "status": "error", "answer": "", "error": BUSY},
+               {"call": "p1.explorer2", "status": "empty", "answer": "", "error": ""}]},
+    {"panel": "p2", "model": "bench/fast", "shape": "quick", "shape_by": "set on the collection", "system": "Be brief.",
+     "prompt": "Spam or not: WIN A PRIZE", "seats": [{"call": "p2.member1", "status": "error", "answer": "", "error": QUOTA},
+                                                     {"call": "p2.member2", "status": "error", "answer": "", "error": ODD}]},
+    {"panel": "p3", "model": "bench/vision", "shape": "vision", "shape_by": "the request carries images",
+     "prompt": "What is printed here? [image: data URL, 12 KB]",
+     "seats": [{"call": "p3.member1", "status": "ok", "answer": "A cat.", "error": ""}]},
+    # Chosen by a decision model: labels nothing.
+    {"panel": "p4", "model": "bench/core", "shape": "general", "shape_by": "set on the collection",
+     "prompt": "A request the gateway's pick sent here.", "picked": {"collection": "bench/core"}, "seats": []},
+    {"panel": "p5", "model": "bench/dear", "shape": "general", "shape_by": "no explorer seats to fill",
+     "prompt": "An ad hoc collection's request.", "seats": []},
+    # The shape a decision model chose is not the collection's own.
+    {"panel": "p6", "model": "bench/core", "shape": "review", "shape_by": "dm2, 0.99", "prompt": "Is this lock safe?",
+     "seats": [{"call": "p6.member1", "status": "ok", "answer": "Yes.", "error": ""}]}])
+jsonl("metrics/offloads.jsonl", [
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": True, "note": "right", "call": "p1.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": False, "note": "a cat? no", "call": "p3.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": False, "note": "it is not", "call": "p6.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": True, "note": "ui test: in-place verdict",
+     "call": "p1.explorer"}])
+write("perf/history.csv", "timestamp,model,prompt,answered,error,detail\n2026-11-08T05:00:00,m1,tiny,0,1,<<TIMED OUT>>\n")
+
 import decide  # noqa: E402
 
+# ---- error text, and the type guard -----------------------------------------------
+
 
-def http_error(body: bytes) -> urllib.error.HTTPError:
-    return urllib.error.HTTPError("http://x", 400, "bad", {}, io.BytesIO(body))
+def http_error(body: bytes, code: int = 400) -> urllib.error.HTTPError:
+    return urllib.error.HTTPError("http://x", code, "bad", {}, io.BytesIO(body))
 
 
 # Every shape providers send must come back as text, never raise.
@@ -18,12 +91,207 @@ assert "passthrough" in decide.error_text(http_error(b'{"detail": "Direct API pa
 assert decide.error_text(http_error(b'{"error": null, "message": "down"}')).startswith("400")
 assert decide.error_text(http_error(b"not json")) == "400"
 
-# ask() refuses a type the entry does not list, before any call is made.
-decide.load = lambda: {"respan": {"served": "r", "via": "nowhere", "question_types": ["noul"]}}
+# ask() refuses a type the entry does not list, before any call is made (and logs nothing).
 try:
-    decide.ask("respan", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
+    decide.ask("yesno", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
 except ValueError as exc:
     assert "does not take choice" in str(exc)
 else:
     raise AssertionError("choice should have been refused")
+assert not decide.LOG.exists()
+
+# ---- the stub provider: dm1 answers the truth, dm2 is rate limited, the rest say yes / the first option --
+
+SEEN: list[dict] = []
+TRUTH: dict[str, object] = {}
+
+
+def urlopen(request, timeout=0):
+    body = json.loads(request.data)
+    SEEN.append({"url": request.full_url, **body})
+    if body["model"] == "vendor/d2:free":
+        raise http_error(b'{"error": {"message": "Rate limit exceeded: free-models-per-min"}}', 429)
+    name, q = next(iter(body["questions"].items()))
+    truth = TRUTH.get(body["state"]) if body["model"] == "d1" else None
+    if q["type"] == "noul":
+        p = 0.9 if truth in (True, None) else 0.1
+        answer = {"type": "noul", "noul": p}
+    else:
+        options = list(q["criteria"])
+        pick = truth if truth in options else options[0]
+        probs = {o: (0.9 if o == pick else 0.1 / (len(options) - 1)) for o in options}
+        answer = {"type": "choice", "choice": pick, "probabilities": probs, "confidence": 0.85}
+    return io.BytesIO(json.dumps({"answers": {name: answer}}).encode())
+
+
+decide.urllib.request.urlopen = urlopen
+
+# ---- every call is logged, and an error's body survives the log -------------------------------
+
+out = decide.ask("dm1", "hello", {"q": decide.PROBES["noul"]}, use="test")
+assert out["answers"]["q"]["noul"] == 0.9 and SEEN[-1]["url"].endswith("/openai/systemone")
+row = decide.log_rows()[-1]
+assert (row["alias"], row["served"], row["use"], row["types"], row["chars"]) == ("dm1", "d1", "test", {"q": "noul"}, 5)
+assert row["answers"]["q"]["noul"] == 0.9 and row["error"] == "" and isinstance(row["seconds"], float), row
+before = len(SEEN)
+try:
+    decide.ask("dm2", "hello", {"q": decide.PROBES["noul"]}, use="test", retry=False)
+except urllib.error.HTTPError as exc:
+    assert "free-models-per-min" in decide.error_text(exc)          # read once by the log, kept for the caller
+else:
+    raise AssertionError("dm2 should have failed")
+assert len(SEEN) == before + 1, "retry=False: a 429 is not retried"
+row = decide.log_rows()[-1]
+assert row["alias"] == "dm2" and row["answers"] is None and "429 Rate limit" in row["error"], row
+
+# ---- shapes: nothing private, the status kept -------------------------------------------------
+
+s = decide.shape("Error code: 429 - key sk-or-v1-abcdef0123456789 for a.b@example.com at"
+                 " https://x.example/v1?key=secret, trace 3fa2c9d81b, retry in 37 s")
+assert "429" in s and "<email>" in s and "<url>" in s and "retry in N s" in s, s
+assert not any(w in s for w in ("secret", "example.com", "abcdef0123", "3fa2c9d81b")), s
+assert decide.shape("word " * 400).count("…") == 1                    # start and end kept
+
+# ---- the sets, from the data written above ----------------------------------------------------
+
+sets = decide.build_sets()
+labels = {k: sorted(str(i["label"]) for i in v["items"]) for k, v in sets.items()}
+assert labels["errors"] == ["busy", "empty", "quota", "timeout"], labels           # the gremlins: unrecognised
+assert labels["collection"] == ["core", "core", "fast", "vision"], labels           # not the pick's, not ad hoc
+assert labels["useful"] == ["False", "True"], labels                                 # text only, no ui test
+assert set(sets) == {"errors", "collection", "useful"}                               # no shape set
+assert set(sets["collection"]["question"]["criteria"]) == {"core", "fast", "vision"}
+assert sets["collection"]["question"]["criteria"]["fast"] == "No description. Tags: fast."
+assert "Be brief.\n\nSpam or not" in next(i["state"] for i in sets["collection"]["items"] if i["label"] == "fast")
+assert next(i for i in sets["useful"]["items"] if i["label"])["state"].startswith("THE REQUEST\n\nReview this")
+assert not any("3fa2c9d81b" in i["state"] for i in sets["errors"]["items"])
+for v in sets.values():
+    for i in v["items"]:
+        TRUTH[i["state"]] = i["label"]
+
+# ---- who is asked, before any result: the first onboarded that takes the type, not ad hoc ------
+
+assert decide.best("errors") == "dm2" and decide.best("useful") == "dm2" and decide.best("role") == "dm2"
+# Per request (the gateway's picks, the notice check) the :free dm2 is spared while dm1 takes the question.
+assert decide.best("shape") == "dm1" and decide.best("collection") == "dm1" and decide.best("notice") == "dm1"
+who, why = decide.choose("shape")
+assert who == "dm1" and "the default" in why and ":free models spared" in why and "collection set" in why, why
+write("config/decision-models.json", {"dm2": MODELS["dm2"]})
+assert decide.best("shape") == "dm2"                                  # the only one: the free budget it is
+write("config/decision-models.json", {"dear": {"served": "dd", "via": "hub", "question_types": ["choice"]}})
+assert decide.best("shape") is None                                   # ad hoc only: nobody
+answer, by, why = decide.pick("shape", "x", decide.SHAPES, "test")
+assert (answer, by, why) == ({}, "", "no decision model onboarded")
+write("config/decision-models.json", MODELS)
+
+# ---- the measures, by hand ----------------------------------------------------------------------
+
+choice = [
+    {"set": "s", "label": "a", "seconds": 1.0, "at": "1",
+     "answers": {"s": {"choice": "a", "probabilities": {"a": 0.9, "b": 0.1}, "confidence": 0.8}}},
+    {"set": "s", "label": "a", "seconds": 3.0, "at": "2",
+     "answers": {"s": {"choice": "b", "probabilities": {"a": 0.3, "b": 0.7}, "confidence": 0.4}}},
+    {"set": "s", "label": "a", "seconds": 2.0, "at": "3", "answers": None, "error": QUOTA}]
+m = decide.measure(choice)
+assert (m["asked"], m["answered"], m["correct"], m["accuracy"]) == (3, 2, 1, 0.5), m
+assert (m["brier"], m["log_loss"], m["ece"], m["macro_f1"], m["seconds"]) == (0.5, 0.6547, 0.4, 0.333, 2.0), m
+assert m["confident"] == {"cutoff": 0.6, "coverage": 0.5, "accuracy": 1.0} and m["failures"] == {"quota": 1}, m
+assert [(b["from"], b["n"], b["right"]) for b in m["calibration"]] == [(0.6, 1, 0.0), (0.8, 1, 1.0)], m
+noul = [{"set": "u", "label": True, "answers": {"u": {"noul": 0.8}}},
+        {"set": "u", "label": False, "answers": {"u": {"noul": 0.7}}}]
+m = decide.measure(noul)
+assert (m["accuracy"], m["brier"], "macro_f1" in m) == (0.5, 0.265, False), m
+assert decide.measure([{"set": "u", "label": True, "answers": {"u": {"noul": "?"}}}])["failures"] == {"no answer": 1}
+# A null confidence is no confidence: the answer still counts, at the probability it gave its choice.
+m = decide.measure([{"set": "s", "label": "a", "answers": {"s": {"choice": "a", "probabilities": {"a": 0.9, "b": 0.1},
+                                                                 "confidence": None}}}])
+assert m["answered"] == 1 and m["confident"]["coverage"] == 1.0, m
+# An endpoint gone from the config is a failed call, logged like any other.
+try:
+    decide.call("gone", "x", "s", {"q": decide.PROBES["noul"]}, use="test")
+except KeyError:
+    pass
+assert decide.log_rows()[-1]["alias"] == "x" and "KeyError" in decide.log_rows()[-1]["error"]
+
+# ---- a dry run asks nobody; a pass is round-robin, paced, and stops a model at its first failure --
+
+plan = decide.bench(per=1, dry=True, out=lambda *_: None)
+assert set(plan) == {"dm2", "dm1", "yesno"}, plan                     # never the ad hoc one
+assert [s for s, _ in plan["yesno"]] == ["useful"] and [s for s, _ in plan["dm1"]] == list(decide.SETS), plan
+SEEN.clear()
+slept: list[float] = []
+res = decide.bench(per=1, gap=7, sleep=slept.append, out=lambda *_: None)
+assert [q["model"] for q in SEEN] == ["vendor/d2:free", "d1", "y", "d1", "d1"], [q["model"] for q in SEEN]
+assert slept == [7] * 4, slept
+assert set(res["models"]["dm1"]) == set(decide.SETS) and all(r["correct"] == 1 for r in res["models"]["dm1"].values())
+assert res["models"]["dm2"]["errors"]["failures"] == {"rate": 1}, res["models"]["dm2"]
+assert res["sets"]["errors"]["size"] == 4 and res["min_items"] == decide.MIN_ITEMS, res
+assert json.loads(decide.RESULTS.read_text(encoding="utf-8"))["models"]["yesno"]["useful"]["correct"] == 1
+# One item each is under MIN_ITEMS: nobody is ranked, the defaults stand.
+assert decide.best("errors") == "dm2" and "none has 8 items on the errors set" in decide.choose("errors")[1]
+decide.MIN_ITEMS = 1
+assert decide.best("errors") == "dm1" and decide.best("role") == "dm1"           # 1 of 1 beats 0 of 1
+assert decide.choose("errors")[1] == "best of 2 ranked on the errors set: 1 right of 1 asked"
+assert decide.best("shape") == "dm1" and decide.best("useful") == "dm1"           # a tie with yesno: config order
+decide.MIN_ITEMS = 8
+bench_rows = [r for r in decide.log_rows() if r.get("use") == "bench"]
+assert len(bench_rows) == 5 and all(r["item"] and "label" in r for r in bench_rows), bench_rows
+# A second pass asks each model the items it was asked longest ago: another collection item now.
+plan2 = decide.bench(per=1, dry=True, out=lambda *_: None)
+assert dict(plan2["dm1"])["collection"] != dict(plan["dm1"])["collection"], (plan, plan2)
+
+# ---- the pick asks the best, and logs the use ----------------------------------------------------
+
+answer, by, why = decide.pick("shape", "Review this function for bugs.", decide.SHAPES, "pick_shape")
+assert by == "dm1" and not why and answer["choice"] in decide.SHAPES
+assert decide.log_rows()[-1]["use"] == "pick_shape"
+
+# ---- unrecognised error shapes: asked once each, cached, read without a call ----------------------
+
+import run  # noqa: E402
+
+decide.MIN_ITEMS = 1                                                  # dm1 ranked first on errors from here
+SEEN.clear()
+TRUTH[decide.shape(ODD)] = "unreachable"                              # what the stub's dm1 will say
+assert run.uptime_class(ODD) == ("unrecognised", "nothing matched")
+assert decide.classify_errors(limit=3, sleep=slept.append) == 1 and len(SEEN) == 1   # only the gremlins
+assert SEEN[0]["state"] == decide.shape(ODD) and SEEN[0]["model"] == "d1"
+assert decide.cached_error_class(ODD) == ("unreachable", "decided by dm1 at 0.85"), decide.cached_error_class(ODD)
+assert decide.classify_errors(limit=3) == 0                           # asked once per shape
+assert decide.cached_error_class("something else entirely") is None
+# uptime_class reads the decision (no call), the benchmark's labels never do.
+assert run.uptime_class(ODD) == ("unreachable", "decided by dm1 at 0.85") and len(SEEN) == 1
+assert run.uptime_class(ODD, decided=False) == ("unrecognised", "nothing matched")
+assert decide.build_sets(["errors"])["errors"]["items"] == sets["errors"]["items"]
+# A decided `empty` is never used: an error is not an answer, and empty counts as up.
+import locked  # noqa: E402
+locked.update(decide.DECIDED, lambda d: d["errors"][decide.shape(ODD)].__setitem__("class", "empty"))
+assert run.uptime_class(ODD)[0] == "unrecognised"
+locked.update(decide.DECIDED, lambda d: d["errors"][decide.shape(ODD)].__setitem__("class", "unreachable"))
+
+# ---- the provider-notice fallback: asked only for a seat's short reply that looks like one ----------
+
+SEEN.clear()
+NOTE = "Your plan's balance is low; top up soon to keep going."
+assert run.provider_notice("ready") is None and run.provider_notice("ready", ask=True) is None
+assert run.provider_notice("The limit here is " + "x " * 200, ask=True) is None      # long: an answer
+assert run.provider_notice(NOTE) is None and not SEEN                 # a page or a score never asks
+assert run.provider_notice("you have run out of credits") == "run out of credits" and not SEEN   # the phrases first
+assert run.provider_notice(NOTE, ask=True) == "decided by dm1 at 0.90" and len(SEEN) == 1
+assert SEEN[0]["model"] == "d1" and SEEN[0]["questions"]["notice"]["type"] == "noul"   # per request: not :free
+assert decide.log_rows()[-1]["use"] == "notice"
+assert run.provider_notice(NOTE, ask=True) == "decided by dm1 at 0.90" and len(SEEN) == 1   # kept: asked once
+assert run.provider_notice(NOTE) == "decided by dm1 at 0.90"                         # and read anywhere
+assert run.failure_class(NOTE) == "entitlement"
+
+# ---- a role suggestion is kept, never set --------------------------------------------------------
+
+hint = decide.suggest_role("m2", "model: vendor/m2\ncontext window: 8192\ndescription: A small coder.")
+assert (hint["role"], hint["confidence"], hint["by"]) == ("general", 0.85, "dm1"), hint   # the stub's first option
+assert "decision" not in SEEN[-1]["questions"]["role"]["criteria"], SEEN[-1]
+kept = json.loads(decide.DECIDED.read_text(encoding="utf-8"))
+assert kept["roles"]["m2"]["role"] == hint["role"] and "unreachable" == kept["errors"][decide.shape(ODD)]["class"]
+assert json.loads((TMP / "config" / "model-roles.json").read_text(encoding="utf-8"))["models"] == {"m1": "general"}
+
+shutil.rmtree(TMP, ignore_errors=True)
 print("ok")

package/metrics/wrong.py

@@ -75,13 +75,30 @@ def judge(spec: dict, text: str, invented: list) -> tuple[str | None, bool]:
     return None, False
 
 
-def table() -> dict[str, dict]:
+def _row() -> dict:
+    return {"answers": 0, "wrong": 0, "by": {k: [0, 0] for k in KINDS}, "reviews": 0, "review_false": 0,
+            "hunt": dict.fromkeys(HUNT, 0)}
+
+
+def table(scored: dict[str, dict] | None = None) -> dict[str, dict]:
+    """scores(), or the `scored` the caller kept from it, with each model's
+    reviews from the offloads added."""
+    import copy
     import offload
+    out = collections.defaultdict(_row, copy.deepcopy(scores() if scored is None else scored))
+    for e in offload.entries():
+        if e.get("useful") is not None:
+            row = out[e["model"]]
+            row["reviews"] += 1
+            row["review_false"] += len(e.get("false") or [])
+    return dict(out)
+
+
+def scores() -> dict[str, dict]:
+    """Every suite answer judged, by model: the costly part (it reads them all)."""
     import run
     from aliases import current
-    out: dict[str, dict] = collections.defaultdict(
-        lambda: {"answers": 0, "wrong": 0, "by": {k: [0, 0] for k in KINDS}, "reviews": 0, "review_false": 0,
-                 "hunt": dict.fromkeys(HUNT, 0)})
+    out: dict[str, dict] = collections.defaultdict(_row)
     for path in sorted(TASKS.glob("*/task.json")):
         spec = json.loads(path.read_text(encoding="utf-8"))
         # Reading images has its own record (serve.vision_rows): a trap taken
@@ -104,11 +121,6 @@ def table() -> dict[str, dict]:
             if wrong:
                 row["wrong"] += 1
                 row["by"][kind][0] += 1
-    for e in offload.entries():
-        if e.get("useful") is not None:
-            row = out[e["model"]]
-            row["reviews"] += 1
-            row["review_false"] += len(e.get("false") or [])
     return dict(out)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.5.5",
+  "version": "0.6.0",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",
@@ -49,6 +49,8 @@
     "repoqa/*.py",
     "repoqa/*.md",
     "ui/",
+    ".github/logo.svg",
+    ".github/logo-dark.svg",
     "examples/",
     "VERSION",
     "NOTICE",