@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.0

package/metrics/test_decide.py

@@ -1,14 +1,87 @@
-"""python metrics/test_decide.py — error parsing and the question-type guard, no network."""
+"""python metrics/test_decide.py — the decision client and its log, the benchmark's
+sets, measures and pass, who is asked, the error-shape cache and the role
+suggestion. No network: urlopen is a stub. BENCHWARMERS_HOME is a temp folder,
+set before anything is imported."""
 import io
+import json
+import os
+import pathlib
+import shutil
 import sys
+import tempfile
 import urllib.error
 
-sys.path.insert(0, __file__.rsplit("\\", 1)[0].rsplit("/", 1)[0])
+TMP = pathlib.Path(tempfile.mkdtemp(prefix="bench-decide-"))
+for sub in ("config", "metrics", "perf", "runs/gateway", "llm"):
+    (TMP / sub).mkdir(parents=True)
+os.environ.update(BENCHWARMERS_HOME=str(TMP), LLM_USER_PATH=str(TMP / "llm"), BENCHWARMERS_KEY_HUB="sk-test")
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path[:0] = [str(HERE), str(HERE.parent / "suite")]
+
+
+def write(path: str, data) -> None:
+    text = data if isinstance(data, str) else json.dumps(data)
+    (TMP / path).write_text(text, encoding="utf-8")
+
+
+def jsonl(path: str, rows: list[dict]) -> None:
+    write(path, "".join(json.dumps(r) + "\n" for r in rows))
+
+
+write("config/endpoints.json", {"hub": {"base": "http://127.0.0.1:9/api", "key": "hub", "default": True,
+                                        "decisions": "http://127.0.0.1:9/openai/{path}"}})
+# dm2 first: before any result it is the one asked (the first that takes choice, as before),
+# except per request, where its :free budget is spared while dm1 can answer.
+MODELS = {
+    "dm2": {"served": "vendor/d2:free", "via": "hub", "question_types": ["noul", "choice", "score"]},
+    "dm1": {"served": "d1", "via": "hub", "path": "systemone", "question_types": ["noul", "choice", "score"]},
+    "yesno": {"served": "y", "via": "hub", "question_types": ["noul"]},
+    "dear": {"served": "dd", "via": "hub", "question_types": ["noul", "choice"]}}
+write("config/decision-models.json", MODELS)
+write("config/model-roles.json", {"models": {"m1": "general"}, "ad_hoc": ["dear"]})
+write("config/collections.json", {
+    "core": {"members": ["m1"], "description": "General second opinions.", "tags": ["general"], "shape": "general"},
+    "fast": {"members": ["m1"], "description": "", "tags": ["fast"], "shape": "quick"},
+    "vision": {"members": ["m1"], "description": "Photos with questions.", "tags": ["vision"]},
+    "dear": {"members": ["m1"], "description": "Costly.", "ad_hoc": True}})
+BUSY = "Error: Error code: 503 - The model is temporarily unavailable due to high demand. (Trace ID: 3fa2c9d81b)"
+QUOTA = "Error: Error code: 429 - Rate limit exceeded: free-models-per-day. Add 10 credits"
+ODD = "Error: the gremlins ate it"
+jsonl("runs/gateway/calls.jsonl", [
+    {"panel": "p1", "model": "bench/core", "shape": "general", "shape_by": "set on the collection", "system": "",
+     "prompt": "Review this function for bugs.",
+     "seats": [{"call": "p1.member1", "status": "ok", "answer": "It double-frees on line 3.", "error": ""},
+               {"call": "p1.explorer", "status": "error", "answer": "", "error": BUSY},
+               {"call": "p1.explorer2", "status": "empty", "answer": "", "error": ""}]},
+    {"panel": "p2", "model": "bench/fast", "shape": "quick", "shape_by": "set on the collection", "system": "Be brief.",
+     "prompt": "Spam or not: WIN A PRIZE", "seats": [{"call": "p2.member1", "status": "error", "answer": "", "error": QUOTA},
+                                                     {"call": "p2.member2", "status": "error", "answer": "", "error": ODD}]},
+    {"panel": "p3", "model": "bench/vision", "shape": "vision", "shape_by": "the request carries images",
+     "prompt": "What is printed here? [image: data URL, 12 KB]",
+     "seats": [{"call": "p3.member1", "status": "ok", "answer": "A cat.", "error": ""}]},
+    # Chosen by a decision model: labels nothing.
+    {"panel": "p4", "model": "bench/core", "shape": "general", "shape_by": "set on the collection",
+     "prompt": "A request the gateway's pick sent here.", "picked": {"collection": "bench/core"}, "seats": []},
+    {"panel": "p5", "model": "bench/dear", "shape": "general", "shape_by": "no explorer seats to fill",
+     "prompt": "An ad hoc collection's request.", "seats": []},
+    # The shape a decision model chose is not the collection's own.
+    {"panel": "p6", "model": "bench/core", "shape": "review", "shape_by": "dm2, 0.99", "prompt": "Is this lock safe?",
+     "seats": [{"call": "p6.member1", "status": "ok", "answer": "Yes.", "error": ""}]}])
+jsonl("metrics/offloads.jsonl", [
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": True, "note": "right", "call": "p1.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": False, "note": "a cat? no", "call": "p3.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": False, "note": "it is not", "call": "p6.member1"},
+    {"date": "2026-11-08", "model": "m1", "task": "t", "useful": True, "note": "ui test: in-place verdict",
+     "call": "p1.explorer"}])
+write("perf/history.csv", "timestamp,model,prompt,answered,error,detail\n2026-11-08T05:00:00,m1,tiny,0,1,<<TIMED OUT>>\n")
+
 import decide  # noqa: E402
 
+# ---- error text, and the type guard -----------------------------------------------
+
 
-def http_error(body: bytes) -> urllib.error.HTTPError:
-    return urllib.error.HTTPError("http://x", 400, "bad", {}, io.BytesIO(body))
+def http_error(body: bytes, code: int = 400) -> urllib.error.HTTPError:
+    return urllib.error.HTTPError("http://x", code, "bad", {}, io.BytesIO(body))
 
 
 # Every shape providers send must come back as text, never raise.
@@ -18,12 +91,207 @@ assert "passthrough" in decide.error_text(http_error(b'{"detail": "Direct API pa
 assert decide.error_text(http_error(b'{"error": null, "message": "down"}')).startswith("400")
 assert decide.error_text(http_error(b"not json")) == "400"
 
-# ask() refuses a type the entry does not list, before any call is made.
-decide.load = lambda: {"respan": {"served": "r", "via": "nowhere", "question_types": ["noul"]}}
+# ask() refuses a type the entry does not list, before any call is made (and logs nothing).
 try:
-    decide.ask("respan", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
+    decide.ask("yesno", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
 except ValueError as exc:
     assert "does not take choice" in str(exc)
 else:
     raise AssertionError("choice should have been refused")
+assert not decide.LOG.exists()
+
+# ---- the stub provider: dm1 answers the truth, dm2 is rate limited, the rest say yes / the first option --
+
+SEEN: list[dict] = []
+TRUTH: dict[str, object] = {}
+
+
+def urlopen(request, timeout=0):
+    body = json.loads(request.data)
+    SEEN.append({"url": request.full_url, **body})
+    if body["model"] == "vendor/d2:free":
+        raise http_error(b'{"error": {"message": "Rate limit exceeded: free-models-per-min"}}', 429)
+    name, q = next(iter(body["questions"].items()))
+    truth = TRUTH.get(body["state"]) if body["model"] == "d1" else None
+    if q["type"] == "noul":
+        p = 0.9 if truth in (True, None) else 0.1
+        answer = {"type": "noul", "noul": p}
+    else:
+        options = list(q["criteria"])
+        pick = truth if truth in options else options[0]
+        probs = {o: (0.9 if o == pick else 0.1 / (len(options) - 1)) for o in options}
+        answer = {"type": "choice", "choice": pick, "probabilities": probs, "confidence": 0.85}
+    return io.BytesIO(json.dumps({"answers": {name: answer}}).encode())
+
+
+decide.urllib.request.urlopen = urlopen
+
+# ---- every call is logged, and an error's body survives the log -------------------------------
+
+out = decide.ask("dm1", "hello", {"q": decide.PROBES["noul"]}, use="test")
+assert out["answers"]["q"]["noul"] == 0.9 and SEEN[-1]["url"].endswith("/openai/systemone")
+row = decide.log_rows()[-1]
+assert (row["alias"], row["served"], row["use"], row["types"], row["chars"]) == ("dm1", "d1", "test", {"q": "noul"}, 5)
+assert row["answers"]["q"]["noul"] == 0.9 and row["error"] == "" and isinstance(row["seconds"], float), row
+before = len(SEEN)
+try:
+    decide.ask("dm2", "hello", {"q": decide.PROBES["noul"]}, use="test", retry=False)
+except urllib.error.HTTPError as exc:
+    assert "free-models-per-min" in decide.error_text(exc)          # read once by the log, kept for the caller
+else:
+    raise AssertionError("dm2 should have failed")
+assert len(SEEN) == before + 1, "retry=False: a 429 is not retried"
+row = decide.log_rows()[-1]
+assert row["alias"] == "dm2" and row["answers"] is None and "429 Rate limit" in row["error"], row
+
+# ---- shapes: nothing private, the status kept -------------------------------------------------
+
+s = decide.shape("Error code: 429 - key sk-or-v1-abcdef0123456789 for a.b@example.com at"
+                 " https://x.example/v1?key=secret, trace 3fa2c9d81b, retry in 37 s")
+assert "429" in s and "<email>" in s and "<url>" in s and "retry in N s" in s, s
+assert not any(w in s for w in ("secret", "example.com", "abcdef0123", "3fa2c9d81b")), s
+assert decide.shape("word " * 400).count("…") == 1                    # start and end kept
+
+# ---- the sets, from the data written above ----------------------------------------------------
+
+sets = decide.build_sets()
+labels = {k: sorted(str(i["label"]) for i in v["items"]) for k, v in sets.items()}
+assert labels["errors"] == ["busy", "empty", "quota", "timeout"], labels           # the gremlins: unrecognised
+assert labels["collection"] == ["core", "core", "fast", "vision"], labels           # not the pick's, not ad hoc
+assert labels["useful"] == ["False", "True"], labels                                 # text only, no ui test
+assert set(sets) == {"errors", "collection", "useful"}                               # no shape set
+assert set(sets["collection"]["question"]["criteria"]) == {"core", "fast", "vision"}
+assert sets["collection"]["question"]["criteria"]["fast"] == "No description. Tags: fast."
+assert "Be brief.\n\nSpam or not" in next(i["state"] for i in sets["collection"]["items"] if i["label"] == "fast")
+assert next(i for i in sets["useful"]["items"] if i["label"])["state"].startswith("THE REQUEST\n\nReview this")
+assert not any("3fa2c9d81b" in i["state"] for i in sets["errors"]["items"])
+for v in sets.values():
+    for i in v["items"]:
+        TRUTH[i["state"]] = i["label"]
+
+# ---- who is asked, before any result: the first onboarded that takes the type, not ad hoc ------
+
+assert decide.best("errors") == "dm2" and decide.best("useful") == "dm2" and decide.best("role") == "dm2"
+# Per request (the gateway's picks, the notice check) the :free dm2 is spared while dm1 takes the question.
+assert decide.best("shape") == "dm1" and decide.best("collection") == "dm1" and decide.best("notice") == "dm1"
+who, why = decide.choose("shape")
+assert who == "dm1" and "the default" in why and ":free models spared" in why and "collection set" in why, why
+write("config/decision-models.json", {"dm2": MODELS["dm2"]})
+assert decide.best("shape") == "dm2"                                  # the only one: the free budget it is
+write("config/decision-models.json", {"dear": {"served": "dd", "via": "hub", "question_types": ["choice"]}})
+assert decide.best("shape") is None                                   # ad hoc only: nobody
+answer, by, why = decide.pick("shape", "x", decide.SHAPES, "test")
+assert (answer, by, why) == ({}, "", "no decision model onboarded")
+write("config/decision-models.json", MODELS)
+
+# ---- the measures, by hand ----------------------------------------------------------------------
+
+choice = [
+    {"set": "s", "label": "a", "seconds": 1.0, "at": "1",
+     "answers": {"s": {"choice": "a", "probabilities": {"a": 0.9, "b": 0.1}, "confidence": 0.8}}},
+    {"set": "s", "label": "a", "seconds": 3.0, "at": "2",
+     "answers": {"s": {"choice": "b", "probabilities": {"a": 0.3, "b": 0.7}, "confidence": 0.4}}},
+    {"set": "s", "label": "a", "seconds": 2.0, "at": "3", "answers": None, "error": QUOTA}]
+m = decide.measure(choice)
+assert (m["asked"], m["answered"], m["correct"], m["accuracy"]) == (3, 2, 1, 0.5), m
+assert (m["brier"], m["log_loss"], m["ece"], m["macro_f1"], m["seconds"]) == (0.5, 0.6547, 0.4, 0.333, 2.0), m
+assert m["confident"] == {"cutoff": 0.6, "coverage": 0.5, "accuracy": 1.0} and m["failures"] == {"quota": 1}, m
+assert [(b["from"], b["n"], b["right"]) for b in m["calibration"]] == [(0.6, 1, 0.0), (0.8, 1, 1.0)], m
+noul = [{"set": "u", "label": True, "answers": {"u": {"noul": 0.8}}},
+        {"set": "u", "label": False, "answers": {"u": {"noul": 0.7}}}]
+m = decide.measure(noul)
+assert (m["accuracy"], m["brier"], "macro_f1" in m) == (0.5, 0.265, False), m
+assert decide.measure([{"set": "u", "label": True, "answers": {"u": {"noul": "?"}}}])["failures"] == {"no answer": 1}
+# A null confidence is no confidence: the answer still counts, at the probability it gave its choice.
+m = decide.measure([{"set": "s", "label": "a", "answers": {"s": {"choice": "a", "probabilities": {"a": 0.9, "b": 0.1},
+                                                                 "confidence": None}}}])
+assert m["answered"] == 1 and m["confident"]["coverage"] == 1.0, m
+# An endpoint gone from the config is a failed call, logged like any other.
+try:
+    decide.call("gone", "x", "s", {"q": decide.PROBES["noul"]}, use="test")
+except KeyError:
+    pass
+assert decide.log_rows()[-1]["alias"] == "x" and "KeyError" in decide.log_rows()[-1]["error"]
+
+# ---- a dry run asks nobody; a pass is round-robin, paced, and stops a model at its first failure --
+
+plan = decide.bench(per=1, dry=True, out=lambda *_: None)
+assert set(plan) == {"dm2", "dm1", "yesno"}, plan                     # never the ad hoc one
+assert [s for s, _ in plan["yesno"]] == ["useful"] and [s for s, _ in plan["dm1"]] == list(decide.SETS), plan
+SEEN.clear()
+slept: list[float] = []
+res = decide.bench(per=1, gap=7, sleep=slept.append, out=lambda *_: None)
+assert [q["model"] for q in SEEN] == ["vendor/d2:free", "d1", "y", "d1", "d1"], [q["model"] for q in SEEN]
+assert slept == [7] * 4, slept
+assert set(res["models"]["dm1"]) == set(decide.SETS) and all(r["correct"] == 1 for r in res["models"]["dm1"].values())
+assert res["models"]["dm2"]["errors"]["failures"] == {"rate": 1}, res["models"]["dm2"]
+assert res["sets"]["errors"]["size"] == 4 and res["min_items"] == decide.MIN_ITEMS, res
+assert json.loads(decide.RESULTS.read_text(encoding="utf-8"))["models"]["yesno"]["useful"]["correct"] == 1
+# One item each is under MIN_ITEMS: nobody is ranked, the defaults stand.
+assert decide.best("errors") == "dm2" and "none has 8 items on the errors set" in decide.choose("errors")[1]
+decide.MIN_ITEMS = 1
+assert decide.best("errors") == "dm1" and decide.best("role") == "dm1"           # 1 of 1 beats 0 of 1
+assert decide.choose("errors")[1] == "best of 2 ranked on the errors set: 1 right of 1 asked"
+assert decide.best("shape") == "dm1" and decide.best("useful") == "dm1"           # a tie with yesno: config order
+decide.MIN_ITEMS = 8
+bench_rows = [r for r in decide.log_rows() if r.get("use") == "bench"]
+assert len(bench_rows) == 5 and all(r["item"] and "label" in r for r in bench_rows), bench_rows
+# A second pass asks each model the items it was asked longest ago: another collection item now.
+plan2 = decide.bench(per=1, dry=True, out=lambda *_: None)
+assert dict(plan2["dm1"])["collection"] != dict(plan["dm1"])["collection"], (plan, plan2)
+
+# ---- the pick asks the best, and logs the use ----------------------------------------------------
+
+answer, by, why = decide.pick("shape", "Review this function for bugs.", decide.SHAPES, "pick_shape")
+assert by == "dm1" and not why and answer["choice"] in decide.SHAPES
+assert decide.log_rows()[-1]["use"] == "pick_shape"
+
+# ---- unrecognised error shapes: asked once each, cached, read without a call ----------------------
+
+import run  # noqa: E402
+
+decide.MIN_ITEMS = 1                                                  # dm1 ranked first on errors from here
+SEEN.clear()
+TRUTH[decide.shape(ODD)] = "unreachable"                              # what the stub's dm1 will say
+assert run.uptime_class(ODD) == ("unrecognised", "nothing matched")
+assert decide.classify_errors(limit=3, sleep=slept.append) == 1 and len(SEEN) == 1   # only the gremlins
+assert SEEN[0]["state"] == decide.shape(ODD) and SEEN[0]["model"] == "d1"
+assert decide.cached_error_class(ODD) == ("unreachable", "decided by dm1 at 0.85"), decide.cached_error_class(ODD)
+assert decide.classify_errors(limit=3) == 0                           # asked once per shape
+assert decide.cached_error_class("something else entirely") is None
+# uptime_class reads the decision (no call), the benchmark's labels never do.
+assert run.uptime_class(ODD) == ("unreachable", "decided by dm1 at 0.85") and len(SEEN) == 1
+assert run.uptime_class(ODD, decided=False) == ("unrecognised", "nothing matched")
+assert decide.build_sets(["errors"])["errors"]["items"] == sets["errors"]["items"]
+# A decided `empty` is never used: an error is not an answer, and empty counts as up.
+import locked  # noqa: E402
+locked.update(decide.DECIDED, lambda d: d["errors"][decide.shape(ODD)].__setitem__("class", "empty"))
+assert run.uptime_class(ODD)[0] == "unrecognised"
+locked.update(decide.DECIDED, lambda d: d["errors"][decide.shape(ODD)].__setitem__("class", "unreachable"))
+
+# ---- the provider-notice fallback: asked only for a seat's short reply that looks like one ----------
+
+SEEN.clear()
+NOTE = "Your plan's balance is low; top up soon to keep going."
+assert run.provider_notice("ready") is None and run.provider_notice("ready", ask=True) is None
+assert run.provider_notice("The limit here is " + "x " * 200, ask=True) is None      # long: an answer
+assert run.provider_notice(NOTE) is None and not SEEN                 # a page or a score never asks
+assert run.provider_notice("you have run out of credits") == "run out of credits" and not SEEN   # the phrases first
+assert run.provider_notice(NOTE, ask=True) == "decided by dm1 at 0.90" and len(SEEN) == 1
+assert SEEN[0]["model"] == "d1" and SEEN[0]["questions"]["notice"]["type"] == "noul"   # per request: not :free
+assert decide.log_rows()[-1]["use"] == "notice"
+assert run.provider_notice(NOTE, ask=True) == "decided by dm1 at 0.90" and len(SEEN) == 1   # kept: asked once
+assert run.provider_notice(NOTE) == "decided by dm1 at 0.90"                         # and read anywhere
+assert run.failure_class(NOTE) == "entitlement"
+
+# ---- a role suggestion is kept, never set --------------------------------------------------------
+
+hint = decide.suggest_role("m2", "model: vendor/m2\ncontext window: 8192\ndescription: A small coder.")
+assert (hint["role"], hint["confidence"], hint["by"]) == ("general", 0.85, "dm1"), hint   # the stub's first option
+assert "decision" not in SEEN[-1]["questions"]["role"]["criteria"], SEEN[-1]
+kept = json.loads(decide.DECIDED.read_text(encoding="utf-8"))
+assert kept["roles"]["m2"]["role"] == hint["role"] and "unreachable" == kept["errors"][decide.shape(ODD)]["class"]
+assert json.loads((TMP / "config" / "model-roles.json").read_text(encoding="utf-8"))["models"] == {"m1": "general"}
+
+shutil.rmtree(TMP, ignore_errors=True)
 print("ok")

package/metrics/wrong.py

@@ -75,13 +75,30 @@ def judge(spec: dict, text: str, invented: list) -> tuple[str | None, bool]:
     return None, False
 
 
-def table() -> dict[str, dict]:
+def _row() -> dict:
+    return {"answers": 0, "wrong": 0, "by": {k: [0, 0] for k in KINDS}, "reviews": 0, "review_false": 0,
+            "hunt": dict.fromkeys(HUNT, 0)}
+
+
+def table(scored: dict[str, dict] | None = None) -> dict[str, dict]:
+    """scores(), or the `scored` the caller kept from it, with each model's
+    reviews from the offloads added."""
+    import copy
     import offload
+    out = collections.defaultdict(_row, copy.deepcopy(scores() if scored is None else scored))
+    for e in offload.entries():
+        if e.get("useful") is not None:
+            row = out[e["model"]]
+            row["reviews"] += 1
+            row["review_false"] += len(e.get("false") or [])
+    return dict(out)
+
+
+def scores() -> dict[str, dict]:
+    """Every suite answer judged, by model: the costly part (it reads them all)."""
     import run
     from aliases import current
-    out: dict[str, dict] = collections.defaultdict(
-        lambda: {"answers": 0, "wrong": 0, "by": {k: [0, 0] for k in KINDS}, "reviews": 0, "review_false": 0,
-                 "hunt": dict.fromkeys(HUNT, 0)})
+    out: dict[str, dict] = collections.defaultdict(_row)
     for path in sorted(TASKS.glob("*/task.json")):
         spec = json.loads(path.read_text(encoding="utf-8"))
         # Reading images has its own record (serve.vision_rows): a trap taken
@@ -104,11 +121,6 @@ def table() -> dict[str, dict]:
             if wrong:
                 row["wrong"] += 1
                 row["by"][kind][0] += 1
-    for e in offload.entries():
-        if e.get("useful") is not None:
-            row = out[e["model"]]
-            row["reviews"] += 1
-            row["review_false"] += len(e.get("false") or [])
     return dict(out)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.5.6",
+  "version": "0.6.0",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",
@@ -49,6 +49,8 @@
     "repoqa/*.py",
     "repoqa/*.md",
     "ui/",
+    ".github/logo.svg",
+    ".github/logo-dark.svg",
     "examples/",
     "VERSION",
     "NOTICE",

package/scripts/auto_retire.py

@@ -49,8 +49,10 @@ from __future__ import annotations
 import csv
 import datetime as dt
 import json
+import os
 import pathlib
 import sys
+import threading
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT / "metrics"))
@@ -201,11 +203,23 @@ def _current(alias: str, history: dict[str, str]) -> str:
     return alias
 
 
-def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
-    """(perf rows, last non-perf answer per alias, restore date per alias)."""
-    history = locked.read_json(DATA / "config" / "alias-history.json")     # {} until a rename
+# evidence() runs on every dashboard render, and re-read 13 MB of offloads and
+# 1,900 saved answers each time: 1.2 s on the owner's data. Each part is read
+# again only when its file changes.
+_perf: tuple = (None, [])               # ((alias history, history.csv's stat), its rows)
+_outputs: dict[str, tuple] = {}         # saved answer's path -> ((mtime, size), day it answered or None)
+
+
+def _stat(path: pathlib.Path) -> tuple:
+    try:
+        st = path.stat()
+    except OSError:
+        return (str(path),)
+    return str(path), st.st_mtime_ns, st.st_size
+
+
+def _perf_rows(path: pathlib.Path, history: dict[str, str]) -> list[dict]:
     rows = []
-    path = DATA / "perf" / "history.csv"
     if path.exists():
         with path.open(encoding="utf-8", newline="") as fh:
             for r in csv.DictReader(fh):
@@ -215,28 +229,74 @@ def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
                              "failure": r.get("failure") or "",
                              "detail": r.get("detail") or ""})
     rows.sort(key=lambda r: r["date"])
+    return rows
+
+
+_useful_lock = threading.Lock()
+_useful_seen: dict = {"mark": {}, "history": None, "days": {}}
+
+
+def _useful(path: pathlib.Path, history: dict[str, str]) -> dict[str, dt.date]:
+    """alias -> the last day a useful offload says it answered. Only the lines
+    added since the last call are read (the gateway adds some on every call),
+    all of them again after a rename."""
+    with _useful_lock:
+        s = _useful_seen
+        if s["history"] != history:
+            s.update(mark={}, history=history)
+        fresh, lines = locked.appended(path, s["mark"])
+        if fresh:
+            s["days"] = {}
+        days = s["days"]
+        for line in lines:
+            try:
+                o = json.loads(line)
+            except ValueError:
+                continue
+            if o.get("useful"):
+                alias, day = _current(o["model"], history), dt.date.fromisoformat(o["date"])
+                days[alias] = max(days.get(alias, day), day)
+        return dict(days)
+
+
+def _saved_answers():
+    """(model, day) for each saved answer that was a real answer. The listing
+    brings each file's stat (free on Windows); a file is read again only when
+    it changes."""
+    from run import outcome
+    for folder in DATA.glob("suite/tasks/*/output"):     # the saved answers are the install's data
+        try:
+            listing = list(os.scandir(folder))
+        except OSError:
+            continue
+        for e in listing:
+            if not e.name.endswith(".md"):
+                continue
+            st = e.stat()
+            hit = _outputs.get(e.path)
+            if hit is None or hit[0] != (st.st_mtime_ns, st.st_size):
+                text = pathlib.Path(e.path).read_text(encoding="utf-8", errors="replace")
+                hit = _outputs[e.path] = ((st.st_mtime_ns, st.st_size),
+                                          dt.date.fromtimestamp(st.st_mtime) if outcome(text) is None else None)
+            if hit[1]:
+                yield e.name[:-3], hit[1]
 
-    answers: dict[str, dt.date] = {}
 
-    def saw(alias: str, day: dt.date) -> None:
+def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
+    """(perf rows, last non-perf answer per alias, restore date per alias).
+    The rows are shared between calls: read them, never change them."""
+    global _perf
+    history = locked.read_json(DATA / "config" / "alias-history.json")     # {} until a rename
+    path = DATA / "perf" / "history.csv"
+    if _perf[0] != (key := (history, _stat(path))):
+        _perf = (key, _perf_rows(path, history))
+    rows = _perf[1]
+    offloads = DATA / "metrics" / "offloads.jsonl"
+    answers = _useful(offloads, history)
+    for alias, day in _saved_answers():
         alias = _current(alias, history)
         answers[alias] = max(answers.get(alias, day), day)
 
-    offloads = DATA / "metrics" / "offloads.jsonl"
-    for line in (offloads.read_text(encoding="utf-8").splitlines()
-                 if offloads.exists() else []):
-        try:
-            o = json.loads(line)
-        except ValueError:
-            continue
-        if o.get("useful"):
-            saw(o["model"], dt.date.fromisoformat(o["date"]))
-
-    from run import outcome
-    for out in DATA.glob("suite/tasks/*/output/*.md"):    # the saved answers are the install's data
-        if outcome(out.read_text(encoding="utf-8", errors="replace")) is None:
-            saw(out.stem, dt.date.fromtimestamp(out.stat().st_mtime))
-
     restored: dict[str, dt.date] = {}
     changes = DATA / "config" / "changes.jsonl"
     for line in (changes.read_text(encoding="utf-8").splitlines()
@@ -403,6 +463,21 @@ def _selfcheck() -> None:
     refused = [row("pk", n, "signin") for n in (9, 5, 0)]
     p = plan(d, {"pk": "v/pk"}, {"v/pk"}, {}, refused, {}, {}, {"v/pk": "v"})
     assert not p["retire"] and not p["confirm"] and not p["pending"], p
+    # evidence() reads a file again when it changes, and only then
+    global DATA
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmp:
+        DATA = pathlib.Path(tmp)
+        out = DATA / "suite" / "tasks" / "t" / "output" / "m.md"
+        out.parent.mkdir(parents=True)
+        out.write_text("a real answer, long enough to count as one: " * 2, encoding="utf-8")
+        log = DATA / "metrics" / "offloads.jsonl"
+        log.parent.mkdir()
+        log.write_text('{"model": "o", "date": "2026-10-01", "useful": true}\n', encoding="utf-8")
+        assert set(evidence()[1]) == {"m", "o"}
+        out.write_text("<<timed out>>", encoding="utf-8")
+        log.write_text("", encoding="utf-8")
+        assert evidence()[1] == {}
     print("auto_retire: rules hold")
 
 

package/scripts/onboard.py

@@ -21,7 +21,9 @@ WHAT IT DOES, AND WHY IN THIS ORDER
 3. **Register and sync**, then, with an Open WebUI hub, refresh the provider
    tags so the connection behind it is known — which is what the router uses
    to keep a panel independent. Without one the endpoint is the connection.
-   Then refresh the catalog, so its details (context, tools, vision) show now.
+   Then refresh the catalog, so its details (context, tools, vision) show now,
+   and ask a decision model which role suits it from them: kept as a
+   suggestion with its confidence (metrics/decide.py suggest_role), never set.
 4. **Probe availability** before spending anything: two prompt sizes, two
    samples. A model that cannot answer "ready" will not answer a 46 KB task,
    and finding that out here costs four calls instead of six large ones.
@@ -185,6 +187,17 @@ def makes_images(alias: str, served_name: str) -> bool:
     return "image" in out if out else bool(IMAGE_NAMES.search(served_name))
 
 
+def role_facts(alias: str, served_name: str, connection: str) -> str:
+    """What a decision model is told when asked for a new chat model's role:
+    its name, connection and catalog facts, the description last."""
+    import catalog
+    rec = catalog.info(alias)
+    facts = [f"{k.replace('_', ' ')}: {', '.join(map(str, v)) if isinstance(v, list) else v}"
+             for k in catalog.FIELDS if k != "description" and (v := rec.get(k)) not in (None, "", [])]
+    return "\n".join([f"model: {served_name}", f"connection: {connection}", *facts,
+                      f"description: {rec.get('description') or 'none known'}"])
+
+
 def run(*args: str, timeout: int = 3600) -> subprocess.CompletedProcess:
     return subprocess.run([str(PY), *args], cwd=str(ROOT), timeout=timeout,
                           capture_output=True, text=True, encoding="utf-8",
@@ -305,7 +318,13 @@ def main() -> None:
                       lambda roles: roles.setdefault("models", {}).__setitem__(alias, "vision"),
                       ensure_ascii=False)
         print("        role vision: a seat only for requests that carry images")
-    run("scripts/sync_config.py", timeout=300)
+    # The llm CLI's copy of the registry, only while it is the client: the
+    # built-in client reads the data folder's, and choosing the llm CLI on
+    # Settings syncs it whole (serve.save_client).
+    import client as model_client
+    synced = model_client.chosen()[0] == "llm"
+    if synced:
+        run("scripts/sync_config.py", timeout=300)
     # Open WebUI's tags name the connection behind the hub; without a hub the
     # endpoint is the connection, and route.connection_of needs no tags.
     if endpoints.find_hub("openwebui"):
@@ -319,7 +338,8 @@ def main() -> None:
     catalog = run("metrics/catalog.py", timeout=300)
     details = ("details from the catalog" if catalog.returncode == 0
                else f"catalog refresh failed (exit {catalog.returncode}); details at its next run")
-    print(f"  [3/5] synced; connection is {connection}; {details}")
+    print(f"  [3/5] {'synced to the llm CLI' if synced else 'the llm CLI not synced (the built-in client is in use)'};"
+          f" connection is {connection}; {details}")
 
     # An image generator: role image, which no chat shape admits, so it never
     # fills a seat or a collection; a chat probe and the suite would only
@@ -341,6 +361,16 @@ def main() -> None:
                      " never a seat, not probed, no suite.\n")
         return
 
+    # A decision model's suggestion for its role, from what the catalog knows:
+    # kept for the model popup (metrics/decided.json), never set from here.
+    try:
+        hint = decide.suggest_role(alias, role_facts(alias, name, connection))
+        print(f"        role suggested by {hint['by']}: {hint['role']} at {hint['confidence']:.2f}"
+              " (a suggestion; the role is yours to set in its model popup)" if "role" in hint
+              else f"        no role suggestion: {hint['error']}")
+    except Exception as exc:                       # noqa: BLE001 -- a suggestion never stops an onboarding
+        print(f"        no role suggestion: {type(exc).__name__}: {exc}")
+
     # 4. availability
     print("  [4/5] probing availability (4 calls)...")
     since = dt.datetime.now().isoformat(timespec="seconds")
@@ -414,6 +444,17 @@ def _selftest() -> None:
     # The name fallback, for a model the catalog says nothing about.
     assert all(IMAGE_NAMES.search(s) for s in ("gpt-image-2.5-flare", "openai/dall-e-3", "black-forest-labs/flux.2-pro"))
     assert not any(IMAGE_NAMES.search(s) for s in ("gpt-5.2", "reflux-7b", "gemini-3-pro"))
+    # What the role suggestion is told: the catalog's facts, the description last.
+    import catalog
+    real = catalog.info
+    catalog.info = lambda a: {"context_window": 131072, "vision": False, "input_modalities": ["text"],
+                              "tools": None, "description": "A coding model."}
+    try:
+        facts = role_facts("m1", "vendor/m1", "groq").splitlines()
+    finally:
+        catalog.info = real
+    assert facts == ["model: vendor/m1", "connection: groq", "context window: 131072", "vision: False",
+                     "input modalities: text", "description: A coding model."], facts
     print("ok")