@wendellresearch-ai/benchwarmers 0.3.3 → 0.5.0

package/serve.py

@@ -43,6 +43,7 @@ import re
 import subprocess
 import sys
 import threading
+import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 ROOT = pathlib.Path(__file__).resolve().parent
@@ -54,6 +55,7 @@ PORT = int(os.environ.get("BENCH_PORT") or 8122)
 # Once, at import: roster() used to insert two entries on every page render.
 for _d in ("metrics", "suite", "scripts"):
     sys.path.insert(0, str(ROOT / _d))
+import aliases  # noqa: E402
 from aliases import current as current_alias  # noqa: E402
 import connect  # noqa: E402
 import gateway  # noqa: E402  -- its routes are answered on this port too (one port)
@@ -164,15 +166,25 @@ def suite_scores() -> dict:
     The notes are the record, so they are also the source here rather than a
     parallel database that could disagree with them. Later files win, which is
     what makes a re-run show up.
+
+    Not a vision or image model's (suite/run.images_only): the suite is text
+    and code, which it is never seated for, and the answers it gave before it
+    had that role stay in the notes, uncounted. Real work, perf and reviews
+    still count it. Nor the vision tasks (vision-*): the Vision panel reads
+    those (vision_rows).
     """
+    from run import images_only
     row = re.compile(r"^\|\s*([\w-]+)\s*\|\s*`([^`]+)`\s*\|\s*([^|]+?)\s*\|"
                      r"\s*([^|]*?)\s*\|\s*([\d.]+)\s*\|")
     scores: dict[tuple[str, str], dict] = {}
+    skip = images_only()
     for path in sorted(DATA.glob("runs/suite-*.md")):
         for line in read(path).splitlines():
             if (m := row.match(line.strip())):
                 task, model, found, invented, seconds = m.groups()
                 model = current_alias(model)
+                if model in skip or task.startswith("vision-"):
+                    continue
                 scores[(task, model)] = {
                     "found": found, "invented": invented.strip(" —-"),
                     "seconds": float(seconds)}
@@ -209,6 +221,17 @@ def tag_for(llm_id: str) -> str:
     return "" if tag == "?" else tag
 
 
+def provider_label(tag: str) -> str:
+    """A connection tag as a short provider name for a chip: a proxy's vendor
+    (cli-proxy-api/codex: codex), else the host without its api./
+    router. prefix, -api suffix and TLD (router.requesty.ai: requesty)."""
+    if "/" in tag:
+        return tag.split("/", 1)[1]
+    host = re.sub(r"^(api|apihub|router)\.", "", tag)
+    host = host.rsplit(".", 1)[0] if "." in host else host
+    return host.removesuffix("-api") or tag
+
+
 def retired() -> dict[str, dict]:
     """Aliases removed from the bench by hand, with when and why."""
     return load_json(DATA / "config" / "retired.json")
@@ -444,6 +467,7 @@ def change_model(alias: str, new_alias: str | None, new_role: str | None,
         history = load_json(HISTORY_PATH)
         history[alias] = new_alias
         _write_atomic(HISTORY_PATH, _dump(history))
+        aliases.forget()                       # its one-second cache, so the next read has the rename
         changed.append("config/alias-history.json")
         # 3. configs keyed by alias
         roles = roles_config()
@@ -773,15 +797,28 @@ def restore_model(alias: str, who: str = "ui") -> dict:
 DISMISSED_PATH = DATA / "config" / "dismissed.json"
 
 
+_modalities_index: tuple = (None, {})
+
+
 def _modelsdev_modalities() -> dict[str, dict]:
     """models.dev's cached modalities by comparable model name
-    (sources.norm_model), read once per picker opening, not once per row."""
+    (sources.norm_model), not once per row, and built again only when the
+    daily fetch rewrites its copy (a second of every picker opening)."""
+    global _modalities_index
     import sources
+    try:
+        st = (sources.CACHE / "modelsdev.json").stat()
+        key = (st.st_mtime_ns, st.st_size)
+    except OSError:
+        key = None
+    if key and _modalities_index[0] == key:
+        return _modalities_index[1]
     out = {}
     for p in sources.cached("modelsdev").values():
         for k, m in ((p or {}).get("models") or {}).items():
             if (m.get("modalities") or {}).get("input"):
                 out.setdefault(sources.norm_model(k), m["modalities"])
+    _modalities_index = (key, out)
     return out
 
 
@@ -797,11 +834,19 @@ def _modalities(entry: dict, served: str, index: dict) -> dict:
     return {"inputs": m.get("input") or [], "outputs": m.get("output") or []}
 
 
-def onboard_candidates() -> dict:
+# What each endpoint answered, for LISTED_FOR seconds: name -> (when, its
+# config, live, conns). Only the asking is kept; what is on the bench, retired
+# or dismissed is taken off fresh on every opening.
+LISTED_FOR = 3600                  # an hour: lists rarely change sooner, and Refresh is one click
+_listed: dict[str, tuple] = {}
+
+
+def onboard_candidates(fresh: bool = False) -> dict:
     """What every endpoint serves that the bench could onboard, for the picker.
 
-    Served, minus registered, minus retired, minus dismissed (those come back
-    separately so a dismissal can be undone), each endpoint asked separately
+    Served, minus registered, minus retired, minus queued or onboarding now,
+    minus dismissed (those come back separately so a dismissal can be undone),
+    each endpoint asked separately
     so one being down only empties its own group. The live calls this page
     makes happen only when the picker opens: a dashboard that needs the
     network to render would be down exactly when you want to look at it.
@@ -811,8 +856,13 @@ def onboard_candidates() -> dict:
     keys): provider-tags.json is written by that script on its schedule, so a
     model added to Open WebUI since was "unknown connection" until it ran.
     The file is the fallback when the hub's connections cannot be read.
+
+    An endpoint's answer is reused for LISTED_FOR seconds, unless `fresh` (the
+    picker's Refresh) or its entry in endpoints.json changed since; a failure
+    is never kept. Open WebUI alone can take ten seconds to list.
     """
-    names = list(endpoints.load())
+    config = endpoints.load()
+    names = list(config)
     if not names:
         return {"ok": False, "errors": ["no endpoint yet: add one first (Endpoints, then Manage, then"
                                         " Add an endpoint), and what it serves is listed here"]}
@@ -820,6 +870,9 @@ def onboard_candidates() -> dict:
              | {endpoints.snapshot_key(r.get("served", ""), r.get("endpoint") or endpoints.default())
                 for r in retired().values()}
              | {endpoints.snapshot_key(d["served"], d["endpoint"]) for d in decide.load().values()})
+    with _OQ_LOCK:                                  # on its way: Start would refuse it anyway
+        taken |= {endpoints.snapshot_key(i["served"], i["endpoint"]) for i in _OQ["items"]
+                  if i["state"] in ("queued", "running")}
     tags = load_json(DATA / "config" / "provider-tags.json")
     dismissed = load_json(DISMISSED_PATH)
     first_seen = load_json(DATA / "config" / "served-snapshot.json").get("last_seen", {})
@@ -827,10 +880,13 @@ def onboard_candidates() -> dict:
     from concurrent.futures import ThreadPoolExecutor
 
     def ask(name: str):
+        kept = _listed.get(name)
+        if not fresh and kept and kept[1] == config[name] and time.time() - kept[0] < LISTED_FOR:
+            return kept[2], kept[3], kept[0]
         try:
             live = endpoints.models(name)
         except Exception as exc:                       # noqa: BLE001
-            return exc, []
+            return exc, [], 0
         conns = []
         if endpoints.hub(name) == "openwebui":
             import provider_tags
@@ -838,11 +894,14 @@ def onboard_candidates() -> dict:
                 conns = provider_tags.connection_names(name)
             except Exception:                          # noqa: BLE001 -- the tags file answers instead
                 pass
-        return live, conns
+        _listed[name] = (time.time(), config[name], live, conns)
+        return live, conns, _listed[name][0]
     with ThreadPoolExecutor(max_workers=len(names)) as pool:   # all at once, not in turn
         answers = dict(zip(names, pool.map(ask, names)))
+    # The oldest answer shown, so the picker can say how old its list is.
+    listed = min((at for _, _, at in answers.values() if at), default=time.time())
     mods = _modelsdev_modalities()
-    for name, (live, conns) in answers.items():
+    for name, (live, conns, _) in answers.items():
         if isinstance(live, Exception):
             errors.append(f"cannot reach {name}: {live}")
             continue
@@ -869,7 +928,8 @@ def onboard_candidates() -> dict:
             (gone if key in dismissed else open_).append(row)
     if not open_ and not gone and errors:
         return {"ok": False, "errors": errors}
-    return {"ok": True, "errors": errors, "candidates": open_, "dismissed": gone}
+    return {"ok": True, "errors": errors, "candidates": open_, "dismissed": gone,
+            "age": round(time.time() - listed)}
 
 
 def dismiss(served, undo: bool = False, who: str = "ui") -> dict:
@@ -1412,12 +1472,18 @@ def collections_view() -> dict:
     # What the tags check members against (bench.js tagBlocks): capabilities
     # (None: the catalog does not say), context, price and role. A shape's
     # tag admits the roles that shape admits, as route.py does for seats.
+    # The member picker draws one row per model, its providers as chips:
+    # `family` is route.family_of, the match that keeps one model out of two
+    # seats, and `provider` the chip's name for its connection.
+    import route
     roles = model_roles.load()
     facts = {}
     for a in _chat_aliases():
         f = catalog.info(a)
+        tag = directory[a].get("tag") or directory[a].get("endpoint") or ""
         facts[a] = {"caps": {c: f.get(c) for c in catalog.CAPS}, "context": f.get("context_window"),
-                    "price": price_of(a, served[a], f), "role": (roles.get("models") or {}).get(a, "general")}
+                    "price": price_of(a, served[a], f), "role": (roles.get("models") or {}).get(a, "general"),
+                    "family": route.family_of(a), "provider": provider_label(tag), "connection": tag}
     return {"ok": True, "errors": [], "collections": rows, "settings": s,
             "shapes": shapes, "models": _chat_aliases(), "served": served, "tag_suggestions": suggestions,
             "facts": facts, "shape_roles": {k: v for k, v in roles["shapes"].items() if k in shapes},
@@ -1522,13 +1588,33 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
 
 # How much an onboarding spends. Full: the 4-call probe, then the suite now.
 # Quick: the probe only; the scheduled `suite --next` passes fill the suite in,
-# six tasks a run, so the same calls are spread out, not saved. Quick + ad
-# hoc: the probe only, and the bench never calls it on its own: the only one
-# that saves calls, for a metered, capped or trial key.
-ONBOARD_MODES = {"full": [], "quick": ["--skip-suite"], "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
-
-
-def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full") -> dict:
+# six tasks a run, so the same calls are spread out, not saved. Ad hoc: the
+# bench never calls it on its own afterwards, for a metered, capped or trial
+# key; Full + ad hoc spends the suite once now so routing has scores for it,
+# Quick + ad hoc only the probe, and no scheduled pass ever fills its suite in
+# (suite/pace.py leaves ad hoc models out). Apart from the mode,
+# `vision` (the picker's Vision model box) gives it role vision as it is
+# registered: a seat only for requests that carry images, and its suite is the
+# vision tasks alone (suite/run.applies), now or through the scheduled passes.
+ONBOARD_MODES = {"full": [], "full-adhoc": ["--ad-hoc"], "quick": ["--skip-suite"],
+                 "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
+
+
+def onboard_choice(prefix: str) -> str:
+    """The mode and the Vision box, as the Onboard dialog (o-) and Discover's Add confirm (ask-) show them."""
+    return (f"<label class='check'>Onboard as <select id='{prefix}-mode'>"
+            "<option value='full'>Full: probe, then the suite now</option>"
+            "<option value='full-adhoc'>Full + ad hoc: probe and the suite now; then the bench never calls it"
+            " on its own</option>"
+            "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
+            "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
+            " (metered, capped or trial keys)</option></select></label>"
+            f"<label class='check'><input type='checkbox' id='{prefix}-vision'> Vision model: a seat only for"
+            " questions about images, and its suite is the vision tasks</label>")
+
+
+def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full",
+                  vision: bool = False) -> dict:
     """Kick off `scripts/onboard.py` for a served model, detached, logging to runs/.
 
     The script does its own checks in the right order (is it served, derive
@@ -1538,7 +1624,7 @@ def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str =
     hands the rest to the script. It runs for minutes and spends calls, so it
     runs in the background and the page tails its log.
     """
-    return _spawn_onboard(served, endpoint, who, mode)[0]
+    return _spawn_onboard(served, endpoint, who, mode, vision)[0]
 
 
 def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str]:
@@ -1567,7 +1653,7 @@ def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str
 
 
 def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
-                   mode: str = "full") -> tuple[dict, subprocess.Popen | None]:
+                   mode: str = "full", vision: bool = False) -> tuple[dict, subprocess.Popen | None]:
     """start_onboard's result, and the process (None when refused) for a caller that waits on it."""
     served = (served or "").strip()
     endpoint = (endpoint or "").strip() or endpoints.default()
@@ -1579,7 +1665,7 @@ def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
     log = DATA / "runs" / f"onboard-{stamp}-{safe}.log"
     from run import bench_python
     cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint,
-           *ONBOARD_MODES[mode]]
+           *ONBOARD_MODES[mode], *(["--vision"] if vision else [])]
     # The log handle goes straight to the child, unbuffered, so the page sees
     # each step as it prints. The old `cmd /c ... > log & echo` wrapper with
     # DETACHED_PROCESS sent the output to a console window instead, so every
@@ -1648,19 +1734,19 @@ def queue_onboard(items, who: str = "ui") -> dict:
         endpoint = str(it.get("endpoint") or "").strip() or endpoints.default()
         mode = str(it.get("mode") or "full")
         errors += _onboard_refusal(served, endpoint, mode)
-        picked.append((served, endpoint, mode))
+        picked.append((served, endpoint, mode, it.get("vision") is True))
     with _OQ_LOCK:
         waiting = {(i["served"], i["endpoint"]) for i in _OQ["items"] if i["state"] in ("queued", "running")}
-        errors += [f"{s} from {e} is already in the queue" for s, e, _ in picked if (s, e) in waiting]
-        if len({(s, e) for s, e, _ in picked}) < len(picked):
+        errors += [f"{s} from {e} is already in the queue" for s, e, *_ in picked if (s, e) in waiting]
+        if len({(s, e) for s, e, *_ in picked}) < len(picked):
             errors.append("a model is picked twice")
         if errors:
             return {"ok": False, "errors": errors}
         if _OQ["worker"] is None:
             _OQ["items"] = []
         _OQ["stop"] = False                              # added after a Stop: these run
-        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "state": "queued", "alias": None, "log": None,
-                          "exit": None, "note": ""} for s, e, m in picked]
+        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "vision": v, "state": "queued", "alias": None,
+                          "log": None, "exit": None, "note": ""} for s, e, m, v in picked]
         if _OQ["worker"] is None:
             _OQ["worker"] = threading.Thread(target=_onboard_worker, args=(who,), daemon=True)
             _OQ["worker"].start()
@@ -1712,7 +1798,8 @@ def _onboard_worker(who: str) -> None:
                 return
             nxt["state"] = "running"
         try:
-            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"))
+            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"),
+                                       nxt.get("vision", False))
             if proc is None:                             # refused now: the bench changed since it was queued
                 state, note, code = "failed", "; ".join(res["errors"]), None
             else:
@@ -1730,13 +1817,17 @@ def _onboard_worker(who: str) -> None:
             nxt.update(state=state, note=note, exit=code)
 
 
-def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "") -> dict:
+def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "",
+                 mode: str = "full", vision: bool = False) -> dict:
     """From the Discover tab: put a model into a hub as it is otherwise done by
     hand, through the adapter of the hub `endpoint` is (endpoints.adapter:
     Open WebUI's connection Model IDs and a public, tagged record with a logo;
     a LiteLLM deployment; a CLI Proxy API provider's models list), then the
-    normal onboarding. Only for a connection the hub already has. No endpoint
+    normal onboarding, in the mode and with the Vision box the confirm chose
+    (ONBOARD_MODES). Only for a connection the hub already has. No endpoint
     means Open WebUI's, as before there were other hubs."""
+    if mode not in ONBOARD_MODES:                      # before the hub has it, not after
+        return {"ok": False, "errors": [f"mode must be one of {', '.join(ONBOARD_MODES)}; got {mode!r}"]}
     if not re.fullmatch(r"[\w./:@+-]{1,200}", model or ""):
         return {"ok": False, "errors": ["the model id is the provider's id, e.g. vendor/model:free"]}
     if not re.fullmatch(r"[\w.-]{1,80}", connection or ""):
@@ -1755,7 +1846,7 @@ def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "
         return {"ok": False, "errors": [f"{label} refused: {type(exc).__name__}: {exc}"]}
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "",
                 f"{which}-add", None, f"{served} on {connection}")
-    started = queue_onboard([{"served": served, "endpoint": hub.endpoint()}], who)
+    started = queue_onboard([{"served": served, "endpoint": hub.endpoint(), "mode": mode, "vision": vision}], who)
     return {**started, "served": served, "endpoint": hub.endpoint(), "steps": steps,
             "errors": started["errors"] + ([] if started["ok"] else
                                           [f"{label} now has {served}; onboard it from the Onboard dialog"])}
@@ -1914,14 +2005,26 @@ def problems() -> str:
     KIND = {"retiring": "err", "retired": "err", "connection": "warn",
             "registration": "warn", "plan": "warn"}
     items = []
+    live = set(names.values())
     for key, a in sorted(alerts.items(), key=lambda kv: kv[1].get("first_seen", "")):
         served = key.split(":", 2)[2] if key.count(":") >= 2 else ""
         alias = a.get("alias") or names.get(served, "")
+        # The alerts file is rewritten by the daily check; until then an alert
+        # whose models have all left the bench (Retire all, or one by one) has
+        # nothing left to act on.
+        if a.get("aliases") and not live & set(a["aliases"]):
+            continue
+        if a.get("kind") == "retiring" and alias and alias not in live:
+            continue
         try:
             age = (today - datetime.date.fromisoformat(a["first_seen"])).days
         except (KeyError, ValueError):
             age = 0
-        why = " · ".join(filter(None, [a.get("action", ""),
+        # Which model and where it comes from, first: the message alone left that to be looked up.
+        served_as = next((s for s, al in names.items() if al == alias), served) if alias else ""
+        where = (f"{alias} · {served_as or '?'} via {tag_for(alias) or a.get('endpoint') or '?'}"
+                 if alias else "")
+        why = " · ".join(filter(None, [where, a.get("action", ""),
                                        "open today" if age == 0 else f"open {age} days"]))
         acts = []
         if a.get("aliases"):
@@ -1938,7 +2041,10 @@ def problems() -> str:
         items.append((key, a.get("kind", "?"), KIND.get(a.get("kind"), "warn"),
                       a.get("message", ""), why, "".join(acts)))
     cutoff = (today - datetime.timedelta(days=7)).isoformat()
-    auto = [(f"retired:{alias}:{r['at']}", "", "", f"{alias} · {r['at'][:10]}",
+    tags = load_json(DATA / "config" / "provider-tags.json")      # a retired model's connection, by served name
+    auto = [(f"retired:{alias}:{r['at']}", "", "",
+             f"{alias} · {r.get('served') or '?'} via {tags.get(r.get('served', ''), '').strip('?') or r.get('endpoint') or endpoints.default()}"
+             f" · {r['at'][:10]}",
              r.get("why", "").removeprefix("auto: "),
              f"<button class='btn small' data-act='restore' data-alias='{esc(alias)}'>Restore</button>")
             for alias, r in sorted(retired().items(), key=lambda kv: kv[1].get("at", ""), reverse=True)
@@ -2154,14 +2260,15 @@ def on_bench() -> set[str]:
     return set(registered().values())
 
 
-def chart_card(title: str, sub: str, chart_id: str, table: str, key: str, extra: str = "") -> str:
+def chart_card(title: str, sub: str, chart_id: str, table: str, key: str, extra: str = "",
+               retired: bool = True) -> str:
     """A Benchmarks panel: the chart first (drawn by bench.js with Observable
     Plot), the table folded under it, the key under that."""
     return (f"<div class='panel'><h2>{title}</h2><p class='sub'>{sub}</p>{extra}"
             f"<div class='chart' id='{chart_id}'></div>"
             "<details class='key'><summary>Table</summary><div class='toolbar'><input type='search'"
             " data-filter placeholder='Filter by model or connection' aria-label='Filter rows'>"
-            "<label><input type='checkbox' data-retired> Show retired</label></div>"
+            + ("<label><input type='checkbox' data-retired> Show retired</label>" if retired else "") + "</div>"
             f"<div class='scroll'>{table}</div></details>"
             f"<details class='key'><summary>How to read this</summary>{key}</details></div>")
 
@@ -2174,13 +2281,34 @@ def _family(task: str) -> str:
     return "hand-built"
 
 
+_wrong: tuple = (0.0, {})
+
+
+def wrong_table() -> dict:
+    """wrong.table(), shared by the three cards of one render: it scores every
+    suite answer, and asking three times was a second of each reload."""
+    global _wrong
+    import wrong
+    from run import images_only
+    seen = _wrong
+    if time.monotonic() - seen[0] > 5:
+        table = wrong.table()
+        # A vision or image model keeps its reviews (real work, judged) but not
+        # its code-task answers: the suite is not what it is for (suite_scores).
+        for alias in images_only() & set(table):
+            r = table[alias]
+            r.update(answers=0, wrong=0, by={k: [0, 0] for k in r["by"]}, hunt=dict.fromkeys(r["hunt"], 0))
+        seen = _wrong = (time.monotonic(), table)    # stamped once scored, not before
+    return seen[1]
+
+
 def charts_data() -> dict:
     """Everything the Benchmarks charts draw, for bench.js. Registered models
     only: the tables keep retired ones behind "Show retired", a chart has no toggle."""
     import route
     import wrong
     live = on_bench()
-    recall, wr = route.suite_recall(), wrong.table()
+    recall, wr = route.suite_recall(), wrong_table()
     overview = [{"model": m, "recall": recall[m][0] / recall[m][1], "wrong": wr[m]["wrong"] / wr[m]["answers"],
                  "found": f"{recall[m][0]}/{recall[m][1]}", "wrongN": f"{wr[m]['wrong']}/{wr[m]['answers']}"}
                 for m in sorted(live) if recall.get(m) and recall[m][1] and wr.get(m, {}).get("answers")]
@@ -2198,7 +2326,7 @@ def charts_data() -> dict:
     return {"overview": overview, "heat": heat, "families": ["hand-built", "claims", "seeded, function",
                                                              "seeded, whole file", "real bugs"],
             "wrong": bars, "wrongTotals": totals, "kinds": list(wrong.KINDS), "hunt": hunt,
-            "work": work_rows(), **perf_charts()}
+            "work": work_rows(), "vision": vision_rows(), **perf_charts()}
 
 
 def work_rows() -> list[dict]:
@@ -2215,6 +2343,184 @@ def work_rows() -> list[dict]:
             for m, s in sorted(summary().items()) if m in live and s["judged"]]
 
 
+VISION_STATES = ("right", "misread", "left out", "trap taken", "no answer")
+VISION_SEATS = ("checked, helped", "checked, did not help", "useful, no claims", "no use, no claims",
+                "not judged", "no answer")
+_vision_work: tuple = (None, {})
+
+
+def vision_rows() -> dict:
+    """The Vision panel's evidence, in two kinds kept apart.
+
+    TASKS: the vision tasks' saved answers (suite/tasks/vision-*), scored by
+    the suite's own scorer and then field by field: right, misread (the line
+    is there with another value), left out, trap taken (a value for something
+    the picture does not show, or a switch that is off called on), no answer.
+    The keys exist by construction (suite/vision_tasks.py), so this is the
+    controlled comparison.
+
+    REAL WORK: every seat of the gateway's vision panels (runs/gateway/
+    calls.jsonl, shape vision) and how it was judged (metrics/offloads.jsonl).
+    A verdict that lists claims was checked claim by claim against the
+    picture; one without claims says only useful or not, which for thicken's
+    face corners means it agreed with another seat. Agreement is not a check
+    (the bench's own rule: seats fail together), so the two are never added.
+    """
+    import run
+    live = on_bench()
+    heat, tasks = [], {}
+    for path in sorted((DATA / "suite" / "tasks").glob("vision-*/task.json")):
+        spec = json.loads(read(path))
+        if spec.get("draft"):
+            continue
+        names = list(dict.fromkeys(c["label"].split(":")[0] for c in spec["must_find"]))
+        traps = {c["label"].split(":")[0] for c in spec.get("must_not_claim", [])}
+        tasks[spec["id"]] = {"title": spec["title"], "fields": names, "family": _vision_family(spec)}
+        for answer in sorted((path.parent / "output").glob("*.md")):
+            alias = current_alias(answer.stem)
+            if alias not in live:
+                continue
+            text = read(answer)
+            s = run.score(spec, text)
+            missed = {lbl.split(":")[0] for lbl in s["missed"]}
+            invented = {lbl.split(":")[0] for lbl in s["invented"]}
+            for name in names:
+                there = re.search(rf"(?m)^[^\w\n]*{re.escape(name).replace('_', '[ _]')}[^\w\n]*:", text.lower())
+                state = ("no answer" if s["kind"] != "answered" else "trap taken" if name in invented
+                         else "right" if name not in missed else "misread" if there else "left out")
+                heat.append({"model": alias, "task": spec["id"], "field": name, "key": f"{spec['id']}·{name}",
+                             "state": state, "trap": name in traps})
+    models: dict[str, dict] = {}
+    for h in heat:
+        m = models.setdefault(h["model"], {"asked": set(), "answered": set(), "fields": 0, "right": 0,
+                                           "traps": 0, "taken": 0})
+        m["asked"].add(h["task"])
+        if h["state"] != "no answer":
+            m["answered"].add(h["task"])
+            m["fields"] += 1
+            m["right"] += h["state"] == "right"
+            m["traps"] += h["trap"]
+            m["taken"] += h["state"] == "trap taken"
+    summary = {a: {"asked": len(m["asked"]), "answered": len(m["answered"]), "fields": m["fields"],
+                   "right": m["right"], "traps": m["traps"], "taken": m["taken"]} for a, m in models.items()}
+    return {"tasks": tasks, "heat": heat, "summary": summary, "states": list(VISION_STATES),
+            "seatKinds": list(VISION_SEATS), "work": _vision_work_rows(live), "held": run.vision_held()}
+
+
+VISION_FAMILIES = (("vision-cord-", "receipts"), ("vision-screenqa-", "screens"), ("vision-tablevqa-", "tables"),
+                   ("vision-pope-", "objects"))
+
+
+def _vision_family(spec: dict) -> str:
+    """The grid's group: a public source (suite/vision_sample.py), the made
+    pictures (suite/vision_tasks.py), or the install's own (written by hand)."""
+    if spec.get("made_by", "").startswith("suite/vision_tasks.py"):
+        return "made"
+    return next((name for prefix, name in VISION_FAMILIES if spec["id"].startswith(prefix)), "own")
+
+
+def _vision_work_rows(live: set[str]) -> dict:
+    """alias -> {seat kind: count, claims right, claims wrong, seconds}; read
+    again only when the gateway's log or the verdicts change (5 MB a render)."""
+    global _vision_work
+    calls, verdicts = DATA / "runs" / "gateway" / "calls.jsonl", DATA / "metrics" / "offloads.jsonl"
+    try:
+        key = tuple((p.stat().st_mtime_ns, p.stat().st_size) for p in (calls, verdicts)) + (frozenset(live),)
+    except OSError:
+        return {}
+    if _vision_work[0] == key:
+        return _vision_work[1]
+    from offload import entries
+    judged = {e["call"]: e for e in entries() if e.get("call") and e.get("useful") is not None}
+    out: dict[str, dict] = {}
+    for raw in read(calls).splitlines():
+        try:
+            panel = json.loads(raw)
+        except ValueError:
+            continue
+        if panel.get("shape") != "vision":
+            continue
+        for s in panel.get("seats", []):
+            alias = current_alias(s.get("alias", ""))
+            if alias not in live:
+                continue
+            r = out.setdefault(alias, {**dict.fromkeys(VISION_SEATS, 0), "real": 0, "false": 0, "seconds": []})
+            v = judged.get(s.get("call"))
+            if s.get("status") != "ok" or not (s.get("answer") or "").strip():
+                kind = "no answer"
+            elif v is None:
+                kind = "not judged"
+            elif v.get("real") or v.get("false"):
+                kind = "checked, helped" if v["useful"] else "checked, did not help"
+                r["real"] += len(v.get("real") or [])
+                r["false"] += len(v.get("false") or [])
+            else:
+                kind = "useful, no claims" if v["useful"] else "no use, no claims"
+            r[kind] += 1
+            if kind != "no answer" and s.get("seconds"):
+                r["seconds"].append(float(s["seconds"]))
+    for r in out.values():
+        secs = sorted(r.pop("seconds"))
+        r["seconds"] = secs[len(secs) // 2] if secs else None
+    _vision_work = (key, out)
+    return out
+
+
+def vision_card() -> str:
+    """Reading images: the vision tasks, then real work on images."""
+    v = vision_rows()
+    s, w = v["summary"], v["work"]
+    body = ["<table><thead><tr>", th("Model"), th("Tasks answered", True), th("Fields read right", True),
+            th("Traps taken", True), th("Image seats", True), th("Checked claims right", True),
+            th("Checked claims wrong", True), th("Useful, no claims", True), th("Not judged", True),
+            th("Median seconds", True), "</tr></thead><tbody>"]
+
+    def cell(n, d):
+        return (f"<td class='num' data-sort='{n / d:.3f}'>{n}/{d}</td>" if d
+                else "<td class='num muted' data-sort='-1'>—</td>")
+    for m in sorted(set(s) | set(w)):
+        t, r = s.get(m, {}), w.get(m, {})
+        seats = sum(r.get(k, 0) for k in VISION_SEATS)
+        claims = r.get("real", 0) + r.get("false", 0)
+        body.append(f"{tr(m)}<td>{named(m)}</td>" + cell(t.get("answered", 0), t.get("asked", 0))
+                    + cell(t.get("right", 0), t.get("fields", 0)) + cell(t.get("taken", 0), t.get("traps", 0))
+                    + f"<td class='num'>{seats or '—'}</td>" + cell(r.get("real", 0), claims)
+                    + cell(r.get("false", 0), claims) + f"<td class='num'>{r.get('useful, no claims', 0) or '—'}</td>"
+                    + f"<td class='num'>{r.get('not judged', 0) or '—'}</td>"
+                    + (f"<td class='num'>{r['seconds']:.1f}</td>" if r.get("seconds") else "<td class='num muted'>—</td>")
+                    + "</tr>")
+    body.append("</tbody></table>")
+    key = """<dl>
+      <dt>The vision tasks</dt><dd>Grouped by where the pictures come from: <b>made</b> with their answer
+        key (<code>suite/vision_tasks.py</code>), your <b>own</b> photos keyed by hand, and items sampled
+        from public datasets with people's answers (<code>suite/vision_sample.py</code>): photographed
+        <b>receipts</b>, app <b>screens</b>, <b>tables</b>, and <b>objects</b> in photos. Each field is
+        <b>right</b>, <b>misread</b> (another value), <b>left out</b>, or a <b>trap taken</b>: a value for
+        something the picture does not show, where the right answer is <code>not shown</code> (a dot marks
+        those fields). They run for every model that takes images; a vision model gets only these, never
+        the code tasks.</dd>
+      <dt>Real work on images</dt><dd>Every seat of a vision panel through the gateway. <b>Checked</b> means
+        its verdict lists claims, each compared with the picture by hand: the claims right and wrong
+        come from these alone. <b>Useful, no claims</b> is a verdict without them, which on thicken's
+        photos means the seat agreed with another seat. Agreement is not a check, since seats fail
+        together, so it is never counted as right.</dd>
+      <dt>The number beside a name</dt><dd>How much there is: fields asked on the tasks, seats on real work.</dd>
+    </dl>"""
+    held = (f"<p class='hint'>The vision tasks wait until {esc(v['held'])}: no run picks them before then"
+            " (<code>vision_from</code> in <code>config/suite.json</code>; remove it to start them now).</p>"
+            if v["held"] else "")
+    return ("<div class='panel'><h2>Vision</h2><p class='sub'>Reading images: fields read right on pictures with"
+            " a known answer, then real work on images and how much of it was actually checked.</p>" + held +
+            "<h3 class='chart-head'>On the vision tasks, field by field</h3><div class='chart' id='chart-vision-fields'></div>"
+            "<h3 class='chart-head'>On real work with images, by how each seat was judged</h3>"
+            "<div class='chart' id='chart-vision-work'></div>"
+            "<details class='key'><summary>Table</summary><div class='toolbar'><input type='search'"
+            " data-filter placeholder='Filter by model or connection' aria-label='Filter rows'>"
+            "<label><input type='checkbox' data-retired> Show retired</label></div>"
+            f"<div class='scroll'>{''.join(body)}</div></details>"
+            f"<details class='key'><summary>How to read this</summary>{key}</details></div>")
+
+
 def perf_charts() -> dict:
     """Reliability over the last 14 days of probes, and speed on real tasks
     against how much a model finds, for the Performance panel of Benchmarks."""
@@ -2239,8 +2545,7 @@ def perf_charts() -> dict:
 
 def bughunt_card() -> str:
     """Seeded and real-bug tasks against their answer keys."""
-    import wrong
-    rows = {m: r["hunt"] for m, r in wrong.table().items() if m in current() and any(r["hunt"].values())}
+    rows = {m: r["hunt"] for m, r in wrong_table().items() if m in current() and any(r["hunt"].values())}
     body = ["<table><thead><tr>", th("Model"), th("Found", True), th("Wrong line", True), th("Missed", True),
             th("Clean said clean", True), th("False alarm", True), "</tr></thead><tbody>"]
     for m in sorted(rows, key=lambda m: -rows[m]["found"]):
@@ -2342,6 +2647,8 @@ def suite_card() -> str:
 
 
 EXTERNAL = (("LiveBench", "https://livebench.ai/", "contamination-limited questions, refreshed monthly"),
+            ("FreeLLMAPI", "https://freellmapi.co/models",
+             "every free tier: limits, ranks and quirks per model (Discover leads with it)"),
             ("models.dev", "https://models.dev/", "specs and prices per provider (the catalog reads it daily)"),
             ("BullshitBench", "https://petergpt.github.io/bullshit-benchmark/viewer/index.next.html",
              "pushing back on nonsense; the idea behind confidently wrong"))
@@ -2360,8 +2667,9 @@ def external_card() -> str:
 def wrong_card() -> str:
     """metrics/wrong.py: how often each model asserts something false."""
     import wrong
-    rows = wrong.table()
-    live = current()
+    from run import images_only
+    rows = wrong_table()
+    live, images = current(), images_only()
     body = ["<table><thead><tr>", th("Model"), th("Confidently wrong", True),
             *[th(k.capitalize(), True) for k in wrong.KINDS], th("Reviews", True), "</tr></thead><tbody>"]
 
@@ -2378,7 +2686,8 @@ def wrong_card() -> str:
                      f" style='width:{max(4, round(rate * 100))}%'></span></span>"
                      f"<span class='mono'>{r['wrong']}/{r['answers']}</span></span>")
         else:
-            rate, meter = -1, "<span class='muted'>no tasks yet</span>"
+            rate, meter = -1, ("<span class='muted'>reads images: no code tasks</span>" if alias in images
+                               else "<span class='muted'>no tasks yet</span>")
         body.append(f"<tr><td>{named(alias)}</td><td data-sort='{rate:.3f}'>{meter}</td>"
                     + "".join(cell(*r["by"][k]) for k in wrong.KINDS)
                     + (f"<td class='num' data-sort='{r['review_false']}'>{r['review_false']} in {r['reviews']}</td>"
@@ -2404,7 +2713,8 @@ def wrong_card() -> str:
 
 def benchmarks_tab() -> str:
     data = json.dumps(charts_data(), ensure_ascii=False).replace("</", "<\\/")
-    return (overview_card() + bughunt_card() + work_card() + wrong_card() + suite_card() + perf_card() + external_card()
+    return (overview_card() + bughunt_card() + work_card() + wrong_card() + suite_card() + vision_card()
+            + perf_card() + external_card()
             + f"<script type='application/json' id='charts-data'>{data}</script>")
 
 
@@ -2537,7 +2847,7 @@ def gateway_card(limit: int = 30) -> str:
         judged = {}
     owed = seats_total = 0
     body = ["<table><thead><tr>", th("When"), th("Collection"), th("Asked by"), th("Explorer shape"),
-            th("Seats"), th("Owed", True), "</tr></thead><tbody>"]
+            th("Seats"), th("Time", True), "</tr></thead><tbody>"]
     for r in records:
         chips, row_owed = [], 0
         for seat in r.get("seats", []):
@@ -2557,13 +2867,20 @@ def gateway_card(limit: int = 30) -> str:
         chips += [f"<span class='chip muted' title='{esc('not asked: ' + str(k.get('why', '')))}'>"
                   f"{esc(k.get('alias', ''))} –</span>" for k in r.get("skipped", []) if isinstance(k, dict)]
         owed += row_owed
+        # The request's own time: its seats ran at once, so the slowest of
+        # them, then the merge (a merger seat answers after the rest).
+        secs = [(s.get("seat"), float(s.get("seconds") or 0)) for s in r.get("seats", [])]
+        panel = max((t for seat, t in secs if seat != "merger"), default=0.0)
+        merge = sum(t for seat, t in secs if seat == "merger")
+        took = (f"<td class='num' data-sort='{panel + merge:.1f}' title='slowest seat {panel:.1f}s"
+                f"{f' + merge {merge:.1f}s' if merge else ''}'>{panel + merge:.0f}s</td>" if secs
+                else "<td class='num muted' data-sort='-1'>—</td>")
         body.append(
             f"<tr><td class='mono' data-sort='{esc(r.get('at', ''))}'>{esc(r.get('at', '')[:16].replace('T', ' '))}</td>"
             f"<td class='mono'>{esc(r.get('model', ''))}</td>"
             f"<td class='mono'>{esc(r.get('key') or '—')}</td>"
             f"<td title='{esc(r.get('shape_by', ''))}'>{esc(r.get('shape', ''))}</td>"
-            f"<td>{' '.join(chips)}</td>"
-            f"<td class='num{' muted' if not row_owed else ''}'>{row_owed}</td></tr>")
+            f"<td>{' '.join(chips)}</td>{took}</tr>")
     if not records:
         body.append("<tr><td colspan='6' class='muted'>No gateway requests yet. Call a collection"
                     f" at <code>{esc(gateway_url())}</code>.</td></tr>")
@@ -2578,7 +2895,10 @@ def gateway_card(limit: int = 30) -> str:
         carried none. Hover it for why.</dd>
       <dt>Asked by</dt><dd>The gateway key that made the request, or <code>local</code>
         when no key is asked for.</dd>
-      <dt>Owed</dt><dd>Seats without a verdict. Each is recorded as an unjudged
+      <dt>Time</dt><dd>How long the caller waited: the seats run at once, so the
+        slowest of them, plus the merge when the reply was merged. Hover for the two
+        parts; a chip's own time is on the chip.</dd>
+      <dt>Verdicts owed</dt><dd>A chip with no mark. Each is recorded as an unjudged
         offload, so rotation still counts it; a verdict tells the router whether
         it helped. The orchestrator that called can also record one:
         <code>python metrics/offload.py verdict &lt;call id&gt; --useful</code>.</dd>
@@ -2717,7 +3037,7 @@ def discover_model(provider: str, model: str) -> dict:
 
 
 def discover_card() -> str:
-    """Models worth adding, from models.dev and FreeLLMAPI (metrics/discover.py),
+    """Models worth adding, from FreeLLMAPI and models.dev (metrics/discover.py),
     deduped. Read from the daily cache, so the page never waits on the network;
     built once per cache refresh. A click on a row opens its popup (bench.js),
     which asks /api/discover-model for the rest: its routes are where Add is."""
@@ -2725,7 +3045,8 @@ def discover_card() -> str:
     rows = [r for r in all_rows if r["free"] or r["reach"] in ("connected", "on bench")]
     hidden = len(all_rows) - len(rows)
     reach_css = {"on bench": "ok", "connected": "ok", "easy to add": "warn", "other": ""}
-    body = ["<table><thead><tr>", th("Model"), th("Provider"), th("Reach"), th("Price"), th("Context", True),
+    body = ["<table><thead><tr>", th("Model"), th("Rank", True), th("Provider"), th("Reach"), th("Price"),
+            th("Context", True),
             th("In → out"), th("Can"), th("Released"), th("Limits"), "<th></th></tr></thead><tbody>"]
     for r in rows:
         caps = [c for c, on in (("tools", r["tools"]), ("vision", r["vision"]),
@@ -2734,8 +3055,13 @@ def discover_card() -> str:
         price = ("free" if r["free"] else
                  f"{cost.get('input', '?')} / {cost.get('output', '?')}" if cost else "paid")
         lim = r.get("limits") or {}
-        lim_text = " · ".join(f"{v:,} {k}" for k, v in lim.items()) if lim else ""
+        lim_text = " · ".join([f"{v:,} {k}" for k, v in lim.items()]
+                              + ([f"{r['budget']} tokens a month"] if r.get("budget") else []))
         ctx = r.get("context")
+        rank = (f"<td class='num' data-sort='{r['rank']}' title='FreeLLMAPI: smartest #{r['rank']}"
+                + (f", fastest #{r['speed']}" if r.get("speed") else "")
+                + (f", {esc(r['size'])}" if r.get("size") else "") + f"'>{r['rank']}</td>"
+                if r.get("rank") is not None else "<td class='num muted' data-sort='99999'>—</td>")
         add = ("<button class='btn small' type='button'>Add…</button>"
                if any(not x["on_bench_as"] for x in r["routes"]) else "")
         body.append(
@@ -2746,8 +3072,8 @@ def discover_card() -> str:
             + f"<span class='conn'>{esc(r['model'])}"
             + (f" · on bench as {esc(r['on_bench_as'])}" if r.get("on_bench_as") else "")
             + (f" · also on bench as {esc(', '.join(r['also_on']))}" if r.get("also_on") else "")
-            + "</span></td>"
-            f"<td title='{esc(r['provider'])}'>{esc(r['provider_name'])}</td>"
+            + "</span></td>" + rank
+            + f"<td title='{esc(r['provider'])}'>{esc(r['provider_name'])}</td>"
             f"<td data-sort='{['on bench', 'connected', 'easy to add', 'other'].index(r['reach'])}'>"
             f"{chip(r['reach'], reach_css[r['reach']], r.get('api') or '')}</td>"
             f"<td title='per million tokens, input / output'>{esc(price)}</td>"
@@ -2764,31 +3090,40 @@ def discover_card() -> str:
                " provider or reach' aria-label='Filter rows'>"
                "<label><input type='checkbox' data-paid> Show paid (connected providers)</label></div>")
     key = """<dl>
+      <dt>Rank</dt><dd>FreeLLMAPI's smartest-first order (1 is the smartest it lists); within
+        each reach the list follows it. Hover for its fastest-first rank and size. A dash where
+        only models.dev lists the model.</dd>
       <dt>Provider</dt><dd>Who serves the model: a row per provider, so the same model
-        from OpenRouter and from NVIDIA is two rows. Hover for its models.dev id.</dd>
+        from OpenRouter and from NVIDIA is two rows. Hover for its id.</dd>
       <dt>Reach</dt><dd><em>on bench</em> already registered; <em>connected</em> this
         install has a route to the provider (an Open WebUI connection, or an endpoint of its
         own); <em>easy to add</em> an OpenAI-compatible API, so one new connection away (hover
         for its base URL); <em>other</em> a native API or an unvetted reseller.</dd>
-      <dt>Price</dt><dd>Free if models.dev prices it at 0 or FreeLLMAPI lists it (a catalog of
-        free tiers): models.dev shows the PAID price even where a free tier exists. Otherwise
+      <dt>Price</dt><dd>Free if FreeLLMAPI lists it (a catalog of free tiers) or models.dev
+        prices it at 0: models.dev shows the PAID price even where a free tier exists. Otherwise
         input / output per million tokens.</dd>
+      <dt>Limits</dt><dd>The free tier's, from FreeLLMAPI: requests and tokens per minute (rpm,
+        tpm) and per day (rpd, tpd), and roughly how many tokens a month that allows.</dd>
       <dt>New</dt><dd>First seen in either list in the last 7 days.</dd>
       <dt>Add</dt><dd>Click a row: its popup has everything known about the model and every
         route this install has to it, each with <em>Add via this</em>. <em>Via Open WebUI</em>
         adds the model to that connection's Model IDs (when it lists them), creates its model
         record public, tagged like the connection's other models and with their logo, then runs
-        the bench's onboarding (register, 4-call probe, suite), so it is in the chats too.
+        the bench's onboarding as the confirm chooses (Onboard as, Vision model), so it is in
+        the chats too.
         <em>Via LiteLLM</em> or <em>CLI Proxy API</em> adds it to that hub's connection (one more
         deployment, or the provider's models list); those hubs are asked when the popup opens.
         <em>Direct</em> onboards it on the provider's own endpoint: bench only. A new provider
         needs a connection or an endpoint first; hover its <em>easy to add</em> chip for the
         base URL.</dd>
-      <dt>Sources</dt><dd>models.dev and/or FreeLLMAPI's free catalog (named in the popup); a
-        model one provider offers in both is shown once. Both are refreshed at most daily.</dd>
+      <dt>Sources</dt><dd>FreeLLMAPI's free catalog first: where it lists a model, its name,
+        provider, context, tools, limits, ranks and quirks lead, and the popup links its
+        freellmapi.co page. models.dev fills the rest and alone has the paid models, prices,
+        base URLs and modalities. A model one provider offers in both is shown once. Both are
+        refreshed at most daily.</dd>
     </dl>"""
-    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from models.dev"
-            f" and FreeLLMAPI, deduped; {hidden:,} paid models on providers you have no route"
+    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI"
+            f" and models.dev, deduped; {hidden:,} paid models on providers you have no route"
             f" to are left out. Click a model for everything known about it and the routes it can be"
             f" added by.</p>{toolbar}"
             f"<div class='scroll'>{''.join(body)}</div>"
@@ -2832,8 +3167,11 @@ def collections_tab() -> str:
         # Not data-filter: that binds to the whole panel. The same match (any
         # text, case-insensitive), on the member list only; ticked ones stay shown.
         "<label class='field' for='c-member-filter'>Members</label>"
+        # The panel as it will sit, drawn as the Collections list draws it
+        # (bench.js recount): a seat per ticked member, explorer seats dashed.
+        "<div id='c-seats' class='seats' aria-live='polite'></div>"
         "<input type='search' id='c-member-filter' class='member-filter' spellcheck='false' autocomplete='off'"
-        " placeholder='Filter by alias or served name' aria-controls='c-members' aria-describedby='c-count'>"
+        " placeholder='Filter by model, provider or served name' aria-controls='c-members' aria-describedby='c-count'>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
         "<label class='field' for='c-shape'>Explorer shape</label><select id='c-shape'></select>"
@@ -3044,12 +3382,71 @@ def access_card() -> str:
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td>{esc(v)}</td>"
                    f"<td class='mono set-by'>{'<br>'.join(esc(s) for s in src.split(', '))}</td></tr>"
                    for k, v, src in rows)
+    running, disk = on_disk()
+    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
+           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
     return ("<div class='panel'><h2>Server and access</h2><p class='muted'>How this install is reached and who"
             " may call it. Set in the environment, read at start: change one and restart. Shown here, never"
             " set here, since a page that could change its own login could open itself to anyone. No secret"
             " is shown, only whether one is set.</p>"
             f"<table class='facts'><thead><tr>{th('Setting')}{th('Now')}{th('Set by')}</tr></thead>"
-            f"<tbody>{body}</tbody></table></div>")
+            f"<tbody>{body}</tbody></table>"
+            # restart(): stopped and started again by serve.py's own parent process.
+            f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
+            " for an update or a setting"
+            " above, whatever runs it (a service, the launcher, a container or a terminal). It waits for"
+            " onboarding, a scheduled job and gateway calls in progress, or cuts them off if you insist; the"
+            " page comes back in a few seconds.</p>"
+            "<p><button class='btn' type='button' id='restart'>Restart Benchwarmers</button>"
+            " <span id='restart-out' role='status'></span></p></div>")
+
+
+RESTART = 75                       # the exit code serve.py's parent reads as "start me again"
+SERVER = None                      # the running ThreadingHTTPServer, set in __main__
+_answering = [0]                   # gateway calls being answered now
+_answering_lock = threading.Lock()
+
+
+def _follow_parent() -> None:
+    """End with serve.py's parent however it went (a service stopped, a
+    terminal closed), so no orphan keeps the port. Its process is waited on,
+    not a pipe read: on Windows a thread blocked reading stdin stalls every
+    subprocess started meanwhile (git, the onboarding, the jobs)."""
+    ppid = int(os.environ.get("BENCH_PARENT") or 0)
+    if not ppid:
+        return
+    if os.name == "nt":
+        import ctypes
+        k32 = ctypes.windll.kernel32
+        handle = k32.OpenProcess(0x00100000, False, ppid)          # SYNCHRONIZE
+        if not handle:
+            return                                                  # cannot watch it: stay up
+        k32.WaitForSingleObject(handle, 0xFFFFFFFF)                 # until it ends
+    else:
+        while os.getppid() == ppid:                                 # a new parent: the old one ended
+            time.sleep(2)
+    os._exit(0)
+
+
+def restart(force: bool = False, who: str = "ui") -> dict:
+    """End this dashboard and start it again on the code now on disk: after
+    an update, or a setting read at start. serve.py runs it as a child of its
+    own parent process (see __main__), which starts it again, so this works
+    however serve.py was started: Servy, the npm or pip launcher, Docker or a
+    terminal. Refused while something would be cut off, unless `force`."""
+    import scheduler
+    if os.environ.get("BENCH_SERVED") != "1" or SERVER is None:
+        return {"ok": False, "errors": ["this dashboard was not started by serve.py's own parent process,"
+                                        " so nothing would start it again: restart it the way you started it"]}
+    busy = ([f"onboarding is running ({sum(i['state'] in ('queued', 'running') for i in _OQ['items'])} left):"
+             " Stop it in the queue, or wait"] if _OQ["worker"] is not None else []) \
+        + ([f"a scheduled job is running: {', '.join(scheduler.RUNNING)}"] if scheduler.RUNNING else []) \
+        + ([f"{_answering[0]} gateway call(s) are being answered"] if _answering[0] else [])
+    if busy and not force:
+        return {"ok": False, "busy": True, "errors": busy}
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "", "restart", *on_disk())
+    threading.Timer(0.5, SERVER.shutdown).start()      # after this reply has gone
+    return {"ok": True, "errors": [], "cut": busy}
 
 
 def save_scheduler(mode: str, jobs, who: str = "ui") -> dict:
@@ -3340,27 +3737,39 @@ def models_page() -> str:
             + "".join(cards))
 
 
+def quota_tiles() -> list[dict]:
+    """metrics/quotaview.py's tiles, each named as the collection popup names
+    its provider, unless the live read named the credentials behind it."""
+    import quotaview
+    return [dict(t, name=t["name"] or provider_label(t["tag"])) for t in quotaview.tiles()]
+
+
+def quota_op(which: str, tag) -> dict:
+    """Refresh or Check now on one tile: {ok, errors, note, tile}; "order"
+    saves the tiles' order (a list of tags, empty for the default sort)."""
+    import quotaview
+    if which == "order":
+        return quotaview.save_order(tag)
+    if tag not in {t["id"] for t in quota_tiles()}:
+        return {"ok": False, "errors": [f"no tile called {tag!r}"]}
+    try:
+        r = (quotaview.refresh if which == "refresh" else quotaview.check)(tag)
+    except (ValueError, OSError) as exc:              # OSError: URLError and timeouts
+        return {"ok": False, "errors": [str(exc)]}
+    if not r.get("ok"):
+        return {"ok": False, "errors": r.get("errors") or ["it failed"]}
+    return {"ok": True, "errors": [], "note": r.get("note", ""),
+            "tile": next((t for t in quota_tiles() if t["id"] == tag), None)}
+
+
 def quotas_page() -> str:
-    """What limits each connection, who is resting after one, what providers
-    last said was left (the built-in client's quota records, metrics/quota.py)
-    and what the bench itself sent (its usage ledger, metrics/ledger.py)."""
+    """A tile per provider: what is left and when it resets, shaped by how the
+    provider limits (metrics/quotaview.py), who is resting, Refresh and Check
+    now; the resets of the next week; what the bench sent (its usage ledger,
+    metrics/ledger.py); every limit known. bench.js draws tiles and charts."""
     import learn
     import ledger
-    import quota
     now = datetime.datetime.now().astimezone()
-    rec_rows = []
-    for r in sorted(quota.live(), key=lambda r: (r.get("connection", ""), r.get("alias", ""), r.get("metric", ""))):
-        try:
-            age = int((now - datetime.datetime.fromisoformat(r["observed_at"])).total_seconds() // 60)
-            reset = datetime.datetime.fromisoformat(r["reset_at"]).astimezone().strftime("%Y-%m-%d %H:%M")
-        except (KeyError, TypeError, ValueError):
-            continue
-        num = lambda v: f"{v:,}" if isinstance(v, int) else "—"          # noqa: E731
-        rec_rows.append(f"<tr><td>{esc(r.get('connection', ''))}</td><td>{named(r.get('alias', ''))}</td>"
-                        f"<td>{esc(r.get('metric', ''))} / {esc(r.get('window', ''))}</td>"
-                        f"<td class='num'>{num(r.get('remaining'))}</td><td class='num'>{num(r.get('limit'))}</td>"
-                        f"<td class='mono'>{esc(reset)}</td>"
-                        f"<td>{esc(r.get('source', ''))}, {age} min ago</td></tr>")
     t = now.timestamp()
     sent: dict = {}
     for row in ledger.rows(min(t - 86400, ledger.month_start(t))):
@@ -3372,15 +3781,10 @@ def quotas_page() -> str:
             s[2], s[3] = s[2] + n, s[3] + tok
     sent_rows = [f"<tr><td>{esc(c)}</td>" + "".join(f"<td class='num'>{v:,}</td>" for v in s) + "</tr>"
                  for c, s in sorted(sent.items())]
-    cool_rows = []
-    for alias, c in sorted(learn.cooldowns().items(), key=lambda kv: kv[1].get("until", "")):
-        try:
-            left = max(0, int((datetime.datetime.fromisoformat(c["until"]) - now).total_seconds() // 60))
-        except (KeyError, ValueError, TypeError):
-            left = 0
-        cool_rows.append(f"<tr><td>{named(alias)}</td><td>{esc(catalog.connection_of(alias))}</td>"
-                         f"<td class='num' data-sort='{left}'>{left} min</td>"
-                         f"<td>{'stated by the provider' if c.get('stated') else 'default wait'}</td></tr>")
+    data = json.dumps({"tiles": quota_tiles(),
+                       "sent": [{"provider": provider_label(c) if c != "?" else "?", "day": s[0], "month": s[2]}
+                                for c, s in sent.items() if s[2]]},
+                      ensure_ascii=False).replace("</", "<\\/")
     entered = catalog.connection_limits()
     keys = catalog.CONNECTION_LIMIT_KEYS
     lim_rows = [f"<tr><td>{esc(conn)}</td>" + "".join(
@@ -3395,40 +3799,42 @@ def quotas_page() -> str:
                                 + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
                                           else "<td class='num muted'>—</td>" for kk in keys)
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
+    sent_table = ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
+                  + th("Requests, this month", True) + th("Tokens, this month", True)
+                  + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
+                  if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
+    sent_key = ("<p>One row per provider, most requests this month first. The pale bar is this month's"
+                " requests, the dark one inside it the last 24 hours'; the label reads 24 h / month."
+                " Retries count. Calls through the llm CLI are not in the ledger.</p>")
     return (
-        "<div class='panel'><h2>Quota management</h2><p class='sub'>What is left on each connection and"
-        " when it resets, from the rate-limit headers and 429 replies the built-in client sees, and what"
-        " the bench sent itself, from its usage ledger; who is resting after a rate limit, and every"
-        " limit known. The built-in client keeps to all of these. Calls through the llm CLI are in"
-        " neither record. Quotas of OAuth credentials stay in CLI Proxy API's own console.</p></div>"
-        "<div class='panel'><h3 class='chart-head'>What providers said is left · " + str(len(rec_rows)) + "</h3>"
-        + ("<table><thead><tr>" + th("Connection") + th("Model") + th("Counts") + th("Left", True)
-           + th("Of", True) + th("Resets") + th("From") + f"</tr></thead><tbody>{''.join(rec_rows)}</tbody></table>"
-           if rec_rows else "<p class='muted'>Nothing in force: no provider has stated a limit whose reset"
-                            " is still ahead.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>What the bench sent</h3><p class='sub'>Requests"
-        " (retries included) and tokens per connection, from the usage ledger.</p>"
-        + ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
-           + th("Requests, this month", True) + th("Tokens, this month", True)
-           + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
-           if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>Resting after a rate limit · " + str(len(cool_rows)) + "</h3>"
-        + ("<table><thead><tr>" + th("Model") + th("Connection") + th("Back in", True) + th("Wait")
-           + f"</tr></thead><tbody>{''.join(cool_rows)}</tbody></table>" if cool_rows
-           else "<p class='muted'>None: every model is free to call.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>Known limits</h3><p class='sub'>How much each connection"
-        " or model allows (tokens or requests per minute or day), so the bench can space its calls."
-        " Two sources: what you enter per connection (Endpoints, Rate limits per connection), and what a"
-        " provider states in a rate-limit error, which the bench writes down by itself. The list grows on"
-        " its own as models meet their limits. A model's own limits (its popup) win, and learned ones"
-        " only ever lower a limit.</p>"
+        "<div class='panel'><h2>Quota</h2><p class='sub'>What is left with each provider and when it"
+        " resets, and who is resting after a rate limit. CLI Proxy API's credentials and OpenRouter are"
+        " read live by Refresh, which spends nothing; every other tile is what the provider last said"
+        " in its replies to the bench, and Check now, where it is offered, asks with one request."
+        " Drag a tile, or move it with the arrow keys, to keep it where you want it.</p>"
+        "<div class='list-actions'><button class='btn' id='q-all'>Refresh all live</button>"
+        "<button class='btn' id='q-reorder'>Sort: out first, then soonest reset</button>"
+        "<span class='done' id='q-done' aria-live='polite'></span>"
+        "<span class='errors' id='q-errors' aria-live='polite'></span></div>"
+        "<div class='q-tiles' id='q-tiles'></div></div>"
+        "<div class='panel'><h3 class='chart-head'>Resets, next 7 days</h3><p class='sub'>A dot where a"
+        " limit resets (a daily one every day), a bar while a model rests, the line is now.</p>"
+        "<div class='chart' id='chart-resets'></div></div>"
+        + chart_card("What the bench sent", "Requests per provider: this month, and the last 24 hours inside it.",
+                     "chart-sent", sent_table, sent_key, retired=False)
+        + "<details class='panel'><summary><b>Known limits</b></summary><p class='sub'>How much each"
+        " connection or model allows (tokens or requests per minute or day), so the bench can space its"
+        " calls. Two sources: what you enter per connection (Endpoints, Rate limits per connection), and"
+        " what a provider states in a rate-limit error, which the bench writes down by itself. A model's"
+        " own limits (its popup) win, and learned ones only ever lower a limit.</p>"
         + ("<table><thead><tr>" + th("Connection or model") + "".join(th(k, True) for k in keys) + th("From")
            + f"</tr></thead><tbody>{''.join(lim_rows)}</tbody></table>" if lim_rows
            else "<p class='muted'>None known yet.</p>")
-        + "</div>")
+        + "</details>"
+        "<dialog id='q-models' aria-labelledby='q-models-h'><h3 id='q-models-h'></h3>"
+        "<p class='sub' id='q-models-sub'></p><div class='scroll' id='q-models-list'></div>"
+        "<div class='list-actions'><button class='btn' id='q-models-close'>Close</button></div></dialog>"
+        f"<script type='application/json' id='quota-data'>{data}</script>")
 
 
 def _log_rows(path: pathlib.Path, limit: int = 200) -> list[str]:
@@ -3500,6 +3906,14 @@ RELEASE, RELEASE_DATE, COMMIT = version()
 VERSION = f"{RELEASE} {RELEASE_DATE}".strip()           # the X-Benchwarmers-Version header
 
 
+def on_disk() -> tuple[str, str]:
+    """(running, on disk): what this process started from and what a restart
+    would start, by commit, or by the VERSION file where there is no git."""
+    if COMMIT:
+        return COMMIT, _git("log", "-1", "--format=%h") or COMMIT
+    return RELEASE, read(ROOT / "VERSION").strip()
+
+
 def info_page() -> str:
     import scheduler
     import client as model_client
@@ -3608,6 +4022,7 @@ def editor() -> str:
         # The page's own confirm (bench.js ask): window.confirm can be blocked.
         "<dialog id='ask-dialog' aria-labelledby='ask-text'>"
         "<p id='ask-text' style='white-space:pre-line'></p>"
+        f"<div id='ask-onboard' hidden>{onboard_choice('ask')}</div>"     # Discover's Add only
         "<div class='actions'><button class='btn' id='ask-no' type='button'>Cancel</button>"
         "<button class='btn primary' id='ask-yes' type='button'>OK</button></div></dialog>"
         # onboarding
@@ -3620,13 +4035,12 @@ def editor() -> str:
         " with progress in the header. A model from somewhere else? Add its endpoint first.</p>"
         "<input type='search' id='o-find' placeholder='Search models or connections'"
         " aria-label='Search models'>"
+        # How old the list is (serve.LISTED_FOR), and asking again now.
+        "<p class='muted' id='o-age' hidden><span></span>"
+        " <button class='linkish' type='button' id='o-refresh'>Refresh</button></p>"
         "<div class='picker' id='o-picker' role='group' aria-label='Served models'></div>"
         # serve.ONBOARD_MODES: how much each onboarding spends.
-        "<label class='check'>Onboard as <select id='o-mode'>"
-        "<option value='full'>Full: probe, then the suite now</option>"
-        "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
-        "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
-        " (metered, capped or trial keys)</option></select></label>"
+        f"{onboard_choice('o')}"
         "</div>"
         "<div id='o-run' hidden><p class='served' id='o-result'></p><ol class='steps' id='o-steps'></ol>"
         "<details><summary class='muted'>Full log</summary><pre class='log' id='o-log'></pre></details></div>"
@@ -3761,8 +4175,8 @@ class Handler(gateway.Handler):
             log = parse_qs(urlparse(self.path).query).get("log", [""])[0]
             self._json(200, onboard_status(log))
             return
-        if self.path == "/api/candidates":
-            self._json(200, onboard_candidates())
+        if self.path in ("/api/candidates", "/api/candidates?fresh=1"):
+            self._json(200, onboard_candidates(fresh=self.path.endswith("fresh=1")))
             return
         if self.path == "/api/onboard-queue":
             self._json(200, onboard_queue())
@@ -3866,7 +4280,13 @@ class Handler(gateway.Handler):
         """
         path = self.path.split("?", 1)[0]
         if guard.gateway_route(path):
-            return super().do_POST()
+            with _answering_lock:
+                _answering[0] += 1                     # restart() waits for these
+            try:
+                return super().do_POST()
+            finally:
+                with _answering_lock:
+                    _answering[0] -= 1
         if path == "/login":
             return self._login()
         ops = {"/api/model": lambda p: change_model(str(p.get("alias", "")),
@@ -3878,10 +4298,13 @@ class Handler(gateway.Handler):
                "/api/retry": lambda p: retry_model(str(p.get("alias") or "")),
                "/api/discover/add": lambda p: discover_add(str(p.get("connection") or ""),
                                                            str(p.get("model") or ""),
-                                                           endpoint=str(p.get("endpoint") or "")),
+                                                           endpoint=str(p.get("endpoint") or ""),
+                                                           mode=str(p.get("mode") or "full"),
+                                                           vision=p.get("vision") is True),
                "/api/onboard": lambda p: start_onboard(str(p.get("served", "")),
                                                        str(p.get("endpoint", "")),
-                                                       mode=str(p.get("mode") or "full")),
+                                                       mode=str(p.get("mode") or "full"),
+                                                       vision=p.get("vision") is True),
                "/api/onboard-queue": lambda p: queue_onboard(p.get("items")),
                "/api/onboard-queue/stop": lambda p: stop_onboard_queue(),
                "/api/onboard-queue/clear": lambda p: clear_onboard_queue(),
@@ -3922,7 +4345,11 @@ class Handler(gateway.Handler):
                "/api/gateway-key": lambda p: make_gateway_key(str(p.get("name") or ""), p.get("collections"),
                                                               p.get("rpm")),
                "/api/gateway-key/remove": lambda p: revoke_gateway_key(str(p.get("name") or "")),
-               "/api/theme": lambda p: save_theme(str(p.get("theme") or ""))}
+               "/api/theme": lambda p: save_theme(str(p.get("theme") or "")),
+               "/api/quota/refresh": lambda p: quota_op("refresh", str(p.get("tag") or "")),
+               "/api/quota/check": lambda p: quota_op("check", str(p.get("tag") or "")),
+               "/api/quota/order": lambda p: quota_op("order", p.get("order")),
+               "/api/restart": lambda p: restart(p.get("force") is True)}
         # SAME ORIGIN ONLY. Any page open in the browser can POST a plain form
         # to 127.0.0.1, and this used to parse whatever arrived as JSON -- so a
         # website could retire models, and now could set a key. The checks are
@@ -4019,6 +4446,19 @@ class Handler(gateway.Handler):
 
 
 if __name__ == "__main__":
+    if os.environ.get("BENCH_SERVED") != "1":
+        # The dashboard runs as this process's child, so Restart (restart())
+        # can end it and have it started again on the code now on disk,
+        # whatever started serve.py. The child ends with this one (_follow_parent).
+        env = {**os.environ, "BENCH_SERVED": "1", "BENCH_PARENT": str(os.getpid())}
+        try:
+            while (code := subprocess.call([sys.executable, os.path.abspath(__file__), *sys.argv[1:]],
+                                           env=env, stdin=subprocess.DEVNULL)) == RESTART:
+                print("Benchwarmers: restarting on the code now on disk", flush=True)
+        except KeyboardInterrupt:
+            code = 0
+        sys.exit(code)
+    threading.Thread(target=_follow_parent, daemon=True).start()
     address = guard.bind()
     guard.startup(address)                             # beyond this machine, only with an admin token
     shown = "localhost" if address == "127.0.0.1" else address
@@ -4027,4 +4467,6 @@ if __name__ == "__main__":
     paths.make()                                       # a fresh data folder's layout, before any writer
     import scheduler
     scheduler.start()                                  # off unless config/scheduler.json says otherwise
-    ThreadingHTTPServer((address, PORT), Handler).serve_forever()
+    SERVER = ThreadingHTTPServer((address, PORT), Handler)
+    SERVER.serve_forever()                             # returns only when restart() shuts it down
+    sys.exit(RESTART)