@wendellresearch-ai/benchwarmers 0.6.1 → 0.6.3

package/VERSION

@@ -1 +1 @@
-0.6.1
+0.6.3

package/metrics/route.py

@@ -148,12 +148,14 @@ def _full_tag_of(alias: str) -> str:
     import endpoints
     # Tags are keyed by the served name. A decision model is not in the llm
     # yaml, so its served name comes from config/decision-models.json.
-    served = (_registered_names().get(alias)
-              or (_json(DATA / "config" / "decision-models.json").get(alias) or {}).get("served")
-              or alias)
+    decision = _json(DATA / "config" / "decision-models.json").get(alias) or {}
+    served = _registered_names().get(alias) or decision.get("served") or alias
     tag = _json(DATA / "config" / "provider-tags.json").get(served)
     if not tag:
-        name = endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+        # A decision model is in no registration either: its own endpoint is its `via`
+        # (Cloudflare's clef, onboarded from a direct cloudflare endpoint).
+        name = (endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+                or decision.get("via") or decision.get("endpoint"))
         tag = name if name and not endpoints.hub(name) else None
     return tag or "?"
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/provider_tags.py

@@ -68,13 +68,29 @@ def get(hub: str, path: str) -> dict:
         return json.load(response)
 
 
+def _host(url: str) -> str:
+    return (url or "").split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0].lower()
+
+
 def connection_names(hub: str) -> list[str]:
-    """One short name per connection index. URLs only — never the keys."""
+    """One short name per connection index. URLs only — never the keys.
+
+    A provider reached both through the hub and on a direct endpoint of its
+    own (Cloudflare: chat through Open WebUI, decisions only direct) is ONE
+    connection: the hub's connection to that host takes the direct endpoint's
+    name, which is what its directly registered models are tagged (main). Any
+    provider, by host; other hubs (CLI Proxy API) are not direct endpoints."""
     urls = get(hub, "/openai/config").get("OPENAI_API_BASE_URLS", [])
+    direct: dict[str, str] = {}
+    for name, entry in endpoints.load().items():
+        if name != hub and not endpoints.hub(name):
+            direct.setdefault(_host(entry.get("base", "")), name)
     names = []
     for url in urls:
-        host = url.split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0]
-        if host.replace(".", "").isdigit():
+        host = _host(url)
+        if host in direct and host not in (endpoints.connections().get("hosts") or {}):
+            names.append(direct[host])
+        elif host.replace(".", "").isdigit():
             # A BARE IP NAMES NOTHING, and guessing from it was worse than
             # useless: a proxy labelled `lan-proxy` is described by where it
             # sits rather than what it is, and Open WebUI does not expose

package/scripts/test_no_owui.py

@@ -236,6 +236,19 @@ def owner(fake: str) -> None:
     assert tags["fast-1"] == "groq" and tags["llm7.codestral"] == "llm7-api" and tags["vendor/x:free"] == "openrouter", tags
     assert "never-read" not in json.dumps(tags)
     assert route.connection_of("fast-1") == "groq" and route.connection_of("x-free") == "openrouter"
+    # A provider also on a direct endpoint of its own is one connection: the hub's takes the endpoint's name.
+    real_load, both = endpoints.load, dict(endpoints.load())
+    both["groq-direct"] = {"base": "https://api.groq.com/openai/v1", "key": "groq-direct"}
+    endpoints.load = lambda *a, **k: both
+    try:
+        assert provider_tags.connection_names("openwebui") == ["groq-direct", "llm7-api"]
+    finally:
+        endpoints.load = real_load
+    # A decision model onboarded from the direct endpoint: in no registration, untagged, its `via` is its connection.
+    write("decision-models.json", {"dec-1": {"served": "vendor/decide:free", "endpoint": "openrouter", "via": "openrouter"},
+                                   "dec-hub": {"served": "hub-decide", "endpoint": "openwebui", "via": "openwebui"}})
+    assert route.connection_of("dec-1") == "openrouter" and route.connection_of("dec-hub") == "?"
+    (TMP / "config" / "decision-models.json").unlink()
 
     # The default hub is recorded whole; the direct endpoint, its registered models only.
     assert check_deprecations.served() == {"fast-1", "llm7.codestral", "fresh-2", "mercury-decide",

package/serve.py

@@ -39,6 +39,7 @@ import datetime
 import functools
 import html
 import json
+import math
 import os
 import pathlib
 import re
@@ -2639,7 +2640,7 @@ def charts_data() -> dict:
                          "rate": int(hit[1]) / max(1, int(hit[2])) if hit else None, "invented": bool(v["invented"])})
     bars = [{"model": m, "kind": k, "rate": w / r["answers"], "n": f"{w}/{n}"}
             for m, r in sorted(wr.items()) if m in live and r["answers"] for k, (w, n) in r["by"].items() if w]
-    totals = {m: {"rate": r["wrong"] / r["answers"], "answers": r["answers"]}
+    totals = {m: {"rate": r["wrong"] / r["answers"], "answers": r["answers"], "wrong": r["wrong"]}
               for m, r in wr.items() if m in live and r["answers"]}
     hunt = [{"model": m, **r["hunt"]} for m, r in sorted(wr.items()) if m in live and any(r["hunt"].values())]
     return {"overview": overview, "heat": heat, "families": ["hand-built", "claims", "seeded, function",
@@ -2900,16 +2901,10 @@ def vision_card() -> str:
 
 
 def perf_charts() -> dict:
-    """Reliability over the last 14 days of probes, and speed on real tasks
-    against how much a model finds, for the Performance panel of Benchmarks."""
+    """Speed on real tasks against how much a model finds, for the Performance
+    panel of Benchmarks. (Whether probes answered is Uptime's.)"""
     import route
     live = on_bench()
-    since = (datetime.date.today() - datetime.timedelta(days=14)).isoformat()
-    outcomes: dict[tuple[str, str], int] = {}
-    for r in perf_rows():
-        if r["model"] in live and r.get("timestamp", "") >= since:
-            o = "answered" if int(r.get("answered") or 0) > 0 else (r.get("failure") or "error")
-            outcomes[(r["model"], o)] = outcomes.get((r["model"], o), 0) + 1
     seconds: dict[str, list[float]] = {}
     for (_task, m), v in suite_scores().items():
         if m in live and re.match(r"\d+/\d+", v["found"]):
@@ -2917,8 +2912,7 @@ def perf_charts() -> dict:
     recall = route.suite_recall()
     speed = [{"model": m, "seconds": sorted(s)[len(s) // 2], "recall": recall[m][0] / recall[m][1], "tasks": len(s)}
              for m, s in sorted(seconds.items()) if recall.get(m) and recall[m][1]]
-    return {"reliability": [{"model": m, "outcome": o, "count": n} for (m, o), n in sorted(outcomes.items())],
-            "speed": speed}
+    return {"speed": speed}
 
 
 # ---- Trend: does a model score lower, or higher, than it used to? ----------
@@ -3545,9 +3539,15 @@ DECIDE_KIND = {"errors": "choice of 6", "collection": "choice of the collections
 def decision_data() -> dict:
     """For bench.js: each model's calibration bins per set, [said, right, n]."""
     res = decide.results()
-    return {"sets": [[s, DECIDE_SETS[s]] for s in decide.SETS],
+    models = sorted((res.get("models") or {}).items())
+    return {"sets": [[s, DECIDE_SETS[s]] for s in decide.SETS], "low": decide.MIN_ITEMS,
+            # Right of asked per model and set: the bars, as the tables count it.
+            "scores": [{"model": a, "set": s, "correct": r.get("correct", 0), "asked": r["asked"],
+                        "answered": r.get("answered", 0), "brier": r.get("brier")}
+                       for a, by in models for s, r in by.items()
+                       if s in decide.SETS and isinstance(r, dict) and r.get("asked")],
             "points": [{"model": a, "set": s, "said": b["said"], "right": b["right"], "n": b["n"]}
-                       for a, by in sorted((res.get("models") or {}).items()) for s, r in by.items()
+                       for a, by in models for s, r in by.items()
                        if s in decide.SETS for b in r.get("calibration") or []]}
 
 
@@ -3614,8 +3614,7 @@ def decision_card() -> str:
     extra = (f"<h3 class='chart-head'>Who each use asks now</h3><ul class='links'>{''.join(uses)}</ul>"
              f"<p class='hint'>{'Measured up to ' + esc(when[:16].replace('T', ' ')) + '.' if when else 'Not measured yet.'}"
              " The scheduler's <code>decisions</code> job asks each model one question a set a day;"
-             " <code>python metrics/decide.py bench</code> runs a pass by hand.</p>"
-             "<h3 class='chart-head'>Calibration: said against right</h3>")
+             " <code>python metrics/decide.py bench</code> runs a pass by hand.</p>")
     key = f"""<dl>
       <dt>The sets</dt><dd>Questions with known answers, built from the bench's own data, no hand labelling.
         <b>Error classes</b>: failed calls' error text (redacted: no keys, ids or emails) and the class the
@@ -3627,8 +3626,12 @@ def decision_card() -> str:
         picked, so always saying the commonest option scores low.</dd>
       <dt>Brier</dt><dd>How far its probabilities sat from the truth: 0 is perfect; for a choice it is
         summed over the options, so up to 2. Lower is better, and a sure wrong answer costs most.</dd>
-      <dt>Calibration</dt><dd>The chart: answers grouped by the probability the model gave its answer
-        (x), against how often those answers were right (y). On the diagonal, its 80% answers come true
+      <dt>The bars</dt><dd>Right of asked for each model on each set, from 0%. A faint bar is a model
+        under {low} items on that set: shown, not ranked yet. Hover a bar for its Brier.</dd>
+      <dt>Calibration</dt><dd>The second chart, once a model answers at two or more levels of sureness
+        on a set: answers grouped by the probability the model gave its answer
+        (x), against how often those answers were right (y); both axes start just under the lowest
+        answer, not at 0%. On the diagonal, its 80% answers come true
         80% of the time; under it, it is surer than it should be. A bigger dot is more answers. The gap is
         the average distance from the diagonal, weighted by answers.</dd>
       <dt>At {decide.CONFIDENT}</dt><dd>The gateway acts on a choice only when the model is this sure: how
@@ -3669,26 +3672,45 @@ def bughunt_card() -> str:
         grey said there was no bug.</dd>
       <dt>Left of the line</dt><dd>The same files untouched: red is a false alarm, claiming a
         bug in clean code. The longer the red, the less to trust its "there's a bug here".</dd>
-      <dt>Order</dt><dd>Found share minus false-alarm share, best at the top.</dd>
+      <dt>Order</dt><dd>On what is proven, best at the top: the low end of the found share's likely
+        range (95%) less the high end of the false alarms'. A few tasks give wide ranges, so they
+        rank low; a model never shown clean code could be raising alarms on all of it.</dd>
+      <dt>Faint bars</dt><dd>Under 10 tasks: shown, but little to go on.</dd>
     </dl>"""
     return chart_card("Bug hunt", "Every task with an answer key: bugs found against false alarms on the"
                       " same files left clean. Right is good, left is bad.", "chart-hunt", "".join(body), key)
 
 
+def _wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
+    """The 95% Wilson interval of k successes in n, as bench.js wilson() draws it."""
+    if not n:
+        return 0.0, 1.0
+    p, z2 = k / n, z * z
+    mid, half = p + z2 / (2 * n), z * math.sqrt(p * (1 - p) / n + z2 / (4 * n * n))
+    return max(0.0, (mid - half) / (1 + z2 / n)), min(1.0, (mid + half) / (1 + z2 / n))
+
+
 def work_card() -> str:
     """How often real offloads helped."""
     rows = work_rows()
     body = ["<table><thead><tr>", th("Model"), th("Offloads", True), th("Judged", True), th("Helped", True),
-            th("Verified findings", True), th("False claims", True), "</tr></thead><tbody>"]
-    for r in sorted(rows, key=lambda r: -r["useful"] / r["judged"]):
+            th("Likely between", True), th("Verified findings", True), th("False claims", True), "</tr></thead><tbody>"]
+    for r in sorted(rows, key=lambda r: (-_wilson(r["useful"], r["judged"])[0], -r["judged"])):
+        lo, hi = _wilson(r["useful"], r["judged"])
         body.append(f"<tr><td>{named(r['model'])}</td><td class='num'>{r['offloads']}</td><td class='num'>{r['judged']}</td>"
                     f"<td class='num' data-sort='{r['useful'] / r['judged']:.3f}'>{r['useful']}/{r['judged']}</td>"
+                    f"<td class='num' data-sort='{lo:.3f}'>{lo * 100:.0f}–{hi * 100:.0f}%</td>"
                     f"<td class='num'>{r['real']}</td><td class='num'>{r['false']}</td></tr>")
     body.append("</tbody></table>")
     key = """<dl>
       <dt>Helped</dt><dd>Of the offloads judged, the share recorded useful: real work (reviews,
         enumerations) handed to the model and checked afterwards. Benchmarks never count here.</dd>
-      <dt>The number beside a name</dt><dd>How many were judged. 3 of 3 is weaker evidence than 40 of 50.</dd>
+      <dt>The dot and its line</dt><dd>The dot is the share that helped, bigger for more judged. The
+        line is where the true share likely sits (a 95% Wilson interval): long for 3 of 3, short for
+        40 of 50. Green when even its low end is over half, red when even its high end is under,
+        grey when there are too few to say.</dd>
+      <dt>Order</dt><dd>By the low end of that line, so volume counts: 40 of 50 ranks above 3 of 3.</dd>
+      <dt>The dashed line</dt><dd>Every model's judged offloads together, for comparison.</dd>
       <dt>Findings and false claims</dt><dd>Verified one by one in reviews, recorded since 2026-10-17.</dd>
     </dl>"""
     return chart_card("Real work", "When a model was given real work and the result was checked: how often"
@@ -3698,7 +3720,8 @@ def work_card() -> str:
 def overview_card() -> str:
     return (f"<div class='panel'>{SHARE}<h2>Who to trust with a review</h2><p class='sub'>Each dot is a model."
             " Right: finds more of the real bugs in our own code. Up: makes fewer things up (confidently"
-            " wrong on clean code, wrong lines, false claims, traps). Top right is where a reviewer should be.</p>"
+            " wrong on clean code, wrong lines, false claims, traps). Top right is where a reviewer should be."
+            " The models in or near the box, and the three that make most up, are named; hover for the rest.</p>"
             "<div class='chart' id='chart-overview'></div></div>")
 
 
@@ -3811,6 +3834,10 @@ def wrong_card() -> str:
       <dt>Reviews</dt><dd>False claims recorded in verdicts (<code>offload.py verdict|add
         --false "..."</code>), over judged offloads. Recorded in structured form only since
         2026-10-17, so older reviews show none.</dd>
+      <dt>The thin line</dt><dd>Where the true share likely sits (95% range). Long for a few
+        answers: 0 of 2 is not a clean record, and the line shows how bad it could be.</dd>
+      <dt>Order</dt><dd>Worst proven at the top: by the low end of that line. Under 10 answers a
+        model's bar is faint.</dd>
     </dl>"""
     return chart_card("Confidently wrong", "Who asserts things that are not true: false alarms on clean"
                       " code, wrong lines, false claims confirmed, traps taken. Shorter is better.",
@@ -3874,17 +3901,14 @@ def perf_card() -> str:
         when it is slow enough to change how you work.</dd>
     </dl>"""
     key = key.replace("</dl>", """
-      <dt>Did it answer?</dt><dd>Each probe of the last 14 days, by what came back: an answer, a
-        rate limit (capacity), a timeout, or a failure the bench classifies (structural, paywalled,
-        entitlement, other).</dd>
+      <dt>Did it answer?</dt><dd>Is on Uptime, by day, with our own quota and setup problems left out.</dd>
       <dt>Speed vs quality</dt><dd>Median seconds per answer on our own suite tasks (real prompts,
-        not the one-word probe) against the share of findings located. Top left is fast and good.</dd>
+        not the one-word probe) against the share of findings located. Top left is fast and good:
+        the green box is the faster half and the top quarter of findings.</dd>
     </dl>""")
     return chart_card("Performance", "What the serving path does — not what the models are worth. One"
                       " reading is weather; the history is climate.", "chart-speed", body, key,
-                      extra="<h3 class='chart-head'>Did it answer? Last 14 days of probes</h3>"
-                            "<div class='chart' id='chart-reliability'></div>"
-                            "<h3 class='chart-head'>Speed vs quality on our own tasks</h3>")
+                      extra="<h3 class='chart-head'>Speed vs quality on our own tasks</h3>")
 
 
 def favicon(theme: str) -> str:
@@ -4300,13 +4324,21 @@ def collections_tab() -> str:
         " the paid ones. Click a suggestion or type your own: comma-separated lower-case words, up to 8.</p>"
         # Not data-filter: that binds to the whole panel. The same match (any
         # text, case-insensitive), on the member list only; the seats show who is ticked.
-        "<label class='field' for='c-member-filter'>Members</label>"
+        "<div class='members-head'><label class='field' for='c-member-filter'>Members</label>"
+        # Model details as they are now (a fact set in a model's popup, a refresh of the
+        # catalog), the ticks and the search kept; also done when a model popup opened from here closes.
+        "<button class='btn small' type='button' id='c-refresh' title='Fetch the models and their details again;"
+        " what is ticked stays ticked'>Refresh model details</button>"
+        "<span class='done' id='c-refresh-out' role='status'></span></div>"
         # The panel as it will sit, drawn as the Collections list draws it
         # (bench.js recount): a seat per ticked member, explorer seats dashed.
         "<div id='c-seats' class='seats' aria-live='polite'></div>"
         "<div class='member-find'><input type='search' id='c-member-filter' class='member-filter' spellcheck='false'"
         " autocomplete='off' placeholder='Filter by model, provider or served name' aria-controls='c-members'"
-        " aria-describedby='c-count'><select id='c-member-provider' aria-label='Provider' aria-controls='c-members'>"
+        " aria-describedby='c-count'><select id='c-member-show' aria-label='Show' aria-controls='c-members'>"
+        "<option value=''>Every model</option><option value='ticked'>Ticked</option>"
+        "<option value='fits'>Fits the tags</option></select>"
+        "<select id='c-member-provider' aria-label='Provider' aria-controls='c-members'>"
         "<option value=''>Every provider</option></select></div>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
@@ -4956,11 +4988,12 @@ def move_card() -> str:
     """Settings' Export and import (export_zip, import_zip; bench.js)."""
     return ("<div class='panel form-panel'><h2>Export and import</h2><p class='muted'>Take this install to another"
             " machine, or keep a copy of it: one .zip file, imported on the other install's Settings page.</p>"
-            "<div class='pick'><label class='check'><input type='radio' name='x-scope' value='settings' checked"
+            # No wrapper: `.pick` is the onboarding picker's 3-column grid row, and squeezed these into it.
+            "<label class='check'><input type='radio' name='x-scope' value='settings' checked"
             " data-kept> <span><b>Settings only</b>: endpoints, collections, roles, model options, the schedule,"
             " gateway keys, the theme (<code>config/</code>)</span></label>"
             "<label class='check'><input type='radio' name='x-scope' value='all' data-kept> <span><b>Everything</b>:"
-            " the settings, and every result, history, log and saved answer the bench judges by</span></label></div>"
+            " the settings, and every result, history, log and saved answer the bench judges by</span></label>"
             "<label class='check'><input type='checkbox' id='x-keys' data-kept> <span>With the endpoints' API keys."
             " The file then holds them: keep it like a password, and delete it once imported.</span></label>"
             "<div class='actions'><button class='btn' type='button' id='x-export'>Download export</button></div>"
@@ -5005,23 +5038,28 @@ def access_card() -> str:
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td>{esc(v)}</td>"
                    f"<td class='mono set-by'>{'<br>'.join(esc(s) for s in src.split(', '))}</td></tr>"
                    for k, v, src in rows)
-    running, disk = on_disk()
-    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
-           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
     return ("<div class='panel'><h2>Server and access</h2><p class='muted'>How this install is reached and who"
             " may call it. Set in the environment, read at start: change one and restart. Shown here, never"
             " set here, since a page that could change its own login could open itself to anyone. No secret"
             " is shown, only whether one is set.</p>"
             f"<table class='facts'><thead><tr>{th('Setting')}{th('Now')}{th('Set by')}</tr></thead>"
-            f"<tbody>{body}</tbody></table>"
-            # restart(): stopped and started again by serve.py's own parent process.
-            f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
-            " for an update or a setting"
-            " above, whatever runs it (a service, the launcher, a container or a terminal). It waits for"
-            " onboarding, a scheduled job and gateway calls in progress, or cuts them off if you insist; the"
-            " page comes back in a few seconds.</p>"
-            "<p><button class='btn' type='button' id='restart'>Restart Benchwarmers</button>"
-            " <span id='restart-out' role='status'></span></p></div>")
+            f"<tbody>{body}</tbody></table>{restart_block('restart', 'a setting above')}</div>")
+
+
+def restart_block(id_: str, what: str) -> str:
+    """The Restart heading, what runs now, and its button (bench.js restart()).
+    On Settings (Server and access) and Info, each with its own ids, since both
+    tabs can be in the page at once."""
+    running, disk = on_disk()
+    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
+           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
+    # restart(): stopped and started again by serve.py's own parent process.
+    return (f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
+            f" for an update or {what}, whatever runs it (a service, the launcher, a container or a terminal)."
+            " It waits for onboarding, a scheduled job and gateway calls in progress, or cuts them off if you"
+            " insist; the page comes back in a few seconds.</p>"
+            f"<p><button class='btn' type='button' id='{id_}'>Restart Benchwarmers</button>"
+            f" <span id='{id_}-out' role='status'></span></p>")
 
 
 RESTART = 75                       # the exit code serve.py's parent reads as "start me again"
@@ -5325,6 +5363,28 @@ def endpoints_page() -> str:
         "<div class='done' id='l-done'></div></div>")
 
 
+def _reply_cell(by_prompt: dict) -> str:
+    """Models' Reply: the daily probe's median seconds to a one-word answer (tiny),
+    with the 18 KB prompt's (large) and tokens a second on hover. Was on Benchmarks'
+    Performance, beside whether it answered, which is Uptime's now."""
+    def p50(r):
+        try:
+            return float(r["p50_s"]) if r and int(r.get("answered") or 0) > 0 else None
+        except (KeyError, TypeError, ValueError):
+            return None
+    tiny, large = p50(by_prompt.get("tiny")), p50(by_prompt.get("large"))
+    if tiny is None and large is None:
+        return "<td class='num muted' data-sort='999999'>—</td>"
+    shown = tiny if tiny is not None else large
+    why = "; ".join(b for b in (
+        f"one-word prompt {tiny:.1f}s" if tiny is not None else "",
+        f"18 KB prompt {large:.1f}s" if large is not None else "",
+        f"{(by_prompt.get('large') or {}).get('tokens_per_s')} tok/s"
+        if (by_prompt.get("large") or {}).get("tokens_per_s") else "") if b)
+    return (f"<td class='num' data-sort='{shown:.2f}' title='median, the daily probe: {esc(why)}'>"
+            f"{shown:.1f}s</td>")
+
+
 def models_page() -> str:
     """Every model on the bench by the connection it comes from."""
     cards = []
@@ -5337,7 +5397,12 @@ def models_page() -> str:
     # Every model, decision models too (connections_view is chat models only,
     # for what a collection may hold): they are on the bench like any other.
     by: dict[str, list[str]] = {}
-    for a in model_directory():
+    directory = model_directory()
+    # The daily probe's latest reading per model and prompt (perf/history.csv), for Reply.
+    speed: dict[str, dict] = {}
+    for r in perf_rows():
+        speed.setdefault(r["model"], {})[r.get("prompt", "")] = r
+    for a in directory:
         by.setdefault(catalog.connection_of(a), []).append(a)
     for c in ({"name": n, "models": sorted(m)} for n, m in sorted(by.items())):
         rows = []
@@ -5348,25 +5413,32 @@ def models_page() -> str:
             ep = through.get(a, "?")
             kind = (" " + chip("decision", "", "a decision model: answers typed questions, not chat")
                     if a in decisions else "")
+            if directory[a].get("ad_hoc"):
+                kind += " " + chip("ad hoc", "adhoc", "ad hoc only: never chosen by the bench on its own, only when you"
+                                   " name it (a collection member, with, --model)")
             rows.append(f"<tr><td><a href='#' class='model' data-alias='{esc(a)}'>"
                         f"<span class='alias'>{esc(a)}</span></a>{kind}</td>"
                         f"<td data-sort='{esc(ep)}'>{chip(ep, 'via', f'reached through the {ep} endpoint')}</td>"
                         f"<td class='num' data-sort='{f.get('context_window') or 0}'>{num(f.get('context_window'))}</td>"
                         f"<td class='num' data-sort='{f.get('max_output') or 0}'>{num(f.get('max_output'))}</td>"
                         f"<td>{esc(', '.join(f.get('input_modalities') or []) or '—')}</td>"
+                        + _reply_cell(speed.get(a, {}))
                         + "".join(f"<td class='num'>{num(lim.get(k))}</td>" for k in catalog.LIMIT_KEYS) + "</tr>")
         # One column plan for every card, so the cards line up under each other
         # (each table sized its own columns, so they drifted from card to card).
         cols = "<colgroup><col class='c-model'><col class='c-via'><col class='c-n'><col class='c-n'>" \
-               "<col class='c-input'>" + "<col class='c-lim'>" * len(catalog.LIMIT_KEYS) + "</colgroup>"
-        cards.append(f"<div class='panel'><h3 class='chart-head'>{esc(c['name'])} · {len(c['models'])} models</h3>"
+               "<col class='c-input'><col class='c-reply'>" + "<col class='c-lim'>" * len(catalog.LIMIT_KEYS) + "</colgroup>"
+        # Each connection folds; the page's one box filters them all (bench.js applyFilter, data-conn).
+        cards.append(f"<details class='panel model-card' open data-conn='{esc(c['name'])}'><summary>"
+                     f"<h3 class='chart-head'>{esc(c['name'])} · {len(c['models'])} models</h3></summary>"
                      f"<table class='model-cards'>{cols}<thead><tr>" + th("Model") + th("Through")
                      + th("Context", True) + th("Max output", True)
-                     + th("Input") + "".join(th(k, True) for k in catalog.LIMIT_KEYS)
-                     + f"</tr></thead><tbody>{''.join(rows)}</tbody></table></div>")
+                     + th("Input") + th("Reply", True) + "".join(th(k, True) for k in catalog.LIMIT_KEYS)
+                     + f"</tr></thead><tbody>{''.join(rows)}</tbody></table></details>")
     return ("<div class='panel'><h2>Models</h2><p class='sub'>Every model on the bench by the connection it"
             " comes from, with the endpoint it is reached through and what is known of its limits. A name"
-            " opens the model.</p></div>"
+            " opens the model.</p><div class='toolbar'><input type='search' data-filter='page'"
+            " placeholder='Find a model or connection' aria-label='Filter models'></div></div>"
             + "".join(cards))
 
 
@@ -5604,6 +5676,7 @@ def info_page() -> str:
         ("Theme", saved_theme())]]
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td class='mono'>{v}</td></tr>" for k, v in rows)
     return (f"<div class='panel'><h2>Info</h2><table class='facts'><tbody>{body}</tbody></table>"
+            f"{restart_block('restart-info', 'a setting')}"
             "<p class='muted'>Benchwarmers · by Wendell Research · MIT licence"
             " (<code>LICENSE</code>, credits in <code>NOTICE</code>). Every response carries"
             " <code>X-Benchwarmers-Version</code>.</p></div>")
@@ -5668,8 +5741,7 @@ def editor() -> str:
         + "".join(f"<label>{k} <input type='number' min='1' data-k='{k}' placeholder='—'></label>"
                   for k in ("rpm", "rpd", "tpm", "tpd"))
         + "</div><div class='src' id='s-limits'></div></div>"
-        "</div><div class='errors' id='f-errors' role='alert'></div><div class='done' id='f-done'></div>"
-        "<div class='actions'><button class='btn' id='f-save' type='button'>Save facts</button></div>"
+        "</div><div class='errors' id='f-errors' role='alert'></div>"
         "</details>"
         # How the bench calls it (serve.save_model_options): every caller, gateway, suite and probes.
         "<details id='m-opts' class='facts-box'><summary>Call options</summary>"
@@ -5679,8 +5751,7 @@ def editor() -> str:
         " makes to it. A reasoning model thinks out of the same budget and answers with an empty string"
         " when it runs out, which looks like a dead model: raise it then. Blank leaves it to the"
         " endpoint. <span id='co-ceiling'></span></p>"
-        "<div class='errors' id='co-errors' role='alert'></div><div class='done' id='co-done'></div>"
-        "<div class='actions'><button class='btn' id='co-save' type='button'>Save options</button></div>"
+        "<div class='errors' id='co-errors' role='alert'></div>"
         "</details>"
         "<label class='check'><input type='checkbox' id='m-adhoc'> Ad hoc only</label>"
         "<p class='rules'>Never chosen by the bench on its own: no explorer seat, no router"

package/ui/bench.css

@@ -293,10 +293,11 @@ td.num, th.num { text-align: right; }
 table.model-cards { table-layout: fixed; min-width: 52rem; }
 .panel:has(> table.model-cards) { overflow-x: auto; }        /* scrolls on a narrow window instead of wrapping numbers */
 table.model-cards td.num { white-space: nowrap; }
-table.model-cards col.c-model { width: 24%; }
-table.model-cards col.c-via { width: 13%; }
+table.model-cards col.c-model { width: 23%; }
+table.model-cards col.c-via { width: 11%; }
 table.model-cards col.c-n { width: 10%; }
-table.model-cards col.c-input { width: 19%; }
+table.model-cards col.c-input { width: 15%; }
+table.model-cards col.c-reply { width: 7%; }
 table.model-cards col.c-lim { width: 6%; }
 table.model-cards td { overflow-wrap: anywhere; }
 tbody tr:hover > td { background: var(--step-1); }
@@ -319,6 +320,7 @@ a.model:hover .alias { box-shadow: inset 0 -.3em 0 var(--signal); }
 .chip.err { background: var(--err-ground); color: var(--err); }
 .chip.new { background: var(--sky); color: var(--ink-deep); }
 /* an endpoint a model is reached through: outlined, so it reads as a place, not a state */
+.chip.adhoc { background: color-mix(in oklab, var(--sky) 55%, var(--paper-2)); color: var(--ink-deep); }
 .chip.via { background: transparent; border: 1px solid var(--line); color: var(--ink); font-family: var(--font-mono); }
 button.chip { font-family: inherit; border: 0; cursor: pointer; }
 button.chip:hover { outline: 1px solid currentColor; }
@@ -334,6 +336,8 @@ td.why { white-space: nowrap; }                     /* Uptime's down / left out,
 dialog { border: 1px solid var(--ink); border-radius: var(--radius); padding: 1.25rem 1.4rem;
          width: min(40rem, calc(100vw - 2rem)); max-height: calc(100vh - 2rem);
          background: var(--paper-2); color: var(--ink); box-shadow: 6px 6px 0 var(--ink); }
+/* The collection popup holds the member table: wider, so names and chips sit on one line. */
+dialog#c-form { width: min(54rem, calc(100vw - 2rem)); }
 dialog::backdrop { background: color-mix(in oklab, var(--ink-deep) 45%, transparent); }
 dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
 :is(dialog, .form-panel) .served { color: var(--muted); font-size: .9375rem; margin: 0 0 .9rem; overflow-wrap: anywhere; }
@@ -344,11 +348,24 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
 :is(dialog, .form-panel) .rules { font-size: .875rem; color: var(--muted); margin: .35rem 0 0; }
 :is(dialog, .form-panel) label.check { display: flex; align-items: center; gap: .5rem; margin-top: 1rem; cursor: pointer; }
 :is(dialog, .form-panel) fieldset { border: 0; padding: 0; margin: 1rem 0 0; }
-:is(dialog, .form-panel) .members { max-height: 18rem; overflow: auto; border-top: 1px solid var(--line); }
-/* One row per model: its name and what it holds, then a chip per provider it
-   is on, each ticking that alias (bench.js fillForm). */
-:is(dialog, .form-panel) .members .m-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .2rem 1rem;
-                  align-items: center; padding: .4rem 0; border-bottom: 1px solid var(--line); }
+:is(dialog, .form-panel) .members { max-height: min(30rem, 55vh); overflow: auto; border: 1px solid var(--line);
+                  border-radius: var(--radius); }
+/* A table to scan: the model (its name opens it), context, what it can do, then a
+   chip per provider it is on, each ticking that alias (bench.js drawMembers). */
+:is(dialog, .form-panel) .members :is(.m-row, .m-head) { display: grid; gap: .2rem .8rem; align-items: center;
+                  grid-template-columns: minmax(9rem, 1.4fr) 3.6rem minmax(0, 1fr) minmax(0, 1.3fr); padding: .4rem .6rem; }
+:is(dialog, .form-panel) .members .m-head { position: sticky; top: 0; z-index: 1; background: var(--paper-2);
+                  border-bottom: 1px solid var(--ink); font-size: .75rem; font-weight: 600; color: var(--muted); }
+:is(dialog, .form-panel) .members .m-row { border-bottom: 1px solid var(--line); }
+:is(dialog, .form-panel) .members .m-row:nth-child(odd) { background: color-mix(in oklab, var(--band) 45%, transparent); }
+:is(dialog, .form-panel) .members .m-row:has(input:checked) { box-shadow: inset 3px 0 0 var(--bench-seat, var(--brand)); }
+:is(dialog, .form-panel) .members .m-name a.model { color: var(--ink); text-decoration: none; font-size: .875rem; }
+:is(dialog, .form-panel) .members .m-name a.model:hover { text-decoration: underline; }
+:is(dialog, .form-panel) .members .m-ctx { font-family: var(--font-mono); font-size: .8125rem; color: var(--muted); text-align: right; }
+:is(dialog, .form-panel) .members .m-can { display: flex; flex-wrap: wrap; gap: .2rem; }
+:is(dialog, .form-panel) .members .m-can .chip { font-size: .6875rem; padding: 0 .4rem; }
+#c-form .members-head { display: flex; align-items: baseline; gap: .6rem; flex-wrap: wrap; }
+#c-form .members-head .field { margin-right: auto; }
 :is(dialog, .form-panel) .members .m-name { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0 .5rem;
                   font-size: .875rem; overflow-wrap: anywhere; }
 :is(dialog, .form-panel) .members .m-facts { color: var(--muted); font-size: .8125rem; }
@@ -380,6 +397,8 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
 :is(dialog, .form-panel) .profile .src { font-size: .75rem; color: var(--muted); }
 :is(dialog, .form-panel) .settings { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 1rem; }
 @media (max-width: 40rem) { :is(dialog, .form-panel) :is(.settings, .members .m-row) { grid-template-columns: minmax(0, 1fr); }
+                            :is(dialog, .form-panel) .members .m-head { display: none; }
+                            :is(dialog, .form-panel) .members .m-ctx { text-align: left; }
                             :is(dialog, .form-panel) .members .m-routes { justify-content: flex-start; } }
 :is(dialog, .form-panel) fieldset label.check { margin-top: .4rem; flex-wrap: wrap; }
 :is(dialog, .form-panel) fieldset label.check .muted { font-size: .875rem; }
@@ -519,6 +538,9 @@ pre.log.answer { max-height: 40vh; }
 .chart { margin: .5rem 0 .75rem; overflow-x: auto; }
 .chart.muted { font-size: .875rem; }
 h3.chart-head { font-size: 1rem; margin: 1rem 0 .2rem; }
+.chart-label { font-size: .875rem; font-weight: 600; margin: .9rem 0 .1rem; }
+/* Models: each connection a fold, its heading the summary */
+.model-card > summary h3.chart-head { display: inline; margin: 0; }
 /* the Collections tab: the dialog's form, kept to a readable width */
 .panel.form-panel > :not(#c-list, table) { max-width: 46rem; }
 /* a list of things you open (collections), with its New button above */