@wendellresearch-ai/benchwarmers 0.6.2 → 0.6.4

package/serve.py

@@ -39,6 +39,7 @@ import datetime
 import functools
 import html
 import json
+import math
 import os
 import pathlib
 import re
@@ -2253,27 +2254,6 @@ def open_count(groups: dict) -> int:
     return sum(len(g["rows"]) + len(g["held"]) for g in groups.values())
 
 
-def and_list(parts: list[str]) -> str:
-    return " and ".join(parts) if len(parts) < 3 else ", ".join(parts[:-1]) + " and " + parts[-1]
-
-
-def actions_line(groups: dict) -> str:
-    """At a glance's one line about Action needed, when anything is open:
-    what is waiting and how soon, and the way to its page."""
-    rows = [r for g in groups.values() for r in g["rows"]]
-    held = sum(len(g["held"]) for g in groups.values())
-    if not rows and not held:
-        return ""
-    models = sum(1 for r in rows if r["alias"])
-    what = and_list([f"{n} {word}{'s' * (n != 1)}" for n, word in
-                     ((models, "model"), (held, "connection"), (len(rows) - models, "notice")) if n])
-    soon = min(r["days"] for r in rows) if rows else 99
-    many = sum(r["days"] == soon for r in rows)
-    when = (f"; {many} retire{'s' * (many == 1)} in {soon} day{'s' * (soon != 1)}" if soon < 99 else "")
-    return (f"<p class='a-line'><strong>Action needed</strong> <span>{what}{when}.</span>"
-            " <button type='button' class='linkish' data-go='needs'>Review →</button></p>")
-
-
 def actions_page(groups: dict, auto: list) -> str:
     """Action needed, its own page since 2026-11-02: it sat above every page
     and grew to a screen and more of red. Grouped by connection, because
@@ -2372,167 +2352,6 @@ def price_of(alias: str, served: str, f: dict) -> str:
     return "paid" if sources.is_free(f.get("cost")) is False else "unknown"
 
 
-def facet(name: str, label: str, values) -> str:
-    """A filter by one column's value, beside a panel's search box (bench.js
-    matches it against each row's data-<name>)."""
-    opts = "".join(f"<option value='{esc(v)}'>{esc(v)}</option>" for v in sorted(set(values) - {""}))
-    return (f"<label>{esc(label)} <select data-facet='{esc(name)}'><option value=''>any</option>"
-            f"{opts}</select></label>")
-
-
-def facts_cells(alias: str, f: dict | None = None) -> str:
-    """Context window and capabilities, from metrics/catalog.py; a value you
-    entered is marked, and an unknown one says so rather than looking empty."""
-    f = f or catalog.info(alias)
-    ctx = f.get("context_window")
-    src = f["sources"].get("context_window", "")
-    ctx_cell = (f"<td class='num' data-sort='{ctx}' title='{esc(src)}'>"
-                f"{ctx // 1000:,}k{'*' if src == 'you' else ''}</td>" if ctx
-                else "<td class='num muted' data-sort='-1' title='unknown: fill it in the model popup'>?</td>")
-    caps = [CAP_LABELS[c] for c in catalog.CAPS if f.get(c)]
-    return ctx_cell + (f"<td class='caps'>{' '.join(chip(c, '') for c in caps)}</td>" if caps
-                       else "<td class='muted'>—</td>")
-
-
-def roster() -> str:
-    """Every model on one table: its state, what it has done, what it scored.
-
-    Replaces a grid of offload cards that answered only "what was it used
-    for". Status first, because the first question on opening this page is
-    whether anything is broken; real work second; benchmarks after."""
-    try:
-        from offload import summary
-        work = summary()
-    except Exception:                              # noqa: BLE001
-        work = {}
-    try:
-        import auto_retire
-        rows, answers, _ = auto_retire.evidence()
-    except Exception:                              # noqa: BLE001
-        rows, answers = [], {}
-    last = dict(answers)
-    for r in rows:
-        if r["answered"]:
-            last[r["alias"]] = max(last.get(r["alias"], r["date"]), r["date"])
-    suite: dict[str, list[int]] = {}
-    for (_task, model), hit in suite_scores().items():
-        if (m := re.match(r"(\d+)/(\d+)", hit["found"])):
-            got = suite.setdefault(model, [0, 0, 0])
-            got[0] += int(m.group(1)); got[1] += int(m.group(2)); got[2] += 1
-    tiny = {r["model"]: float(r["p50_s"]) for r in perf_rows()
-            if r["prompt"] == "tiny" and int(r.get("answered") or 0) > 0}
-    status = model_status()
-    new = recently_onboarded()
-    stuck = onboard_issues(status)
-    directory = model_directory()
-    severity = {"err": 0, "warn": 1, "": 2, "ok": 3}
-
-    up = uptime_rows()["models"]
-    body = ["<table><thead><tr>", th("Model"), th("Status"), th("Uptime 7d", True), th("Role"),
-            th("Context", True), th("Can"),
-            th("Offloads", True), th("Helped"), th("Suite", True),
-            th("Reply time", True), th("Last answer"), "</tr></thead><tbody>"]
-    # The model browser: each row says its connection, status, role and price
-    # for the toolbar's filters, and the filters offer what the rows hold.
-    offered: dict[str, list[str]] = {"connection": [], "status": [], "role": [], "price": []}
-
-    def browse(**values) -> str:
-        for k, v in values.items():
-            offered[k].append(v)
-        return "".join(f" data-{k}='{esc(v)}'" for k, v in values.items())
-    for alias in sorted(status, key=lambda a: (-work.get(a, {}).get("offloads", 0), a)):
-        label, css, why = status[alias]
-        f = catalog.info(alias)
-        served = directory.get(alias, {}).get("served") or retired().get(alias, {}).get("served", "")
-        w = work.get(alias, {})
-        judged, useful = w.get("judged", 0), w.get("useful", 0)
-        if judged:
-            rate = useful / judged
-            helped = (f"<span class='helped'><span class='meter'><span"
-                      f"{' class=low' if rate < .5 else ''} style='width:{max(4, round(rate * 100))}%'>"
-                      f"</span></span><span class='mono'>{useful}/{judged}</span></span>")
-        else:
-            rate, helped = -1, "<span class='muted'>not judged</span>"
-        s = suite.get(alias)
-        suite_cell = (f"<td class='num' data-sort='{s[0] / s[1]:.3f}'"
-                      f" title='{s[2]} task{'s' * (s[2] != 1)}'>{s[0]}/{s[1]}</td>"
-                      if s and s[1] else "<td class='num muted' data-sort='-1'>—</td>")
-        p50 = tiny.get(alias)
-        seen = last.get(alias)
-        role = directory.get(alias, {}).get("role") or retired().get(alias, {}).get("role", "")
-        conn = catalog.connection_of(alias)
-        if conn == "?":                    # retired: no registration left to say, but its record does
-            conn = retired().get(alias, {}).get("tag", "").split("/")[0] or "unknown"
-        attrs = browse(connection=conn, status=re.sub(r" in \d+d$", "", label),
-                       role=role, price=price_of(alias, served, f))
-        body.append(
-            f"{tr(alias, attrs)}<td>{named(alias)}</td>"
-            f"<td data-sort='{severity[css]}'>{chip(label, css, why)}"
-            + (" " + chip("new", "new", "onboarded this week") if alias in new else "")
-            + (" " + chip("onboard stopped", stuck[alias]["css"],
-                          f"onboarding stopped at the {stuck[alias].get('step', 'probe')}:"
-                          f" {stuck[alias]['label']}. Open the model for the error.")
-               if alias in stuck else "")
-            + "</td>" + uptime_cell(up.get(alias)) + f"<td>{esc(role)}</td>{facts_cells(alias, f)}"
-            f"<td class='num'>{w.get('offloads', 0)}</td>"
-            f"<td data-sort='{rate:.3f}'>{helped}</td>{suite_cell}"
-            + (f"<td class='num' data-sort='{p50}'>{p50:.1f}s</td>" if p50 is not None
-               else "<td class='num muted' data-sort='99999'>—</td>")
-            + (f"<td class='mono' data-sort='{seen}'>{seen}</td>" if seen
-               else "<td class='muted' data-sort='0'>never</td>")
-            + "</tr>")
-    # Decision models are not in the llm yaml, so no probe, suite or reply time
-    # applies; the role column says what they are and what they answer.
-    for alias, d in sorted(decide.load().items()):
-        w = work.get(alias, {})
-        body.append(
-            "<tr" + browse(connection=d.get("via", ""), status="no probe", role="decision",
-                           price="free" if ":free" in d["served"] else "unknown") + ">"
-            f"<td>{named(alias)}</td>"
-            "<td data-sort='2'><span class='muted' title='decision models have no"
-            " availability probe yet'>—</span>"
-            + (" " + chip("new", "new", "onboarded this week") if d.get("onboarded", "") >=
-               (datetime.date.today() - datetime.timedelta(days=7)).isoformat() else "")
-            + "</td>" + uptime_cell(up.get(alias)) + f"<td>decision · {esc(', '.join(d['question_types']))}</td>"
-            "<td class='num muted' data-sort='-1'>—</td><td class='muted'>—</td>"
-            f"<td class='num'>{w.get('offloads', 0)}</td>"
-            "<td data-sort='-1'><span class='muted'>not judged</span></td>"
-            "<td class='num muted' data-sort='-1'>—</td><td class='num muted' data-sort='99999'>—</td>"
-            "<td class='muted' data-sort='0'>—</td></tr>")
-    body.append("</tbody></table>")
-    key = """<dl>
-      <dt>Status</dt><dd>From the latest probe. <em>healthy</em> answered;
-        <em>throttled</em> and <em>slow</em> are rate limits and timeouts, which
-        never retire a model inside 7 days; <em>retiring in Nd</em> is the
-        countdown before the bench retires it on its own. Hover a chip for the
-        provider's own words.</dd>
-      <dt>Uptime 7d</dt><dd>Of its calls in the last 7 days that say anything about the provider,
-        the share answered: busy, unreachable and timed out count against it; our own quota,
-        pacing and setup problems are left out. Benchmarks has it a day at a time.</dd>
-      <dt>Offloads, Helped</dt><dd>Real work handed over, and how much of what
-        was judged actually helped. Benchmarks never count here.</dd>
-      <dt>Suite</dt><dd>Verified findings located across the internal tasks:
-        the tier that decides routing.</dd>
-      <dt>Reply time</dt><dd>Median seconds to answer a one-word prompt: the
-        serving path's overhead, not the model's worth.</dd>
-      <dt>Last answer</dt><dd>The latest real answer from any source: probe,
-        suite or a useful offload. Seven days without one retires a model.</dd>
-      <dt>decision</dt><dd>A decision model: it answers typed questions (yes/no,
-        choice, score) with probabilities through a Decisions API, never chat.
-        The role column lists the question types it takes. No probe, suite or
-        reply time applies to it.</dd>
-      <dt>Filters</dt><dd>Connection is where a model's calls go (its Open WebUI
-        tag, or its own endpoint). Price: <em>free</em> for a <code>:free</code>
-        slug, a local model, a list price of 0 or a free-tier listing;
-        <em>paid</em> for a subscription or a list price; else unknown. Combine
-        them with the search box.</dd>
-    </dl>"""
-    facets = "".join(facet(k, k.title(), v) for k, v in offered.items())
-    return card("At a glance", "Every model on the bench: whether it works right "
-                "now, what it has been used for, and what it scored.",
-                "".join(body), key, facets)
-
-
 def on_bench() -> set[str]:
     """Aliases registered now: what the charts draw."""
     return set(registered().values())
@@ -2639,7 +2458,7 @@ def charts_data() -> dict:
                          "rate": int(hit[1]) / max(1, int(hit[2])) if hit else None, "invented": bool(v["invented"])})
     bars = [{"model": m, "kind": k, "rate": w / r["answers"], "n": f"{w}/{n}"}
             for m, r in sorted(wr.items()) if m in live and r["answers"] for k, (w, n) in r["by"].items() if w]
-    totals = {m: {"rate": r["wrong"] / r["answers"], "answers": r["answers"]}
+    totals = {m: {"rate": r["wrong"] / r["answers"], "answers": r["answers"], "wrong": r["wrong"]}
               for m, r in wr.items() if m in live and r["answers"]}
     hunt = [{"model": m, **r["hunt"]} for m, r in sorted(wr.items()) if m in live and any(r["hunt"].values())]
     return {"overview": overview, "heat": heat, "families": ["hand-built", "claims", "seeded, function",
@@ -2900,16 +2719,10 @@ def vision_card() -> str:
 
 
 def perf_charts() -> dict:
-    """Reliability over the last 14 days of probes, and speed on real tasks
-    against how much a model finds, for the Performance panel of Benchmarks."""
+    """Speed on real tasks against how much a model finds, for the Performance
+    panel of Benchmarks. (Whether probes answered is Uptime's.)"""
     import route
     live = on_bench()
-    since = (datetime.date.today() - datetime.timedelta(days=14)).isoformat()
-    outcomes: dict[tuple[str, str], int] = {}
-    for r in perf_rows():
-        if r["model"] in live and r.get("timestamp", "") >= since:
-            o = "answered" if int(r.get("answered") or 0) > 0 else (r.get("failure") or "error")
-            outcomes[(r["model"], o)] = outcomes.get((r["model"], o), 0) + 1
     seconds: dict[str, list[float]] = {}
     for (_task, m), v in suite_scores().items():
         if m in live and re.match(r"\d+/\d+", v["found"]):
@@ -2917,8 +2730,7 @@ def perf_charts() -> dict:
     recall = route.suite_recall()
     speed = [{"model": m, "seconds": sorted(s)[len(s) // 2], "recall": recall[m][0] / recall[m][1], "tasks": len(s)}
              for m, s in sorted(seconds.items()) if recall.get(m) and recall[m][1]]
-    return {"reliability": [{"model": m, "outcome": o, "count": n} for (m, o), n in sorted(outcomes.items())],
-            "speed": speed}
+    return {"speed": speed}
 
 
 # ---- Trend: does a model score lower, or higher, than it used to? ----------
@@ -3384,15 +3196,6 @@ def _uptime_window(cells: list, days: int) -> dict:
             "median": seconds[len(seconds) // 2] if seconds else None}
 
 
-def uptime_cell(cells: list | None, days: int = UPTIME_DAYS[0]) -> str:
-    """At a glance's Uptime column: answered of counted over the last `days`."""
-    t = _uptime_window(cells or [], days)
-    if t["pct"] is None:
-        return "<td class='num muted' data-sort='-1' title='no counted calls'>—</td>"
-    return (f"<td class='num{' err' if t['pct'] < .8 else ''}' data-sort='{t['pct']:.4f}'"
-            f" title='{t['up']} of {t['up'] + t['down']} answered'>{t['pct'] * 100:.1f}%</td>")
-
-
 def _why(kinds: dict, which) -> str:
     return ", ".join(f"{k} {n}" for k, n in sorted(kinds.items(), key=lambda kv: -kv[1]) if which(k))
 
@@ -3545,9 +3348,15 @@ DECIDE_KIND = {"errors": "choice of 6", "collection": "choice of the collections
 def decision_data() -> dict:
     """For bench.js: each model's calibration bins per set, [said, right, n]."""
     res = decide.results()
-    return {"sets": [[s, DECIDE_SETS[s]] for s in decide.SETS],
+    models = sorted((res.get("models") or {}).items())
+    return {"sets": [[s, DECIDE_SETS[s]] for s in decide.SETS], "low": decide.MIN_ITEMS,
+            # Right of asked per model and set: the bars, as the tables count it.
+            "scores": [{"model": a, "set": s, "correct": r.get("correct", 0), "asked": r["asked"],
+                        "answered": r.get("answered", 0), "brier": r.get("brier")}
+                       for a, by in models for s, r in by.items()
+                       if s in decide.SETS and isinstance(r, dict) and r.get("asked")],
             "points": [{"model": a, "set": s, "said": b["said"], "right": b["right"], "n": b["n"]}
-                       for a, by in sorted((res.get("models") or {}).items()) for s, r in by.items()
+                       for a, by in models for s, r in by.items()
                        if s in decide.SETS for b in r.get("calibration") or []]}
 
 
@@ -3614,8 +3423,7 @@ def decision_card() -> str:
     extra = (f"<h3 class='chart-head'>Who each use asks now</h3><ul class='links'>{''.join(uses)}</ul>"
              f"<p class='hint'>{'Measured up to ' + esc(when[:16].replace('T', ' ')) + '.' if when else 'Not measured yet.'}"
              " The scheduler's <code>decisions</code> job asks each model one question a set a day;"
-             " <code>python metrics/decide.py bench</code> runs a pass by hand.</p>"
-             "<h3 class='chart-head'>Calibration: said against right</h3>")
+             " <code>python metrics/decide.py bench</code> runs a pass by hand.</p>")
     key = f"""<dl>
       <dt>The sets</dt><dd>Questions with known answers, built from the bench's own data, no hand labelling.
         <b>Error classes</b>: failed calls' error text (redacted: no keys, ids or emails) and the class the
@@ -3627,8 +3435,12 @@ def decision_card() -> str:
         picked, so always saying the commonest option scores low.</dd>
       <dt>Brier</dt><dd>How far its probabilities sat from the truth: 0 is perfect; for a choice it is
         summed over the options, so up to 2. Lower is better, and a sure wrong answer costs most.</dd>
-      <dt>Calibration</dt><dd>The chart: answers grouped by the probability the model gave its answer
-        (x), against how often those answers were right (y). On the diagonal, its 80% answers come true
+      <dt>The bars</dt><dd>Right of asked for each model on each set, from 0%. A faint bar is a model
+        under {low} items on that set: shown, not ranked yet. Hover a bar for its Brier.</dd>
+      <dt>Calibration</dt><dd>The second chart, once a model answers at two or more levels of sureness
+        on a set: answers grouped by the probability the model gave its answer
+        (x), against how often those answers were right (y); both axes start just under the lowest
+        answer, not at 0%. On the diagonal, its 80% answers come true
         80% of the time; under it, it is surer than it should be. A bigger dot is more answers. The gap is
         the average distance from the diagonal, weighted by answers.</dd>
       <dt>At {decide.CONFIDENT}</dt><dd>The gateway acts on a choice only when the model is this sure: how
@@ -3669,26 +3481,45 @@ def bughunt_card() -> str:
         grey said there was no bug.</dd>
       <dt>Left of the line</dt><dd>The same files untouched: red is a false alarm, claiming a
         bug in clean code. The longer the red, the less to trust its "there's a bug here".</dd>
-      <dt>Order</dt><dd>Found share minus false-alarm share, best at the top.</dd>
+      <dt>Order</dt><dd>On what is proven, best at the top: the low end of the found share's likely
+        range (95%) less the high end of the false alarms'. A few tasks give wide ranges, so they
+        rank low; a model never shown clean code could be raising alarms on all of it.</dd>
+      <dt>Faint bars</dt><dd>Under 10 tasks: shown, but little to go on.</dd>
     </dl>"""
     return chart_card("Bug hunt", "Every task with an answer key: bugs found against false alarms on the"
                       " same files left clean. Right is good, left is bad.", "chart-hunt", "".join(body), key)
 
 
+def _wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
+    """The 95% Wilson interval of k successes in n, as bench.js wilson() draws it."""
+    if not n:
+        return 0.0, 1.0
+    p, z2 = k / n, z * z
+    mid, half = p + z2 / (2 * n), z * math.sqrt(p * (1 - p) / n + z2 / (4 * n * n))
+    return max(0.0, (mid - half) / (1 + z2 / n)), min(1.0, (mid + half) / (1 + z2 / n))
+
+
 def work_card() -> str:
     """How often real offloads helped."""
     rows = work_rows()
     body = ["<table><thead><tr>", th("Model"), th("Offloads", True), th("Judged", True), th("Helped", True),
-            th("Verified findings", True), th("False claims", True), "</tr></thead><tbody>"]
-    for r in sorted(rows, key=lambda r: -r["useful"] / r["judged"]):
+            th("Likely between", True), th("Verified findings", True), th("False claims", True), "</tr></thead><tbody>"]
+    for r in sorted(rows, key=lambda r: (-_wilson(r["useful"], r["judged"])[0], -r["judged"])):
+        lo, hi = _wilson(r["useful"], r["judged"])
         body.append(f"<tr><td>{named(r['model'])}</td><td class='num'>{r['offloads']}</td><td class='num'>{r['judged']}</td>"
                     f"<td class='num' data-sort='{r['useful'] / r['judged']:.3f}'>{r['useful']}/{r['judged']}</td>"
+                    f"<td class='num' data-sort='{lo:.3f}'>{lo * 100:.0f}–{hi * 100:.0f}%</td>"
                     f"<td class='num'>{r['real']}</td><td class='num'>{r['false']}</td></tr>")
     body.append("</tbody></table>")
     key = """<dl>
       <dt>Helped</dt><dd>Of the offloads judged, the share recorded useful: real work (reviews,
         enumerations) handed to the model and checked afterwards. Benchmarks never count here.</dd>
-      <dt>The number beside a name</dt><dd>How many were judged. 3 of 3 is weaker evidence than 40 of 50.</dd>
+      <dt>The dot and its line</dt><dd>The dot is the share that helped, bigger for more judged. The
+        line is where the true share likely sits (a 95% Wilson interval): long for 3 of 3, short for
+        40 of 50. Green when even its low end is over half, red when even its high end is under,
+        grey when there are too few to say.</dd>
+      <dt>Order</dt><dd>By the low end of that line, so volume counts: 40 of 50 ranks above 3 of 3.</dd>
+      <dt>The dashed line</dt><dd>Every model's judged offloads together, for comparison.</dd>
       <dt>Findings and false claims</dt><dd>Verified one by one in reviews, recorded since 2026-10-17.</dd>
     </dl>"""
     return chart_card("Real work", "When a model was given real work and the result was checked: how often"
@@ -3698,7 +3529,8 @@ def work_card() -> str:
 def overview_card() -> str:
     return (f"<div class='panel'>{SHARE}<h2>Who to trust with a review</h2><p class='sub'>Each dot is a model."
             " Right: finds more of the real bugs in our own code. Up: makes fewer things up (confidently"
-            " wrong on clean code, wrong lines, false claims, traps). Top right is where a reviewer should be.</p>"
+            " wrong on clean code, wrong lines, false claims, traps). Top right is where a reviewer should be."
+            " The models in or near the box, and the three that make most up, are named; hover for the rest.</p>"
             "<div class='chart' id='chart-overview'></div></div>")
 
 
@@ -3811,6 +3643,10 @@ def wrong_card() -> str:
       <dt>Reviews</dt><dd>False claims recorded in verdicts (<code>offload.py verdict|add
         --false "..."</code>), over judged offloads. Recorded in structured form only since
         2026-10-17, so older reviews show none.</dd>
+      <dt>The thin line</dt><dd>Where the true share likely sits (95% range). Long for a few
+        answers: 0 of 2 is not a clean record, and the line shows how bad it could be.</dd>
+      <dt>Order</dt><dd>Worst proven at the top: by the low end of that line. Under 10 answers a
+        model's bar is faint.</dd>
     </dl>"""
     return chart_card("Confidently wrong", "Who asserts things that are not true: false alarms on clean"
                       " code, wrong lines, false claims confirmed, traps taken. Shorter is better.",
@@ -3874,17 +3710,14 @@ def perf_card() -> str:
         when it is slow enough to change how you work.</dd>
     </dl>"""
     key = key.replace("</dl>", """
-      <dt>Did it answer?</dt><dd>Each probe of the last 14 days, by what came back: an answer, a
-        rate limit (capacity), a timeout, or a failure the bench classifies (structural, paywalled,
-        entitlement, other).</dd>
+      <dt>Did it answer?</dt><dd>Is on Uptime, by day, with our own quota and setup problems left out.</dd>
       <dt>Speed vs quality</dt><dd>Median seconds per answer on our own suite tasks (real prompts,
-        not the one-word probe) against the share of findings located. Top left is fast and good.</dd>
+        not the one-word probe) against the share of findings located. Top left is fast and good:
+        the green box is the faster half and the top quarter of findings.</dd>
     </dl>""")
     return chart_card("Performance", "What the serving path does — not what the models are worth. One"
                       " reading is weather; the history is climate.", "chart-speed", body, key,
-                      extra="<h3 class='chart-head'>Did it answer? Last 14 days of probes</h3>"
-                            "<div class='chart' id='chart-reliability'></div>"
-                            "<h3 class='chart-head'>Speed vs quality on our own tasks</h3>")
+                      extra="<h3 class='chart-head'>Speed vs quality on our own tasks</h3>")
 
 
 def favicon(theme: str) -> str:
@@ -3901,12 +3734,12 @@ def favicon(theme: str) -> str:
     return f"<link rel='icon' type='image/svg+xml' href='data:image/svg+xml,{quote(svg)}'>"
 
 
-def render(theme: str | None = None, shown: str = "glance") -> str:
+def render(theme: str | None = None, shown: str = "suite") -> str:
     with locked.snapshot():                    # each file read once a render (metrics/locked.py)
         return _page(theme, shown)
 
 
-def _page(theme: str | None, shown: str = "glance") -> str:
+def _page(theme: str | None, shown: str = "suite") -> str:
     theme = theme if theme in themes() else saved_theme()      # a ?theme preview, else the saved one
     return (
         f"<!doctype html><html lang='en' data-theme='{esc(theme)}'><meta charset='utf-8'>"
@@ -4300,13 +4133,21 @@ def collections_tab() -> str:
         " the paid ones. Click a suggestion or type your own: comma-separated lower-case words, up to 8.</p>"
         # Not data-filter: that binds to the whole panel. The same match (any
         # text, case-insensitive), on the member list only; the seats show who is ticked.
-        "<label class='field' for='c-member-filter'>Members</label>"
+        "<div class='members-head'><label class='field' for='c-member-filter'>Members</label>"
+        # Model details as they are now (a fact set in a model's popup, a refresh of the
+        # catalog), the ticks and the search kept; also done when a model popup opened from here closes.
+        "<button class='btn small' type='button' id='c-refresh' title='Fetch the models and their details again;"
+        " what is ticked stays ticked'>Refresh model details</button>"
+        "<span class='done' id='c-refresh-out' role='status'></span></div>"
         # The panel as it will sit, drawn as the Collections list draws it
         # (bench.js recount): a seat per ticked member, explorer seats dashed.
         "<div id='c-seats' class='seats' aria-live='polite'></div>"
         "<div class='member-find'><input type='search' id='c-member-filter' class='member-filter' spellcheck='false'"
         " autocomplete='off' placeholder='Filter by model, provider or served name' aria-controls='c-members'"
-        " aria-describedby='c-count'><select id='c-member-provider' aria-label='Provider' aria-controls='c-members'>"
+        " aria-describedby='c-count'><select id='c-member-show' aria-label='Show' aria-controls='c-members'>"
+        "<option value=''>Every model</option><option value='ticked'>Ticked</option>"
+        "<option value='fits'>Fits the tags</option></select>"
+        "<select id='c-member-provider' aria-label='Provider' aria-controls='c-members'>"
         "<option value=''>Every provider</option></select></div>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
@@ -4374,9 +4215,10 @@ def collections_tab() -> str:
 # A grouped sidebar since 2026-10-19 (RELEASE-PLAN "The dashboard"): eight
 # pages were too many for a row of tabs. Collections holds the gateway's
 # requests under the collections they ran; Benchmarks holds Performance.
-SECTIONS = (("Observe", (("glance", "At a glance", "roster"),
+# At a glance (every model's state and uptime) was retired 2026-11-10: Benchmarks,
+# Uptime and Models said it all again. Benchmarks is the home page.
+SECTIONS = (("Observe", (("suite", "Benchmarks", "benchmarks_tab"),
                          ("needs", "Action needed", "actions_page"),
-                         ("suite", "Benchmarks", "benchmarks_tab"),
                          ("quotas", "Quota management", "quotas_page"),
                          ("logs", "Logs", "logs_page"))),
             ("Gateway", (("endpoints", "Endpoints", "endpoints_page"),
@@ -4401,7 +4243,9 @@ def collections_and_gateway() -> str:
 # names an environment variable, or the app asks, or (an MCP header in Zed,
 # Cline or goose, which take no variable) KEY_HOLDER stays for the reader.
 # Only blocks marked `key` (shell lines, fields typed into an app) get the key
-# made on the page, swapped in for KEY_HOLDER by bench.js.
+# made on the page, swapped in for KEY_HOLDER by bench.js. CLIENTS.md is written
+# from client_setups (scripts/clients_doc.py): after a change here, run it, as
+# check.py fails while the file differs.
 KEY_HOLDER = "<your gateway key>"
 KEY_ENV = "BENCHWARMERS_KEY"         # the plugin's and the CLI's name for it too
 CLIENTS_READ = "2026-11-08"
@@ -4697,20 +4541,20 @@ def _snip_html(group: str, i: int, s: dict) -> str:
 def pane(key: str, actions: tuple | None = None) -> str:
     """One tab's page, as its section holds it. The page renders the tab shown
     (tabs); bench.js fetches the others from GET /api/pane when first opened."""
-    if key in ("needs", "glance"):
+    if key == "needs":
         groups, auto = actions or open_actions()
-        return actions_page(groups, auto) if key == "needs" else actions_line(groups) + roster()
+        return actions_page(groups, auto)
     return globals()[next(fn for k, _, fn in TABS if k == key)]()
 
 
-def tabs(shown: str = "glance") -> str:
-    # Read once a render: Action needed's count in the sidebar, and its page or
-    # At a glance's line about it when one of those is shown. Only the tab
-    # shown is rendered (the bench_tab cookie, set by bench.js): rendering every
-    # one took seconds. The rest are empty until opened (data-lazy).
+def tabs(shown: str = "suite") -> str:
+    # Read once a render: Action needed's count in the sidebar, and its page when
+    # shown. Only the tab shown is rendered (the bench_tab cookie, set by bench.js):
+    # rendering every one took seconds. The rest are empty until opened (data-lazy).
+    # A tab that is gone (a cookie still saying glance) shows the home page.
     actions = open_actions()
     count = {"needs": open_count(actions[0])}
-    shown = shown if any(k == shown for k, _, _ in TABS) else "glance"
+    shown = shown if any(k == shown for k, _, _ in TABS) else "suite"
 
     def button(key: str, label: str) -> str:
         n = count.get(key)
@@ -5331,6 +5175,28 @@ def endpoints_page() -> str:
         "<div class='done' id='l-done'></div></div>")
 
 
+def _reply_cell(by_prompt: dict) -> str:
+    """Models' Reply: the daily probe's median seconds to a one-word answer (tiny),
+    with the 18 KB prompt's (large) and tokens a second on hover. Was on Benchmarks'
+    Performance, beside whether it answered, which is Uptime's now."""
+    def p50(r):
+        try:
+            return float(r["p50_s"]) if r and int(r.get("answered") or 0) > 0 else None
+        except (KeyError, TypeError, ValueError):
+            return None
+    tiny, large = p50(by_prompt.get("tiny")), p50(by_prompt.get("large"))
+    if tiny is None and large is None:
+        return "<td class='num muted' data-sort='999999'>—</td>"
+    shown = tiny if tiny is not None else large
+    why = "; ".join(b for b in (
+        f"one-word prompt {tiny:.1f}s" if tiny is not None else "",
+        f"18 KB prompt {large:.1f}s" if large is not None else "",
+        f"{(by_prompt.get('large') or {}).get('tokens_per_s')} tok/s"
+        if (by_prompt.get("large") or {}).get("tokens_per_s") else "") if b)
+    return (f"<td class='num' data-sort='{shown:.2f}' title='median, the daily probe: {esc(why)}'>"
+            f"{shown:.1f}s</td>")
+
+
 def models_page() -> str:
     """Every model on the bench by the connection it comes from."""
     cards = []
@@ -5344,6 +5210,10 @@ def models_page() -> str:
     # for what a collection may hold): they are on the bench like any other.
     by: dict[str, list[str]] = {}
     directory = model_directory()
+    # The daily probe's latest reading per model and prompt (perf/history.csv), for Reply.
+    speed: dict[str, dict] = {}
+    for r in perf_rows():
+        speed.setdefault(r["model"], {})[r.get("prompt", "")] = r
     for a in directory:
         by.setdefault(catalog.connection_of(a), []).append(a)
     for c in ({"name": n, "models": sorted(m)} for n, m in sorted(by.items())):
@@ -5356,7 +5226,7 @@ def models_page() -> str:
             kind = (" " + chip("decision", "", "a decision model: answers typed questions, not chat")
                     if a in decisions else "")
             if directory[a].get("ad_hoc"):
-                kind += " " + chip("ad hoc", "", "ad hoc only: never chosen by the bench on its own, only when you"
+                kind += " " + chip("ad hoc", "adhoc", "ad hoc only: never chosen by the bench on its own, only when you"
                                    " name it (a collection member, with, --model)")
             rows.append(f"<tr><td><a href='#' class='model' data-alias='{esc(a)}'>"
                         f"<span class='alias'>{esc(a)}</span></a>{kind}</td>"
@@ -5364,19 +5234,27 @@ def models_page() -> str:
                         f"<td class='num' data-sort='{f.get('context_window') or 0}'>{num(f.get('context_window'))}</td>"
                         f"<td class='num' data-sort='{f.get('max_output') or 0}'>{num(f.get('max_output'))}</td>"
                         f"<td>{esc(', '.join(f.get('input_modalities') or []) or '—')}</td>"
+                        + _reply_cell(speed.get(a, {}))
                         + "".join(f"<td class='num'>{num(lim.get(k))}</td>" for k in catalog.LIMIT_KEYS) + "</tr>")
         # One column plan for every card, so the cards line up under each other
         # (each table sized its own columns, so they drifted from card to card).
         cols = "<colgroup><col class='c-model'><col class='c-via'><col class='c-n'><col class='c-n'>" \
-               "<col class='c-input'>" + "<col class='c-lim'>" * len(catalog.LIMIT_KEYS) + "</colgroup>"
-        cards.append(f"<div class='panel'><h3 class='chart-head'>{esc(c['name'])} · {len(c['models'])} models</h3>"
+               "<col class='c-input'><col class='c-reply'>" + "<col class='c-lim'>" * len(catalog.LIMIT_KEYS) + "</colgroup>"
+        # Each connection folds; the page's one box filters them all (bench.js applyFilter, data-conn).
+        cards.append(f"<details class='panel model-card' open data-conn='{esc(c['name'])}'><summary>"
+                     f"<h3 class='chart-head'>{esc(c['name'])} · {len(c['models'])} models</h3></summary>"
                      f"<table class='model-cards'>{cols}<thead><tr>" + th("Model") + th("Through")
                      + th("Context", True) + th("Max output", True)
-                     + th("Input") + "".join(th(k, True) for k in catalog.LIMIT_KEYS)
-                     + f"</tr></thead><tbody>{''.join(rows)}</tbody></table></div>")
+                     + th("Input") + th("Reply", True) + "".join(th(k, True) for k in catalog.LIMIT_KEYS)
+                     + f"</tr></thead><tbody>{''.join(rows)}</tbody></table></details>")
     return ("<div class='panel'><h2>Models</h2><p class='sub'>Every model on the bench by the connection it"
             " comes from, with the endpoint it is reached through and what is known of its limits. A name"
-            " opens the model.</p></div>"
+            " opens the model.</p><div class='toolbar'><input type='search' data-filter='page'"
+            " placeholder='Find a model or connection' aria-label='Filter models'>"
+            # One labelled toggle, as accordions do it (GOV.UK's "Show all sections"): Collapse all
+            # while every table is open, else Expand all (bench.js foldButtons keeps it right).
+            "<button class='fold-all' type='button' data-fold-all='close' aria-expanded='true'>"
+            "<span class='fold-chev' aria-hidden='true'></span>Collapse all</button></div></div>"
             + "".join(cards))
 
 
@@ -5553,11 +5431,16 @@ def logs_page() -> str:
                       for p in sorted((DATA / "runs").glob("onboard-*.log"), reverse=True)[:30])
 
     def section(title, body, n, open_=False):
-        return (f"<details class='panel'{' open' if open_ else ''}><summary><b>{esc(title)}</b>"
-                f" <span class='muted'>· {n}</span></summary>{body}</details>")
+        # CSV: this log alone, its own columns, the filter applied (bench.js); for a spreadsheet.
+        return (f"<details class='panel'{' open' if open_ else ''} data-log='{esc(title)}'><summary><b>{esc(title)}</b>"
+                f" <span class='muted'>· {n}</span> <button class='btn small' type='button' data-csv"
+                f" title='Save this log, as filtered, as a CSV file'>CSV</button></summary>{body}</details>")
     return (
-        "<div class='panel'><input data-filter='page' placeholder='Filter every log on this page'"
-        " aria-label='Filter logs'></div>"
+        "<div class='panel'><div class='toolbar'><input data-filter='page' placeholder='Filter every log on this page'"
+        " aria-label='Filter logs'>"
+        # bench.js: what the page shows, filter applied, as one .txt to read offline.
+        "<button class='btn small' type='button' id='logs-export' title='Save the logs on this page, as"
+        " filtered, to a text file'>Export as text</button></div></div>"
         + section("Gateway calls", "<table><thead><tr>" + th("When") + th("Collection") + th("Shape")
                   + th("Seats") + f"</tr></thead><tbody>{''.join(gw)}</tbody></table>", len(gw), True)
         + section("Changes", "<table><thead><tr>" + th("When") + th("By") + th("Model") + th("What")
@@ -5876,8 +5759,7 @@ class Handler(gateway.Handler):
         if self.path == "/api/needs":                  # after an action there: that page, not the whole one
             with locked.snapshot():
                 groups, auto = open_actions()
-                self._json(200, {"html": actions_page(groups, auto), "line": actions_line(groups),
-                                 "count": open_count(groups)})
+                self._json(200, {"html": actions_page(groups, auto), "count": open_count(groups)})
             return
         if path == "/api/pane":                        # a tab's page, when first opened (bench.js load)
             key = q.get("tab", [""])[0]
@@ -5996,7 +5878,7 @@ class Handler(gateway.Handler):
         theme = q.get("theme", [None])[0]               # a preview; else the saved one
         shown = re.search(r"(?:^|;)\s*bench_tab=(\w+)", self.headers.get("Cookie") or "")   # the tab last shown
         try:
-            page = render(theme, shown.group(1) if shown else "glance")
+            page = render(theme, shown.group(1) if shown else "suite")
         except RuntimeError as exc:                    # endpoints.json unreadable: say which and why
             page = f"<!doctype html><meta charset='utf-8'><title>Benchwarmers</title><p>{esc(exc)}</p>"
         self._page(page)                               # gzipped beyond this machine (_length)