@wendellresearch-ai/benchwarmers 0.4.0 → 0.5.0

package/serve.py

@@ -221,6 +221,17 @@ def tag_for(llm_id: str) -> str:
     return "" if tag == "?" else tag
 
 
+def provider_label(tag: str) -> str:
+    """A connection tag as a short provider name for a chip: a proxy's vendor
+    (cli-proxy-api/codex: codex), else the host without its api./
+    router. prefix, -api suffix and TLD (router.requesty.ai: requesty)."""
+    if "/" in tag:
+        return tag.split("/", 1)[1]
+    host = re.sub(r"^(api|apihub|router)\.", "", tag)
+    host = host.rsplit(".", 1)[0] if "." in host else host
+    return host.removesuffix("-api") or tag
+
+
 def retired() -> dict[str, dict]:
     """Aliases removed from the bench by hand, with when and why."""
     return load_json(DATA / "config" / "retired.json")
@@ -826,15 +837,16 @@ def _modalities(entry: dict, served: str, index: dict) -> dict:
 # What each endpoint answered, for LISTED_FOR seconds: name -> (when, its
 # config, live, conns). Only the asking is kept; what is on the bench, retired
 # or dismissed is taken off fresh on every opening.
-LISTED_FOR = 300
+LISTED_FOR = 3600                  # an hour: lists rarely change sooner, and Refresh is one click
 _listed: dict[str, tuple] = {}
 
 
 def onboard_candidates(fresh: bool = False) -> dict:
     """What every endpoint serves that the bench could onboard, for the picker.
 
-    Served, minus registered, minus retired, minus dismissed (those come back
-    separately so a dismissal can be undone), each endpoint asked separately
+    Served, minus registered, minus retired, minus queued or onboarding now,
+    minus dismissed (those come back separately so a dismissal can be undone),
+    each endpoint asked separately
     so one being down only empties its own group. The live calls this page
     makes happen only when the picker opens: a dashboard that needs the
     network to render would be down exactly when you want to look at it.
@@ -858,6 +870,9 @@ def onboard_candidates(fresh: bool = False) -> dict:
              | {endpoints.snapshot_key(r.get("served", ""), r.get("endpoint") or endpoints.default())
                 for r in retired().values()}
              | {endpoints.snapshot_key(d["served"], d["endpoint"]) for d in decide.load().values()})
+    with _OQ_LOCK:                                  # on its way: Start would refuse it anyway
+        taken |= {endpoints.snapshot_key(i["served"], i["endpoint"]) for i in _OQ["items"]
+                  if i["state"] in ("queued", "running")}
     tags = load_json(DATA / "config" / "provider-tags.json")
     dismissed = load_json(DISMISSED_PATH)
     first_seen = load_json(DATA / "config" / "served-snapshot.json").get("last_seen", {})
@@ -1457,12 +1472,18 @@ def collections_view() -> dict:
     # What the tags check members against (bench.js tagBlocks): capabilities
     # (None: the catalog does not say), context, price and role. A shape's
     # tag admits the roles that shape admits, as route.py does for seats.
+    # The member picker draws one row per model, its providers as chips:
+    # `family` is route.family_of, the match that keeps one model out of two
+    # seats, and `provider` the chip's name for its connection.
+    import route
     roles = model_roles.load()
     facts = {}
     for a in _chat_aliases():
         f = catalog.info(a)
+        tag = directory[a].get("tag") or directory[a].get("endpoint") or ""
         facts[a] = {"caps": {c: f.get(c) for c in catalog.CAPS}, "context": f.get("context_window"),
-                    "price": price_of(a, served[a], f), "role": (roles.get("models") or {}).get(a, "general")}
+                    "price": price_of(a, served[a], f), "role": (roles.get("models") or {}).get(a, "general"),
+                    "family": route.family_of(a), "provider": provider_label(tag), "connection": tag}
     return {"ok": True, "errors": [], "collections": rows, "settings": s,
             "shapes": shapes, "models": _chat_aliases(), "served": served, "tag_suggestions": suggestions,
             "facts": facts, "shape_roles": {k: v for k, v in roles["shapes"].items() if k in shapes},
@@ -1567,13 +1588,29 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
 
 # How much an onboarding spends. Full: the 4-call probe, then the suite now.
 # Quick: the probe only; the scheduled `suite --next` passes fill the suite in,
-# six tasks a run, so the same calls are spread out, not saved. Quick + ad
-# hoc: the probe only, and the bench never calls it on its own: the only one
-# that saves calls, for a metered, capped or trial key. Apart from the mode,
+# six tasks a run, so the same calls are spread out, not saved. Ad hoc: the
+# bench never calls it on its own afterwards, for a metered, capped or trial
+# key; Full + ad hoc spends the suite once now so routing has scores for it,
+# Quick + ad hoc only the probe, and no scheduled pass ever fills its suite in
+# (suite/pace.py leaves ad hoc models out). Apart from the mode,
 # `vision` (the picker's Vision model box) gives it role vision as it is
 # registered: a seat only for requests that carry images, and its suite is the
 # vision tasks alone (suite/run.applies), now or through the scheduled passes.
-ONBOARD_MODES = {"full": [], "quick": ["--skip-suite"], "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
+ONBOARD_MODES = {"full": [], "full-adhoc": ["--ad-hoc"], "quick": ["--skip-suite"],
+                 "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
+
+
+def onboard_choice(prefix: str) -> str:
+    """The mode and the Vision box, as the Onboard dialog (o-) and Discover's Add confirm (ask-) show them."""
+    return (f"<label class='check'>Onboard as <select id='{prefix}-mode'>"
+            "<option value='full'>Full: probe, then the suite now</option>"
+            "<option value='full-adhoc'>Full + ad hoc: probe and the suite now; then the bench never calls it"
+            " on its own</option>"
+            "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
+            "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
+            " (metered, capped or trial keys)</option></select></label>"
+            f"<label class='check'><input type='checkbox' id='{prefix}-vision'> Vision model: a seat only for"
+            " questions about images, and its suite is the vision tasks</label>")
 
 
 def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full",
@@ -1780,13 +1817,17 @@ def _onboard_worker(who: str) -> None:
             nxt.update(state=state, note=note, exit=code)
 
 
-def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "") -> dict:
+def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "",
+                 mode: str = "full", vision: bool = False) -> dict:
     """From the Discover tab: put a model into a hub as it is otherwise done by
     hand, through the adapter of the hub `endpoint` is (endpoints.adapter:
     Open WebUI's connection Model IDs and a public, tagged record with a logo;
     a LiteLLM deployment; a CLI Proxy API provider's models list), then the
-    normal onboarding. Only for a connection the hub already has. No endpoint
+    normal onboarding, in the mode and with the Vision box the confirm chose
+    (ONBOARD_MODES). Only for a connection the hub already has. No endpoint
     means Open WebUI's, as before there were other hubs."""
+    if mode not in ONBOARD_MODES:                      # before the hub has it, not after
+        return {"ok": False, "errors": [f"mode must be one of {', '.join(ONBOARD_MODES)}; got {mode!r}"]}
     if not re.fullmatch(r"[\w./:@+-]{1,200}", model or ""):
         return {"ok": False, "errors": ["the model id is the provider's id, e.g. vendor/model:free"]}
     if not re.fullmatch(r"[\w.-]{1,80}", connection or ""):
@@ -1805,7 +1846,7 @@ def discover_add(connection: str, model: str, who: str = "ui", endpoint: str = "
         return {"ok": False, "errors": [f"{label} refused: {type(exc).__name__}: {exc}"]}
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "",
                 f"{which}-add", None, f"{served} on {connection}")
-    started = queue_onboard([{"served": served, "endpoint": hub.endpoint()}], who)
+    started = queue_onboard([{"served": served, "endpoint": hub.endpoint(), "mode": mode, "vision": vision}], who)
     return {**started, "served": served, "endpoint": hub.endpoint(), "steps": steps,
             "errors": started["errors"] + ([] if started["ok"] else
                                           [f"{label} now has {served}; onboard it from the Onboard dialog"])}
@@ -1964,14 +2005,26 @@ def problems() -> str:
     KIND = {"retiring": "err", "retired": "err", "connection": "warn",
             "registration": "warn", "plan": "warn"}
     items = []
+    live = set(names.values())
     for key, a in sorted(alerts.items(), key=lambda kv: kv[1].get("first_seen", "")):
         served = key.split(":", 2)[2] if key.count(":") >= 2 else ""
         alias = a.get("alias") or names.get(served, "")
+        # The alerts file is rewritten by the daily check; until then an alert
+        # whose models have all left the bench (Retire all, or one by one) has
+        # nothing left to act on.
+        if a.get("aliases") and not live & set(a["aliases"]):
+            continue
+        if a.get("kind") == "retiring" and alias and alias not in live:
+            continue
         try:
             age = (today - datetime.date.fromisoformat(a["first_seen"])).days
         except (KeyError, ValueError):
             age = 0
-        why = " · ".join(filter(None, [a.get("action", ""),
+        # Which model and where it comes from, first: the message alone left that to be looked up.
+        served_as = next((s for s, al in names.items() if al == alias), served) if alias else ""
+        where = (f"{alias} · {served_as or '?'} via {tag_for(alias) or a.get('endpoint') or '?'}"
+                 if alias else "")
+        why = " · ".join(filter(None, [where, a.get("action", ""),
                                        "open today" if age == 0 else f"open {age} days"]))
         acts = []
         if a.get("aliases"):
@@ -1988,7 +2041,10 @@ def problems() -> str:
         items.append((key, a.get("kind", "?"), KIND.get(a.get("kind"), "warn"),
                       a.get("message", ""), why, "".join(acts)))
     cutoff = (today - datetime.timedelta(days=7)).isoformat()
-    auto = [(f"retired:{alias}:{r['at']}", "", "", f"{alias} · {r['at'][:10]}",
+    tags = load_json(DATA / "config" / "provider-tags.json")      # a retired model's connection, by served name
+    auto = [(f"retired:{alias}:{r['at']}", "", "",
+             f"{alias} · {r.get('served') or '?'} via {tags.get(r.get('served', ''), '').strip('?') or r.get('endpoint') or endpoints.default()}"
+             f" · {r['at'][:10]}",
              r.get("why", "").removeprefix("auto: "),
              f"<button class='btn small' data-act='restore' data-alias='{esc(alias)}'>Restore</button>")
             for alias, r in sorted(retired().items(), key=lambda kv: kv[1].get("at", ""), reverse=True)
@@ -2204,14 +2260,15 @@ def on_bench() -> set[str]:
     return set(registered().values())
 
 
-def chart_card(title: str, sub: str, chart_id: str, table: str, key: str, extra: str = "") -> str:
+def chart_card(title: str, sub: str, chart_id: str, table: str, key: str, extra: str = "",
+               retired: bool = True) -> str:
     """A Benchmarks panel: the chart first (drawn by bench.js with Observable
     Plot), the table folded under it, the key under that."""
     return (f"<div class='panel'><h2>{title}</h2><p class='sub'>{sub}</p>{extra}"
             f"<div class='chart' id='{chart_id}'></div>"
             "<details class='key'><summary>Table</summary><div class='toolbar'><input type='search'"
             " data-filter placeholder='Filter by model or connection' aria-label='Filter rows'>"
-            "<label><input type='checkbox' data-retired> Show retired</label></div>"
+            + ("<label><input type='checkbox' data-retired> Show retired</label>" if retired else "") + "</div>"
             f"<div class='scroll'>{table}</div></details>"
             f"<details class='key'><summary>How to read this</summary>{key}</details></div>")
 
@@ -2590,6 +2647,8 @@ def suite_card() -> str:
 
 
 EXTERNAL = (("LiveBench", "https://livebench.ai/", "contamination-limited questions, refreshed monthly"),
+            ("FreeLLMAPI", "https://freellmapi.co/models",
+             "every free tier: limits, ranks and quirks per model (Discover leads with it)"),
             ("models.dev", "https://models.dev/", "specs and prices per provider (the catalog reads it daily)"),
             ("BullshitBench", "https://petergpt.github.io/bullshit-benchmark/viewer/index.next.html",
              "pushing back on nonsense; the idea behind confidently wrong"))
@@ -2788,7 +2847,7 @@ def gateway_card(limit: int = 30) -> str:
         judged = {}
     owed = seats_total = 0
     body = ["<table><thead><tr>", th("When"), th("Collection"), th("Asked by"), th("Explorer shape"),
-            th("Seats"), th("Owed", True), "</tr></thead><tbody>"]
+            th("Seats"), th("Time", True), "</tr></thead><tbody>"]
     for r in records:
         chips, row_owed = [], 0
         for seat in r.get("seats", []):
@@ -2808,13 +2867,20 @@ def gateway_card(limit: int = 30) -> str:
         chips += [f"<span class='chip muted' title='{esc('not asked: ' + str(k.get('why', '')))}'>"
                   f"{esc(k.get('alias', ''))} –</span>" for k in r.get("skipped", []) if isinstance(k, dict)]
         owed += row_owed
+        # The request's own time: its seats ran at once, so the slowest of
+        # them, then the merge (a merger seat answers after the rest).
+        secs = [(s.get("seat"), float(s.get("seconds") or 0)) for s in r.get("seats", [])]
+        panel = max((t for seat, t in secs if seat != "merger"), default=0.0)
+        merge = sum(t for seat, t in secs if seat == "merger")
+        took = (f"<td class='num' data-sort='{panel + merge:.1f}' title='slowest seat {panel:.1f}s"
+                f"{f' + merge {merge:.1f}s' if merge else ''}'>{panel + merge:.0f}s</td>" if secs
+                else "<td class='num muted' data-sort='-1'>—</td>")
         body.append(
             f"<tr><td class='mono' data-sort='{esc(r.get('at', ''))}'>{esc(r.get('at', '')[:16].replace('T', ' '))}</td>"
             f"<td class='mono'>{esc(r.get('model', ''))}</td>"
             f"<td class='mono'>{esc(r.get('key') or '—')}</td>"
             f"<td title='{esc(r.get('shape_by', ''))}'>{esc(r.get('shape', ''))}</td>"
-            f"<td>{' '.join(chips)}</td>"
-            f"<td class='num{' muted' if not row_owed else ''}'>{row_owed}</td></tr>")
+            f"<td>{' '.join(chips)}</td>{took}</tr>")
     if not records:
         body.append("<tr><td colspan='6' class='muted'>No gateway requests yet. Call a collection"
                     f" at <code>{esc(gateway_url())}</code>.</td></tr>")
@@ -2829,7 +2895,10 @@ def gateway_card(limit: int = 30) -> str:
         carried none. Hover it for why.</dd>
       <dt>Asked by</dt><dd>The gateway key that made the request, or <code>local</code>
         when no key is asked for.</dd>
-      <dt>Owed</dt><dd>Seats without a verdict. Each is recorded as an unjudged
+      <dt>Time</dt><dd>How long the caller waited: the seats run at once, so the
+        slowest of them, plus the merge when the reply was merged. Hover for the two
+        parts; a chip's own time is on the chip.</dd>
+      <dt>Verdicts owed</dt><dd>A chip with no mark. Each is recorded as an unjudged
         offload, so rotation still counts it; a verdict tells the router whether
         it helped. The orchestrator that called can also record one:
         <code>python metrics/offload.py verdict &lt;call id&gt; --useful</code>.</dd>
@@ -2968,7 +3037,7 @@ def discover_model(provider: str, model: str) -> dict:
 
 
 def discover_card() -> str:
-    """Models worth adding, from models.dev and FreeLLMAPI (metrics/discover.py),
+    """Models worth adding, from FreeLLMAPI and models.dev (metrics/discover.py),
     deduped. Read from the daily cache, so the page never waits on the network;
     built once per cache refresh. A click on a row opens its popup (bench.js),
     which asks /api/discover-model for the rest: its routes are where Add is."""
@@ -2976,7 +3045,8 @@ def discover_card() -> str:
     rows = [r for r in all_rows if r["free"] or r["reach"] in ("connected", "on bench")]
     hidden = len(all_rows) - len(rows)
     reach_css = {"on bench": "ok", "connected": "ok", "easy to add": "warn", "other": ""}
-    body = ["<table><thead><tr>", th("Model"), th("Provider"), th("Reach"), th("Price"), th("Context", True),
+    body = ["<table><thead><tr>", th("Model"), th("Rank", True), th("Provider"), th("Reach"), th("Price"),
+            th("Context", True),
             th("In → out"), th("Can"), th("Released"), th("Limits"), "<th></th></tr></thead><tbody>"]
     for r in rows:
         caps = [c for c, on in (("tools", r["tools"]), ("vision", r["vision"]),
@@ -2985,8 +3055,13 @@ def discover_card() -> str:
         price = ("free" if r["free"] else
                  f"{cost.get('input', '?')} / {cost.get('output', '?')}" if cost else "paid")
         lim = r.get("limits") or {}
-        lim_text = " · ".join(f"{v:,} {k}" for k, v in lim.items()) if lim else ""
+        lim_text = " · ".join([f"{v:,} {k}" for k, v in lim.items()]
+                              + ([f"{r['budget']} tokens a month"] if r.get("budget") else []))
         ctx = r.get("context")
+        rank = (f"<td class='num' data-sort='{r['rank']}' title='FreeLLMAPI: smartest #{r['rank']}"
+                + (f", fastest #{r['speed']}" if r.get("speed") else "")
+                + (f", {esc(r['size'])}" if r.get("size") else "") + f"'>{r['rank']}</td>"
+                if r.get("rank") is not None else "<td class='num muted' data-sort='99999'>—</td>")
         add = ("<button class='btn small' type='button'>Add…</button>"
                if any(not x["on_bench_as"] for x in r["routes"]) else "")
         body.append(
@@ -2997,8 +3072,8 @@ def discover_card() -> str:
             + f"<span class='conn'>{esc(r['model'])}"
             + (f" · on bench as {esc(r['on_bench_as'])}" if r.get("on_bench_as") else "")
             + (f" · also on bench as {esc(', '.join(r['also_on']))}" if r.get("also_on") else "")
-            + "</span></td>"
-            f"<td title='{esc(r['provider'])}'>{esc(r['provider_name'])}</td>"
+            + "</span></td>" + rank
+            + f"<td title='{esc(r['provider'])}'>{esc(r['provider_name'])}</td>"
             f"<td data-sort='{['on bench', 'connected', 'easy to add', 'other'].index(r['reach'])}'>"
             f"{chip(r['reach'], reach_css[r['reach']], r.get('api') or '')}</td>"
             f"<td title='per million tokens, input / output'>{esc(price)}</td>"
@@ -3015,31 +3090,40 @@ def discover_card() -> str:
                " provider or reach' aria-label='Filter rows'>"
                "<label><input type='checkbox' data-paid> Show paid (connected providers)</label></div>")
     key = """<dl>
+      <dt>Rank</dt><dd>FreeLLMAPI's smartest-first order (1 is the smartest it lists); within
+        each reach the list follows it. Hover for its fastest-first rank and size. A dash where
+        only models.dev lists the model.</dd>
       <dt>Provider</dt><dd>Who serves the model: a row per provider, so the same model
-        from OpenRouter and from NVIDIA is two rows. Hover for its models.dev id.</dd>
+        from OpenRouter and from NVIDIA is two rows. Hover for its id.</dd>
       <dt>Reach</dt><dd><em>on bench</em> already registered; <em>connected</em> this
         install has a route to the provider (an Open WebUI connection, or an endpoint of its
         own); <em>easy to add</em> an OpenAI-compatible API, so one new connection away (hover
         for its base URL); <em>other</em> a native API or an unvetted reseller.</dd>
-      <dt>Price</dt><dd>Free if models.dev prices it at 0 or FreeLLMAPI lists it (a catalog of
-        free tiers): models.dev shows the PAID price even where a free tier exists. Otherwise
+      <dt>Price</dt><dd>Free if FreeLLMAPI lists it (a catalog of free tiers) or models.dev
+        prices it at 0: models.dev shows the PAID price even where a free tier exists. Otherwise
         input / output per million tokens.</dd>
+      <dt>Limits</dt><dd>The free tier's, from FreeLLMAPI: requests and tokens per minute (rpm,
+        tpm) and per day (rpd, tpd), and roughly how many tokens a month that allows.</dd>
       <dt>New</dt><dd>First seen in either list in the last 7 days.</dd>
       <dt>Add</dt><dd>Click a row: its popup has everything known about the model and every
         route this install has to it, each with <em>Add via this</em>. <em>Via Open WebUI</em>
         adds the model to that connection's Model IDs (when it lists them), creates its model
         record public, tagged like the connection's other models and with their logo, then runs
-        the bench's onboarding (register, 4-call probe, suite), so it is in the chats too.
+        the bench's onboarding as the confirm chooses (Onboard as, Vision model), so it is in
+        the chats too.
         <em>Via LiteLLM</em> or <em>CLI Proxy API</em> adds it to that hub's connection (one more
         deployment, or the provider's models list); those hubs are asked when the popup opens.
         <em>Direct</em> onboards it on the provider's own endpoint: bench only. A new provider
         needs a connection or an endpoint first; hover its <em>easy to add</em> chip for the
         base URL.</dd>
-      <dt>Sources</dt><dd>models.dev and/or FreeLLMAPI's free catalog (named in the popup); a
-        model one provider offers in both is shown once. Both are refreshed at most daily.</dd>
+      <dt>Sources</dt><dd>FreeLLMAPI's free catalog first: where it lists a model, its name,
+        provider, context, tools, limits, ranks and quirks lead, and the popup links its
+        freellmapi.co page. models.dev fills the rest and alone has the paid models, prices,
+        base URLs and modalities. A model one provider offers in both is shown once. Both are
+        refreshed at most daily.</dd>
     </dl>"""
-    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from models.dev"
-            f" and FreeLLMAPI, deduped; {hidden:,} paid models on providers you have no route"
+    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI"
+            f" and models.dev, deduped; {hidden:,} paid models on providers you have no route"
             f" to are left out. Click a model for everything known about it and the routes it can be"
             f" added by.</p>{toolbar}"
             f"<div class='scroll'>{''.join(body)}</div>"
@@ -3083,8 +3167,11 @@ def collections_tab() -> str:
         # Not data-filter: that binds to the whole panel. The same match (any
         # text, case-insensitive), on the member list only; ticked ones stay shown.
         "<label class='field' for='c-member-filter'>Members</label>"
+        # The panel as it will sit, drawn as the Collections list draws it
+        # (bench.js recount): a seat per ticked member, explorer seats dashed.
+        "<div id='c-seats' class='seats' aria-live='polite'></div>"
         "<input type='search' id='c-member-filter' class='member-filter' spellcheck='false' autocomplete='off'"
-        " placeholder='Filter by alias or served name' aria-controls='c-members' aria-describedby='c-count'>"
+        " placeholder='Filter by model, provider or served name' aria-controls='c-members' aria-describedby='c-count'>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
         "<label class='field' for='c-shape'>Explorer shape</label><select id='c-shape'></select>"
@@ -3295,12 +3382,71 @@ def access_card() -> str:
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td>{esc(v)}</td>"
                    f"<td class='mono set-by'>{'<br>'.join(esc(s) for s in src.split(', '))}</td></tr>"
                    for k, v, src in rows)
+    running, disk = on_disk()
+    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
+           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
     return ("<div class='panel'><h2>Server and access</h2><p class='muted'>How this install is reached and who"
             " may call it. Set in the environment, read at start: change one and restart. Shown here, never"
             " set here, since a page that could change its own login could open itself to anyone. No secret"
             " is shown, only whether one is set.</p>"
             f"<table class='facts'><thead><tr>{th('Setting')}{th('Now')}{th('Set by')}</tr></thead>"
-            f"<tbody>{body}</tbody></table></div>")
+            f"<tbody>{body}</tbody></table>"
+            # restart(): stopped and started again by serve.py's own parent process.
+            f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
+            " for an update or a setting"
+            " above, whatever runs it (a service, the launcher, a container or a terminal). It waits for"
+            " onboarding, a scheduled job and gateway calls in progress, or cuts them off if you insist; the"
+            " page comes back in a few seconds.</p>"
+            "<p><button class='btn' type='button' id='restart'>Restart Benchwarmers</button>"
+            " <span id='restart-out' role='status'></span></p></div>")
+
+
+RESTART = 75                       # the exit code serve.py's parent reads as "start me again"
+SERVER = None                      # the running ThreadingHTTPServer, set in __main__
+_answering = [0]                   # gateway calls being answered now
+_answering_lock = threading.Lock()
+
+
+def _follow_parent() -> None:
+    """End with serve.py's parent however it went (a service stopped, a
+    terminal closed), so no orphan keeps the port. Its process is waited on,
+    not a pipe read: on Windows a thread blocked reading stdin stalls every
+    subprocess started meanwhile (git, the onboarding, the jobs)."""
+    ppid = int(os.environ.get("BENCH_PARENT") or 0)
+    if not ppid:
+        return
+    if os.name == "nt":
+        import ctypes
+        k32 = ctypes.windll.kernel32
+        handle = k32.OpenProcess(0x00100000, False, ppid)          # SYNCHRONIZE
+        if not handle:
+            return                                                  # cannot watch it: stay up
+        k32.WaitForSingleObject(handle, 0xFFFFFFFF)                 # until it ends
+    else:
+        while os.getppid() == ppid:                                 # a new parent: the old one ended
+            time.sleep(2)
+    os._exit(0)
+
+
+def restart(force: bool = False, who: str = "ui") -> dict:
+    """End this dashboard and start it again on the code now on disk: after
+    an update, or a setting read at start. serve.py runs it as a child of its
+    own parent process (see __main__), which starts it again, so this works
+    however serve.py was started: Servy, the npm or pip launcher, Docker or a
+    terminal. Refused while something would be cut off, unless `force`."""
+    import scheduler
+    if os.environ.get("BENCH_SERVED") != "1" or SERVER is None:
+        return {"ok": False, "errors": ["this dashboard was not started by serve.py's own parent process,"
+                                        " so nothing would start it again: restart it the way you started it"]}
+    busy = ([f"onboarding is running ({sum(i['state'] in ('queued', 'running') for i in _OQ['items'])} left):"
+             " Stop it in the queue, or wait"] if _OQ["worker"] is not None else []) \
+        + ([f"a scheduled job is running: {', '.join(scheduler.RUNNING)}"] if scheduler.RUNNING else []) \
+        + ([f"{_answering[0]} gateway call(s) are being answered"] if _answering[0] else [])
+    if busy and not force:
+        return {"ok": False, "busy": True, "errors": busy}
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "", "restart", *on_disk())
+    threading.Timer(0.5, SERVER.shutdown).start()      # after this reply has gone
+    return {"ok": True, "errors": [], "cut": busy}
 
 
 def save_scheduler(mode: str, jobs, who: str = "ui") -> dict:
@@ -3591,27 +3737,39 @@ def models_page() -> str:
             + "".join(cards))
 
 
+def quota_tiles() -> list[dict]:
+    """metrics/quotaview.py's tiles, each named as the collection popup names
+    its provider, unless the live read named the credentials behind it."""
+    import quotaview
+    return [dict(t, name=t["name"] or provider_label(t["tag"])) for t in quotaview.tiles()]
+
+
+def quota_op(which: str, tag) -> dict:
+    """Refresh or Check now on one tile: {ok, errors, note, tile}; "order"
+    saves the tiles' order (a list of tags, empty for the default sort)."""
+    import quotaview
+    if which == "order":
+        return quotaview.save_order(tag)
+    if tag not in {t["id"] for t in quota_tiles()}:
+        return {"ok": False, "errors": [f"no tile called {tag!r}"]}
+    try:
+        r = (quotaview.refresh if which == "refresh" else quotaview.check)(tag)
+    except (ValueError, OSError) as exc:              # OSError: URLError and timeouts
+        return {"ok": False, "errors": [str(exc)]}
+    if not r.get("ok"):
+        return {"ok": False, "errors": r.get("errors") or ["it failed"]}
+    return {"ok": True, "errors": [], "note": r.get("note", ""),
+            "tile": next((t for t in quota_tiles() if t["id"] == tag), None)}
+
+
 def quotas_page() -> str:
-    """What limits each connection, who is resting after one, what providers
-    last said was left (the built-in client's quota records, metrics/quota.py)
-    and what the bench itself sent (its usage ledger, metrics/ledger.py)."""
+    """A tile per provider: what is left and when it resets, shaped by how the
+    provider limits (metrics/quotaview.py), who is resting, Refresh and Check
+    now; the resets of the next week; what the bench sent (its usage ledger,
+    metrics/ledger.py); every limit known. bench.js draws tiles and charts."""
     import learn
     import ledger
-    import quota
     now = datetime.datetime.now().astimezone()
-    rec_rows = []
-    for r in sorted(quota.live(), key=lambda r: (r.get("connection", ""), r.get("alias", ""), r.get("metric", ""))):
-        try:
-            age = int((now - datetime.datetime.fromisoformat(r["observed_at"])).total_seconds() // 60)
-            reset = datetime.datetime.fromisoformat(r["reset_at"]).astimezone().strftime("%Y-%m-%d %H:%M")
-        except (KeyError, TypeError, ValueError):
-            continue
-        num = lambda v: f"{v:,}" if isinstance(v, int) else "—"          # noqa: E731
-        rec_rows.append(f"<tr><td>{esc(r.get('connection', ''))}</td><td>{named(r.get('alias', ''))}</td>"
-                        f"<td>{esc(r.get('metric', ''))} / {esc(r.get('window', ''))}</td>"
-                        f"<td class='num'>{num(r.get('remaining'))}</td><td class='num'>{num(r.get('limit'))}</td>"
-                        f"<td class='mono'>{esc(reset)}</td>"
-                        f"<td>{esc(r.get('source', ''))}, {age} min ago</td></tr>")
     t = now.timestamp()
     sent: dict = {}
     for row in ledger.rows(min(t - 86400, ledger.month_start(t))):
@@ -3623,15 +3781,10 @@ def quotas_page() -> str:
             s[2], s[3] = s[2] + n, s[3] + tok
     sent_rows = [f"<tr><td>{esc(c)}</td>" + "".join(f"<td class='num'>{v:,}</td>" for v in s) + "</tr>"
                  for c, s in sorted(sent.items())]
-    cool_rows = []
-    for alias, c in sorted(learn.cooldowns().items(), key=lambda kv: kv[1].get("until", "")):
-        try:
-            left = max(0, int((datetime.datetime.fromisoformat(c["until"]) - now).total_seconds() // 60))
-        except (KeyError, ValueError, TypeError):
-            left = 0
-        cool_rows.append(f"<tr><td>{named(alias)}</td><td>{esc(catalog.connection_of(alias))}</td>"
-                         f"<td class='num' data-sort='{left}'>{left} min</td>"
-                         f"<td>{'stated by the provider' if c.get('stated') else 'default wait'}</td></tr>")
+    data = json.dumps({"tiles": quota_tiles(),
+                       "sent": [{"provider": provider_label(c) if c != "?" else "?", "day": s[0], "month": s[2]}
+                                for c, s in sent.items() if s[2]]},
+                      ensure_ascii=False).replace("</", "<\\/")
     entered = catalog.connection_limits()
     keys = catalog.CONNECTION_LIMIT_KEYS
     lim_rows = [f"<tr><td>{esc(conn)}</td>" + "".join(
@@ -3646,40 +3799,42 @@ def quotas_page() -> str:
                                 + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
                                           else "<td class='num muted'>—</td>" for kk in keys)
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
+    sent_table = ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
+                  + th("Requests, this month", True) + th("Tokens, this month", True)
+                  + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
+                  if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
+    sent_key = ("<p>One row per provider, most requests this month first. The pale bar is this month's"
+                " requests, the dark one inside it the last 24 hours'; the label reads 24 h / month."
+                " Retries count. Calls through the llm CLI are not in the ledger.</p>")
     return (
-        "<div class='panel'><h2>Quota management</h2><p class='sub'>What is left on each connection and"
-        " when it resets, from the rate-limit headers and 429 replies the built-in client sees, and what"
-        " the bench sent itself, from its usage ledger; who is resting after a rate limit, and every"
-        " limit known. The built-in client keeps to all of these. Calls through the llm CLI are in"
-        " neither record. Quotas of OAuth credentials stay in CLI Proxy API's own console.</p></div>"
-        "<div class='panel'><h3 class='chart-head'>What providers said is left · " + str(len(rec_rows)) + "</h3>"
-        + ("<table><thead><tr>" + th("Connection") + th("Model") + th("Counts") + th("Left", True)
-           + th("Of", True) + th("Resets") + th("From") + f"</tr></thead><tbody>{''.join(rec_rows)}</tbody></table>"
-           if rec_rows else "<p class='muted'>Nothing in force: no provider has stated a limit whose reset"
-                            " is still ahead.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>What the bench sent</h3><p class='sub'>Requests"
-        " (retries included) and tokens per connection, from the usage ledger.</p>"
-        + ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
-           + th("Requests, this month", True) + th("Tokens, this month", True)
-           + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
-           if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>Resting after a rate limit · " + str(len(cool_rows)) + "</h3>"
-        + ("<table><thead><tr>" + th("Model") + th("Connection") + th("Back in", True) + th("Wait")
-           + f"</tr></thead><tbody>{''.join(cool_rows)}</tbody></table>" if cool_rows
-           else "<p class='muted'>None: every model is free to call.</p>")
-        + "</div>"
-        "<div class='panel'><h3 class='chart-head'>Known limits</h3><p class='sub'>How much each connection"
-        " or model allows (tokens or requests per minute or day), so the bench can space its calls."
-        " Two sources: what you enter per connection (Endpoints, Rate limits per connection), and what a"
-        " provider states in a rate-limit error, which the bench writes down by itself. The list grows on"
-        " its own as models meet their limits. A model's own limits (its popup) win, and learned ones"
-        " only ever lower a limit.</p>"
+        "<div class='panel'><h2>Quota</h2><p class='sub'>What is left with each provider and when it"
+        " resets, and who is resting after a rate limit. CLI Proxy API's credentials and OpenRouter are"
+        " read live by Refresh, which spends nothing; every other tile is what the provider last said"
+        " in its replies to the bench, and Check now, where it is offered, asks with one request."
+        " Drag a tile, or move it with the arrow keys, to keep it where you want it.</p>"
+        "<div class='list-actions'><button class='btn' id='q-all'>Refresh all live</button>"
+        "<button class='btn' id='q-reorder'>Sort: out first, then soonest reset</button>"
+        "<span class='done' id='q-done' aria-live='polite'></span>"
+        "<span class='errors' id='q-errors' aria-live='polite'></span></div>"
+        "<div class='q-tiles' id='q-tiles'></div></div>"
+        "<div class='panel'><h3 class='chart-head'>Resets, next 7 days</h3><p class='sub'>A dot where a"
+        " limit resets (a daily one every day), a bar while a model rests, the line is now.</p>"
+        "<div class='chart' id='chart-resets'></div></div>"
+        + chart_card("What the bench sent", "Requests per provider: this month, and the last 24 hours inside it.",
+                     "chart-sent", sent_table, sent_key, retired=False)
+        + "<details class='panel'><summary><b>Known limits</b></summary><p class='sub'>How much each"
+        " connection or model allows (tokens or requests per minute or day), so the bench can space its"
+        " calls. Two sources: what you enter per connection (Endpoints, Rate limits per connection), and"
+        " what a provider states in a rate-limit error, which the bench writes down by itself. A model's"
+        " own limits (its popup) win, and learned ones only ever lower a limit.</p>"
         + ("<table><thead><tr>" + th("Connection or model") + "".join(th(k, True) for k in keys) + th("From")
            + f"</tr></thead><tbody>{''.join(lim_rows)}</tbody></table>" if lim_rows
            else "<p class='muted'>None known yet.</p>")
-        + "</div>")
+        + "</details>"
+        "<dialog id='q-models' aria-labelledby='q-models-h'><h3 id='q-models-h'></h3>"
+        "<p class='sub' id='q-models-sub'></p><div class='scroll' id='q-models-list'></div>"
+        "<div class='list-actions'><button class='btn' id='q-models-close'>Close</button></div></dialog>"
+        f"<script type='application/json' id='quota-data'>{data}</script>")
 
 
 def _log_rows(path: pathlib.Path, limit: int = 200) -> list[str]:
@@ -3751,6 +3906,14 @@ RELEASE, RELEASE_DATE, COMMIT = version()
 VERSION = f"{RELEASE} {RELEASE_DATE}".strip()           # the X-Benchwarmers-Version header
 
 
+def on_disk() -> tuple[str, str]:
+    """(running, on disk): what this process started from and what a restart
+    would start, by commit, or by the VERSION file where there is no git."""
+    if COMMIT:
+        return COMMIT, _git("log", "-1", "--format=%h") or COMMIT
+    return RELEASE, read(ROOT / "VERSION").strip()
+
+
 def info_page() -> str:
     import scheduler
     import client as model_client
@@ -3859,6 +4022,7 @@ def editor() -> str:
         # The page's own confirm (bench.js ask): window.confirm can be blocked.
         "<dialog id='ask-dialog' aria-labelledby='ask-text'>"
         "<p id='ask-text' style='white-space:pre-line'></p>"
+        f"<div id='ask-onboard' hidden>{onboard_choice('ask')}</div>"     # Discover's Add only
         "<div class='actions'><button class='btn' id='ask-no' type='button'>Cancel</button>"
         "<button class='btn primary' id='ask-yes' type='button'>OK</button></div></dialog>"
         # onboarding
@@ -3876,13 +4040,7 @@ def editor() -> str:
         " <button class='linkish' type='button' id='o-refresh'>Refresh</button></p>"
         "<div class='picker' id='o-picker' role='group' aria-label='Served models'></div>"
         # serve.ONBOARD_MODES: how much each onboarding spends.
-        "<label class='check'>Onboard as <select id='o-mode'>"
-        "<option value='full'>Full: probe, then the suite now</option>"
-        "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
-        "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
-        " (metered, capped or trial keys)</option></select></label>"
-        "<label class='check'><input type='checkbox' id='o-vision'> Vision model: a seat only for questions"
-        " about images, and its suite is the vision tasks</label>"
+        f"{onboard_choice('o')}"
         "</div>"
         "<div id='o-run' hidden><p class='served' id='o-result'></p><ol class='steps' id='o-steps'></ol>"
         "<details><summary class='muted'>Full log</summary><pre class='log' id='o-log'></pre></details></div>"
@@ -4122,7 +4280,13 @@ class Handler(gateway.Handler):
         """
         path = self.path.split("?", 1)[0]
         if guard.gateway_route(path):
-            return super().do_POST()
+            with _answering_lock:
+                _answering[0] += 1                     # restart() waits for these
+            try:
+                return super().do_POST()
+            finally:
+                with _answering_lock:
+                    _answering[0] -= 1
         if path == "/login":
             return self._login()
         ops = {"/api/model": lambda p: change_model(str(p.get("alias", "")),
@@ -4134,7 +4298,9 @@ class Handler(gateway.Handler):
                "/api/retry": lambda p: retry_model(str(p.get("alias") or "")),
                "/api/discover/add": lambda p: discover_add(str(p.get("connection") or ""),
                                                            str(p.get("model") or ""),
-                                                           endpoint=str(p.get("endpoint") or "")),
+                                                           endpoint=str(p.get("endpoint") or ""),
+                                                           mode=str(p.get("mode") or "full"),
+                                                           vision=p.get("vision") is True),
                "/api/onboard": lambda p: start_onboard(str(p.get("served", "")),
                                                        str(p.get("endpoint", "")),
                                                        mode=str(p.get("mode") or "full"),
@@ -4179,7 +4345,11 @@ class Handler(gateway.Handler):
                "/api/gateway-key": lambda p: make_gateway_key(str(p.get("name") or ""), p.get("collections"),
                                                               p.get("rpm")),
                "/api/gateway-key/remove": lambda p: revoke_gateway_key(str(p.get("name") or "")),
-               "/api/theme": lambda p: save_theme(str(p.get("theme") or ""))}
+               "/api/theme": lambda p: save_theme(str(p.get("theme") or "")),
+               "/api/quota/refresh": lambda p: quota_op("refresh", str(p.get("tag") or "")),
+               "/api/quota/check": lambda p: quota_op("check", str(p.get("tag") or "")),
+               "/api/quota/order": lambda p: quota_op("order", p.get("order")),
+               "/api/restart": lambda p: restart(p.get("force") is True)}
         # SAME ORIGIN ONLY. Any page open in the browser can POST a plain form
         # to 127.0.0.1, and this used to parse whatever arrived as JSON -- so a
         # website could retire models, and now could set a key. The checks are
@@ -4276,6 +4446,19 @@ class Handler(gateway.Handler):
 
 
 if __name__ == "__main__":
+    if os.environ.get("BENCH_SERVED") != "1":
+        # The dashboard runs as this process's child, so Restart (restart())
+        # can end it and have it started again on the code now on disk,
+        # whatever started serve.py. The child ends with this one (_follow_parent).
+        env = {**os.environ, "BENCH_SERVED": "1", "BENCH_PARENT": str(os.getpid())}
+        try:
+            while (code := subprocess.call([sys.executable, os.path.abspath(__file__), *sys.argv[1:]],
+                                           env=env, stdin=subprocess.DEVNULL)) == RESTART:
+                print("Benchwarmers: restarting on the code now on disk", flush=True)
+        except KeyboardInterrupt:
+            code = 0
+        sys.exit(code)
+    threading.Thread(target=_follow_parent, daemon=True).start()
     address = guard.bind()
     guard.startup(address)                             # beyond this machine, only with an admin token
     shown = "localhost" if address == "127.0.0.1" else address
@@ -4284,4 +4467,6 @@ if __name__ == "__main__":
     paths.make()                                       # a fresh data folder's layout, before any writer
     import scheduler
     scheduler.start()                                  # off unless config/scheduler.json says otherwise
-    ThreadingHTTPServer((address, PORT), Handler).serve_forever()
+    SERVER = ThreadingHTTPServer((address, PORT), Handler)
+    SERVER.serve_forever()                             # returns only when restart() shuts it down
+    sys.exit(RESTART)