@wendellresearch-ai/benchwarmers 0.2.1 → 0.3.0

package/serve.py

@@ -683,7 +683,7 @@ def retry_model(alias: str, who: str = "ui") -> dict:
     with (DATA / "perf" / "history.csv").open("a", encoding="utf-8", newline="") as fh:
         csv.writer(fh).writerow([datetime.datetime.now().isoformat(timespec="seconds"), alias, "retry", 1,
                                  f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}",
-                                 0.0, 1, 0, 0, 0, "", "retry from the dashboard"])
+                                 0.0, 1, 0, 0, 0, "", "retry from the dashboard", ""])
     import alerts as _alerts
     served = {a: s for s, a in names.items()}.get(alias, "")
     # Exactly this model's: auto_retire.py raises model:pending:<served> with its alias.
@@ -963,7 +963,7 @@ def free_name(stem: str) -> str:
 
 def save_endpoint(name: str, base: str, secret: str, who: str = "ui", extra: dict | None = None,
                   new: bool = False, kind: str = "", hub: str = "", admin: str = "",
-                  test_only: bool = False) -> dict:
+                  test_only: bool = False, api: str = "") -> dict:
     """Add an endpoint, or change one's key. Tested before anything is saved;
     `test_only` (the form's Test button) stops there and saves nothing.
 
@@ -980,11 +980,16 @@ def save_endpoint(name: str, base: str, secret: str, who: str = "ui", extra: dic
     key where it is not the API key (LiteLLM's master key, CLI Proxy API's
     management key), is kept beside it under endpoints.admin_name; nothing
     tests it here, since a refused CLI Proxy API key counts toward a ban.
+    `api` (endpoints.APIS), on a new endpoint: what it speaks, when not chat
+    completions; Anthropic's and Gemini's native APIs take their own key
+    header, which the test sends too.
     """
     name, base, secret = (name or "").strip().lower(), (base or "").strip().rstrip("/"), (secret or "").strip()
-    kind, hub, admin = (kind or "").strip(), (hub or "").strip(), (admin or "").strip()
+    kind, hub, admin, api = (kind or "").strip(), (hub or "").strip(), (admin or "").strip(), (api or "").strip()
     data = endpoints.load()
     errors = []
+    if api and (api not in endpoints.APIS or api == "responses"):    # responses: Connect sets it, with its sign-in
+        errors.append(f"api: one of {', '.join(a for a in endpoints.APIS if a != 'responses')}")
     if kind and kind not in endpoints.KINDS:
         errors.append(f"kind: one of {', '.join(endpoints.KINDS)}")
     elif kind == "hub" and hub not in endpoints.HUBS:
@@ -1011,6 +1016,8 @@ def save_endpoint(name: str, base: str, secret: str, who: str = "ui", extra: dic
         # with its stale refresh token and overwrite the typed key.
         entry.pop("connect", None)
     entry.update(extra, base=base)
+    if api and new:
+        entry.pop("api", None) if api == "chat" else entry.update(api=api)
     if kind:
         entry.pop("hub", None)
         entry.update(kind=kind, **({"hub": hub} if kind == "hub" else {}))
@@ -1244,10 +1251,11 @@ def connect_page(out: dict) -> str:
             f"<p><a href='{html.escape(home)}'>Back to Benchwarmers</a>. You can close this tab.</p>")
 
 
-def _clean_limits(raw) -> tuple[dict, list[str]]:
-    """rpm/rpd/tpm/tpd as positive whole numbers; blank clears one."""
+def _clean_limits(raw, keys=catalog.LIMIT_KEYS) -> tuple[dict, list[str]]:
+    """rpm/rpd/tpm/tpd (and a connection's rpmo) as positive whole numbers;
+    blank clears one."""
     out, errors = {}, []
-    for k in catalog.LIMIT_KEYS:
+    for k in keys:
         v = (raw or {}).get(k)
         if v in (None, ""):
             continue
@@ -1355,7 +1363,7 @@ def save_connection_limits(connection: str, limits: dict, who: str = "ui") -> di
     if connection not in known:
         return {"ok": False, "errors": [f"{connection!r} is not a connection any model uses;"
                                         f" known: {', '.join(sorted(known))}"]}
-    clean, errors = _clean_limits(limits)
+    clean, errors = _clean_limits(limits, catalog.CONNECTION_LIMIT_KEYS)
     if errors:
         return {"ok": False, "errors": errors}
     data = catalog._read(catalog.LIMITS)
@@ -1376,7 +1384,8 @@ def connections_view() -> dict:
     for a in _chat_aliases():
         by.setdefault(catalog.connection_of(a), []).append(a)
     lim = catalog.connection_limits()
-    return {"ok": True, "errors": [], "keys": list(catalog.LIMIT_KEYS),
+    return {"ok": True, "errors": [], "keys": list(catalog.CONNECTION_LIMIT_KEYS),
+            "labels": {k: catalog.LIMIT_LABELS[k] for k in catalog.CONNECTION_LIMIT_KEYS},
             "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {})}
                             for c, m in sorted(by.items())]}
 
@@ -1511,7 +1520,15 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
     return {"ok": True, "errors": [], "changed": ["config/gateway.json"]}
 
 
-def start_onboard(served: str, endpoint: str = "", who: str = "ui") -> dict:
+# How much an onboarding spends. Full: the 4-call probe, then the suite now.
+# Quick: the probe only; the scheduled `suite --next` passes fill the suite in,
+# six tasks a run, so the same calls are spread out, not saved. Quick + ad
+# hoc: the probe only, and the bench never calls it on its own: the only one
+# that saves calls, for a metered, capped or trial key.
+ONBOARD_MODES = {"full": [], "quick": ["--skip-suite"], "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
+
+
+def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full") -> dict:
     """Kick off `scripts/onboard.py` for a served model, detached, logging to runs/.
 
     The script does its own checks in the right order (is it served, derive
@@ -1521,12 +1538,12 @@ def start_onboard(served: str, endpoint: str = "", who: str = "ui") -> dict:
     hands the rest to the script. It runs for minutes and spends calls, so it
     runs in the background and the page tails its log.
     """
-    return _spawn_onboard(served, endpoint, who)[0]
+    return _spawn_onboard(served, endpoint, who, mode)[0]
 
 
-def _onboard_refusal(served: str, endpoint: str) -> list[str]:
+def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str]:
     """What start_onboard refuses without a call (its docstring)."""
-    errors = []
+    errors = [] if mode in ONBOARD_MODES else [f"mode must be one of {', '.join(ONBOARD_MODES)}; got {mode!r}"]
     # It goes into a shell command line below, so only the characters model
     # ids actually use -- not merely "no spaces".
     if not re.fullmatch(r"[\w./:@+-]{1,200}", served):
@@ -1549,18 +1566,20 @@ def _onboard_refusal(served: str, endpoint: str) -> list[str]:
     return errors
 
 
-def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui") -> tuple[dict, subprocess.Popen | None]:
+def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
+                   mode: str = "full") -> tuple[dict, subprocess.Popen | None]:
     """start_onboard's result, and the process (None when refused) for a caller that waits on it."""
     served = (served or "").strip()
     endpoint = (endpoint or "").strip() or endpoints.default()
-    errors = _onboard_refusal(served, endpoint)
+    errors = _onboard_refusal(served, endpoint, mode)
     if errors:
         return {"ok": False, "errors": errors}, None
     stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
     safe = re.sub(r"[^A-Za-z0-9._-]+", "-", served)
     log = DATA / "runs" / f"onboard-{stamp}-{safe}.log"
     from run import bench_python
-    cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint]
+    cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint,
+           *ONBOARD_MODES[mode]]
     # The log handle goes straight to the child, unbuffered, so the page sees
     # each step as it prints. The old `cmd /c ... > log & echo` wrapper with
     # DETACHED_PROCESS sent the output to a console window instead, so every
@@ -1583,7 +1602,7 @@ def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui") -> tuple[di
 
     threading.Thread(target=sentinel, daemon=True).start()
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
-                who, "", "onboard", None, served)
+                who, "", "onboard" + ("" if mode == "full" else f" ({mode})"), None, served)
     return {"ok": True, "errors": [], "log": log.name}, proc
 
 
@@ -1627,20 +1646,21 @@ def queue_onboard(items, who: str = "ui") -> dict:
         it = it if isinstance(it, dict) else {}
         served = str(it.get("served") or "").strip()
         endpoint = str(it.get("endpoint") or "").strip() or endpoints.default()
-        errors += _onboard_refusal(served, endpoint)
-        picked.append((served, endpoint))
+        mode = str(it.get("mode") or "full")
+        errors += _onboard_refusal(served, endpoint, mode)
+        picked.append((served, endpoint, mode))
     with _OQ_LOCK:
         waiting = {(i["served"], i["endpoint"]) for i in _OQ["items"] if i["state"] in ("queued", "running")}
-        errors += [f"{s} from {e} is already in the queue" for s, e in picked if (s, e) in waiting]
-        if len(set(picked)) < len(picked):
+        errors += [f"{s} from {e} is already in the queue" for s, e, _ in picked if (s, e) in waiting]
+        if len({(s, e) for s, e, _ in picked}) < len(picked):
             errors.append("a model is picked twice")
         if errors:
             return {"ok": False, "errors": errors}
         if _OQ["worker"] is None:
             _OQ["items"] = []
         _OQ["stop"] = False                              # added after a Stop: these run
-        _OQ["items"] += [{"served": s, "endpoint": e, "state": "queued", "alias": None, "log": None,
-                          "exit": None, "note": ""} for s, e in picked]
+        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "state": "queued", "alias": None, "log": None,
+                          "exit": None, "note": ""} for s, e, m in picked]
         if _OQ["worker"] is None:
             _OQ["worker"] = threading.Thread(target=_onboard_worker, args=(who,), daemon=True)
             _OQ["worker"].start()
@@ -1692,7 +1712,7 @@ def _onboard_worker(who: str) -> None:
                 return
             nxt["state"] = "running"
         try:
-            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who)
+            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"))
             if proc is None:                             # refused now: the bench changed since it was queued
                 state, note, code = "failed", "; ".join(res["errors"]), None
             else:
@@ -2398,7 +2418,7 @@ def perf_card() -> str:
                   if r["model"] in live}
         widest = max((float(r["p50_s"]) for r in latest.values()), default=1) or 1
         body = ["<table><thead><tr>" + th("Model") + th("Prompt") + th("p50", True)
-                + th("p95", True) + th("ch/s", True) + th("Outcomes")
+                + th("p95", True) + th("ch/s", True) + th("tok/s", True) + th("Outcomes")
                 + "<th>p50, relative</th></tr></thead><tbody>"]
         for (model, prompt), row in sorted(latest.items()):
             p50 = float(row["p50_s"])
@@ -2411,6 +2431,7 @@ def perf_card() -> str:
                 f"<td class='num'>{p50:.1f}s</td>"
                 f"<td class='num'>{float(row['p95_s']):.1f}s</td>"
                 f"<td class='num'>{esc(row['chars_per_s'])}</td>"
+                f"<td class='num'>{esc(row.get('tokens_per_s') or '—')}</td>"
                 f"<td class='{'err' if bad else ''}'>{esc(oc)}</td>"
                 f"<td><span class='meter' style='width:{width}px' "
                 f"title='p50 {p50:.1f}s'><span style='width:100%'></span></span></td></tr>")
@@ -2428,6 +2449,9 @@ def perf_card() -> str:
         questions.</dd>
       <dt>ch/s</dt><dd>Characters per second, from the large prompt only. A
         proxy for tokens/s, affected by how verbose a model is.</dd>
+      <dt>tok/s</dt><dd>Output tokens per second from the provider's own count, first token to
+        last, large prompt only. Only where the built-in client made the call and the provider
+        reported usage; blank otherwise, never estimated.</dd>
       <dt>Outcomes</dt><dd>answered / refused / error / timeout.</dd>
       <dt>Weight it LEAST</dt><dd>Seconds are a wash. Speed only vetoes a model
         when it is slow enough to change how you work.</dd>
@@ -2549,8 +2573,9 @@ def gateway_card(limit: int = 30) -> str:
         explorers, then the merger if the reply was merged. Green answered, amber came back empty, red failed. ✓ judged
         useful, ✗ judged not useful; no mark means the verdict is still owed.
         Hover a chip for its seat, timing and call id; click it to read the answer
-        and record a verdict. A grey chip with a dash is a member not asked because the
-        request carried images it cannot take; hover it for why.</dd>
+        and record a verdict. A grey chip with a dash is a member not asked: the request
+        carried images it cannot take, or it is a vision specialist and the request
+        carried none. Hover it for why.</dd>
       <dt>Asked by</dt><dd>The gateway key that made the request, or <code>local</code>
         when no key is asked for.</dd>
       <dt>Owed</dt><dd>Seats without a verdict. Each is recorded as an unjudged
@@ -2562,7 +2587,49 @@ def gateway_card(limit: int = 30) -> str:
     </dl>"""
     return card("Gateway", f"The last {len(records)} bench-gateway requests:"
                 f" {seats_total} seats, {owed} verdict{'s' * (owed != 1)} owed.",
-                "".join(body), key)
+                "".join(body) + route_table(judged), key)
+
+
+def route_table(judged: dict, limit: int = 30) -> str:
+    """Route-mode conversations (runs/gateway/routes.jsonl, a line a turn),
+    one row each, apart from the panels: an agent turn is not a seat, and a
+    conversation's verdict is optional, so none of this counts as Owed.
+    ponytail: reads the whole log; read its tail once it grows large."""
+    convs: dict[str, dict] = {}
+    for line in read(DATA / "runs" / "gateway" / "routes.jsonl").splitlines():
+        try:
+            t = json.loads(line)
+        except ValueError:
+            continue
+        c = convs.setdefault(str(t.get("conversation", "")), {
+            "at": t.get("at", ""), "model": t.get("model", ""), "key": t.get("key", ""), "task": t.get("task", ""),
+            "turns": 0, "tools": 0, "failovers": 0, "models": {}})
+        c["turns"] += 1
+        c["tools"] += t.get("tool_calls") or 0
+        c["failovers"] += len(t.get("tried") or [])
+        c.update(last=t.get("at", ""), status=t.get("status", ""), error=t.get("error", ""))
+        if t.get("alias"):
+            c["models"].setdefault(t["alias"], t.get("call", ""))
+    if not convs:
+        return ""
+    rows = ["<h3>Route conversations</h3><table><thead><tr>", th("When"), th("Collection"), th("Asked by"),
+            th("First message"), th("Answered by"), th("Turns", True), th("Tool calls", True),
+            th("Failovers", True), th("Last turn"), "</tr></thead><tbody>"]
+    for conv, c in sorted(convs.items(), key=lambda kv: kv[1]["last"], reverse=True)[:limit]:
+        chips = " ".join(f"<span class='chip' title='{esc(call + ' · ' + {True: 'useful', False: 'not useful', None: 'no verdict (optional)'}[judged.get(call)])}'>"
+                         f"{esc(alias)}{ {True: ' ✓', False: ' ✗', None: ''}[judged.get(call)]}</span>"
+                         for alias, call in c["models"].items())
+        css = {"ok": "ok", "empty": "warn", "error": "err"}.get(c["status"], "")
+        rows.append(
+            f"<tr><td class='mono' data-sort='{esc(c['at'])}' title='{esc(conv)}'>{esc(c['at'][:16].replace('T', ' '))}</td>"
+            f"<td class='mono'>{esc(c['model'])}</td><td class='mono'>{esc(c['key'] or '—')}</td>"
+            f"<td>{esc(c['task'][:60])}</td><td>{chips or '—'}</td><td class='num'>{c['turns']}</td>"
+            f"<td class='num'>{c['tools']}</td><td class='num{' muted' if not c['failovers'] else ''}'>{c['failovers']}</td>"
+            f"<td><span class='chip {css}' title='{esc(c['error'])}'>{esc(c['status'])}</span></td></tr>")
+    rows.append("</tbody></table><p class='muted'>Each conversation is one unjudged offload per model that"
+                " answered it, under the call id on its chip; a verdict is optional:"
+                " <code>python metrics/offload.py verdict &lt;call id&gt; --useful</code>.</p>")
+    return "".join(rows)
 
 
 def gateway_seat(call: str) -> dict:
@@ -2773,11 +2840,13 @@ def collections_tab() -> str:
         "<p class='rules'>Which models may fill the explorer seats. Automatic asks a decision"
         " model to label each prompt when one is onboarded, else general.</p>"
         "<label class='field' for='c-reply'>Reply</label><select id='c-reply'>"
-        "<option value='all'>every answer</option><option value='merged'>merged</option></select>"
+        "<option value='all'>every answer</option><option value='merged'>merged</option>"
+        "<option value='route'>route (one member, for agents)</option></select>"
         "<p class='rules'>Merged: after the panel, one model writes a single answer saying where the"
         " seats agreed and who said what, for a chat UI that shows one answer; every seat is still"
-        " called and recorded. A caller can ask for either with <code>bench/&lt;name&gt;:merged</code>"
-        " or <code>:all</code>.</p>"
+        " called and recorded. Route: no panel; one member, picked by its record, gets the request"
+        " with its tools, for an agent such as Claude Code or Codex. A caller can ask for any with"
+        " <code>bench/&lt;name&gt;:merged</code>, <code>:all</code> or <code>:route</code>.</p>"
         # Only a merged reply has a merger; the choice is kept while hidden.
         "<div id='c-merger-row' hidden><label class='field' for='c-merger'>Merger</label>"
         "<select id='c-merger'></select>"
@@ -3219,7 +3288,9 @@ def endpoints_page() -> str:
         "<div class='panel form-panel'><h2>Rate limits per connection</h2>"
         "<p class='sub'>What a key behind each connection allows, from the provider's docs or"
         " freellmapi.co/models. Models on that connection share it; a model's own limits (its"
-        " popup) win, and limits the bench learns from errors only ever lower these.</p>"
+        " popup) win, and limits the bench learns from errors only ever lower these. rpmo is calls"
+        " a calendar month (UTC), for a trial key's monthly cap. The built-in client keeps to them:"
+        " when one is used up it waits if the call can still finish in time, else it does not send.</p>"
         "<div id='l-list'></div><div class='errors' id='l-errors' role='alert'></div>"
         "<div class='done' id='l-done'></div></div>")
 
@@ -3270,11 +3341,37 @@ def models_page() -> str:
 
 
 def quotas_page() -> str:
-    """What limits each connection, and who is resting after one. The live
-    numbers (what is left, when it resets) come with the built-in client's
-    quota records, RELEASE-PLAN Phase 2b."""
+    """What limits each connection, who is resting after one, what providers
+    last said was left (the built-in client's quota records, metrics/quota.py)
+    and what the bench itself sent (its usage ledger, metrics/ledger.py)."""
     import learn
+    import ledger
+    import quota
     now = datetime.datetime.now().astimezone()
+    rec_rows = []
+    for r in sorted(quota.live(), key=lambda r: (r.get("connection", ""), r.get("alias", ""), r.get("metric", ""))):
+        try:
+            age = int((now - datetime.datetime.fromisoformat(r["observed_at"])).total_seconds() // 60)
+            reset = datetime.datetime.fromisoformat(r["reset_at"]).astimezone().strftime("%Y-%m-%d %H:%M")
+        except (KeyError, TypeError, ValueError):
+            continue
+        num = lambda v: f"{v:,}" if isinstance(v, int) else "—"          # noqa: E731
+        rec_rows.append(f"<tr><td>{esc(r.get('connection', ''))}</td><td>{named(r.get('alias', ''))}</td>"
+                        f"<td>{esc(r.get('metric', ''))} / {esc(r.get('window', ''))}</td>"
+                        f"<td class='num'>{num(r.get('remaining'))}</td><td class='num'>{num(r.get('limit'))}</td>"
+                        f"<td class='mono'>{esc(reset)}</td>"
+                        f"<td>{esc(r.get('source', ''))}, {age} min ago</td></tr>")
+    t = now.timestamp()
+    sent: dict = {}
+    for row in ledger.rows(min(t - 86400, ledger.month_start(t))):
+        s = sent.setdefault(row.get("connection") or "?", [0, 0, 0, 0])
+        n, tok = row.get("attempts") or 0, (row.get("tokens_in") or 0) + (row.get("tokens_out") or 0)
+        if row["t"] >= t - 86400:
+            s[0], s[1] = s[0] + n, s[1] + tok
+        if row["t"] >= ledger.month_start(t):
+            s[2], s[3] = s[2] + n, s[3] + tok
+    sent_rows = [f"<tr><td>{esc(c)}</td>" + "".join(f"<td class='num'>{v:,}</td>" for v in s) + "</tr>"
+                 for c, s in sorted(sent.items())]
     cool_rows = []
     for alias, c in sorted(learn.cooldowns().items(), key=lambda kv: kv[1].get("until", "")):
         try:
@@ -3285,7 +3382,7 @@ def quotas_page() -> str:
                          f"<td class='num' data-sort='{left}'>{left} min</td>"
                          f"<td>{'stated by the provider' if c.get('stated') else 'default wait'}</td></tr>")
     entered = catalog.connection_limits()
-    keys = catalog.LIMIT_KEYS
+    keys = catalog.CONNECTION_LIMIT_KEYS
     lim_rows = [f"<tr><td>{esc(conn)}</td>" + "".join(
                     f"<td class='num'>{lim[k]:,}</td>" if lim.get(k) else "<td class='num muted'>—</td>" for k in keys)
                 + "<td>you</td></tr>" for conn, lim in sorted(entered.items()) if any(lim.get(k) for k in keys)]
@@ -3300,10 +3397,23 @@ def quotas_page() -> str:
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
     return (
         "<div class='panel'><h2>Quota management</h2><p class='sub'>What is left on each connection and"
-        " when it resets, read from provider endpoints, rate-limit headers and the bench's own call"
-        " ledger, arrives with the built-in client's quota records (RELEASE-PLAN Phase 2b). Until then:"
-        " who is resting after a rate limit, and every limit known. Quotas of OAuth credentials stay"
-        " in CLI Proxy API's own console.</p></div>"
+        " when it resets, from the rate-limit headers and 429 replies the built-in client sees, and what"
+        " the bench sent itself, from its usage ledger; who is resting after a rate limit, and every"
+        " limit known. The built-in client keeps to all of these. Calls through the llm CLI are in"
+        " neither record. Quotas of OAuth credentials stay in CLI Proxy API's own console.</p></div>"
+        "<div class='panel'><h3 class='chart-head'>What providers said is left · " + str(len(rec_rows)) + "</h3>"
+        + ("<table><thead><tr>" + th("Connection") + th("Model") + th("Counts") + th("Left", True)
+           + th("Of", True) + th("Resets") + th("From") + f"</tr></thead><tbody>{''.join(rec_rows)}</tbody></table>"
+           if rec_rows else "<p class='muted'>Nothing in force: no provider has stated a limit whose reset"
+                            " is still ahead.</p>")
+        + "</div>"
+        "<div class='panel'><h3 class='chart-head'>What the bench sent</h3><p class='sub'>Requests"
+        " (retries included) and tokens per connection, from the usage ledger.</p>"
+        + ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
+           + th("Requests, this month", True) + th("Tokens, this month", True)
+           + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
+           if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
+        + "</div>"
         "<div class='panel'><h3 class='chart-head'>Resting after a rate limit · " + str(len(cool_rows)) + "</h3>"
         + ("<table><thead><tr>" + th("Model") + th("Connection") + th("Back in", True) + th("Wait")
            + f"</tr></thead><tbody>{''.join(cool_rows)}</tbody></table>" if cool_rows
@@ -3476,8 +3586,10 @@ def editor() -> str:
         "<div class='actions'><button class='btn' id='co-save' type='button'>Save options</button></div>"
         "</details>"
         "<label class='check'><input type='checkbox' id='m-adhoc'> Ad hoc only</label>"
-        "<p class='rules'>For expensive or capped models: never seated automatically, never"
-        " swept, never probed daily. Used only when named.</p>"
+        "<p class='rules'>Never chosen by the bench on its own: no explorer seat, no router"
+        " pick, no scheduled suite, perf or availability runs. Still called whenever you choose"
+        " it: as a collection member (or merger), with <code>with</code>, or with"
+        " <code>--model</code>. For metered, capped or trial keys.</p>"
         "</div>"
         "<div class='errors' id='m-errors' role='alert'></div><div class='done' id='m-done'></div>"
         "<div class='actions'><button class='btn' id='m-close'>Close</button>"
@@ -3509,6 +3621,12 @@ def editor() -> str:
         "<input type='search' id='o-find' placeholder='Search models or connections'"
         " aria-label='Search models'>"
         "<div class='picker' id='o-picker' role='group' aria-label='Served models'></div>"
+        # serve.ONBOARD_MODES: how much each onboarding spends.
+        "<label class='check'>Onboard as <select id='o-mode'>"
+        "<option value='full'>Full: probe, then the suite now</option>"
+        "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
+        "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
+        " (metered, capped or trial keys)</option></select></label>"
         "</div>"
         "<div id='o-run' hidden><p class='served' id='o-result'></p><ol class='steps' id='o-steps'></ol>"
         "<details><summary class='muted'>Full log</summary><pre class='log' id='o-log'></pre></details></div>"
@@ -3548,7 +3666,14 @@ def editor() -> str:
         + "</select><p class='rules'>Open WebUI's base is its <code>/api</code>, e.g."
         " <code>http://localhost:3000/api</code>, with an admin's API key. LiteLLM's and CLI Proxy"
         " API's are their <code>/v1</code>, e.g. <code>http://localhost:4000/v1</code> and"
-        " <code>http://localhost:8317/v1</code>.</p></div></div>"
+        " <code>http://localhost:8317/v1</code>.</p></div>"
+        # What it speaks (endpoints.APIS): the built-in client builds the native requests.
+        "<label class='field' for='e-api'>API</label><select id='e-api'>"
+        + "".join(f"<option value='{k}'>{esc(label)}</option>" for k, label in endpoints.APIS.items() if k != "responses")
+        + "</select><p class='rules'>Chat completions unless the provider has none for your key. Native"
+        " Anthropic's base is <code>https://api.anthropic.com/v1</code>, Gemini's"
+        " <code>https://generativelanguage.googleapis.com/v1beta</code>; their models are called by the"
+        " built-in client, and route mode does not pick them.</p></div>"
         "<label class='field' for='e-key'>API key</label>"
         "<input id='e-key' type='password' autocomplete='off' placeholder='leave blank to keep the current one'>"
         "<p class='rules'>Stored where <code>llm keys set</code> stores it, never in the"
@@ -3755,7 +3880,8 @@ class Handler(gateway.Handler):
                                                            str(p.get("model") or ""),
                                                            endpoint=str(p.get("endpoint") or "")),
                "/api/onboard": lambda p: start_onboard(str(p.get("served", "")),
-                                                       str(p.get("endpoint", ""))),
+                                                       str(p.get("endpoint", "")),
+                                                       mode=str(p.get("mode") or "full")),
                "/api/onboard-queue": lambda p: queue_onboard(p.get("items")),
                "/api/onboard-queue/stop": lambda p: stop_onboard_queue(),
                "/api/onboard-queue/clear": lambda p: clear_onboard_queue(),
@@ -3765,7 +3891,7 @@ class Handler(gateway.Handler):
                                                         str(p.get("key", "")), new=bool(p.get("new")),
                                                         kind=str(p.get("kind") or ""), hub=str(p.get("hub") or ""),
                                                         admin=str(p.get("admin") or ""),
-                                                        test_only=bool(p.get("test"))),
+                                                        test_only=bool(p.get("test")), api=str(p.get("api") or "")),
                "/api/endpoint/remove": lambda p: remove_endpoint(str(p.get("name", "")),
                                                                  with_models=bool(p.get("with_models"))),
                "/api/connect/start": lambda p: connect_start(str(p.get("provider") or ""),

package/ui/bench.js

@@ -573,7 +573,8 @@
   // model at a time. The popup closes: the header's chip shows the batch, and
   // its Details the queue. Refused, it stays open with the reasons.
   o.start.addEventListener('click', function () {
-    var items = Object.keys(chosen).map(function (k) { return chosen[k]; });
+    var mode = $('o-mode').value;
+    var items = Object.keys(chosen).map(function (k) { return Object.assign({mode: mode}, chosen[k]); });
     if (!items.length) return;
     o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true; o.start.textContent = 'Starting…';
     post('/api/onboard-queue', {items: items}).then(function (res) {
@@ -1016,7 +1017,7 @@
         left.appendChild(el('div', {class: 'base'}, c.models.join(', ')));
         var inputs = el('div', {class: 'limits'});
         res.keys.forEach(function (k) {
-          var lab = el('label', {}, k + ' ');
+          var lab = el('label', {title: (res.labels || {})[k] || k}, k + ' ');
           var box = el('input', {type: 'number', min: '1', 'data-k': k, placeholder: '—'});
           if (c.limits[k]) box.value = c.limits[k];
           lab.appendChild(box); inputs.appendChild(lab);
@@ -1092,7 +1093,9 @@
   function keyBody() {
     var body = {name: e.name.value, base: e.base.value, key: e.key.value, new: e.adding,
                 admin: $('e-admin-row').hidden ? '' : e.admin.value};
-    if (e.adding) { body.kind = e.kind.value; body.hub = e.kind.value === 'hub' ? e.hub.value : ''; }   // Change key keeps its kind
+    if (e.adding) {                     // Change key keeps its kind and API
+      body.kind = e.kind.value; body.hub = e.kind.value === 'hub' ? e.hub.value : ''; body.api = $('e-api').value;
+    }
     return body;
   }
   $('e-add').addEventListener('click', function () { openEndpoint('add'); });