@wendellresearch-ai/benchwarmers 0.2.1 → 0.2.2

package/serve.py

@@ -683,7 +683,7 @@ def retry_model(alias: str, who: str = "ui") -> dict:
     with (DATA / "perf" / "history.csv").open("a", encoding="utf-8", newline="") as fh:
         csv.writer(fh).writerow([datetime.datetime.now().isoformat(timespec="seconds"), alias, "retry", 1,
                                  f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}",
-                                 0.0, 1, 0, 0, 0, "", "retry from the dashboard"])
+                                 0.0, 1, 0, 0, 0, "", "retry from the dashboard", ""])
     import alerts as _alerts
     served = {a: s for s, a in names.items()}.get(alias, "")
     # Exactly this model's: auto_retire.py raises model:pending:<served> with its alias.
@@ -1244,10 +1244,11 @@ def connect_page(out: dict) -> str:
             f"<p><a href='{html.escape(home)}'>Back to Benchwarmers</a>. You can close this tab.</p>")
 
 
-def _clean_limits(raw) -> tuple[dict, list[str]]:
-    """rpm/rpd/tpm/tpd as positive whole numbers; blank clears one."""
+def _clean_limits(raw, keys=catalog.LIMIT_KEYS) -> tuple[dict, list[str]]:
+    """rpm/rpd/tpm/tpd (and a connection's rpmo) as positive whole numbers;
+    blank clears one."""
     out, errors = {}, []
-    for k in catalog.LIMIT_KEYS:
+    for k in keys:
         v = (raw or {}).get(k)
         if v in (None, ""):
             continue
@@ -1355,7 +1356,7 @@ def save_connection_limits(connection: str, limits: dict, who: str = "ui") -> di
     if connection not in known:
         return {"ok": False, "errors": [f"{connection!r} is not a connection any model uses;"
                                         f" known: {', '.join(sorted(known))}"]}
-    clean, errors = _clean_limits(limits)
+    clean, errors = _clean_limits(limits, catalog.CONNECTION_LIMIT_KEYS)
     if errors:
         return {"ok": False, "errors": errors}
     data = catalog._read(catalog.LIMITS)
@@ -1376,7 +1377,8 @@ def connections_view() -> dict:
     for a in _chat_aliases():
         by.setdefault(catalog.connection_of(a), []).append(a)
     lim = catalog.connection_limits()
-    return {"ok": True, "errors": [], "keys": list(catalog.LIMIT_KEYS),
+    return {"ok": True, "errors": [], "keys": list(catalog.CONNECTION_LIMIT_KEYS),
+            "labels": {k: catalog.LIMIT_LABELS[k] for k in catalog.CONNECTION_LIMIT_KEYS},
             "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {})}
                             for c, m in sorted(by.items())]}
 
@@ -1511,7 +1513,15 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
     return {"ok": True, "errors": [], "changed": ["config/gateway.json"]}
 
 
-def start_onboard(served: str, endpoint: str = "", who: str = "ui") -> dict:
+# How much an onboarding spends. Full: the 4-call probe, then the suite now.
+# Quick: the probe only; the scheduled `suite --next` passes fill the suite in,
+# six tasks a run, so the same calls are spread out, not saved. Quick + ad
+# hoc: the probe only, and the bench never calls it on its own: the only one
+# that saves calls, for a metered, capped or trial key.
+ONBOARD_MODES = {"full": [], "quick": ["--skip-suite"], "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
+
+
+def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full") -> dict:
     """Kick off `scripts/onboard.py` for a served model, detached, logging to runs/.
 
     The script does its own checks in the right order (is it served, derive
@@ -1521,12 +1531,12 @@ def start_onboard(served: str, endpoint: str = "", who: str = "ui") -> dict:
     hands the rest to the script. It runs for minutes and spends calls, so it
     runs in the background and the page tails its log.
     """
-    return _spawn_onboard(served, endpoint, who)[0]
+    return _spawn_onboard(served, endpoint, who, mode)[0]
 
 
-def _onboard_refusal(served: str, endpoint: str) -> list[str]:
+def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str]:
     """What start_onboard refuses without a call (its docstring)."""
-    errors = []
+    errors = [] if mode in ONBOARD_MODES else [f"mode must be one of {', '.join(ONBOARD_MODES)}; got {mode!r}"]
     # It goes into a shell command line below, so only the characters model
     # ids actually use -- not merely "no spaces".
     if not re.fullmatch(r"[\w./:@+-]{1,200}", served):
@@ -1549,18 +1559,20 @@ def _onboard_refusal(served: str, endpoint: str) -> list[str]:
     return errors
 
 
-def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui") -> tuple[dict, subprocess.Popen | None]:
+def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
+                   mode: str = "full") -> tuple[dict, subprocess.Popen | None]:
     """start_onboard's result, and the process (None when refused) for a caller that waits on it."""
     served = (served or "").strip()
     endpoint = (endpoint or "").strip() or endpoints.default()
-    errors = _onboard_refusal(served, endpoint)
+    errors = _onboard_refusal(served, endpoint, mode)
     if errors:
         return {"ok": False, "errors": errors}, None
     stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
     safe = re.sub(r"[^A-Za-z0-9._-]+", "-", served)
     log = DATA / "runs" / f"onboard-{stamp}-{safe}.log"
     from run import bench_python
-    cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint]
+    cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint,
+           *ONBOARD_MODES[mode]]
     # The log handle goes straight to the child, unbuffered, so the page sees
     # each step as it prints. The old `cmd /c ... > log & echo` wrapper with
     # DETACHED_PROCESS sent the output to a console window instead, so every
@@ -1583,7 +1595,7 @@ def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui") -> tuple[di
 
     threading.Thread(target=sentinel, daemon=True).start()
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
-                who, "", "onboard", None, served)
+                who, "", "onboard" + ("" if mode == "full" else f" ({mode})"), None, served)
     return {"ok": True, "errors": [], "log": log.name}, proc
 
 
@@ -1627,20 +1639,21 @@ def queue_onboard(items, who: str = "ui") -> dict:
         it = it if isinstance(it, dict) else {}
         served = str(it.get("served") or "").strip()
         endpoint = str(it.get("endpoint") or "").strip() or endpoints.default()
-        errors += _onboard_refusal(served, endpoint)
-        picked.append((served, endpoint))
+        mode = str(it.get("mode") or "full")
+        errors += _onboard_refusal(served, endpoint, mode)
+        picked.append((served, endpoint, mode))
     with _OQ_LOCK:
         waiting = {(i["served"], i["endpoint"]) for i in _OQ["items"] if i["state"] in ("queued", "running")}
-        errors += [f"{s} from {e} is already in the queue" for s, e in picked if (s, e) in waiting]
-        if len(set(picked)) < len(picked):
+        errors += [f"{s} from {e} is already in the queue" for s, e, _ in picked if (s, e) in waiting]
+        if len({(s, e) for s, e, _ in picked}) < len(picked):
             errors.append("a model is picked twice")
         if errors:
             return {"ok": False, "errors": errors}
         if _OQ["worker"] is None:
             _OQ["items"] = []
         _OQ["stop"] = False                              # added after a Stop: these run
-        _OQ["items"] += [{"served": s, "endpoint": e, "state": "queued", "alias": None, "log": None,
-                          "exit": None, "note": ""} for s, e in picked]
+        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "state": "queued", "alias": None, "log": None,
+                          "exit": None, "note": ""} for s, e, m in picked]
         if _OQ["worker"] is None:
             _OQ["worker"] = threading.Thread(target=_onboard_worker, args=(who,), daemon=True)
             _OQ["worker"].start()
@@ -1692,7 +1705,7 @@ def _onboard_worker(who: str) -> None:
                 return
             nxt["state"] = "running"
         try:
-            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who)
+            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"))
             if proc is None:                             # refused now: the bench changed since it was queued
                 state, note, code = "failed", "; ".join(res["errors"]), None
             else:
@@ -2398,7 +2411,7 @@ def perf_card() -> str:
                   if r["model"] in live}
         widest = max((float(r["p50_s"]) for r in latest.values()), default=1) or 1
         body = ["<table><thead><tr>" + th("Model") + th("Prompt") + th("p50", True)
-                + th("p95", True) + th("ch/s", True) + th("Outcomes")
+                + th("p95", True) + th("ch/s", True) + th("tok/s", True) + th("Outcomes")
                 + "<th>p50, relative</th></tr></thead><tbody>"]
         for (model, prompt), row in sorted(latest.items()):
             p50 = float(row["p50_s"])
@@ -2411,6 +2424,7 @@ def perf_card() -> str:
                 f"<td class='num'>{p50:.1f}s</td>"
                 f"<td class='num'>{float(row['p95_s']):.1f}s</td>"
                 f"<td class='num'>{esc(row['chars_per_s'])}</td>"
+                f"<td class='num'>{esc(row.get('tokens_per_s') or '—')}</td>"
                 f"<td class='{'err' if bad else ''}'>{esc(oc)}</td>"
                 f"<td><span class='meter' style='width:{width}px' "
                 f"title='p50 {p50:.1f}s'><span style='width:100%'></span></span></td></tr>")
@@ -2428,6 +2442,9 @@ def perf_card() -> str:
         questions.</dd>
       <dt>ch/s</dt><dd>Characters per second, from the large prompt only. A
         proxy for tokens/s, affected by how verbose a model is.</dd>
+      <dt>tok/s</dt><dd>Output tokens per second from the provider's own count, first token to
+        last, large prompt only. Only where the built-in client made the call and the provider
+        reported usage; blank otherwise, never estimated.</dd>
       <dt>Outcomes</dt><dd>answered / refused / error / timeout.</dd>
       <dt>Weight it LEAST</dt><dd>Seconds are a wash. Speed only vetoes a model
         when it is slow enough to change how you work.</dd>
@@ -2549,8 +2566,9 @@ def gateway_card(limit: int = 30) -> str:
         explorers, then the merger if the reply was merged. Green answered, amber came back empty, red failed. ✓ judged
         useful, ✗ judged not useful; no mark means the verdict is still owed.
         Hover a chip for its seat, timing and call id; click it to read the answer
-        and record a verdict. A grey chip with a dash is a member not asked because the
-        request carried images it cannot take; hover it for why.</dd>
+        and record a verdict. A grey chip with a dash is a member not asked: the request
+        carried images it cannot take, or it is a vision specialist and the request
+        carried none. Hover it for why.</dd>
       <dt>Asked by</dt><dd>The gateway key that made the request, or <code>local</code>
         when no key is asked for.</dd>
       <dt>Owed</dt><dd>Seats without a verdict. Each is recorded as an unjudged
@@ -3219,7 +3237,9 @@ def endpoints_page() -> str:
         "<div class='panel form-panel'><h2>Rate limits per connection</h2>"
         "<p class='sub'>What a key behind each connection allows, from the provider's docs or"
         " freellmapi.co/models. Models on that connection share it; a model's own limits (its"
-        " popup) win, and limits the bench learns from errors only ever lower these.</p>"
+        " popup) win, and limits the bench learns from errors only ever lower these. rpmo is calls"
+        " a calendar month (UTC), for a trial key's monthly cap. The built-in client keeps to them:"
+        " when one is used up it waits if the call can still finish in time, else it does not send.</p>"
         "<div id='l-list'></div><div class='errors' id='l-errors' role='alert'></div>"
         "<div class='done' id='l-done'></div></div>")
 
@@ -3270,11 +3290,37 @@ def models_page() -> str:
 
 
 def quotas_page() -> str:
-    """What limits each connection, and who is resting after one. The live
-    numbers (what is left, when it resets) come with the built-in client's
-    quota records, RELEASE-PLAN Phase 2b."""
+    """What limits each connection, who is resting after one, what providers
+    last said was left (the built-in client's quota records, metrics/quota.py)
+    and what the bench itself sent (its usage ledger, metrics/ledger.py)."""
     import learn
+    import ledger
+    import quota
     now = datetime.datetime.now().astimezone()
+    rec_rows = []
+    for r in sorted(quota.live(), key=lambda r: (r.get("connection", ""), r.get("alias", ""), r.get("metric", ""))):
+        try:
+            age = int((now - datetime.datetime.fromisoformat(r["observed_at"])).total_seconds() // 60)
+            reset = datetime.datetime.fromisoformat(r["reset_at"]).astimezone().strftime("%Y-%m-%d %H:%M")
+        except (KeyError, TypeError, ValueError):
+            continue
+        num = lambda v: f"{v:,}" if isinstance(v, int) else "—"          # noqa: E731
+        rec_rows.append(f"<tr><td>{esc(r.get('connection', ''))}</td><td>{named(r.get('alias', ''))}</td>"
+                        f"<td>{esc(r.get('metric', ''))} / {esc(r.get('window', ''))}</td>"
+                        f"<td class='num'>{num(r.get('remaining'))}</td><td class='num'>{num(r.get('limit'))}</td>"
+                        f"<td class='mono'>{esc(reset)}</td>"
+                        f"<td>{esc(r.get('source', ''))}, {age} min ago</td></tr>")
+    t = now.timestamp()
+    sent: dict = {}
+    for row in ledger.rows(min(t - 86400, ledger.month_start(t))):
+        s = sent.setdefault(row.get("connection") or "?", [0, 0, 0, 0])
+        n, tok = row.get("attempts") or 0, (row.get("tokens_in") or 0) + (row.get("tokens_out") or 0)
+        if row["t"] >= t - 86400:
+            s[0], s[1] = s[0] + n, s[1] + tok
+        if row["t"] >= ledger.month_start(t):
+            s[2], s[3] = s[2] + n, s[3] + tok
+    sent_rows = [f"<tr><td>{esc(c)}</td>" + "".join(f"<td class='num'>{v:,}</td>" for v in s) + "</tr>"
+                 for c, s in sorted(sent.items())]
     cool_rows = []
     for alias, c in sorted(learn.cooldowns().items(), key=lambda kv: kv[1].get("until", "")):
         try:
@@ -3285,7 +3331,7 @@ def quotas_page() -> str:
                          f"<td class='num' data-sort='{left}'>{left} min</td>"
                          f"<td>{'stated by the provider' if c.get('stated') else 'default wait'}</td></tr>")
     entered = catalog.connection_limits()
-    keys = catalog.LIMIT_KEYS
+    keys = catalog.CONNECTION_LIMIT_KEYS
     lim_rows = [f"<tr><td>{esc(conn)}</td>" + "".join(
                     f"<td class='num'>{lim[k]:,}</td>" if lim.get(k) else "<td class='num muted'>—</td>" for k in keys)
                 + "<td>you</td></tr>" for conn, lim in sorted(entered.items()) if any(lim.get(k) for k in keys)]
@@ -3300,10 +3346,23 @@ def quotas_page() -> str:
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
     return (
         "<div class='panel'><h2>Quota management</h2><p class='sub'>What is left on each connection and"
-        " when it resets, read from provider endpoints, rate-limit headers and the bench's own call"
-        " ledger, arrives with the built-in client's quota records (RELEASE-PLAN Phase 2b). Until then:"
-        " who is resting after a rate limit, and every limit known. Quotas of OAuth credentials stay"
-        " in CLI Proxy API's own console.</p></div>"
+        " when it resets, from the rate-limit headers and 429 replies the built-in client sees, and what"
+        " the bench sent itself, from its usage ledger; who is resting after a rate limit, and every"
+        " limit known. The built-in client keeps to all of these. Calls through the llm CLI are in"
+        " neither record. Quotas of OAuth credentials stay in CLI Proxy API's own console.</p></div>"
+        "<div class='panel'><h3 class='chart-head'>What providers said is left · " + str(len(rec_rows)) + "</h3>"
+        + ("<table><thead><tr>" + th("Connection") + th("Model") + th("Counts") + th("Left", True)
+           + th("Of", True) + th("Resets") + th("From") + f"</tr></thead><tbody>{''.join(rec_rows)}</tbody></table>"
+           if rec_rows else "<p class='muted'>Nothing in force: no provider has stated a limit whose reset"
+                            " is still ahead.</p>")
+        + "</div>"
+        "<div class='panel'><h3 class='chart-head'>What the bench sent</h3><p class='sub'>Requests"
+        " (retries included) and tokens per connection, from the usage ledger.</p>"
+        + ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
+           + th("Requests, this month", True) + th("Tokens, this month", True)
+           + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
+           if sent_rows else "<p class='muted'>No call in the ledger this month.</p>")
+        + "</div>"
         "<div class='panel'><h3 class='chart-head'>Resting after a rate limit · " + str(len(cool_rows)) + "</h3>"
         + ("<table><thead><tr>" + th("Model") + th("Connection") + th("Back in", True) + th("Wait")
            + f"</tr></thead><tbody>{''.join(cool_rows)}</tbody></table>" if cool_rows
@@ -3476,8 +3535,10 @@ def editor() -> str:
         "<div class='actions'><button class='btn' id='co-save' type='button'>Save options</button></div>"
         "</details>"
         "<label class='check'><input type='checkbox' id='m-adhoc'> Ad hoc only</label>"
-        "<p class='rules'>For expensive or capped models: never seated automatically, never"
-        " swept, never probed daily. Used only when named.</p>"
+        "<p class='rules'>Never chosen by the bench on its own: no explorer seat, no router"
+        " pick, no scheduled suite, perf or availability runs. Still called whenever you choose"
+        " it: as a collection member (or merger), with <code>with</code>, or with"
+        " <code>--model</code>. For metered, capped or trial keys.</p>"
         "</div>"
         "<div class='errors' id='m-errors' role='alert'></div><div class='done' id='m-done'></div>"
         "<div class='actions'><button class='btn' id='m-close'>Close</button>"
@@ -3509,6 +3570,12 @@ def editor() -> str:
         "<input type='search' id='o-find' placeholder='Search models or connections'"
         " aria-label='Search models'>"
         "<div class='picker' id='o-picker' role='group' aria-label='Served models'></div>"
+        # serve.ONBOARD_MODES: how much each onboarding spends.
+        "<label class='check'>Onboard as <select id='o-mode'>"
+        "<option value='full'>Full: probe, then the suite now</option>"
+        "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
+        "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
+        " (metered, capped or trial keys)</option></select></label>"
         "</div>"
         "<div id='o-run' hidden><p class='served' id='o-result'></p><ol class='steps' id='o-steps'></ol>"
         "<details><summary class='muted'>Full log</summary><pre class='log' id='o-log'></pre></details></div>"
@@ -3755,7 +3822,8 @@ class Handler(gateway.Handler):
                                                            str(p.get("model") or ""),
                                                            endpoint=str(p.get("endpoint") or "")),
                "/api/onboard": lambda p: start_onboard(str(p.get("served", "")),
-                                                       str(p.get("endpoint", ""))),
+                                                       str(p.get("endpoint", "")),
+                                                       mode=str(p.get("mode") or "full")),
                "/api/onboard-queue": lambda p: queue_onboard(p.get("items")),
                "/api/onboard-queue/stop": lambda p: stop_onboard_queue(),
                "/api/onboard-queue/clear": lambda p: clear_onboard_queue(),

package/ui/bench.js

@@ -573,7 +573,8 @@
   // model at a time. The popup closes: the header's chip shows the batch, and
   // its Details the queue. Refused, it stays open with the reasons.
   o.start.addEventListener('click', function () {
-    var items = Object.keys(chosen).map(function (k) { return chosen[k]; });
+    var mode = $('o-mode').value;
+    var items = Object.keys(chosen).map(function (k) { return Object.assign({mode: mode}, chosen[k]); });
     if (!items.length) return;
     o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true; o.start.textContent = 'Starting…';
     post('/api/onboard-queue', {items: items}).then(function (res) {
@@ -1016,7 +1017,7 @@
         left.appendChild(el('div', {class: 'base'}, c.models.join(', ')));
         var inputs = el('div', {class: 'limits'});
         res.keys.forEach(function (k) {
-          var lab = el('label', {}, k + ' ');
+          var lab = el('label', {title: (res.labels || {})[k] || k}, k + ' ');
           var box = el('input', {type: 'number', min: '1', 'data-k': k, placeholder: '—'});
           if (c.limits[k]) box.value = c.limits[k];
           lab.appendChild(box); inputs.appendChild(lab);