@wendellresearch-ai/benchwarmers 0.6.0 → 0.6.2

package/USAGE.md

@@ -314,7 +314,8 @@ limit is that model's own:
 - **Pool**: OpenRouter's free requests a day, shared by its models, with a bar
   per key, against the limit OpenRouter stated in a rate-limit reply.
 - **Caps**: a day or month cap, counted from the bench's own calls: one you set
-  on Endpoints (Rate limits per connection), else one the provider states in
+  on Endpoints (Rate limits per connection), else a day cap the bench learned
+  for the connection (below), else one the provider states in
   its replies without saying what is left (a Cohere trial key's 1,000 calls a
   month), shown but not enforced until you set it. Calls made elsewhere with
   the same key are not in the count.
@@ -323,7 +324,14 @@ limit is that model's own:
 
 Nothing is taken from a provider's docs, which can change without notice; a
 limit a provider states in a rate-limit error (UnoRouter's "allows 1
-request(s) every 1 min" for a model) is learned and kept to. One exception,
+request(s) every 1 min" for a model) is learned and kept to. When the error
+names no number (Cloudflare Workers AI's, once the day's free Neurons are
+gone), the requests the bench had sent on that connection in that minute or
+day are taken as the connection's limit: noted the first time, kept to once a
+second refusal lands near the same count, unless you set your own for that
+connection on Endpoints, where each learned limit is shown with a Forget
+button. A learned rate limit not met again for 14 days is forgotten, so a
+raised limit is found. One exception,
 watched: NVIDIA states no limit per key, so its tile reads the file
 build.nvidia.com's own badge reads (10,000 requests a day, up to 40 a minute,
 the same for every account) daily and on Refresh, and counts the bench's

package/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.2

package/metrics/decide.py

@@ -34,6 +34,14 @@ URL, which nothing writes (an Open WebUI hub is given one by hand), and the
 onboarding hints about it show only for an Open WebUI hub (endpoints.hub). OpenRouter's /models omits its decision
 models, so that endpoint names them in `unlisted_models` to make them pickable.
 
+Cloudflare Workers AI answers decisions only on `/ai/run/<model>`, outside its
+OpenAI-compatible `/ai/v1` base, so no hub can relay them: its endpoint's
+`decisions` URL is `.../accounts/<id>/ai/run/{model}`. `{model}` puts the
+served id in the URL and leaves it out of the body (Cloudflare refuses
+`@cf/...` there), and its `{"result": {...}}` wrapper is taken off the reply.
+Its /models answers 405, so that endpoint lists from `models_url`, the
+catalog search (`.../ai/models/search`), read by endpoints.list_models.
+
 WHAT EACH MODEL CAN ANSWER
 --------------------------
 `question_types` lists the types a model accepted when onboarding probed it
@@ -235,7 +243,13 @@ def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
         headers = {"Content-Type": "application/json", **(entry.get("headers") or {})}
         if (secret := endpoints.key(endpoint)):
             headers["Authorization"] = f"Bearer {secret}"
-        body = json.dumps({"model": served, "state": state, "questions": questions}).encode()
+        payload = {"model": served, "state": state, "questions": questions}
+        if "{model}" in url:
+            # Cloudflare names the model in the URL (/ai/run/@cf/cloudflare/clef)
+            # and refuses its own id in the body ("clef", not "@cf/...", is valid there).
+            url = url.replace("{model}", served)
+            del payload["model"]
+        body = json.dumps(payload).encode()
         # Inception answers 429 to calls in quick succession (seen 2026-10-16:
         # the second of three onboarding probes), so a 429 waits and retries.
         for wait in ((5, 15, 0) if retry else (0,)):
@@ -243,6 +257,9 @@ def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
             try:
                 with urllib.request.urlopen(request, timeout=timeout) as response:
                     reply = json.load(response)
+                    # Cloudflare wraps every reply: {"result": {"answers": ...}, "success": true}.
+                    if isinstance(reply, dict) and "answers" not in reply and isinstance(reply.get("result"), dict):
+                        reply = reply["result"]
                     return reply
             except urllib.error.HTTPError as exc:
                 if exc.code != 429 or not wait:
@@ -358,14 +375,23 @@ def refuses_chat(endpoint: str, served: str) -> bool:
         with urllib.request.urlopen(request, timeout=60):
             return False
     except urllib.error.HTTPError as exc:
-        # OpenRouter says "is a decisions model"; other providers word it their
-        # own way, so any refusal that names decisions or systemone counts.
-        text = error_text(exc).lower()
-        return any(w in text for w in ("decisions model", "decision model", "systemone", "/decisions"))
+        return names_decisions(error_text(exc))
     except Exception:                                  # noqa: BLE001
         return False
 
 
+def names_decisions(text: str) -> bool:
+    """A chat refusal that says the model is a decision model. OpenRouter names
+    it ("is a decisions model"); Cloudflare only lists the fields it wanted
+    ("required properties ... 'model,state,questions'", "Required value
+    missing: questions"). Every Decisions API takes `state` and `questions`,
+    so a refusal asking for both counts, whoever words it."""
+    low = (text or "").lower()
+    if any(w in low for w in ("decisions model", "decision model", "systemone", "/decisions")):
+        return True
+    return "questions" in low and ("state" in low or "required" in low)
+
+
 def question_types(endpoint: str, served: str, path: str = "decisions") -> list[str]:
     """Which question types the model accepts, one probe call per type.
 

package/metrics/endpoints.py

@@ -295,7 +295,8 @@ def models(name: str | None = None, timeout: int = 30) -> dict[str, dict]:
 def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, dict]:
     """What an endpoint entry lists, saved or not yet (the dashboard tests one
     before saving it). `models_url` when the list lives elsewhere (GitHub
-    Models: its catalog, a bare JSON list); entries marked `visibility` other
+    Models: its catalog, a bare JSON list; Cloudflare: its catalog search,
+    {"result": [...]} by `name`); entries marked `visibility` other
     than "list" are left out (ChatGPT's plan lists ones it will not serve)."""
     api = entry.get("api") or "chat"
     headers = dict(entry.get("headers") or {})
@@ -313,6 +314,9 @@ def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, d
         # what generateContent serves, by its bare name.
         rows = [{**m, "id": str(m.get("name", "")).removeprefix("models/")} for m in data.get("models", [])
                 if isinstance(m, dict) and "generateContent" in (m.get("supportedGenerationMethods") or ())]
+    elif isinstance(data, dict) and isinstance(data.get("result"), list):
+        # Cloudflare's catalog search: {"result": [{"name": "@cf/...", ...}]}.
+        rows = [{**m, "id": m.get("name")} for m in data["result"] if isinstance(m, dict)]
     else:
         rows = data.get("data", []) if isinstance(data, dict) else data
     return {e["id"]: e for e in rows

package/metrics/learn.py

@@ -13,18 +13,40 @@ learns exactly these at runtime, and so does this: every failed call from the
 perf probe, the suite (and so suite/pace.py) and the gateway passes its error
 text through `record()`.
 
-ONLY EVER TIGHTER
------------------
+ONLY EVER TIGHTER, UNTIL IT EXPIRES
+-----------------------------------
 A learned context window or rate limit is kept only if it is LOWER than what is
 known, and replaces an older learned value only when lower still. An error can
 prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
 says. Values you entered in the model popup (config/model-overrides.json)
 still win: you can always correct a bad parse.
 
-    config/model-learned.json   alias -> {context_window, limits, capabilities, served_as}
-                                with the date and the evidence for each
-    runs/cooldowns.json         alias -> {until, why}: rate-limited, come back later.
-                                Not config: it expires, and is nobody's decision.
+A rate limit can be raised, though (a plan upgrade, a provider's change), and
+no error ever says so. So a learned RATE limit that has not been seen again for
+LEARNED_TTL_S (14 days) is left out wherever it is read: the bench goes back to
+the higher rate, and the provider's next refusal teaches the limit again. The
+same limit seen again refreshes its `confirmed_at`, so one that still holds
+does not expire. A context window does not: a model's window does not grow.
+
+A 429 THAT NAMES NO NUMBER
+--------------------------
+Cloudflare Workers AI says only that the day's free Neurons are used up. The
+bench knows how many requests it sent on that connection in the window before
+the refusal (metrics/ledger.py), and that count is the limit, near enough. It
+is a connection's limit, not a model's: the key's allowance is shared. One
+refusal is a candidate; a second at a count within 20% of it, a window or more
+later (sooner is the same refusal, met by another call), is adopted (the lower
+of the two) and enforced by metrics/pacing.py unless you set your own for that
+connection. Seen again near the same count it is confirmed; a refusal at a
+clearly lower count lowers it at once. Only rpm and rpd, and never from the
+bench's own refusal.
+
+    config/model-learned.json        alias -> {context_window, limits, capabilities, served_as}
+                                     with the date and the evidence for each
+    config/connection-learned.json   connection -> {rpm, rpd}: {value, at, confirmed_at, where,
+                                     evidence}, or the candidates until one is adopted
+    runs/cooldowns.json              alias -> {until, why}: rate-limited, come back later.
+                                     Not config: it expires, and is nobody's decision.
 """
 
 from __future__ import annotations
@@ -40,9 +62,13 @@ import locked
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
 LEARNED = DATA / "config" / "model-learned.json"
+CONN_LEARNED = DATA / "config" / "connection-learned.json"
 COOLDOWNS = DATA / "runs" / "cooldowns.json"
 DEFAULT_COOLDOWN_S = 1800          # a 429 with no stated wait
 MAX_COOLDOWN_S = 24 * 3600
+LEARNED_TTL_S = 14 * 86400         # a learned rate limit not seen again for this long is forgotten
+CONN_AXES = {"rpm": 60, "rpd": 86400}     # what a connection's limit is learned for: its window, seconds
+MIN_COUNT = 3                      # fewer requests in the window than this prove nothing
 
 CONTEXT = [
     r"maximum context length is\s*([\d,]+)",
@@ -71,6 +97,10 @@ ALLOWS = re.compile(r"allows?\s+([\d,]+)\s+requests?(?:\(s\))?\s+every\s+(?:1\s+
 # new-api gateways (Agnes): "maximum 20 requests in 1 minutes" (its
 # middleware/model-rate-limit.go); a window of 1 minute or 1,440 only.
 MAXIMUM = re.compile(r"maximum\s+([\d,]+)\s+requests?\s+in\s+(1|1440)\s+minutes?\b")
+# Which window a 429 with no number is about. A window, not a wait: "try
+# again in 3 minutes" names no per-minute limit.
+DAY = re.compile(r"\bday\b|daily|free allocation|neurons")
+MINUTE = re.compile(r"per[- ]?min(?:ute)?\b|a minute|\brpm\b")
 
 
 def _int(s: str) -> int:
@@ -108,18 +138,64 @@ def parse(text: str) -> dict:
 _read = locked.read_json
 
 
+def _age(stamp, now: dt.datetime) -> float:
+    """Seconds since an ISO stamp; forever when it is missing or unreadable."""
+    try:
+        return (now - dt.datetime.fromisoformat(stamp)).total_seconds()
+    except (TypeError, ValueError):
+        return float("inf")
+
+
+def _live(ev: dict, now: dt.datetime) -> bool:
+    """Seen within LEARNED_TTL_S: an older rate limit may have been raised since."""
+    return _age(ev.get("confirmed_at") or ev.get("at"), now) <= LEARNED_TTL_S
+
+
 def learned() -> dict:
-    return {k: v for k, v in _read(LEARNED).items() if not k.startswith("_")}
+    """alias -> what was learned, without the rate limits that have expired."""
+    now = dt.datetime.now().astimezone()
+    out = {}
+    for alias, e in _read(LEARNED).items():
+        if alias.startswith("_"):
+            continue
+        if isinstance(e, dict) and e.get("limits"):
+            e = {**e, "limits": {k: ev for k, ev in e["limits"].items() if _live(ev, now)}}
+        out[alias] = e
+    return out
+
+
+def connection_learned(pending: bool = False) -> dict:
+    """connection -> {axis: {value, at, confirmed_at, where, evidence}}: the
+    adopted limits that have not expired. pending=True: for the axes with none,
+    the latest candidate instead ({n, at}), seen once and not enforced."""
+    now = dt.datetime.now().astimezone()
+    out: dict = {}
+    for conn, axes in _read(CONN_LEARNED).items():
+        if conn.startswith("_") or not isinstance(axes, dict):
+            continue
+        for axis, e in axes.items():
+            if axis not in CONN_AXES or not isinstance(e, dict):
+                continue
+            adopted = bool(e.get("value")) and _live(e, now)
+            waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
+            if adopted and not pending:
+                out.setdefault(conn, {})[axis] = {k: v for k, v in e.items() if k != "candidates"}
+            elif pending and not adopted and waiting:
+                out.setdefault(conn, {})[axis] = waiting[-1]
+    return out
 
 
 def _known(alias: str, field: str, axis: str | None = None):
-    """The value a learned figure has to beat: the catalog's, else the last learned."""
+    """The value a learned figure has to beat: the catalog's, else the last
+    learned; for a rate limit, its connection's too, yours and the learned."""
     import catalog
     auto = catalog.catalog().get(alias, {})
     mine = learned().get(alias, {})
     if axis:
+        conn = catalog.connection_of(alias)
         vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
-                catalog.connection_limits().get(catalog.connection_of(alias), {}).get(axis)]
+                catalog.connection_limits().get(conn, {}).get(axis),
+                connection_learned().get(conn, {}).get(axis, {}).get("value")]
     else:
         vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
     vals = [v for v in vals if v]
@@ -137,10 +213,13 @@ def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) ->
             entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
             stored["context_window"] = n
         for axis, n in (found.get("limits") or {}).items():
-            if n < (_known(alias, "", axis) or 1 << 62):
+            had = (entry.get("limits") or {}).get(axis) or {}
+            if n < (_known(alias, "", axis) or 1 << 62):   # an expired one is not known: a higher value may replace it
                 entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
                                                         "evidence": evidence}
                 stored[axis] = n
+            elif had.get("value") == n:
+                had["confirmed_at"] = stamp                # still true, so it does not expire
         if stored:
             data[alias] = entry
             data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
@@ -158,6 +237,75 @@ def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
     return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
 
 
+def _axis(text: str, found: dict) -> str | None:
+    """The window a 429 that names no number is about, or None: nothing to
+    learn. Never the bench's own refusal ("not sent", pacing.py): a limit the
+    bench enforced must not be learned back from itself. Nor a token limit: the
+    count is of requests."""
+    low = (text or "").lower()
+    if not found.get("rate_limited") or found.get("limits") or "not sent" in low \
+            or re.search(AXES["tpm"] + "|" + AXES["tpd"], low):
+        return None
+    wait = found.get("retry_after_s")
+    if DAY.search(low) or (wait or 0) >= 3600:
+        return "rpd"
+    # A short wait alone is not enough for rpm: calls in flight are not in the
+    # ledger yet, so a minute's count undercounts, and two passing 429s could
+    # pin a connection at rpm 3 for 14 days. Only a named minute window counts.
+    if MINUTE.search(low):
+        return "rpm"
+    return None
+
+
+def _sent(connection: str, since: float) -> int:
+    """Requests the provider took on a connection since then: the ledger's
+    attempts, without the calls that failed, whose own retries would count the
+    refusal as use."""
+    import ledger
+    return sum(r.get("attempts") or 0 for r in ledger.rows(since)
+               if r.get("connection") == connection and r.get("status") != "error")
+
+
+def _near(n: int, value: int) -> bool:
+    return 0.8 * value <= n <= 1.2 * value
+
+
+def _learn_connection(connection: str, axis: str, n: int, where: str, evidence: str, now: dt.datetime) -> dict:
+    """A connection refused after `n` requests in the axis's window. Returns
+    {axis: value} when that adopted a limit or lowered one."""
+    stamp, span = now.isoformat(timespec="seconds"), CONN_AXES[axis]
+    stored: dict = {}
+
+    def change(data: dict) -> None:
+        axes = data.setdefault(connection, {})
+        e = axes.get(axis) or {}
+        fresh = {"value": n, "at": stamp, "confirmed_at": stamp, "where": where, "evidence": evidence}
+        waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
+        if e.get("value") and _live(e, now):
+            # One lesson a window: while a provider keeps refusing, the count
+            # of what it took only falls, and would drag the limit down with it.
+            if _age(e.get("confirmed_at") or e.get("at"), now) < span:
+                return
+            if n < 0.8 * e["value"]:                   # refused well below it: that is proof
+                axes[axis] = fresh
+                stored[axis] = n
+            elif _near(n, e["value"]):
+                e["confirmed_at"] = stamp
+        elif waiting and _age(waiting[-1]["at"], now) < span:
+            return                                     # the same refusal, met by another call
+        elif agree := [c["n"] for c in waiting if _near(n, c["n"])]:
+            axes[axis] = fresh | {"value": min(agree + [n])}
+            stored[axis] = axes[axis]["value"]
+        else:
+            axes[axis] = {"candidates": (waiting + [{"n": n, "at": stamp}])[-5:]}
+        data.setdefault("_why", "Learned by metrics/learn.py from 429s that named no number: the requests"
+                                " the bench had sent in the window. Enforced unless you set your own"
+                                " (config/connection-limits.json); forgotten 14 days after last seen.")
+
+    locked.update(CONN_LEARNED, change, ensure_ascii=False)
+    return stored
+
+
 def record(alias: str, text: str, where: str) -> dict:
     """Learn from one failed call. Returns what was stored or set, for logging."""
     found = parse(text)
@@ -165,6 +313,14 @@ def record(alias: str, text: str, where: str) -> dict:
     stamp = now.isoformat(timespec="seconds")
     evidence = " ".join((text or "").split())[:200]
     stored = _tighten(alias, found, where, evidence, stamp)
+    if alias and (axis := _axis(text, found)):
+        try:
+            import catalog
+            conn = catalog.connection_of(alias)
+            if conn and (n := _sent(conn, now.timestamp() - CONN_AXES[axis])) >= MIN_COUNT:
+                stored.update(_learn_connection(conn, axis, n, where, evidence, now))
+        except Exception:                              # noqa: BLE001  -- learning never breaks a call
+            pass
     if alias and found.get("rate_limited"):
         wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
         cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
@@ -231,9 +387,104 @@ def _selftest() -> None:
     assert "limits" not in parse("maximum 100 requests in 5 minutes")         # a window no axis holds
     assert parse("model not found") == {"rate_limited": False}
     assert parse("context length of 8k exceeded").get("context_window") is None   # "8" < 512
+
+    # Which window a 429 with no number is about.
+    def axis(text: str):
+        return _axis(text, parse(text))
+    neurons = "Error code: 429 - you have used up your daily free allocation of 10,000 neurons"
+    assert axis(neurons) == "rpd" and "limits" not in parse(neurons)
+    assert axis("Rate limit exceeded: free-models-per-day. Try again in 3 minutes") == "rpd"
+    assert axis("429 Too Many Requests, retry-after: 7200") == "rpd"
+    assert axis("429 Too Many Requests, retry-after: 30") is None        # a short wait alone: no window
+    assert axis("429: too many requests per minute") == "rpm"
+    assert axis("429 Too Many Requests") is None                         # no window named
+    assert axis("Rate limit exceeded. Try again in 3 minutes") is None    # a wait, not a window
+    assert axis("429: rate limit on tokens per minute") is None          # requests are what is counted
+    assert axis("Limit 30, Used 30 (requests per minute) 429") is None   # it gave the number
+    assert axis("model not found") is None
+    # The bench's own refusal is never learned from.
+    refusal = ("Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests"
+               " per day (learned) and 50 were used; try again in 2h")
+    assert axis(refusal) is None and axis(refusal.replace("not sent; ", "")) == "rpd"
+    _selftest_stores(neurons, refusal)
     print("ok")
 
 
+def _selftest_stores(neurons: str, refusal: str) -> None:
+    """The connection store and the expiry, in a temp folder with a stub
+    catalog: nothing of the install's is read or written."""
+    import tempfile
+
+    import catalog
+    import ledger
+    with tempfile.TemporaryDirectory() as folder:
+        tmp = pathlib.Path(folder)
+        globals().update(LEARNED=tmp / "model-learned.json", CONN_LEARNED=tmp / "connection-learned.json",
+                         COOLDOWNS=tmp / "cooldowns.json")
+        ledger.DIR = tmp / "usage"
+        catalog.catalog = catalog.connection_limits = lambda: {}
+        catalog.connection_of = lambda alias: "conn-" + alias
+        now = dt.datetime.now().astimezone()
+
+        def ago(**kw) -> dt.datetime:
+            return now - dt.timedelta(**kw)
+
+        def refused(n: int, when: dt.datetime, conn: str = "prov", ax: str = "rpd") -> dict:
+            return _learn_connection(conn, ax, n, "test", "429", when)
+        # One refusal is a candidate; the same refusal met again changes nothing.
+        assert refused(100, ago(days=5)) == {} and connection_learned() == {}
+        assert connection_learned(pending=True)["prov"]["rpd"]["n"] == 100
+        assert refused(104, ago(days=5, hours=-1)) == {} and connection_learned() == {}
+        # A second count far from the first does not adopt; one within 20% does, the lower of the two.
+        assert refused(300, ago(days=4)) == {} and connection_learned() == {}
+        assert refused(110, ago(days=3)) == {"rpd": 100}
+        got = connection_learned()["prov"]["rpd"]
+        assert got["value"] == 100 and "candidates" not in got and connection_learned(pending=True) == {}, got
+        # Seen again near it: confirmed. Clearly lower: lowered at once. Higher: left alone.
+        assert refused(95, ago(days=2)) == {}
+        got = connection_learned()["prov"]["rpd"]
+        assert got["value"] == 100 and got["confirmed_at"] == ago(days=2).isoformat(timespec="seconds") != got["at"]
+        assert refused(40, ago(days=2, hours=-1)) == {}                      # within the window: the same refusal
+        assert refused(500, ago(days=1)) == {} and connection_learned()["prov"]["rpd"]["value"] == 100
+        assert refused(60, now) == {"rpd": 60} and connection_learned()["prov"]["rpd"]["value"] == 60
+        # Not seen for 15 days: forgotten on read, and a higher limit can be learned.
+        assert refused(20, ago(days=16), "old", "rpm") == {} and refused(20, ago(days=15), "old", "rpm") == {"rpm": 20}
+        assert "old" not in connection_learned()
+        assert refused(50, ago(minutes=5), "old", "rpm") == {} and refused(50, now, "old", "rpm") == {"rpm": 50}
+
+        # A model's own learned limit: the same expiry, and the same limit seen again is confirmed.
+        old = {"value": 5, "at": ago(days=15).isoformat(timespec="seconds"), "where": "test", "evidence": ""}
+        LEARNED.write_text(json.dumps({"m": {"limits": {"rpm": dict(old), "rpd": dict(old)},
+                                             "context_window": dict(old, value=8192)}}), encoding="utf-8")
+        assert learned()["m"]["limits"] == {} and learned()["m"]["context_window"]["value"] == 8192
+        stamp = now.isoformat(timespec="seconds")
+        assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", stamp) == {"rpm": 20}
+        assert _tighten("m", {"limits": {"rpm": 30}}, "test", "", stamp) == {}        # live: only ever tighter
+        assert "confirmed_at" not in learned()["m"]["limits"]["rpm"]
+        again = ago(hours=-1).isoformat(timespec="seconds")
+        assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", again) == {}
+        got = learned()["m"]["limits"]["rpm"]
+        assert got["value"] == 20 and got["at"] == stamp and got["confirmed_at"] == again, got
+        # A model's limit must beat its connection's learned one too.
+        assert refused(10, ago(minutes=5), "conn-m2", "rpm") == {} and refused(10, now, "conn-m2", "rpm") == {"rpm": 10}
+        assert _tighten("m2", {"limits": {"rpm": 60}}, "test", "", stamp) == {}
+        assert _tighten("m2", {"limits": {"rpm": 6}}, "test", "", stamp) == {"rpm": 6}
+
+        # record(): the count is the ledger's, of what the provider took on that connection.
+        def sent(conn: str, n: int, status: str = "ok", attempts: int = 1) -> None:
+            for i in range(n):
+                ledger.append({"t": now.timestamp() - 30 - i, "alias": "x", "connection": conn, "status": status,
+                               "attempts": attempts})
+        sent("conn-r", 4), sent("conn-r", 1, "error", 3), sent("elsewhere", 9), sent("conn-p", 4), sent("conn-few", 2)
+        assert record("r", neurons, "test") == {"cooldown_s": DEFAULT_COOLDOWN_S}     # a candidate: nothing stored yet
+        assert connection_learned(pending=True)["conn-r"]["rpd"]["n"] == 4
+        assert record("few", neurons, "test") and "conn-few" not in connection_learned(pending=True)   # too few to say
+        # The bench's own refusal cools the model as before, and teaches nothing.
+        assert record("p", refusal, "test") == {"cooldown_s": 7200} and "p" in cooldowns()
+        assert "conn-p" not in connection_learned(pending=True)
+
+
+
 def main() -> None:
     sys.stdout.reconfigure(encoding="utf-8", errors="replace")
     if "--selftest" in sys.argv:
@@ -248,6 +499,9 @@ def main() -> None:
         if e.get("served_as"):
             bits.append(f"served as {e['served_as']['served']}")
         print(f"  {alias:<34} {', '.join(bits)}")
+    for conn, axes in sorted(connection_learned().items()):
+        bits = [f"{k} {v['value']}" for k, v in axes.items()]
+        print(f"  {conn:<34} {', '.join(bits)}  (the connection's, from 429s with no number)")
     for alias, c in sorted(cooldowns().items()):
         print(f"  cooling  {alias:<26} until {c['until'][11:16]}  ({c['why'][:70]})")
 

package/metrics/pacing.py

@@ -28,7 +28,9 @@ WHAT IS COUNTED
   deadline is dropped, so a killed process cannot hold a slot.
 - A connection's limits count every model on it (a key's limits are the
   key's); a model's own or learned limits count that model alone, its own
-  beating the learned (as everywhere: your values win).
+  beating the learned (as everywhere: your values win). Where you set none
+  for a connection, the rpm or rpd learn.py worked out from its 429s is kept
+  to (config/connection-learned.json), until it expires.
 - Quota records (metrics/quota.py): what a provider last said was left, less
   the calls sent since, until its reset.
 
@@ -77,6 +79,7 @@ def rules(alias: str, connection: str) -> list[dict]:
     conn = catalog.connection_limits().get(connection) or {}
     mine = (catalog.overrides().get(alias) or {}).get("limits") or {}
     seen = (learn.learned().get(alias) or {}).get("limits") or {}
+    met = learn.connection_learned().get(connection) or {}
     out = []
 
     def ok(n) -> bool:
@@ -85,6 +88,10 @@ def rules(alias: str, connection: str) -> list[dict]:
         if ok(n := conn.get(axis)):
             out.append({"scope": "connection", "axis": axis, "limit": n,
                         "source": f"you, for {connection} (config/connection-limits.json)"})
+        elif ok(n := (met.get(axis) or {}).get("value")):
+            out.append({"scope": "connection", "axis": axis, "limit": n,
+                        "source": f"learned {str(met[axis].get('at', ''))[:10]} from 429s on {connection}"
+                                  " (config/connection-learned.json)"})
         if axis == "rpmo":
             continue
         if ok(n := mine.get(axis)):

package/metrics/quotaview.py

@@ -512,14 +512,21 @@ def _limit_text(lim: dict) -> str:
     return ", ".join(f"{v:,} {words[k]}" for k, v in lim.items() if k in words and v)
 
 
-def caps(host: str, entered: dict, stated: dict, day_rows: list, month_rows, now: float) -> list[dict]:
+def caps(host: str, entered: dict, stated: dict, day_rows: list, month_rows, now: float,
+         learned: dict | None = None) -> list[dict]:
     """The connection's day and month caps as groups of windows, each counted
     from the bench's ledger since midnight UTC or the month's start (what a
-    provider's calendar counts): yours, else `stated` ({axis: (limit, source,
-    detail)}: what its replies say with nothing left given, or NVIDIA's site).
+    provider's calendar counts): yours, else `learned` (learn.py's, from this
+    connection's 429s), else `stated` ({axis: (limit, source, detail)}: what
+    its replies say with nothing left given, or NVIDIA's site).
     `month_rows` is called only when a month cap needs it."""
     mine = {k: (v, "your cap", "Set on Endpoints, Rate limits per connection; the bench keeps to it.")
             for k, v in (entered.get(host) or {}).items() if k in COUNTED and v}
+    for axis, ev in (learned or {}).items():
+        if axis in COUNTED:
+            mine.setdefault(axis, (ev["value"], "learned", f"Worked out {str(ev.get('at'))[:10]} from this connection's"
+                                   " 429s, which named no number; the bench keeps to it until you set your own"
+                                   " on Endpoints, or it has not been met again for 14 days."))
     for axis, (n, source, detail) in stated.items():
         mine.setdefault(axis, (n, source, detail + " Shown, not enforced: set it on Endpoints to have the bench"
                                                    " keep to it."))
@@ -554,7 +561,7 @@ def tiles(now: float | None = None) -> list[dict]:
     resting, learned = learn.cooldowns(), learn.learned()
     day_rows = list(ledger.rows(now - 86400))
     utc_rows = [r for r in day_rows if r["t"] >= _utc_midnight(now)]
-    entered = catalog.connection_limits()
+    entered, met = catalog.connection_limits(), learn.connection_learned()
     regs = __import__("endpoints").registrations()
     month: list = []                                   # this month's ledger, read once if a month cap asks
 
@@ -589,7 +596,7 @@ def tiles(now: float | None = None) -> list[dict]:
         t = {"id": tag, "tag": tag, "host": host, "bench": len(aliases), "kind": "models", "groups": [],
              "pool": [], "notes": [],          # a day or month cap is a meter (caps), a minute one is said
              "conn_limit": _limit_text({k: v for k, v in (entered.get(host) or {}).items() if k in ("rpm", "tpm")}),
-             "caps": caps(host, entered, stated, day_rows, month_rows, now),
+             "caps": caps(host, entered, stated, day_rows, month_rows, now, met.get(host)),
              "models": _model_rows(aliases, rows, resting, limits),
              "refresh": how_refreshed(tag), "plan": "", "name": ""}
         got = live.get(tag) or {}
@@ -743,6 +750,13 @@ def _selftest() -> None:
     g = caps("router.requesty.ai", {"router.requesty.ai": {"rpd": 10, "rpm": 5}}, {"rpd": said}, day, lambda: [], now)
     assert g[0]["label"] == "your cap · counted by the bench" and g[0]["windows"] == [
         {"label": "requests · day", "left": 80, "remaining": 8, "of": 10, "reset_at": _iso(mid + 86400)}], g
+    # A learned day cap (learn.py, from 429s) beats a stated one, and yours beats it; a learned rpm is no meter.
+    met = {"rpd": {"value": 20, "at": "2026-11-10T09:00:00+00:00"}, "rpm": {"value": 4, "at": ""}}
+    g = caps("router.requesty.ai", {}, {"rpd": said}, day, lambda: [], now, met)
+    assert len(g) == 1 and g[0]["label"] == "learned · counted by the bench" and "2026-11-10" in g[0]["title"] \
+        and g[0]["windows"][0]["of"] == 20 and g[0]["windows"][0]["remaining"] == 18, g
+    g = caps("router.requesty.ai", {"router.requesty.ai": {"rpd": 10}}, {}, day, lambda: [], now, met)
+    assert len(g) == 1 and g[0]["label"] == "your cap · counted by the bench", g
     asked = []
     g = caps("api.cohere.ai", {}, {"rpmo": (1000, "its replies", "")}, day, lambda: asked.append(1) or day, now)
     assert asked and g[0]["windows"][0]["remaining"] == 999 and g[0]["windows"][0]["label"] == "requests · month", g

package/metrics/route.py

@@ -148,12 +148,14 @@ def _full_tag_of(alias: str) -> str:
     import endpoints
     # Tags are keyed by the served name. A decision model is not in the llm
     # yaml, so its served name comes from config/decision-models.json.
-    served = (_registered_names().get(alias)
-              or (_json(DATA / "config" / "decision-models.json").get(alias) or {}).get("served")
-              or alias)
+    decision = _json(DATA / "config" / "decision-models.json").get(alias) or {}
+    served = _registered_names().get(alias) or decision.get("served") or alias
     tag = _json(DATA / "config" / "provider-tags.json").get(served)
     if not tag:
-        name = endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+        # A decision model is in no registration either: its own endpoint is its `via`
+        # (Cloudflare's clef, onboarded from a direct cloudflare endpoint).
+        name = (endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+                or decision.get("via") or decision.get("endpoint"))
         tag = name if name and not endpoints.hub(name) else None
     return tag or "?"
 

package/metrics/test_decide.py

@@ -29,7 +29,9 @@ def jsonl(path: str, rows: list[dict]) -> None:
 
 
 write("config/endpoints.json", {"hub": {"base": "http://127.0.0.1:9/api", "key": "hub", "default": True,
-                                        "decisions": "http://127.0.0.1:9/openai/{path}"}})
+                                        "decisions": "http://127.0.0.1:9/openai/{path}"},
+                                "cf": {"base": "http://127.0.0.1:9/ai/v1", "key": "hub",
+                                       "decisions": "http://127.0.0.1:9/ai/run/{model}"}})
 # dm2 first: before any result it is the one asked (the first that takes choice, as before),
 # except per request, where its :free budget is spared while dm1 can answer.
 MODELS = {
@@ -91,6 +93,14 @@ assert "passthrough" in decide.error_text(http_error(b'{"detail": "Direct API pa
 assert decide.error_text(http_error(b'{"error": null, "message": "down"}')).startswith("400")
 assert decide.error_text(http_error(b"not json")) == "400"
 
+# A chat refusal says "decision model" however the provider words it.
+assert decide.names_decisions("400 is a decisions model and cannot be used with chat")             # OpenRouter
+assert decide.names_decisions("400 {'errors': [{'message': \"AiError: Bad input: Error: required properties"
+                              " at '/' are 'model,state,questions' (72ae)\", 'code': 5006}]}")     # Cloudflare
+assert decide.names_decisions("400 Model execution failed (User Input Error): Required value missing: questions.")
+assert not decide.names_decisions("400 messages: Required")       # Cloudflare's v1 refusing a decision body
+assert not decide.names_decisions("429 Rate limit exceeded") and not decide.names_decisions("")
+
 # ask() refuses a type the entry does not list, before any call is made (and logs nothing).
 try:
     decide.ask("yesno", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
@@ -124,6 +134,20 @@ def urlopen(request, timeout=0):
     return io.BytesIO(json.dumps({"answers": {name: answer}}).encode())
 
 
+
+
+# Cloudflare: the model in the URL and not the body, the reply in a "result" wrapper.
+def cf_urlopen(request, timeout=0):
+    body = json.loads(request.data)
+    assert request.full_url == "http://127.0.0.1:9/ai/run/@cf/cloudflare/clef-flash" and "model" not in body, body
+    return io.BytesIO(json.dumps({"result": {"model": "clef-flash", "answers": {"probe": {"type": "noul", "noul": 0.98}}},
+                                  "success": True, "errors": []}).encode())
+
+
+decide.urllib.request.urlopen = cf_urlopen
+assert decide.probe("cf", "@cf/cloudflare/clef-flash") == (None, "decisions")
+assert decide.log_rows()[-1]["answers"]["probe"]["noul"] == 0.98
+decide.LOG.unlink()
 decide.urllib.request.urlopen = urlopen
 
 # ---- every call is logged, and an error's body survives the log -------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/onboard.py

@@ -286,7 +286,13 @@ def main() -> None:
             return
         # The passthrough hints are about Open WebUI; anywhere else a 404 is the provider's.
         owui = endpoints.hub(endpoint) == "openwebui"
-        hint = ("  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
+        # ponytail: Cloudflare told apart by its @cf/ ids; its partner models (no @cf/) get the generic hint.
+        hint = ("  Cloudflare answers decisions only on its /ai/run route, which no hub relays: add"
+                " Cloudflare as a direct endpoint (decisions .../accounts/<id>/ai/run/{model},"
+                " models_url .../ai/models/search?per_page=1000) and onboard it from there.\n"
+                if "@cf/" in name and endpoint != "cloudflare" and "{model}" not in
+                (endpoints.load()[endpoint].get("decisions") or "") else
+                "  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
                 " in its environment.\n" if owui and "passthrough" in why.lower() else
                 "  Its connection has no /decisions under its base. An OpenRouter-hosted decision"
                 " model cannot go through Open WebUI (its Decisions API is outside /api/v1);"

package/scripts/test_no_owui.py

@@ -236,6 +236,11 @@ def owner(fake: str) -> None:
     assert tags["fast-1"] == "groq" and tags["llm7.codestral"] == "llm7-api" and tags["vendor/x:free"] == "openrouter", tags
     assert "never-read" not in json.dumps(tags)
     assert route.connection_of("fast-1") == "groq" and route.connection_of("x-free") == "openrouter"
+    # A decision model onboarded from the direct endpoint: in no registration, untagged, its `via` is its connection.
+    write("decision-models.json", {"dec-1": {"served": "vendor/decide:free", "endpoint": "openrouter", "via": "openrouter"},
+                                   "dec-hub": {"served": "hub-decide", "endpoint": "openwebui", "via": "openwebui"}})
+    assert route.connection_of("dec-1") == "openrouter" and route.connection_of("dec-hub") == "?"
+    (TMP / "config" / "decision-models.json").unlink()
 
     # The default hub is recorded whole; the direct endpoint, its registered models only.
     assert check_deprecations.served() == {"fast-1", "llm7.codestral", "fresh-2", "mercury-decide",

package/scripts/test_quota.py

@@ -20,6 +20,8 @@ not mix. Covered:
   a Gemini 429 body's daily quota learned as a limit;
 - an rpm limit across six threads at once and across two processes (calls in
   flight count), a monthly cap (rpmo), and both refusals classed as capacity;
+- a connection's learned limit (metrics/learn.py) enforced, its refusal not
+  learned from, your own value winning over it, and Forget;
 - the free-tier guard holding a model that went paid, until it is allowed;
 - metrics/cost.py reading the ledger.
 """
@@ -331,6 +333,31 @@ def main() -> int:
     check(not bad["ok"] and serve.save_model_facts("t-ok", {"limits": {"rpmo": 5}}).get("ok") is not None
           and "rpmo" not in ((serve.catalog.overrides().get("t-ok") or {}).get("limits") or {}),
           "a bad rpmo is refused; a model's own limits have none", bad)
+    # A connection's learned limit (learn.py: 429s that named no number) is kept to like one of yours, which wins.
+    import datetime as dt
+    at = dt.datetime.now().astimezone()
+    learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=2))
+    got = learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=1))
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    h = pacing.held("t-month", "monthprov")
+    check(got == {"rpd": 3} and len(rule) == 1 and rule[0]["scope"] == "connection" and rule[0]["limit"] == 3
+          and rule[0]["source"].startswith("learned ") and h is not None and h[0] > 3600 and "learned" in h[1],
+          "a connection's learned rpd is enforced", (got, rule, h))
+    r = client.chat("t-month", "hi", timeout=5)
+    learn.record("t-month", client.output(r, 5), "test")
+    kept = learn.connection_learned()["monthprov"]["rpd"]
+    check("rate limit (not sent" in r["error"] and "connection-learned.json" in r["error"]
+          and kept["confirmed_at"] == kept["at"] and not learn.connection_learned(pending=True),
+          "its refusal names where the limit is from, and is not learned from", (r["error"], kept))
+    view = {c["name"]: c for c in serve.connections_view()["connections"]}
+    serve.save_connection_limits("monthprov", {"rpm": 20, "rpmo": 1000, "rpd": 100})
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    check(view["monthprov"]["learned"] == {"rpd": {k: kept[k] for k in ("value", "at", "confirmed_at")}}
+          and len(rule) == 1 and rule[0]["limit"] == 100 and rule[0]["source"].startswith("you"),
+          "the Endpoints dialog shows it; your own value for that connection wins", (view["monthprov"], rule))
+    gone = serve.forget_connection_learned("monthprov", "rpd")
+    check(gone["ok"] and learn.connection_learned() == {} and not serve.forget_connection_learned("monthprov", "rpd")["ok"],
+          "Forget drops a learned limit", gone)
     page = serve.quotas_page()
     check("id='quota-data'" in page and "fakeprov" in page and "What the bench sent" in page,
           "the Quotas page carries its tiles and the ledger's counts")

package/serve.py

@@ -1544,15 +1544,42 @@ def save_connection_limits(connection: str, limits: dict, who: str = "ui") -> di
     return {"ok": True, "errors": [], "changed": ["config/connection-limits.json"]}
 
 
+def forget_connection_learned(connection: str, axis: str, who: str = "ui") -> dict:
+    """Drop what the bench learned for one connection's axis (metrics/learn.py):
+    its limit and its candidates, so the next 429 starts over."""
+    import learn
+
+    def drop(data: dict):
+        axes = data.get(connection)
+        old = axes.pop(axis, None) if isinstance(axes, dict) else None
+        if axes == {}:
+            del data[connection]
+        return old
+    old = locked.update(learn.CONN_LEARNED, drop, ensure_ascii=False)
+    if old is None:
+        return {"ok": False, "errors": [f"nothing is learned for {axis or '?'} on {connection or '?'}"]}
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
+                who, "", f"learned limits {connection}", {axis: old}, None)
+    return {"ok": True, "errors": [], "changed": ["config/connection-learned.json"]}
+
+
 def connections_view() -> dict:
-    """Each connection the chat models use, its models, and its limits."""
+    """Each connection the chat models use, its models, its limits, and what
+    the bench learned from its 429s (`learned`: enforced where you set none;
+    `pending`: met once, not enforced). Without the 429's text: it can carry
+    an account id."""
+    import learn
     by = {}
     for a in _chat_aliases():
         by.setdefault(catalog.connection_of(a), []).append(a)
     lim = catalog.connection_limits()
+    met, once = learn.connection_learned(), learn.connection_learned(pending=True)
     return {"ok": True, "errors": [], "keys": list(catalog.CONNECTION_LIMIT_KEYS),
             "labels": {k: catalog.LIMIT_LABELS[k] for k in catalog.CONNECTION_LIMIT_KEYS},
-            "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {})}
+            "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {}),
+                             "learned": {k: {f: ev.get(f) for f in ("value", "at", "confirmed_at")}
+                                         for k, ev in met.get(c, {}).items()},
+                             "pending": once.get(c, {})}
                             for c, m in sorted(by.items())]}
 
 
@@ -4929,11 +4956,12 @@ def move_card() -> str:
     """Settings' Export and import (export_zip, import_zip; bench.js)."""
     return ("<div class='panel form-panel'><h2>Export and import</h2><p class='muted'>Take this install to another"
             " machine, or keep a copy of it: one .zip file, imported on the other install's Settings page.</p>"
-            "<div class='pick'><label class='check'><input type='radio' name='x-scope' value='settings' checked"
+            # No wrapper: `.pick` is the onboarding picker's 3-column grid row, and squeezed these into it.
+            "<label class='check'><input type='radio' name='x-scope' value='settings' checked"
             " data-kept> <span><b>Settings only</b>: endpoints, collections, roles, model options, the schedule,"
             " gateway keys, the theme (<code>config/</code>)</span></label>"
             "<label class='check'><input type='radio' name='x-scope' value='all' data-kept> <span><b>Everything</b>:"
-            " the settings, and every result, history, log and saved answer the bench judges by</span></label></div>"
+            " the settings, and every result, history, log and saved answer the bench judges by</span></label>"
             "<label class='check'><input type='checkbox' id='x-keys' data-kept> <span>With the endpoints' API keys."
             " The file then holds them: keep it like a password, and delete it once imported.</span></label>"
             "<div class='actions'><button class='btn' type='button' id='x-export'>Download export</button></div>"
@@ -4978,23 +5006,28 @@ def access_card() -> str:
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td>{esc(v)}</td>"
                    f"<td class='mono set-by'>{'<br>'.join(esc(s) for s in src.split(', '))}</td></tr>"
                    for k, v, src in rows)
-    running, disk = on_disk()
-    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
-           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
     return ("<div class='panel'><h2>Server and access</h2><p class='muted'>How this install is reached and who"
             " may call it. Set in the environment, read at start: change one and restart. Shown here, never"
             " set here, since a page that could change its own login could open itself to anyone. No secret"
             " is shown, only whether one is set.</p>"
             f"<table class='facts'><thead><tr>{th('Setting')}{th('Now')}{th('Set by')}</tr></thead>"
-            f"<tbody>{body}</tbody></table>"
-            # restart(): stopped and started again by serve.py's own parent process.
-            f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
-            " for an update or a setting"
-            " above, whatever runs it (a service, the launcher, a container or a terminal). It waits for"
-            " onboarding, a scheduled job and gateway calls in progress, or cuts them off if you insist; the"
-            " page comes back in a few seconds.</p>"
-            "<p><button class='btn' type='button' id='restart'>Restart Benchwarmers</button>"
-            " <span id='restart-out' role='status'></span></p></div>")
+            f"<tbody>{body}</tbody></table>{restart_block('restart', 'a setting above')}</div>")
+
+
+def restart_block(id_: str, what: str) -> str:
+    """The Restart heading, what runs now, and its button (bench.js restart()).
+    On Settings (Server and access) and Info, each with its own ids, since both
+    tabs can be in the page at once."""
+    running, disk = on_disk()
+    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
+           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
+    # restart(): stopped and started again by serve.py's own parent process.
+    return (f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
+            f" for an update or {what}, whatever runs it (a service, the launcher, a container or a terminal)."
+            " It waits for onboarding, a scheduled job and gateway calls in progress, or cuts them off if you"
+            " insist; the page comes back in a few seconds.</p>"
+            f"<p><button class='btn' type='button' id='{id_}'>Restart Benchwarmers</button>"
+            f" <span id='{id_}-out' role='status'></span></p>")
 
 
 RESTART = 75                       # the exit code serve.py's parent reads as "start me again"
@@ -5288,6 +5321,12 @@ def endpoints_page() -> str:
         " popup) win, and limits the bench learns from errors only ever lower these. rpmo is calls"
         " a calendar month (UTC), for a trial key's monthly cap. The built-in client keeps to them:"
         " when one is used up it waits if the call can still finish in time, else it does not send.</p>"
+        "<p class='sub'>The bench also learns a connection's own rpm and rpd: when a provider refuses"
+        " with a 429 that names no number, the requests the bench had sent in that minute or day are the"
+        " limit, near enough. Met once it is only noted; met again at about the same count, a minute or a"
+        " day later, it is kept to like a limit of yours, unless you set your own here, which always wins."
+        " A learned limit not met again for 14 days is forgotten, so a raised limit is found; Forget drops"
+        " one now.</p>"
         "<div id='l-list'></div><div class='errors' id='l-errors' role='alert'></div>"
         "<div class='done' id='l-done'></div></div>")
 
@@ -5304,7 +5343,8 @@ def models_page() -> str:
     # Every model, decision models too (connections_view is chat models only,
     # for what a collection may hold): they are on the bench like any other.
     by: dict[str, list[str]] = {}
-    for a in model_directory():
+    directory = model_directory()
+    for a in directory:
         by.setdefault(catalog.connection_of(a), []).append(a)
     for c in ({"name": n, "models": sorted(m)} for n, m in sorted(by.items())):
         rows = []
@@ -5315,6 +5355,9 @@ def models_page() -> str:
             ep = through.get(a, "?")
             kind = (" " + chip("decision", "", "a decision model: answers typed questions, not chat")
                     if a in decisions else "")
+            if directory[a].get("ad_hoc"):
+                kind += " " + chip("ad hoc", "", "ad hoc only: never chosen by the bench on its own, only when you"
+                                   " name it (a collection member, with, --model)")
             rows.append(f"<tr><td><a href='#' class='model' data-alias='{esc(a)}'>"
                         f"<span class='alias'>{esc(a)}</span></a>{kind}</td>"
                         f"<td data-sort='{esc(ep)}'>{chip(ep, 'via', f'reached through the {ep} endpoint')}</td>"
@@ -5405,6 +5448,14 @@ def quotas_page() -> str:
                                 + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
                                           else "<td class='num muted'>—</td>" for kk in keys)
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
+    # And per connection, from 429s that named no number, where you set none.
+    for conn, axes in sorted(learn.connection_learned().items()):
+        for k, ev in axes.items():
+            if not (entered.get(conn) or {}).get(k):
+                lim_rows.append(f"<tr><td>{esc(conn)}</td>"
+                                + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
+                                          else "<td class='num muted'>—</td>" for kk in keys)
+                                + f"<td>learned {esc(ev.get('at', '')[:10])} (its 429s)</td></tr>")
     sent_table = ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
                   + th("Requests, this month", True) + th("Tokens, this month", True)
                   + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
@@ -5432,8 +5483,10 @@ def quotas_page() -> str:
         + "<details class='panel'><summary><b>Known limits</b></summary><p class='sub'>How much each"
         " connection or model allows (tokens or requests per minute or day), so the bench can space its"
         " calls. Two sources: what you enter per connection (Endpoints, Rate limits per connection), and"
-        " what a provider states in a rate-limit error, which the bench writes down by itself. A model's"
-        " own limits (its popup) win, and learned ones only ever lower a limit.</p>"
+        " what a provider states in a rate-limit error, which the bench writes down by itself (or, when the"
+        " error names no number, works out for the connection from the requests it had sent). A model's"
+        " own limits (its popup) win, and learned ones only ever lower a limit, until they have not been"
+        " met again for 14 days and are forgotten.</p>"
         + ("<table><thead><tr>" + th("Connection or model") + "".join(th(k, True) for k in keys) + th("From")
            + f"</tr></thead><tbody>{''.join(lim_rows)}</tbody></table>" if lim_rows
            else "<p class='muted'>None known yet.</p>")
@@ -5561,6 +5614,7 @@ def info_page() -> str:
         ("Theme", saved_theme())]]
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td class='mono'>{v}</td></tr>" for k, v in rows)
     return (f"<div class='panel'><h2>Info</h2><table class='facts'><tbody>{body}</tbody></table>"
+            f"{restart_block('restart-info', 'a setting')}"
             "<p class='muted'>Benchwarmers · by Wendell Research · MIT licence"
             " (<code>LICENSE</code>, credits in <code>NOTICE</code>). Every response carries"
             " <code>X-Benchwarmers-Version</code>.</p></div>")
@@ -5625,8 +5679,7 @@ def editor() -> str:
         + "".join(f"<label>{k} <input type='number' min='1' data-k='{k}' placeholder='—'></label>"
                   for k in ("rpm", "rpd", "tpm", "tpd"))
         + "</div><div class='src' id='s-limits'></div></div>"
-        "</div><div class='errors' id='f-errors' role='alert'></div><div class='done' id='f-done'></div>"
-        "<div class='actions'><button class='btn' id='f-save' type='button'>Save facts</button></div>"
+        "</div><div class='errors' id='f-errors' role='alert'></div>"
         "</details>"
         # How the bench calls it (serve.save_model_options): every caller, gateway, suite and probes.
         "<details id='m-opts' class='facts-box'><summary>Call options</summary>"
@@ -5636,8 +5689,7 @@ def editor() -> str:
         " makes to it. A reasoning model thinks out of the same budget and answers with an empty string"
         " when it runs out, which looks like a dead model: raise it then. Blank leaves it to the"
         " endpoint. <span id='co-ceiling'></span></p>"
-        "<div class='errors' id='co-errors' role='alert'></div><div class='done' id='co-done'></div>"
-        "<div class='actions'><button class='btn' id='co-save' type='button'>Save options</button></div>"
+        "<div class='errors' id='co-errors' role='alert'></div>"
         "</details>"
         "<label class='check'><input type='checkbox' id='m-adhoc'> Ad hoc only</label>"
         "<p class='rules'>Never chosen by the bench on its own: no explorer seat, no router"
@@ -6037,6 +6089,8 @@ class Handler(gateway.Handler):
                "/api/own-client-free": lambda p: save_own_client_free(p.get("providers")),
                "/api/connection-limits": lambda p: save_connection_limits(str(p.get("connection") or ""),
                                                                           p.get("limits")),
+               "/api/connection-learned/forget": lambda p: forget_connection_learned(
+                   str(p.get("connection") or ""), str(p.get("axis") or "")),
                "/api/verdict": lambda p: record_verdict(str(p.get("call") or ""), p.get("useful"),
                                                         str(p.get("note") or ""), p.get("real"),
                                                         p.get("false")),

package/ui/bench.js

@@ -55,30 +55,33 @@
   askD.addEventListener('close', function () { answer(false); });   // Escape
 
   // ---- restart (serve.restart): the page reloads once the new one answers ----
-  function restart(force) {
-    var rs = $('restart'), out = $('restart-out');
+  // One button on Settings (restart) and one on Info (restart-info), each with its own status line.
+  function restart(id, force) {
+    var rs = $(id), out = $(id + '-out');
     rs.disabled = true; out.textContent = 'Restarting…';
     post('/api/restart', {force: force}).then(function (res) {
-      if (res.ok) return back(Date.now());
+      if (res.ok) return back(out, Date.now());
       rs.disabled = false; out.textContent = '';
       if (!res.busy) { listErrors(out, res.errors); return; }
       ask('Still in progress: ' + res.errors.join('; ') + '. Restart anyway and cut them off?', 'Restart anyway', true)
-        .then(function (yes) { if (yes) restart(true); });
+        .then(function (yes) { if (yes) restart(id, true); });
     }).catch(function (err) { rs.disabled = false; out.textContent = String(err); });
   }
   // The old one stops half a second after answering; /health answers again once the new one is up.
-  function back(since) {
+  function back(out, since) {
     setTimeout(function () {
       fetch('/health', {cache: 'no-store'}).then(function (r) { if (!r.ok) throw r.status; location.reload(); })
         .catch(function () {
-          if (Date.now() - since < 120000) back(since);
-          else $('restart-out').textContent = 'Not back after two minutes: look at whatever runs it.';
+          if (Date.now() - since < 120000) back(out, since);
+          else out.textContent = 'Not back after two minutes: look at whatever runs it.';
         });
     }, 1500);
   }
-  on('click', 'restart', function () {
-    ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
-      .then(function (yes) { if (yes) restart(false); });
+  ['restart', 'restart-info'].forEach(function (id) {
+    on('click', id, function () {
+      ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
+        .then(function (yes) { if (yes) restart(id, false); });
+    });
   });
 
   // ---- Export and import (serve.export_zip, import_zip) ----------------
@@ -490,7 +493,7 @@
            reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
            qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why'), roleHint: $('m-role-hint')};
   m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
-  var editing = null;
+  var editing = null, shown = {};
 
   function openModel(alias) {
     var model = D.models[alias], gone = (D.retired || {})[alias];
@@ -539,6 +542,8 @@
         + (hint.at || '').slice(0, 10) + '). Only a suggestion: the role is the one you pick above.' : '';
       m.save.disabled = false; m.save.textContent = 'Save';
       m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
+      // What the popup opened with: Save sends only the parts that differ from it.
+      shown = {model: JSON.stringify(modelBody()), facts: factsBody(), opts: optsBody()};
     }
     if (!md.open) md.showModal();      // open already: shown again as it now is (shownAgain)
   }
@@ -577,7 +582,7 @@
     var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
     box.hidden = profile.hidden = decision || !f;
     profile.textContent = '';
-    $('f-errors').textContent = ''; $('f-done').textContent = '';
+    $('f-errors').textContent = '';
     if (box.hidden) return;
     var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
     // At a glance, as models.dev shows a model: what goes in, what comes out.
@@ -621,40 +626,25 @@
     var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
     $('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
   }
-  $('f-save').addEventListener('click', function () {
-    if (!editing) return;
+  // The facts as typed, as /api/model-facts takes them; null when the box is not shown.
+  function factsBody() {
+    if ($('m-facts').hidden) return null;
     var facts = {limits: {}};
     FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
     FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
     if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
     document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
-    $('f-errors').textContent = ''; $('f-done').textContent = '';
-    var alias = editing;
-    post('/api/model-facts', {alias: alias, facts: facts}).then(function (r) {
-      if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
-      // The tables that show facts (At a glance, Models, Quotas) follow once the popup closes (live).
-      shownAgain(alias, 'f-done', 'Saved. The tables show it once this closes.');
-    });
-  });
+    return JSON.stringify(facts);
+  }
   // ---- call options: how the bench asks this model (serve.save_model_options)
   function fillOptions(alias, decision) {
     var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
     $('m-opts').hidden = decision;
-    $('co-errors').textContent = ''; $('co-done').textContent = '';
+    $('co-errors').textContent = '';
     $('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
     $('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
   }
-  $('co-save').addEventListener('click', function () {
-    if (!editing) return;
-    var v = $('co-max').value.trim();
-    $('co-errors').textContent = ''; $('co-done').textContent = 'Saving…';   // the llm CLI is told too: seconds
-    var alias = editing;
-    post('/api/model-options', {alias: alias, max_tokens: v ? Number(v) : null}).then(function (r) {
-      if (!r.ok) { $('co-done').textContent = ''; listErrors($('co-errors'), r.errors); return; }
-      ownChange();                         // only this popup shows them
-      shownAgain(alias, 'co-done', 'Saved' + (r.note ? '; ' + r.note : '') + '.');
-    });
-  });
+  function optsBody() { return $('m-opts').hidden ? null : $('co-max').value.trim(); }
 
   function refresh() {
     m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
@@ -690,22 +680,57 @@
       then(res);
     };
   }
-  m.save.addEventListener('click', function () {
-    if (!editing) return;
-    var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
+  function modelBody() {
+    var body = {new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
     if (!m.qtypes.hidden) {
       body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
         function (c) { return c.checked; }).map(function (c) { return c.value; });
     }
-    (body.new_alias === editing ? Promise.resolve(true)
-      : ask('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
+    return body;
+  }
+  // The one Save: facts, then call options, then the model itself (alias, role, ad hoc,
+  // question types), each only if changed. The model goes last: a rename moves the
+  // facts and options saved under the old alias with it. The first refusal stops it.
+  m.save.addEventListener('click', function () {
+    if (!editing) return;
+    var alias = editing, body = modelBody(), facts = factsBody(), opts = optsBody(), said = [];
+    var steps = [];
+    if (facts !== shown.facts) steps.push(function () {
+      return post('/api/model-facts', {alias: alias, facts: JSON.parse(facts)}).then(function (r) {
+        if (!r.ok) { listErrors($('f-errors'), r.errors); $('m-facts').open = true; throw 'refused'; }
+        said.push('model facts');
+      });
+    });
+    if (opts !== shown.opts) steps.push(function () {   // the llm CLI is told too: seconds
+      return post('/api/model-options', {alias: alias, max_tokens: opts ? Number(opts) : null}).then(function (r) {
+        if (!r.ok) { listErrors($('co-errors'), r.errors); $('m-opts').open = true; throw 'refused'; }
+        ownChange();                         // only this popup shows them
+        said.push('call options' + (r.note ? ' (' + r.note + ')' : ''));
+      });
+    });
+    if (JSON.stringify(body) !== shown.model) steps.push(function () {
+      body.alias = alias;
+      return post('/api/model', body).then(function (r) {
+        if (!r.ok) { listErrors(m.errors, r.errors); throw 'refused'; }
+        said.push('the model (' + done(r).replace(/^Done: |\.$/g, '') + ')');
+        if (body.new_alias !== alias) { m.title.textContent = body.new_alias; refreshPanes(); }
+      });
+    });
+    m.errors.textContent = ''; m.done.textContent = ''; $('f-errors').textContent = ''; $('co-errors').textContent = '';
+    if (!steps.length) { m.done.textContent = 'Nothing changed.'; return; }
+    (body.new_alias === alias ? Promise.resolve(true)
+      : ask('Rename ' + alias + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
     ).then(function (yes) {
       if (!yes) return;
-      m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
-      post('/api/model', body).then(after(m.save, 'Save', function (res) {
-        if (body.new_alias !== body.alias) { m.title.textContent = body.new_alias; refreshPanes(); }
-        shownAgain(body.new_alias, 'm-done', done(res));
-      })).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
+      m.save.disabled = true; m.save.textContent = 'Saving…';
+      steps.reduce(function (p, step) { return p.then(step); }, Promise.resolve()).then(function () {
+        // The tables that show facts follow once the popup closes (live).
+        shownAgain(body.new_alias, 'm-done', 'Saved: ' + said.join(' · ') + '.');
+      }).catch(function (err) {
+        if (err !== 'refused') m.errors.textContent = String(err);
+        if (said.length) m.done.textContent = 'Saved before that: ' + said.join(' · ') + '.';
+        m.save.disabled = false; m.save.textContent = 'Save';
+      });
     });
   });
   // The confirm is the last word: the popup closes at once and the page
@@ -1545,6 +1570,29 @@
           lab.appendChild(box); inputs.appendChild(lab);
         });
         left.appendChild(inputs);
+        // What the bench worked out from this connection's 429s (learn.py):
+        // kept to where no value of yours is set, until Forget or 14 days.
+        res.keys.forEach(function (k) {
+          var ev = (c.learned || {})[k], once = (c.pending || {})[k];
+          if (!ev && !once) return;
+          var on = function (iso) { return new Date(iso).toLocaleDateString(undefined, {day: 'numeric', month: 'short'}); };
+          var note = el('div', {class: 'base'}, ev
+            ? 'learned ' + ev.value + ' ' + k + ' · 429s on ' + on(ev.at)
+              + (ev.confirmed_at && on(ev.confirmed_at) !== on(ev.at) ? ', last on ' + on(ev.confirmed_at) : '')
+              + ' · ' + (c.limits[k] ? 'not enforced: yours wins' : 'enforced (yours wins)') + ' '
+            : 'met once: ' + once.n + ' ' + k + ' · a 429 on ' + on(once.at) + ' · not enforced unless met again ');
+          var forget = el('button', {class: 'btn small', type: 'button',
+                                     'aria-label': 'Forget the learned ' + k + ' for ' + c.name}, 'Forget');
+          forget.addEventListener('click', function () {
+            $('l-errors').textContent = ''; $('l-done').textContent = '';
+            post('/api/connection-learned/forget', {connection: c.name, axis: k}).then(function (r) {
+              if (!r.ok) { listErrors($('l-errors'), r.errors); return; }
+              $('l-done').textContent = 'Forgot the learned ' + k + ' for ' + c.name + '.';
+              loadLimits();
+            });
+          });
+          note.appendChild(forget); left.appendChild(note);
+        });
         row.appendChild(left);
         var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
         save.addEventListener('click', function () {