@wendellresearch-ai/benchwarmers 0.6.0 → 0.6.1

package/USAGE.md

@@ -314,7 +314,8 @@ limit is that model's own:
 - **Pool**: OpenRouter's free requests a day, shared by its models, with a bar
   per key, against the limit OpenRouter stated in a rate-limit reply.
 - **Caps**: a day or month cap, counted from the bench's own calls: one you set
-  on Endpoints (Rate limits per connection), else one the provider states in
+  on Endpoints (Rate limits per connection), else a day cap the bench learned
+  for the connection (below), else one the provider states in
   its replies without saying what is left (a Cohere trial key's 1,000 calls a
   month), shown but not enforced until you set it. Calls made elsewhere with
   the same key are not in the count.
@@ -323,7 +324,14 @@ limit is that model's own:
 
 Nothing is taken from a provider's docs, which can change without notice; a
 limit a provider states in a rate-limit error (UnoRouter's "allows 1
-request(s) every 1 min" for a model) is learned and kept to. One exception,
+request(s) every 1 min" for a model) is learned and kept to. When the error
+names no number (Cloudflare Workers AI's, once the day's free Neurons are
+gone), the requests the bench had sent on that connection in that minute or
+day are taken as the connection's limit: noted the first time, kept to once a
+second refusal lands near the same count, unless you set your own for that
+connection on Endpoints, where each learned limit is shown with a Forget
+button. A learned rate limit not met again for 14 days is forgotten, so a
+raised limit is found. One exception,
 watched: NVIDIA states no limit per key, so its tile reads the file
 build.nvidia.com's own badge reads (10,000 requests a day, up to 40 a minute,
 the same for every account) daily and on Refresh, and counts the bench's

package/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

package/metrics/decide.py

@@ -34,6 +34,14 @@ URL, which nothing writes (an Open WebUI hub is given one by hand), and the
 onboarding hints about it show only for an Open WebUI hub (endpoints.hub). OpenRouter's /models omits its decision
 models, so that endpoint names them in `unlisted_models` to make them pickable.
 
+Cloudflare Workers AI answers decisions only on `/ai/run/<model>`, outside its
+OpenAI-compatible `/ai/v1` base, so no hub can relay them: its endpoint's
+`decisions` URL is `.../accounts/<id>/ai/run/{model}`. `{model}` puts the
+served id in the URL and leaves it out of the body (Cloudflare refuses
+`@cf/...` there), and its `{"result": {...}}` wrapper is taken off the reply.
+Its /models answers 405, so that endpoint lists from `models_url`, the
+catalog search (`.../ai/models/search`), read by endpoints.list_models.
+
 WHAT EACH MODEL CAN ANSWER
 --------------------------
 `question_types` lists the types a model accepted when onboarding probed it
@@ -235,7 +243,13 @@ def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
         headers = {"Content-Type": "application/json", **(entry.get("headers") or {})}
         if (secret := endpoints.key(endpoint)):
             headers["Authorization"] = f"Bearer {secret}"
-        body = json.dumps({"model": served, "state": state, "questions": questions}).encode()
+        payload = {"model": served, "state": state, "questions": questions}
+        if "{model}" in url:
+            # Cloudflare names the model in the URL (/ai/run/@cf/cloudflare/clef)
+            # and refuses its own id in the body ("clef", not "@cf/...", is valid there).
+            url = url.replace("{model}", served)
+            del payload["model"]
+        body = json.dumps(payload).encode()
         # Inception answers 429 to calls in quick succession (seen 2026-10-16:
         # the second of three onboarding probes), so a 429 waits and retries.
         for wait in ((5, 15, 0) if retry else (0,)):
@@ -243,6 +257,9 @@ def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
             try:
                 with urllib.request.urlopen(request, timeout=timeout) as response:
                     reply = json.load(response)
+                    # Cloudflare wraps every reply: {"result": {"answers": ...}, "success": true}.
+                    if isinstance(reply, dict) and "answers" not in reply and isinstance(reply.get("result"), dict):
+                        reply = reply["result"]
                     return reply
             except urllib.error.HTTPError as exc:
                 if exc.code != 429 or not wait:
@@ -358,14 +375,23 @@ def refuses_chat(endpoint: str, served: str) -> bool:
         with urllib.request.urlopen(request, timeout=60):
             return False
     except urllib.error.HTTPError as exc:
-        # OpenRouter says "is a decisions model"; other providers word it their
-        # own way, so any refusal that names decisions or systemone counts.
-        text = error_text(exc).lower()
-        return any(w in text for w in ("decisions model", "decision model", "systemone", "/decisions"))
+        return names_decisions(error_text(exc))
     except Exception:                                  # noqa: BLE001
         return False
 
 
+def names_decisions(text: str) -> bool:
+    """A chat refusal that says the model is a decision model. OpenRouter names
+    it ("is a decisions model"); Cloudflare only lists the fields it wanted
+    ("required properties ... 'model,state,questions'", "Required value
+    missing: questions"). Every Decisions API takes `state` and `questions`,
+    so a refusal asking for both counts, whoever words it."""
+    low = (text or "").lower()
+    if any(w in low for w in ("decisions model", "decision model", "systemone", "/decisions")):
+        return True
+    return "questions" in low and ("state" in low or "required" in low)
+
+
 def question_types(endpoint: str, served: str, path: str = "decisions") -> list[str]:
     """Which question types the model accepts, one probe call per type.
 

package/metrics/endpoints.py

@@ -295,7 +295,8 @@ def models(name: str | None = None, timeout: int = 30) -> dict[str, dict]:
 def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, dict]:
     """What an endpoint entry lists, saved or not yet (the dashboard tests one
     before saving it). `models_url` when the list lives elsewhere (GitHub
-    Models: its catalog, a bare JSON list); entries marked `visibility` other
+    Models: its catalog, a bare JSON list; Cloudflare: its catalog search,
+    {"result": [...]} by `name`); entries marked `visibility` other
     than "list" are left out (ChatGPT's plan lists ones it will not serve)."""
     api = entry.get("api") or "chat"
     headers = dict(entry.get("headers") or {})
@@ -313,6 +314,9 @@ def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, d
         # what generateContent serves, by its bare name.
         rows = [{**m, "id": str(m.get("name", "")).removeprefix("models/")} for m in data.get("models", [])
                 if isinstance(m, dict) and "generateContent" in (m.get("supportedGenerationMethods") or ())]
+    elif isinstance(data, dict) and isinstance(data.get("result"), list):
+        # Cloudflare's catalog search: {"result": [{"name": "@cf/...", ...}]}.
+        rows = [{**m, "id": m.get("name")} for m in data["result"] if isinstance(m, dict)]
     else:
         rows = data.get("data", []) if isinstance(data, dict) else data
     return {e["id"]: e for e in rows

package/metrics/learn.py

@@ -13,18 +13,40 @@ learns exactly these at runtime, and so does this: every failed call from the
 perf probe, the suite (and so suite/pace.py) and the gateway passes its error
 text through `record()`.
 
-ONLY EVER TIGHTER
------------------
+ONLY EVER TIGHTER, UNTIL IT EXPIRES
+-----------------------------------
 A learned context window or rate limit is kept only if it is LOWER than what is
 known, and replaces an older learned value only when lower still. An error can
 prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
 says. Values you entered in the model popup (config/model-overrides.json)
 still win: you can always correct a bad parse.
 
-    config/model-learned.json   alias -> {context_window, limits, capabilities, served_as}
-                                with the date and the evidence for each
-    runs/cooldowns.json         alias -> {until, why}: rate-limited, come back later.
-                                Not config: it expires, and is nobody's decision.
+A rate limit can be raised, though (a plan upgrade, a provider's change), and
+no error ever says so. So a learned RATE limit that has not been seen again for
+LEARNED_TTL_S (14 days) is left out wherever it is read: the bench goes back to
+the higher rate, and the provider's next refusal teaches the limit again. The
+same limit seen again refreshes its `confirmed_at`, so one that still holds
+does not expire. A context window does not: a model's window does not grow.
+
+A 429 THAT NAMES NO NUMBER
+--------------------------
+Cloudflare Workers AI says only that the day's free Neurons are used up. The
+bench knows how many requests it sent on that connection in the window before
+the refusal (metrics/ledger.py), and that count is the limit, near enough. It
+is a connection's limit, not a model's: the key's allowance is shared. One
+refusal is a candidate; a second at a count within 20% of it, a window or more
+later (sooner is the same refusal, met by another call), is adopted (the lower
+of the two) and enforced by metrics/pacing.py unless you set your own for that
+connection. Seen again near the same count it is confirmed; a refusal at a
+clearly lower count lowers it at once. Only rpm and rpd, and never from the
+bench's own refusal.
+
+    config/model-learned.json        alias -> {context_window, limits, capabilities, served_as}
+                                     with the date and the evidence for each
+    config/connection-learned.json   connection -> {rpm, rpd}: {value, at, confirmed_at, where,
+                                     evidence}, or the candidates until one is adopted
+    runs/cooldowns.json              alias -> {until, why}: rate-limited, come back later.
+                                     Not config: it expires, and is nobody's decision.
 """
 
 from __future__ import annotations
@@ -40,9 +62,13 @@ import locked
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
 LEARNED = DATA / "config" / "model-learned.json"
+CONN_LEARNED = DATA / "config" / "connection-learned.json"
 COOLDOWNS = DATA / "runs" / "cooldowns.json"
 DEFAULT_COOLDOWN_S = 1800          # a 429 with no stated wait
 MAX_COOLDOWN_S = 24 * 3600
+LEARNED_TTL_S = 14 * 86400         # a learned rate limit not seen again for this long is forgotten
+CONN_AXES = {"rpm": 60, "rpd": 86400}     # what a connection's limit is learned for: its window, seconds
+MIN_COUNT = 3                      # fewer requests in the window than this prove nothing
 
 CONTEXT = [
     r"maximum context length is\s*([\d,]+)",
@@ -71,6 +97,10 @@ ALLOWS = re.compile(r"allows?\s+([\d,]+)\s+requests?(?:\(s\))?\s+every\s+(?:1\s+
 # new-api gateways (Agnes): "maximum 20 requests in 1 minutes" (its
 # middleware/model-rate-limit.go); a window of 1 minute or 1,440 only.
 MAXIMUM = re.compile(r"maximum\s+([\d,]+)\s+requests?\s+in\s+(1|1440)\s+minutes?\b")
+# Which window a 429 with no number is about. A window, not a wait: "try
+# again in 3 minutes" names no per-minute limit.
+DAY = re.compile(r"\bday\b|daily|free allocation|neurons")
+MINUTE = re.compile(r"per[- ]?min(?:ute)?\b|a minute|\brpm\b")
 
 
 def _int(s: str) -> int:
@@ -108,18 +138,64 @@ def parse(text: str) -> dict:
 _read = locked.read_json
 
 
+def _age(stamp, now: dt.datetime) -> float:
+    """Seconds since an ISO stamp; forever when it is missing or unreadable."""
+    try:
+        return (now - dt.datetime.fromisoformat(stamp)).total_seconds()
+    except (TypeError, ValueError):
+        return float("inf")
+
+
+def _live(ev: dict, now: dt.datetime) -> bool:
+    """Seen within LEARNED_TTL_S: an older rate limit may have been raised since."""
+    return _age(ev.get("confirmed_at") or ev.get("at"), now) <= LEARNED_TTL_S
+
+
 def learned() -> dict:
-    return {k: v for k, v in _read(LEARNED).items() if not k.startswith("_")}
+    """alias -> what was learned, without the rate limits that have expired."""
+    now = dt.datetime.now().astimezone()
+    out = {}
+    for alias, e in _read(LEARNED).items():
+        if alias.startswith("_"):
+            continue
+        if isinstance(e, dict) and e.get("limits"):
+            e = {**e, "limits": {k: ev for k, ev in e["limits"].items() if _live(ev, now)}}
+        out[alias] = e
+    return out
+
+
+def connection_learned(pending: bool = False) -> dict:
+    """connection -> {axis: {value, at, confirmed_at, where, evidence}}: the
+    adopted limits that have not expired. pending=True: for the axes with none,
+    the latest candidate instead ({n, at}), seen once and not enforced."""
+    now = dt.datetime.now().astimezone()
+    out: dict = {}
+    for conn, axes in _read(CONN_LEARNED).items():
+        if conn.startswith("_") or not isinstance(axes, dict):
+            continue
+        for axis, e in axes.items():
+            if axis not in CONN_AXES or not isinstance(e, dict):
+                continue
+            adopted = bool(e.get("value")) and _live(e, now)
+            waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
+            if adopted and not pending:
+                out.setdefault(conn, {})[axis] = {k: v for k, v in e.items() if k != "candidates"}
+            elif pending and not adopted and waiting:
+                out.setdefault(conn, {})[axis] = waiting[-1]
+    return out
 
 
 def _known(alias: str, field: str, axis: str | None = None):
-    """The value a learned figure has to beat: the catalog's, else the last learned."""
+    """The value a learned figure has to beat: the catalog's, else the last
+    learned; for a rate limit, its connection's too, yours and the learned."""
     import catalog
     auto = catalog.catalog().get(alias, {})
     mine = learned().get(alias, {})
     if axis:
+        conn = catalog.connection_of(alias)
         vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
-                catalog.connection_limits().get(catalog.connection_of(alias), {}).get(axis)]
+                catalog.connection_limits().get(conn, {}).get(axis),
+                connection_learned().get(conn, {}).get(axis, {}).get("value")]
     else:
         vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
     vals = [v for v in vals if v]
@@ -137,10 +213,13 @@ def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) ->
             entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
             stored["context_window"] = n
         for axis, n in (found.get("limits") or {}).items():
-            if n < (_known(alias, "", axis) or 1 << 62):
+            had = (entry.get("limits") or {}).get(axis) or {}
+            if n < (_known(alias, "", axis) or 1 << 62):   # an expired one is not known: a higher value may replace it
                 entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
                                                         "evidence": evidence}
                 stored[axis] = n
+            elif had.get("value") == n:
+                had["confirmed_at"] = stamp                # still true, so it does not expire
         if stored:
             data[alias] = entry
             data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
@@ -158,6 +237,75 @@ def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
     return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
 
 
+def _axis(text: str, found: dict) -> str | None:
+    """The window a 429 that names no number is about, or None: nothing to
+    learn. Never the bench's own refusal ("not sent", pacing.py): a limit the
+    bench enforced must not be learned back from itself. Nor a token limit: the
+    count is of requests."""
+    low = (text or "").lower()
+    if not found.get("rate_limited") or found.get("limits") or "not sent" in low \
+            or re.search(AXES["tpm"] + "|" + AXES["tpd"], low):
+        return None
+    wait = found.get("retry_after_s")
+    if DAY.search(low) or (wait or 0) >= 3600:
+        return "rpd"
+    # A short wait alone is not enough for rpm: calls in flight are not in the
+    # ledger yet, so a minute's count undercounts, and two passing 429s could
+    # pin a connection at rpm 3 for 14 days. Only a named minute window counts.
+    if MINUTE.search(low):
+        return "rpm"
+    return None
+
+
+def _sent(connection: str, since: float) -> int:
+    """Requests the provider took on a connection since then: the ledger's
+    attempts, without the calls that failed, whose own retries would count the
+    refusal as use."""
+    import ledger
+    return sum(r.get("attempts") or 0 for r in ledger.rows(since)
+               if r.get("connection") == connection and r.get("status") != "error")
+
+
+def _near(n: int, value: int) -> bool:
+    return 0.8 * value <= n <= 1.2 * value
+
+
+def _learn_connection(connection: str, axis: str, n: int, where: str, evidence: str, now: dt.datetime) -> dict:
+    """A connection refused after `n` requests in the axis's window. Returns
+    {axis: value} when that adopted a limit or lowered one."""
+    stamp, span = now.isoformat(timespec="seconds"), CONN_AXES[axis]
+    stored: dict = {}
+
+    def change(data: dict) -> None:
+        axes = data.setdefault(connection, {})
+        e = axes.get(axis) or {}
+        fresh = {"value": n, "at": stamp, "confirmed_at": stamp, "where": where, "evidence": evidence}
+        waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
+        if e.get("value") and _live(e, now):
+            # One lesson a window: while a provider keeps refusing, the count
+            # of what it took only falls, and would drag the limit down with it.
+            if _age(e.get("confirmed_at") or e.get("at"), now) < span:
+                return
+            if n < 0.8 * e["value"]:                   # refused well below it: that is proof
+                axes[axis] = fresh
+                stored[axis] = n
+            elif _near(n, e["value"]):
+                e["confirmed_at"] = stamp
+        elif waiting and _age(waiting[-1]["at"], now) < span:
+            return                                     # the same refusal, met by another call
+        elif agree := [c["n"] for c in waiting if _near(n, c["n"])]:
+            axes[axis] = fresh | {"value": min(agree + [n])}
+            stored[axis] = axes[axis]["value"]
+        else:
+            axes[axis] = {"candidates": (waiting + [{"n": n, "at": stamp}])[-5:]}
+        data.setdefault("_why", "Learned by metrics/learn.py from 429s that named no number: the requests"
+                                " the bench had sent in the window. Enforced unless you set your own"
+                                " (config/connection-limits.json); forgotten 14 days after last seen.")
+
+    locked.update(CONN_LEARNED, change, ensure_ascii=False)
+    return stored
+
+
 def record(alias: str, text: str, where: str) -> dict:
     """Learn from one failed call. Returns what was stored or set, for logging."""
     found = parse(text)
@@ -165,6 +313,14 @@ def record(alias: str, text: str, where: str) -> dict:
     stamp = now.isoformat(timespec="seconds")
     evidence = " ".join((text or "").split())[:200]
     stored = _tighten(alias, found, where, evidence, stamp)
+    if alias and (axis := _axis(text, found)):
+        try:
+            import catalog
+            conn = catalog.connection_of(alias)
+            if conn and (n := _sent(conn, now.timestamp() - CONN_AXES[axis])) >= MIN_COUNT:
+                stored.update(_learn_connection(conn, axis, n, where, evidence, now))
+        except Exception:                              # noqa: BLE001  -- learning never breaks a call
+            pass
     if alias and found.get("rate_limited"):
         wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
         cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
@@ -231,9 +387,104 @@ def _selftest() -> None:
     assert "limits" not in parse("maximum 100 requests in 5 minutes")         # a window no axis holds
     assert parse("model not found") == {"rate_limited": False}
     assert parse("context length of 8k exceeded").get("context_window") is None   # "8" < 512
+
+    # Which window a 429 with no number is about.
+    def axis(text: str):
+        return _axis(text, parse(text))
+    neurons = "Error code: 429 - you have used up your daily free allocation of 10,000 neurons"
+    assert axis(neurons) == "rpd" and "limits" not in parse(neurons)
+    assert axis("Rate limit exceeded: free-models-per-day. Try again in 3 minutes") == "rpd"
+    assert axis("429 Too Many Requests, retry-after: 7200") == "rpd"
+    assert axis("429 Too Many Requests, retry-after: 30") is None        # a short wait alone: no window
+    assert axis("429: too many requests per minute") == "rpm"
+    assert axis("429 Too Many Requests") is None                         # no window named
+    assert axis("Rate limit exceeded. Try again in 3 minutes") is None    # a wait, not a window
+    assert axis("429: rate limit on tokens per minute") is None          # requests are what is counted
+    assert axis("Limit 30, Used 30 (requests per minute) 429") is None   # it gave the number
+    assert axis("model not found") is None
+    # The bench's own refusal is never learned from.
+    refusal = ("Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests"
+               " per day (learned) and 50 were used; try again in 2h")
+    assert axis(refusal) is None and axis(refusal.replace("not sent; ", "")) == "rpd"
+    _selftest_stores(neurons, refusal)
     print("ok")
 
 
+def _selftest_stores(neurons: str, refusal: str) -> None:
+    """The connection store and the expiry, in a temp folder with a stub
+    catalog: nothing of the install's is read or written."""
+    import tempfile
+
+    import catalog
+    import ledger
+    with tempfile.TemporaryDirectory() as folder:
+        tmp = pathlib.Path(folder)
+        globals().update(LEARNED=tmp / "model-learned.json", CONN_LEARNED=tmp / "connection-learned.json",
+                         COOLDOWNS=tmp / "cooldowns.json")
+        ledger.DIR = tmp / "usage"
+        catalog.catalog = catalog.connection_limits = lambda: {}
+        catalog.connection_of = lambda alias: "conn-" + alias
+        now = dt.datetime.now().astimezone()
+
+        def ago(**kw) -> dt.datetime:
+            return now - dt.timedelta(**kw)
+
+        def refused(n: int, when: dt.datetime, conn: str = "prov", ax: str = "rpd") -> dict:
+            return _learn_connection(conn, ax, n, "test", "429", when)
+        # One refusal is a candidate; the same refusal met again changes nothing.
+        assert refused(100, ago(days=5)) == {} and connection_learned() == {}
+        assert connection_learned(pending=True)["prov"]["rpd"]["n"] == 100
+        assert refused(104, ago(days=5, hours=-1)) == {} and connection_learned() == {}
+        # A second count far from the first does not adopt; one within 20% does, the lower of the two.
+        assert refused(300, ago(days=4)) == {} and connection_learned() == {}
+        assert refused(110, ago(days=3)) == {"rpd": 100}
+        got = connection_learned()["prov"]["rpd"]
+        assert got["value"] == 100 and "candidates" not in got and connection_learned(pending=True) == {}, got
+        # Seen again near it: confirmed. Clearly lower: lowered at once. Higher: left alone.
+        assert refused(95, ago(days=2)) == {}
+        got = connection_learned()["prov"]["rpd"]
+        assert got["value"] == 100 and got["confirmed_at"] == ago(days=2).isoformat(timespec="seconds") != got["at"]
+        assert refused(40, ago(days=2, hours=-1)) == {}                      # within the window: the same refusal
+        assert refused(500, ago(days=1)) == {} and connection_learned()["prov"]["rpd"]["value"] == 100
+        assert refused(60, now) == {"rpd": 60} and connection_learned()["prov"]["rpd"]["value"] == 60
+        # Not seen for 15 days: forgotten on read, and a higher limit can be learned.
+        assert refused(20, ago(days=16), "old", "rpm") == {} and refused(20, ago(days=15), "old", "rpm") == {"rpm": 20}
+        assert "old" not in connection_learned()
+        assert refused(50, ago(minutes=5), "old", "rpm") == {} and refused(50, now, "old", "rpm") == {"rpm": 50}
+
+        # A model's own learned limit: the same expiry, and the same limit seen again is confirmed.
+        old = {"value": 5, "at": ago(days=15).isoformat(timespec="seconds"), "where": "test", "evidence": ""}
+        LEARNED.write_text(json.dumps({"m": {"limits": {"rpm": dict(old), "rpd": dict(old)},
+                                             "context_window": dict(old, value=8192)}}), encoding="utf-8")
+        assert learned()["m"]["limits"] == {} and learned()["m"]["context_window"]["value"] == 8192
+        stamp = now.isoformat(timespec="seconds")
+        assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", stamp) == {"rpm": 20}
+        assert _tighten("m", {"limits": {"rpm": 30}}, "test", "", stamp) == {}        # live: only ever tighter
+        assert "confirmed_at" not in learned()["m"]["limits"]["rpm"]
+        again = ago(hours=-1).isoformat(timespec="seconds")
+        assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", again) == {}
+        got = learned()["m"]["limits"]["rpm"]
+        assert got["value"] == 20 and got["at"] == stamp and got["confirmed_at"] == again, got
+        # A model's limit must beat its connection's learned one too.
+        assert refused(10, ago(minutes=5), "conn-m2", "rpm") == {} and refused(10, now, "conn-m2", "rpm") == {"rpm": 10}
+        assert _tighten("m2", {"limits": {"rpm": 60}}, "test", "", stamp) == {}
+        assert _tighten("m2", {"limits": {"rpm": 6}}, "test", "", stamp) == {"rpm": 6}
+
+        # record(): the count is the ledger's, of what the provider took on that connection.
+        def sent(conn: str, n: int, status: str = "ok", attempts: int = 1) -> None:
+            for i in range(n):
+                ledger.append({"t": now.timestamp() - 30 - i, "alias": "x", "connection": conn, "status": status,
+                               "attempts": attempts})
+        sent("conn-r", 4), sent("conn-r", 1, "error", 3), sent("elsewhere", 9), sent("conn-p", 4), sent("conn-few", 2)
+        assert record("r", neurons, "test") == {"cooldown_s": DEFAULT_COOLDOWN_S}     # a candidate: nothing stored yet
+        assert connection_learned(pending=True)["conn-r"]["rpd"]["n"] == 4
+        assert record("few", neurons, "test") and "conn-few" not in connection_learned(pending=True)   # too few to say
+        # The bench's own refusal cools the model as before, and teaches nothing.
+        assert record("p", refusal, "test") == {"cooldown_s": 7200} and "p" in cooldowns()
+        assert "conn-p" not in connection_learned(pending=True)
+
+
+
 def main() -> None:
     sys.stdout.reconfigure(encoding="utf-8", errors="replace")
     if "--selftest" in sys.argv:
@@ -248,6 +499,9 @@ def main() -> None:
         if e.get("served_as"):
             bits.append(f"served as {e['served_as']['served']}")
         print(f"  {alias:<34} {', '.join(bits)}")
+    for conn, axes in sorted(connection_learned().items()):
+        bits = [f"{k} {v['value']}" for k, v in axes.items()]
+        print(f"  {conn:<34} {', '.join(bits)}  (the connection's, from 429s with no number)")
     for alias, c in sorted(cooldowns().items()):
         print(f"  cooling  {alias:<26} until {c['until'][11:16]}  ({c['why'][:70]})")
 

package/metrics/pacing.py

@@ -28,7 +28,9 @@ WHAT IS COUNTED
   deadline is dropped, so a killed process cannot hold a slot.
 - A connection's limits count every model on it (a key's limits are the
   key's); a model's own or learned limits count that model alone, its own
-  beating the learned (as everywhere: your values win).
+  beating the learned (as everywhere: your values win). Where you set none
+  for a connection, the rpm or rpd learn.py worked out from its 429s is kept
+  to (config/connection-learned.json), until it expires.
 - Quota records (metrics/quota.py): what a provider last said was left, less
   the calls sent since, until its reset.
 
@@ -77,6 +79,7 @@ def rules(alias: str, connection: str) -> list[dict]:
     conn = catalog.connection_limits().get(connection) or {}
     mine = (catalog.overrides().get(alias) or {}).get("limits") or {}
     seen = (learn.learned().get(alias) or {}).get("limits") or {}
+    met = learn.connection_learned().get(connection) or {}
     out = []
 
     def ok(n) -> bool:
@@ -85,6 +88,10 @@ def rules(alias: str, connection: str) -> list[dict]:
         if ok(n := conn.get(axis)):
             out.append({"scope": "connection", "axis": axis, "limit": n,
                         "source": f"you, for {connection} (config/connection-limits.json)"})
+        elif ok(n := (met.get(axis) or {}).get("value")):
+            out.append({"scope": "connection", "axis": axis, "limit": n,
+                        "source": f"learned {str(met[axis].get('at', ''))[:10]} from 429s on {connection}"
+                                  " (config/connection-learned.json)"})
         if axis == "rpmo":
             continue
         if ok(n := mine.get(axis)):

package/metrics/quotaview.py

@@ -512,14 +512,21 @@ def _limit_text(lim: dict) -> str:
     return ", ".join(f"{v:,} {words[k]}" for k, v in lim.items() if k in words and v)
 
 
-def caps(host: str, entered: dict, stated: dict, day_rows: list, month_rows, now: float) -> list[dict]:
+def caps(host: str, entered: dict, stated: dict, day_rows: list, month_rows, now: float,
+         learned: dict | None = None) -> list[dict]:
     """The connection's day and month caps as groups of windows, each counted
     from the bench's ledger since midnight UTC or the month's start (what a
-    provider's calendar counts): yours, else `stated` ({axis: (limit, source,
-    detail)}: what its replies say with nothing left given, or NVIDIA's site).
+    provider's calendar counts): yours, else `learned` (learn.py's, from this
+    connection's 429s), else `stated` ({axis: (limit, source, detail)}: what
+    its replies say with nothing left given, or NVIDIA's site).
     `month_rows` is called only when a month cap needs it."""
     mine = {k: (v, "your cap", "Set on Endpoints, Rate limits per connection; the bench keeps to it.")
             for k, v in (entered.get(host) or {}).items() if k in COUNTED and v}
+    for axis, ev in (learned or {}).items():
+        if axis in COUNTED:
+            mine.setdefault(axis, (ev["value"], "learned", f"Worked out {str(ev.get('at'))[:10]} from this connection's"
+                                   " 429s, which named no number; the bench keeps to it until you set your own"
+                                   " on Endpoints, or it has not been met again for 14 days."))
     for axis, (n, source, detail) in stated.items():
         mine.setdefault(axis, (n, source, detail + " Shown, not enforced: set it on Endpoints to have the bench"
                                                    " keep to it."))
@@ -554,7 +561,7 @@ def tiles(now: float | None = None) -> list[dict]:
     resting, learned = learn.cooldowns(), learn.learned()
     day_rows = list(ledger.rows(now - 86400))
     utc_rows = [r for r in day_rows if r["t"] >= _utc_midnight(now)]
-    entered = catalog.connection_limits()
+    entered, met = catalog.connection_limits(), learn.connection_learned()
     regs = __import__("endpoints").registrations()
     month: list = []                                   # this month's ledger, read once if a month cap asks
 
@@ -589,7 +596,7 @@ def tiles(now: float | None = None) -> list[dict]:
         t = {"id": tag, "tag": tag, "host": host, "bench": len(aliases), "kind": "models", "groups": [],
              "pool": [], "notes": [],          # a day or month cap is a meter (caps), a minute one is said
              "conn_limit": _limit_text({k: v for k, v in (entered.get(host) or {}).items() if k in ("rpm", "tpm")}),
-             "caps": caps(host, entered, stated, day_rows, month_rows, now),
+             "caps": caps(host, entered, stated, day_rows, month_rows, now, met.get(host)),
              "models": _model_rows(aliases, rows, resting, limits),
              "refresh": how_refreshed(tag), "plan": "", "name": ""}
         got = live.get(tag) or {}
@@ -743,6 +750,13 @@ def _selftest() -> None:
     g = caps("router.requesty.ai", {"router.requesty.ai": {"rpd": 10, "rpm": 5}}, {"rpd": said}, day, lambda: [], now)
     assert g[0]["label"] == "your cap · counted by the bench" and g[0]["windows"] == [
         {"label": "requests · day", "left": 80, "remaining": 8, "of": 10, "reset_at": _iso(mid + 86400)}], g
+    # A learned day cap (learn.py, from 429s) beats a stated one, and yours beats it; a learned rpm is no meter.
+    met = {"rpd": {"value": 20, "at": "2026-11-10T09:00:00+00:00"}, "rpm": {"value": 4, "at": ""}}
+    g = caps("router.requesty.ai", {}, {"rpd": said}, day, lambda: [], now, met)
+    assert len(g) == 1 and g[0]["label"] == "learned · counted by the bench" and "2026-11-10" in g[0]["title"] \
+        and g[0]["windows"][0]["of"] == 20 and g[0]["windows"][0]["remaining"] == 18, g
+    g = caps("router.requesty.ai", {"router.requesty.ai": {"rpd": 10}}, {}, day, lambda: [], now, met)
+    assert len(g) == 1 and g[0]["label"] == "your cap · counted by the bench", g
     asked = []
     g = caps("api.cohere.ai", {}, {"rpmo": (1000, "its replies", "")}, day, lambda: asked.append(1) or day, now)
     assert asked and g[0]["windows"][0]["remaining"] == 999 and g[0]["windows"][0]["label"] == "requests · month", g

package/metrics/test_decide.py

@@ -29,7 +29,9 @@ def jsonl(path: str, rows: list[dict]) -> None:
 
 
 write("config/endpoints.json", {"hub": {"base": "http://127.0.0.1:9/api", "key": "hub", "default": True,
-                                        "decisions": "http://127.0.0.1:9/openai/{path}"}})
+                                        "decisions": "http://127.0.0.1:9/openai/{path}"},
+                                "cf": {"base": "http://127.0.0.1:9/ai/v1", "key": "hub",
+                                       "decisions": "http://127.0.0.1:9/ai/run/{model}"}})
 # dm2 first: before any result it is the one asked (the first that takes choice, as before),
 # except per request, where its :free budget is spared while dm1 can answer.
 MODELS = {
@@ -91,6 +93,14 @@ assert "passthrough" in decide.error_text(http_error(b'{"detail": "Direct API pa
 assert decide.error_text(http_error(b'{"error": null, "message": "down"}')).startswith("400")
 assert decide.error_text(http_error(b"not json")) == "400"
 
+# A chat refusal says "decision model" however the provider words it.
+assert decide.names_decisions("400 is a decisions model and cannot be used with chat")             # OpenRouter
+assert decide.names_decisions("400 {'errors': [{'message': \"AiError: Bad input: Error: required properties"
+                              " at '/' are 'model,state,questions' (72ae)\", 'code': 5006}]}")     # Cloudflare
+assert decide.names_decisions("400 Model execution failed (User Input Error): Required value missing: questions.")
+assert not decide.names_decisions("400 messages: Required")       # Cloudflare's v1 refusing a decision body
+assert not decide.names_decisions("429 Rate limit exceeded") and not decide.names_decisions("")
+
 # ask() refuses a type the entry does not list, before any call is made (and logs nothing).
 try:
     decide.ask("yesno", "s", {"q": {"type": "choice", "instructions": "?", "criteria": {"a": None}}})
@@ -124,6 +134,20 @@ def urlopen(request, timeout=0):
     return io.BytesIO(json.dumps({"answers": {name: answer}}).encode())
 
 
+
+
+# Cloudflare: the model in the URL and not the body, the reply in a "result" wrapper.
+def cf_urlopen(request, timeout=0):
+    body = json.loads(request.data)
+    assert request.full_url == "http://127.0.0.1:9/ai/run/@cf/cloudflare/clef-flash" and "model" not in body, body
+    return io.BytesIO(json.dumps({"result": {"model": "clef-flash", "answers": {"probe": {"type": "noul", "noul": 0.98}}},
+                                  "success": True, "errors": []}).encode())
+
+
+decide.urllib.request.urlopen = cf_urlopen
+assert decide.probe("cf", "@cf/cloudflare/clef-flash") == (None, "decisions")
+assert decide.log_rows()[-1]["answers"]["probe"]["noul"] == 0.98
+decide.LOG.unlink()
 decide.urllib.request.urlopen = urlopen
 
 # ---- every call is logged, and an error's body survives the log -------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/onboard.py

@@ -286,7 +286,13 @@ def main() -> None:
             return
         # The passthrough hints are about Open WebUI; anywhere else a 404 is the provider's.
         owui = endpoints.hub(endpoint) == "openwebui"
-        hint = ("  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
+        # ponytail: Cloudflare told apart by its @cf/ ids; its partner models (no @cf/) get the generic hint.
+        hint = ("  Cloudflare answers decisions only on its /ai/run route, which no hub relays: add"
+                " Cloudflare as a direct endpoint (decisions .../accounts/<id>/ai/run/{model},"
+                " models_url .../ai/models/search?per_page=1000) and onboard it from there.\n"
+                if "@cf/" in name and endpoint != "cloudflare" and "{model}" not in
+                (endpoints.load()[endpoint].get("decisions") or "") else
+                "  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
                 " in its environment.\n" if owui and "passthrough" in why.lower() else
                 "  Its connection has no /decisions under its base. An OpenRouter-hosted decision"
                 " model cannot go through Open WebUI (its Decisions API is outside /api/v1);"

package/scripts/test_quota.py

@@ -20,6 +20,8 @@ not mix. Covered:
   a Gemini 429 body's daily quota learned as a limit;
 - an rpm limit across six threads at once and across two processes (calls in
   flight count), a monthly cap (rpmo), and both refusals classed as capacity;
+- a connection's learned limit (metrics/learn.py) enforced, its refusal not
+  learned from, your own value winning over it, and Forget;
 - the free-tier guard holding a model that went paid, until it is allowed;
 - metrics/cost.py reading the ledger.
 """
@@ -331,6 +333,31 @@ def main() -> int:
     check(not bad["ok"] and serve.save_model_facts("t-ok", {"limits": {"rpmo": 5}}).get("ok") is not None
           and "rpmo" not in ((serve.catalog.overrides().get("t-ok") or {}).get("limits") or {}),
           "a bad rpmo is refused; a model's own limits have none", bad)
+    # A connection's learned limit (learn.py: 429s that named no number) is kept to like one of yours, which wins.
+    import datetime as dt
+    at = dt.datetime.now().astimezone()
+    learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=2))
+    got = learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=1))
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    h = pacing.held("t-month", "monthprov")
+    check(got == {"rpd": 3} and len(rule) == 1 and rule[0]["scope"] == "connection" and rule[0]["limit"] == 3
+          and rule[0]["source"].startswith("learned ") and h is not None and h[0] > 3600 and "learned" in h[1],
+          "a connection's learned rpd is enforced", (got, rule, h))
+    r = client.chat("t-month", "hi", timeout=5)
+    learn.record("t-month", client.output(r, 5), "test")
+    kept = learn.connection_learned()["monthprov"]["rpd"]
+    check("rate limit (not sent" in r["error"] and "connection-learned.json" in r["error"]
+          and kept["confirmed_at"] == kept["at"] and not learn.connection_learned(pending=True),
+          "its refusal names where the limit is from, and is not learned from", (r["error"], kept))
+    view = {c["name"]: c for c in serve.connections_view()["connections"]}
+    serve.save_connection_limits("monthprov", {"rpm": 20, "rpmo": 1000, "rpd": 100})
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    check(view["monthprov"]["learned"] == {"rpd": {k: kept[k] for k in ("value", "at", "confirmed_at")}}
+          and len(rule) == 1 and rule[0]["limit"] == 100 and rule[0]["source"].startswith("you"),
+          "the Endpoints dialog shows it; your own value for that connection wins", (view["monthprov"], rule))
+    gone = serve.forget_connection_learned("monthprov", "rpd")
+    check(gone["ok"] and learn.connection_learned() == {} and not serve.forget_connection_learned("monthprov", "rpd")["ok"],
+          "Forget drops a learned limit", gone)
     page = serve.quotas_page()
     check("id='quota-data'" in page and "fakeprov" in page and "What the bench sent" in page,
           "the Quotas page carries its tiles and the ledger's counts")

package/serve.py

@@ -1544,15 +1544,42 @@ def save_connection_limits(connection: str, limits: dict, who: str = "ui") -> di
     return {"ok": True, "errors": [], "changed": ["config/connection-limits.json"]}
 
 
+def forget_connection_learned(connection: str, axis: str, who: str = "ui") -> dict:
+    """Drop what the bench learned for one connection's axis (metrics/learn.py):
+    its limit and its candidates, so the next 429 starts over."""
+    import learn
+
+    def drop(data: dict):
+        axes = data.get(connection)
+        old = axes.pop(axis, None) if isinstance(axes, dict) else None
+        if axes == {}:
+            del data[connection]
+        return old
+    old = locked.update(learn.CONN_LEARNED, drop, ensure_ascii=False)
+    if old is None:
+        return {"ok": False, "errors": [f"nothing is learned for {axis or '?'} on {connection or '?'}"]}
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
+                who, "", f"learned limits {connection}", {axis: old}, None)
+    return {"ok": True, "errors": [], "changed": ["config/connection-learned.json"]}
+
+
 def connections_view() -> dict:
-    """Each connection the chat models use, its models, and its limits."""
+    """Each connection the chat models use, its models, its limits, and what
+    the bench learned from its 429s (`learned`: enforced where you set none;
+    `pending`: met once, not enforced). Without the 429's text: it can carry
+    an account id."""
+    import learn
     by = {}
     for a in _chat_aliases():
         by.setdefault(catalog.connection_of(a), []).append(a)
     lim = catalog.connection_limits()
+    met, once = learn.connection_learned(), learn.connection_learned(pending=True)
     return {"ok": True, "errors": [], "keys": list(catalog.CONNECTION_LIMIT_KEYS),
             "labels": {k: catalog.LIMIT_LABELS[k] for k in catalog.CONNECTION_LIMIT_KEYS},
-            "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {})}
+            "connections": [{"name": c, "models": sorted(m), "limits": lim.get(c, {}),
+                             "learned": {k: {f: ev.get(f) for f in ("value", "at", "confirmed_at")}
+                                         for k, ev in met.get(c, {}).items()},
+                             "pending": once.get(c, {})}
                             for c, m in sorted(by.items())]}
 
 
@@ -5288,6 +5315,12 @@ def endpoints_page() -> str:
         " popup) win, and limits the bench learns from errors only ever lower these. rpmo is calls"
         " a calendar month (UTC), for a trial key's monthly cap. The built-in client keeps to them:"
         " when one is used up it waits if the call can still finish in time, else it does not send.</p>"
+        "<p class='sub'>The bench also learns a connection's own rpm and rpd: when a provider refuses"
+        " with a 429 that names no number, the requests the bench had sent in that minute or day are the"
+        " limit, near enough. Met once it is only noted; met again at about the same count, a minute or a"
+        " day later, it is kept to like a limit of yours, unless you set your own here, which always wins."
+        " A learned limit not met again for 14 days is forgotten, so a raised limit is found; Forget drops"
+        " one now.</p>"
         "<div id='l-list'></div><div class='errors' id='l-errors' role='alert'></div>"
         "<div class='done' id='l-done'></div></div>")
 
@@ -5405,6 +5438,14 @@ def quotas_page() -> str:
                                 + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
                                           else "<td class='num muted'>—</td>" for kk in keys)
                                 + f"<td>learned {esc(ev.get('at', '')[:10])} ({esc(ev.get('where', ''))})</td></tr>")
+    # And per connection, from 429s that named no number, where you set none.
+    for conn, axes in sorted(learn.connection_learned().items()):
+        for k, ev in axes.items():
+            if not (entered.get(conn) or {}).get(k):
+                lim_rows.append(f"<tr><td>{esc(conn)}</td>"
+                                + "".join(f"<td class='num'>{ev['value']:,}</td>" if kk == k
+                                          else "<td class='num muted'>—</td>" for kk in keys)
+                                + f"<td>learned {esc(ev.get('at', '')[:10])} (its 429s)</td></tr>")
     sent_table = ("<table><thead><tr>" + th("Connection") + th("Requests, 24 h", True) + th("Tokens, 24 h", True)
                   + th("Requests, this month", True) + th("Tokens, this month", True)
                   + f"</tr></thead><tbody>{''.join(sent_rows)}</tbody></table>"
@@ -5432,8 +5473,10 @@ def quotas_page() -> str:
         + "<details class='panel'><summary><b>Known limits</b></summary><p class='sub'>How much each"
         " connection or model allows (tokens or requests per minute or day), so the bench can space its"
         " calls. Two sources: what you enter per connection (Endpoints, Rate limits per connection), and"
-        " what a provider states in a rate-limit error, which the bench writes down by itself. A model's"
-        " own limits (its popup) win, and learned ones only ever lower a limit.</p>"
+        " what a provider states in a rate-limit error, which the bench writes down by itself (or, when the"
+        " error names no number, works out for the connection from the requests it had sent). A model's"
+        " own limits (its popup) win, and learned ones only ever lower a limit, until they have not been"
+        " met again for 14 days and are forgotten.</p>"
         + ("<table><thead><tr>" + th("Connection or model") + "".join(th(k, True) for k in keys) + th("From")
            + f"</tr></thead><tbody>{''.join(lim_rows)}</tbody></table>" if lim_rows
            else "<p class='muted'>None known yet.</p>")
@@ -6037,6 +6080,8 @@ class Handler(gateway.Handler):
                "/api/own-client-free": lambda p: save_own_client_free(p.get("providers")),
                "/api/connection-limits": lambda p: save_connection_limits(str(p.get("connection") or ""),
                                                                           p.get("limits")),
+               "/api/connection-learned/forget": lambda p: forget_connection_learned(
+                   str(p.get("connection") or ""), str(p.get("axis") or "")),
                "/api/verdict": lambda p: record_verdict(str(p.get("call") or ""), p.get("useful"),
                                                         str(p.get("note") or ""), p.get("real"),
                                                         p.get("false")),

package/ui/bench.js

@@ -1545,6 +1545,29 @@
           lab.appendChild(box); inputs.appendChild(lab);
         });
         left.appendChild(inputs);
+        // What the bench worked out from this connection's 429s (learn.py):
+        // kept to where no value of yours is set, until Forget or 14 days.
+        res.keys.forEach(function (k) {
+          var ev = (c.learned || {})[k], once = (c.pending || {})[k];
+          if (!ev && !once) return;
+          var on = function (iso) { return new Date(iso).toLocaleDateString(undefined, {day: 'numeric', month: 'short'}); };
+          var note = el('div', {class: 'base'}, ev
+            ? 'learned ' + ev.value + ' ' + k + ' · 429s on ' + on(ev.at)
+              + (ev.confirmed_at && on(ev.confirmed_at) !== on(ev.at) ? ', last on ' + on(ev.confirmed_at) : '')
+              + ' · ' + (c.limits[k] ? 'not enforced: yours wins' : 'enforced (yours wins)') + ' '
+            : 'met once: ' + once.n + ' ' + k + ' · a 429 on ' + on(once.at) + ' · not enforced unless met again ');
+          var forget = el('button', {class: 'btn small', type: 'button',
+                                     'aria-label': 'Forget the learned ' + k + ' for ' + c.name}, 'Forget');
+          forget.addEventListener('click', function () {
+            $('l-errors').textContent = ''; $('l-done').textContent = '';
+            post('/api/connection-learned/forget', {connection: c.name, axis: k}).then(function (r) {
+              if (!r.ok) { listErrors($('l-errors'), r.errors); return; }
+              $('l-done').textContent = 'Forgot the learned ' + k + ' for ' + c.name + '.';
+              loadLimits();
+            });
+          });
+          note.appendChild(forget); left.appendChild(note);
+        });
         row.appendChild(left);
         var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
         save.addEventListener('click', function () {