@wendellresearch-ai/benchwarmers 0.5.0 → 0.5.2

package/metrics/quotaview.py

@@ -12,13 +12,22 @@ A tile takes the shape of what its provider says:
             rate-limit headers the built-in client saw (metrics/quota.py)
     pool    one allowance the models share: OpenRouter's free requests a
             day, a bar per key, counted from the usage ledger
-    models  nothing stated: each bench model's last day, answered or not
+    models  nothing stated: the provider's last day of calls, all together
 
-REFRESH NEVER SPENDS QUOTA. Two providers can be asked what is left for free,
-and are, when Refresh is pressed:
+and any of them its day and month caps (caps): yours, else a limit its replies
+state with nothing left said, counted from the ledger. A model is named only
+where a limit is its own. Nothing comes from a provider's docs: they change
+unseen.
+
+ONE REFRESH, THREE WAYS (how_refreshed). Three providers can be asked what is
+left for free, and are:
 
     OpenRouter      GET /key on a direct OpenRouter endpoint: free tier or
-                    not (50 or 1,000 free requests a day), and its credits
+                    not, and its credits (the free requests' limit comes
+                    from its 429s, never its docs)
+    xKiro           GET /v1/usage: free-model tokens left today (to midnight
+                    UTC) and a plan's spend windows; the key is a direct
+                    endpoint's, else its Open WebUI connection's (_key_for)
     CLI Proxy API   its management route `requests/api-call`, which calls the
                     credential's own quota endpoint with the credential's
                     token, as its console does (Google's retrieveUserQuotaSummary,
@@ -27,10 +36,10 @@ and are, when Refresh is pressed:
                     (endpoints.admin_key, as metrics/cliproxy.py); the proxy is
                     on the LAN and calls only providers the credential uses.
 
-Every other provider's Refresh re-reads the bench's own records. `check`
-is the one that asks: a 16-token call to one of its models, whose reply
-headers quota.py records. Offered only where the provider has sent limit
-headers before, since elsewhere the reply would say nothing.
+Where a provider has sent limit headers before (Groq; Routeway, through Open
+WebUI too, since the bench streams), Refresh is `check`: a 16-token call to
+one of its models, whose reply headers quota.py records. Everywhere else it
+re-reads the bench's own records, since a call would say nothing.
 
 Live reads are kept in runs/quota-live.json, newest per tag.
 """
@@ -65,6 +74,23 @@ CODEX_URL = "https://chatgpt.com/backend-api/wham/usage"
 CODEX_HEAD = {"Authorization": "Bearer $TOKEN$", "Content-Type": "application/json",
               "User-Agent": "codex-tui/0.149.1"}
 WINDOW_NAMES = {"5h": "5 hours", "weekly": "week", "daily": "day", "monthly": "month"}
+# Free to ask: no tokens, outside the rate limits (docs.xkiro.com/api/usage, read 2026-10-29).
+XKIRO_HOST = "api.xkiro.com"
+XKIRO_USAGE = "https://api.xkiro.com/v1/usage"
+# A key may read its own usage with completions permission alone; the org's
+# needs manage (docs.requesty.ai/features/key-management-api, tried 2026-10-30).
+REQUESTY_HOST = "router.requesty.ai"
+REQUESTY_KEYS = "https://api-v2.requesty.ai/v1/manage/apikey"
+REQUESTY_MODELS = "https://router.requesty.ai/v1/models"     # public; a free model is priced at zero
+# The one figure from a provider's published text, not its replies: the file
+# build.nvidia.com reads for its "Your API Rate Limit" badge, the same for
+# everyone. Watched (watch_nvidia): a change or a failure to read it is an alert.
+NVIDIA_COPY = "https://build.nvidia.com/api/runtime/content/copy.yaml"
+# A day or month cap is drawn counted from the ledger, as pacing.py counts:
+# yours from Endpoints (enforced), else one a provider states in its replies
+# without saying what is left (Cohere's x-endpoint-monthly-call-limit; shown,
+# never enforced). Never one from a provider's docs: they change unseen.
+COUNTED = {"rpd": ("requests", "day"), "tpd": ("tokens", "day"), "rpmo": ("requests", "month")}
 
 
 def _iso(t: float | None) -> str | None:
@@ -118,6 +144,24 @@ def codex_groups(body: dict) -> list[dict]:
     return [{"label": "", "windows": wins}] if wins else []
 
 
+def xkiro_groups(body: dict, now: float) -> list[dict]:
+    """/v1/usage -> the free models' tokens today (to midnight UTC), then the
+    plan's spend windows (USD, as strings), where there is a plan."""
+    out, ft = [], body.get("free_tokens") or {}
+    if ft.get("limit_per_day"):
+        lim, rem = int(ft["limit_per_day"]), int(ft.get("remaining") or 0)
+        out.append({"label": "free models", "windows": [
+            {"label": "tokens · day", "left": round(100 * rem / lim), "remaining": rem, "of": lim,
+             "reset_at": _iso(_utc_midnight(now) + 86400)}]})
+    wins = [{"label": _seconds_name(w.get("window_sec")) + " · spend",
+             "left": round(100 * float(w.get("remaining_usd") or 0) / float(w["cap_usd"])),
+             "reset_at": _iso(now + float(w.get("resets_in_sec") or 0))}
+            for w in body.get("windows") or [] if isinstance(w, dict) and float(w.get("cap_usd") or 0) > 0]
+    if wins:
+        out.append({"label": "plan", "windows": wins})
+    return out
+
+
 # ---- live reads --------------------------------------------------------------
 
 def _get_json(url: str, headers: dict, data: bytes | None = None, timeout: int = 20) -> dict:
@@ -147,6 +191,106 @@ def read_openrouter() -> dict:
             "limit": data.get("limit"), "remaining": data.get("limit_remaining"), "usage": data.get("usage")}
 
 
+def _key_for(host: str) -> str:
+    """A key for `host`: a direct endpoint's, else the Open WebUI connection's
+    (read with the hub's admin key, as owui.py reads its connections)."""
+    import endpoints
+    for name, entry in endpoints.load().items():
+        if (urllib.parse.urlsplit(entry.get("base", "")).hostname or "") == host and not endpoints.hub(name):
+            if endpoints.key(name):
+                return endpoints.key(name)
+    import owui
+    cfg = owui._call("/openai/config")
+    for url, key in zip(cfg.get("OPENAI_API_BASE_URLS") or [], cfg.get("OPENAI_API_KEYS") or []):
+        if (urllib.parse.urlsplit(url).hostname or "") == host and key:
+            return key
+    raise ValueError(f"no key for {host}: neither an endpoint nor an Open WebUI connection has one")
+
+
+def read_xkiro() -> dict:
+    """Only the figures are kept: the reply also names the account's owner."""
+    try:                                               # Cloudflare refuses Python's own User-Agent (1010)
+        body = _get_json(XKIRO_USAGE, {"Authorization": f"Bearer {_key_for(XKIRO_HOST)}", "User-Agent": "benchwarmers"})
+    except urllib.error.HTTPError as exc:
+        raise ValueError(f"xKiro answered {exc.code} on /v1/usage") from None
+    return {"groups": xkiro_groups(body, time.time()), "plan": str(body.get("plan") or ""), "notes": []}
+
+
+def requesty_free(usage: dict, models: dict) -> int:
+    """Requests to free models in a usage reply's (first) period. Its
+    model_used drops the provider (`muse-glimmer-30b` for nvidia/muse-glimmer-30b)."""
+    free = {str(m.get("id", "")).split("/")[-1] for m in models.get("data") or []
+            if not float(m.get("input_price") or 0) and not float(m.get("output_price") or 0)}
+    day = next(iter((usage.get("usage") or {}).values()), {})
+    return sum(r.get("total_requests") or 0 for r in day.get("grouped_data") or []
+               if str((r.get("group_by_values") or {}).get("model_used", "")).split("/")[-1] in free)
+
+
+def read_requesty() -> dict:
+    """Requesty's own count of this key's free-model requests since midnight
+    UTC (its usage API, whose periods are UTC dates). Its free limit is not
+    reported anywhere: only a 429 that states one gives the pool its `of`."""
+    head = {"Authorization": f"Bearer {_key_for(REQUESTY_HOST)}", "User-Agent": "benchwarmers"}
+    try:
+        me = _get_json(REQUESTY_KEYS + "/self", head)
+        start = dt.datetime.fromtimestamp(_utc_midnight(time.time()), UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+        usage = _get_json(f"{REQUESTY_KEYS}/{me['id']}/usage?start={start}&resolution=day&group_by=model_used", head)
+        models = _get_json(REQUESTY_MODELS, {"User-Agent": "benchwarmers"})
+    except urllib.error.HTTPError as exc:
+        raise ValueError(f"Requesty answered {exc.code} on its usage API") from None
+    return {"free_used": requesty_free(usage, models), "notes": []}
+
+
+def nvidia_figures(body: dict) -> dict:
+    """copy.yaml (JSON in fact) -> {rpd, rpm_text}; ValueError when its
+    rateLimits or a day's number is missing: the file changed shape."""
+    import re
+    rl = body.get("rateLimits") if isinstance(body, dict) else None
+    if not isinstance(rl, dict):
+        raise ValueError("it no longer has rateLimits")
+    m = re.search(r"([\d,]+)\s+requests?\s+(?:per|a)\s+day", str(rl.get("requestsPerDay") or ""))
+    if not m:
+        raise ValueError(f"its requestsPerDay no longer says a number a day: {str(rl.get('requestsPerDay'))[:60]!r}")
+    return {"rpd": int(m.group(1).replace(",", "")), "rpm_text": str(rl.get("requestsPerMinute") or "")}
+
+
+def _nvidia_on_bench() -> bool:
+    return any("nvidia" in (t + " " + _host(t)).lower() for t in _tags())
+
+
+def watch_nvidia() -> dict:
+    """Read NVIDIA's stated figures and keep the last good read (LIVE
+    "_nvidia"). A change raises an alert you dismiss; a failure (moved,
+    broken, a new shape) raises one that clears once it reads again. Run
+    daily by the catalog job and by the tile's Refresh; nothing without an
+    NVIDIA connection on the bench."""
+    import alerts
+    prev = locked.read_json(LIVE).get("_nvidia") or {}
+    if not _nvidia_on_bench():
+        return prev
+    try:
+        try:
+            body = _get_json(NVIDIA_COPY, {"User-Agent": "benchwarmers"})
+        except urllib.error.HTTPError as exc:
+            raise ValueError(f"build.nvidia.com answered {exc.code}") from None
+        got = dict(nvidia_figures(body), at=_iso(time.time()))
+    except (ValueError, OSError) as exc:
+        since = f" (last read {prev['at'][:10]})" if prev.get("at") else ""
+        alerts.raise_alert("limits:nvidia:unread", "limits", f"NVIDIA's limits file could not be read: {exc}",
+                           action=f"It may have moved from {NVIDIA_COPY}. The NVIDIA tile keeps the figures"
+                                  f" it read before{since}.")
+        locked.update(LIVE, lambda d: d.setdefault("_nvidia", {}).update(broken=str(exc)), ensure_ascii=False)
+        return dict(prev, broken=str(exc))
+    alerts.clear("limits:nvidia:unread")
+    if prev.get("rpd") and (prev["rpd"], prev.get("rpm_text")) != (got["rpd"], got["rpm_text"]):
+        alerts.raise_alert("limits:nvidia:changed", "limits",
+                           f"NVIDIA changed its stated limits: {prev['rpd']:,} a day, {prev.get('rpm_text')!r}"
+                           f" -> {got['rpd']:,} a day, {got['rpm_text']!r}",
+                           action="The NVIDIA tile counts against the new figures.", dismiss=True)
+    locked.update(LIVE, lambda d: d.update({"_nvidia": got}), ensure_ascii=False)
+    return got
+
+
 def _cpa_hub() -> tuple[str, str]:
     """(management base, key) of the CLI Proxy API hub; ValueError when there is none."""
     import endpoints
@@ -235,21 +379,72 @@ def read_cliproxy(tag: str, sibling_vendors: set[str]) -> dict:
 
 
 def refresh(tag: str) -> dict:
-    """Ask the provider when it can be asked for free, keep the answer;
-    {"ok", "how": "live"|"records", "note"}."""
-    if tag == "openrouter" and _openrouter_endpoint():
+    """The tile's one Refresh, which picks its own way (how_refreshed): ask the
+    provider when it can be asked for free; else, where it states limits in its
+    replies, one 16-token call (check); else the bench's own records.
+    {"ok", "how": "live"|"check"|"records", "note"}."""
+    how = how_refreshed(tag)
+    if "nvidia" in (tag + " " + _host(tag)).lower():
+        got = watch_nvidia()
+        return {"ok": True, "how": "live", "note": f"NVIDIA's limits file could not be read: {got['broken']};"
+                " showing the figures read before" if got.get("broken") else ""}
+    if how == "check":
+        r = check(tag)
+        if r.get("ok"):
+            return dict(r, how="check")
+        return {"ok": True, "how": "records", "note": "; ".join(r.get("errors") or []) + "; shown from the bench's own records"}
+    if how == "records":
+        return {"ok": True, "how": "records",
+                "note": "this provider states no limits to ask about: shown from the bench's own records"}
+    if tag == "openrouter":
         got = read_openrouter()
-    elif tag.split("/")[0] == "cli-proxy-api":
+    elif _host(tag) == XKIRO_HOST:
+        got = read_xkiro()
+    elif _host(tag) == REQUESTY_HOST:
+        got = read_requesty()
+    else:
         vendors = {t.split("/", 1)[1] for t in _tags() if t.startswith("cli-proxy-api/")}
         got = read_cliproxy(tag, vendors)
-    else:
-        return {"ok": True, "how": "records",
-                "note": "this provider has no free quota read: shown from the bench's own records"}
     got["at"] = _iso(time.time())
     locked.update(LIVE, lambda d: d.update({tag: got}), ensure_ascii=False)
     return {"ok": True, "how": "live", "note": "; ".join(got.get("notes") or [])}
 
 
+def about(tag: str) -> dict:
+    """How a tile's provider is reached, for its Models popup: {connection,
+    routes: [{via, url}]}, a direct endpoint by that name and each Open WebUI
+    connection named so (scripts/provider_tags.py's names). URLs only:
+    /openai/config returns Open WebUI's keys too, never read here."""
+    import endpoints
+    host, routes = tag.split("/")[0], []
+    if host in endpoints.load() and not endpoints.hub(host):
+        routes.append({"via": f"its own endpoint, {host}", "url": endpoints.base(host)})
+    hub = endpoints.find_hub("openwebui")
+    if hub:
+        sys.path.insert(0, str(ROOT / "scripts"))
+        import provider_tags
+        urls = provider_tags.get(hub, "/openai/config").get("OPENAI_API_BASE_URLS", [])
+        routes += [{"via": f"an Open WebUI connection on {hub}", "url": url}
+                   for name, url in zip(provider_tags.connection_names(hub), urls) if name == host]
+    return {"connection": tag, "routes": routes}
+
+
+def _host(tag: str) -> str:
+    """The connection a tile's models are called through (route.connection_of)."""
+    import route
+    aliases = _tags().get(tag) or []
+    return route.connection_of(aliases[0]) if aliases else tag
+
+
+def how_refreshed(tag: str) -> str:
+    """"live" (a free read), "check" (one short call whose headers say what is
+    left) or "records" (nothing to ask: the bench's own records)."""
+    if (tag.split("/")[0] == "cli-proxy-api" or tag == "openrouter" and _openrouter_endpoint()
+            or _host(tag) in (XKIRO_HOST, REQUESTY_HOST) or "nvidia" in (tag + " " + _host(tag)).lower()):
+        return "live"
+    return "check" if _headers_seen(tag) else "records"
+
+
 def check(tag: str) -> dict:
     """One 16-token call to a model on `tag` that is free to call; the
     built-in client records what its reply headers say (quota.observe)."""
@@ -312,10 +507,41 @@ def _model_rows(aliases, rows, resting, limits) -> list[dict]:
 
 
 def _limit_text(lim: dict) -> str:
-    words = {"rpm": "requests a minute", "rpd": "requests a day", "tpm": "tokens a minute", "tpd": "tokens a day"}
+    words = {"rpm": "requests a minute", "rpd": "requests a day", "tpm": "tokens a minute", "tpd": "tokens a day",
+             "rpmo": "requests a calendar month"}
     return ", ".join(f"{v:,} {words[k]}" for k, v in lim.items() if k in words and v)
 
 
+def caps(host: str, entered: dict, stated: dict, day_rows: list, month_rows, now: float) -> list[dict]:
+    """The connection's day and month caps as groups of windows, each counted
+    from the bench's ledger since midnight UTC or the month's start (what a
+    provider's calendar counts): yours, else `stated` ({axis: (limit, source,
+    detail)}: what its replies say with nothing left given, or NVIDIA's site).
+    `month_rows` is called only when a month cap needs it."""
+    mine = {k: (v, "your cap", "Set on Endpoints, Rate limits per connection; the bench keeps to it.")
+            for k, v in (entered.get(host) or {}).items() if k in COUNTED and v}
+    for axis, (n, source, detail) in stated.items():
+        mine.setdefault(axis, (n, source, detail + " Shown, not enforced: set it on Endpoints to have the bench"
+                                                   " keep to it."))
+    by: dict[str, list] = {}
+    about: dict[str, str] = {}
+    for axis, (n, source, detail) in mine.items():
+        about[source] = detail
+        metric, window = COUNTED[axis]
+        start, end = ((_utc_midnight(now), _utc_midnight(now) + 86400) if window == "day"
+                      else (__import__("ledger").month_start(now), __import__("ledger").next_month(now)))
+        rows = [r for r in (day_rows if window == "day" else month_rows())
+                if r.get("connection") == host and r["t"] >= start]
+        used = sum((r.get("attempts") or 0) if metric == "requests"
+                   else (r.get("tokens_in") or 0) + (r.get("tokens_out") or 0) for r in rows)
+        left = max(0, n - used)
+        by.setdefault(source, []).append({"label": f"{metric} · {window}", "left": round(100 * left / n),
+                                          "remaining": left, "of": n, "reset_at": _iso(end)})
+    return [{"label": f"{source} · counted by the bench", "title": about[source] + " Counted from the bench's"
+             " own calls: calls made elsewhere with the same key are not in it.", "windows": w}
+            for source, w in by.items()]
+
+
 def tiles(now: float | None = None) -> list[dict]:
     """Every tile, in the saved order (out first, then soonest reset, for new ones)."""
     import catalog
@@ -330,6 +556,12 @@ def tiles(now: float | None = None) -> list[dict]:
     utc_rows = [r for r in day_rows if r["t"] >= _utc_midnight(now)]
     entered = catalog.connection_limits()
     regs = __import__("endpoints").registrations()
+    month: list = []                                   # this month's ledger, read once if a month cap asks
+
+    def month_rows() -> list:
+        if not month:
+            month.append(list(ledger.rows(ledger.month_start(now))))
+        return month[0]
     out = []
     for tag, aliases in tags.items():
         host = route.connection_of(aliases[0])
@@ -338,22 +570,48 @@ def tiles(now: float | None = None) -> list[dict]:
                   for a in aliases}
         limits = {a: t for a, t in limits.items() if t}
         rows = [r for r in day_rows if r.get("alias") in mine]
+        # What the replies' headers said, kept a day past each reset (quota.py).
+        # A limit with nothing left said (Cohere's month) is a cap to count.
+        seen_recs = [r for a, by in (locked.read_json(quota.FILE).get(host) or {}).items()
+                     if a in mine and isinstance(by, dict) for r in by.values()
+                     if isinstance(r, dict) and r.get("limit") and r.get("window") != "retry-after"]
+        stated: dict[str, tuple] = {}
+        for r in seen_recs:
+            if r.get("remaining") is None and r["metric"] == "requests" and r.get("window") in ("day", "month"):
+                axis = "rpd" if r["window"] == "day" else "rpmo"
+                if r["limit"] > (stated.get(axis) or (0,))[0]:
+                    stated[axis] = (r["limit"], "its replies", "The provider's replies state this limit but not"
+                                                               " what is left.")
+        nv = (live.get("_nvidia") or {}) if "nvidia" in (host + " " + tag).lower() else {}
+        if nv.get("rpd"):
+            stated.setdefault("rpd", (nv["rpd"], "NVIDIA's site", "The figure build.nvidia.com shows every account"
+                                      f" (not this key's own), read {str(nv.get('at'))[:10]}."))
         t = {"id": tag, "tag": tag, "host": host, "bench": len(aliases), "kind": "models", "groups": [],
-             "pool": [], "notes": [], "conn_limit": _limit_text(entered.get(host) or {}),
+             "pool": [], "notes": [],          # a day or month cap is a meter (caps), a minute one is said
+             "conn_limit": _limit_text({k: v for k, v in (entered.get(host) or {}).items() if k in ("rpm", "tpm")}),
+             "caps": caps(host, entered, stated, day_rows, month_rows, now),
              "models": _model_rows(aliases, rows, resting, limits),
-             "refresh": "records", "check": False, "plan": "", "name": ""}
+             "refresh": how_refreshed(tag), "plan": "", "name": ""}
         got = live.get(tag) or {}
-        if tag.split("/")[0] == "cli-proxy-api":
-            t["refresh"] = "live"
+        if tag.split("/")[0] == "cli-proxy-api" or host == XKIRO_HOST:
             if got.get("groups"):
                 t.update(kind="groups", groups=got["groups"], name=got.get("name", ""), plan=got.get("plan", ""))
         elif tag == "openrouter":
-            t["refresh"] = "live" if _openrouter_endpoint() else "records"
             t["kind"] = "pool"
             t["pool"] = _openrouter_pool(aliases, utc_rows, regs, got, learned, now)
-        if t["kind"] == "models":                       # the headers' windows, when any is in force
-            recs = [r for r in quota.live(host, now=now) if r.get("alias") in mine
-                    and r.get("window") != "retry-after" and r.get("limit")]
+        elif host == REQUESTY_HOST and "free_used" in got and (_ts(got.get("at")) or 0) >= _utc_midnight(now):
+            # Requesty's count; the limit only as a 429 stated it (learned).
+            of = min([v for v in (((learned.get(a) or {}).get("limits") or {}).get("rpd", {}).get("value")
+                                  for a in aliases) if v] or [0]) or None
+            used = got["free_used"]
+            t["kind"] = "pool"
+            t["pool"] = [{"label": "free requests today · this key", "used": used, "of": of,
+                          "left": max(0, round(100 * (of - used) / of)) if of else None, "reset_at": None}]
+        if t["kind"] == "models":                       # the headers' windows, as last seen
+            # A window past its reset has filled again: its limit is known, all
+            # of it left, the next reset not.
+            recs = [r if (_ts(r.get("reset_at")) or 0) > now else dict(r, remaining=r["limit"], reset_at=None)
+                    for r in seen_recs if r.get("remaining") is not None]
             by: dict[str, list] = {}
             for r in recs:
                 by.setdefault(r["alias"], []).append(
@@ -366,26 +624,35 @@ def tiles(now: float | None = None) -> list[dict]:
             seen = [_ts(r.get("observed_at")) or 0 for r in recs]
             if seen:
                 got = {"at": _iso(max(seen)), "how": "seen"}
-        t["check"] = t["refresh"] == "records" and _headers_seen(tag)
+        # Where the tile's figures come from, said on it: exact (an endpoint,
+        # the replies' headers) or a stated limit counted by the bench.
+        t["source"] = ((["its reply headers" if got.get("how") == "seen" else "its quota endpoint"] if t["groups"] else [])
+                       + ([("its usage endpoint" if host == REQUESTY_HOST
+                             else "its stated daily limit, counted by the bench")] if t["pool"] else [])
+                       + [g["label"] for g in t["caps"]]) or ["nothing stated"]
         last = max((r["t"] for r in rows if r.get("status") == "ok"), default=None)
         t["fresh"] = ({"how": "live", "at": got["at"]} if got.get("at") and got.get("how") != "seen"
                       else {"how": "seen", "at": got["at"]} if got.get("at")
                       else {"how": "answered", "at": _iso(last)} if last else {"how": "none", "at": None})
         t["live"] = sum(1 for a in aliases if a not in resting)
-        resets = [_ts(w.get("reset_at")) for g in t["groups"] for w in g["windows"]] \
-            + [_ts(p.get("reset_at")) for p in t["pool"]]
+        windows = [w for g in t["groups"] + t["caps"] for w in g["windows"]]
+        resets = [_ts(w.get("reset_at")) for w in windows] + [_ts(p.get("reset_at")) for p in t["pool"]]
         resets = [r for r in resets if r and r > now]
         t["next_reset"] = _iso(min(resets)) if resets else None
-        t["out"] = (t["live"] == 0 or any(w["left"] <= 0 for g in t["groups"] for w in g["windows"])
+        t["out"] = (t["live"] == 0 or any(w["left"] <= 0 for w in windows)
                     or any(p.get("left") == 0 for p in t["pool"]))
-        t["notes"] = list(got.get("notes") or [])
+        t["notes"] = list(got.get("notes") or []) \
+            + ([f"NVIDIA's site: {nv['rpm_text']}, varying by model and traffic."] if nv.get("rpm_text") else []) \
+            + ([f"NVIDIA's limits file could not be read ({nv['broken']}); these are the figures read"
+                f" {str(nv.get('at'))[:10]}."] if nv.get("broken") else [])
         out.append(t)
     return ordered(out)
 
 
 def _openrouter_pool(aliases, utc_rows, regs, got, learned, now) -> list[dict]:
     """A bar per OpenRouter key the bench uses: free-model requests since
-    00:00 UTC, the bench's own count, against 50 or 1,000 a day."""
+    00:00 UTC, the bench's own count, against the day's limit OpenRouter
+    stated (a 429's), else none."""
     import endpoints
     import quota
     seen_429 = locked.read_json(quota.FILE).get("openrouter") or {}
@@ -396,14 +663,13 @@ def _openrouter_pool(aliases, utc_rows, regs, got, learned, now) -> list[dict]:
     for ep, eps in sorted(by_ep.items()):
         used = sum(r.get("attempts") or 1 for r in utc_rows
                    if r.get("alias") in eps and str(r.get("served", "")).endswith(":free"))
-        # What OpenRouter stated wins (learned, else the last 429's, kept a day
-        # past reset); the key's free tier only guesses, and was seen wrong.
+        # Only what OpenRouter stated (learned, else the last 429's, kept a day
+        # past reset): its docs' 50-or-1,000 by tier changes unseen, and the
+        # tier was seen wrong.
         stated = [((learned.get(a) or {}).get("limits") or {}).get("rpd", {}).get("value") for a in eps]
         stated += [r.get("limit") for a in eps for r in (seen_429.get(a) or {}).values()
                    if isinstance(r, dict) and r.get("window") in ("day", "unknown")]
         limit = min([s for s in stated if s] or [0]) or None
-        if not limit and got.get("endpoint") == ep:
-            limit = 50 if got.get("free_tier") else 1000
         via = "direct key" if not endpoints.hub(ep) else f"via {ep}"
         bar = {"label": f"free requests today · {via}", "used": used, "of": limit,
                "left": max(0, round(100 * (limit - used) / limit)) if limit else None,
@@ -447,6 +713,72 @@ def _selftest() -> None:
     assert c == [{"label": "", "windows": [{"label": "month", "left": 0,
                                             "reset_at": "2026-11-26T15:38:32+00:00"}]}], c
     assert codex_groups({}) == [] and antigravity_groups({}) == []
+    now = 1793306000.0                                   # 2026-10-29 ~19:13 UTC
+    xk = xkiro_groups({"free_tokens": {"used_today": 125000, "limit_per_day": 500000, "remaining": 375000},
+                       "windows": [{"kind": "short", "window_sec": 18000, "spent_usd": "1.500000",
+                                    "cap_usd": "6.000000", "remaining_usd": "4.500000", "resets_in_sec": 600}]}, now)
+    assert xk[0]["windows"][0] == {"label": "tokens · day", "left": 75, "remaining": 375000, "of": 500000,
+                                   "reset_at": "2026-10-30T00:00:00+00:00"}, xk
+    assert xk[1]["windows"][0]["label"] == "5 hours · spend" and xk[1]["windows"][0]["left"] == 75, xk
+    assert xkiro_groups({"free_tokens": {"limit_per_day": None}, "windows": []}, now) == []
+    rq = {"usage": {"2026-10-30": {"total_requests": 10, "grouped_data": [
+        {"group_by_values": {"model_used": "muse-glimmer-30b"}, "total_requests": 2},
+        {"group_by_values": {"model_used": "nemotron-3-nano-omni-30b-a3b-reasoning"}, "total_requests": 7},
+        {"group_by_values": {"model_used": "some-paid-model"}, "total_requests": 1}]}}}
+    ms = {"data": [{"id": "nvidia/muse-glimmer-30b", "input_price": 0, "output_price": 0},
+                   {"id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", "input_price": 0, "output_price": 0},
+                   {"id": "lab/some-paid-model", "input_price": 0.000001, "output_price": 0.000002}]}
+    assert requesty_free(rq, ms) == 9 and requesty_free({}, ms) == 0
+    # Caps: yours beat a stated one on the same axis; counted since midnight UTC or the month's start.
+    mid = _utc_midnight(now)
+    day = [{"connection": "router.requesty.ai", "t": mid + 60, "attempts": 2},
+           {"connection": "router.requesty.ai", "t": mid - 60, "attempts": 5},      # yesterday
+           {"connection": "api.cohere.ai", "t": mid + 60, "attempts": 1, "tokens_in": 10, "tokens_out": 5},
+           {"connection": "router.requesty.ai", "t": mid + 90, "attempts": 0}]       # refused: not sent
+    said = (200, "its replies", "Stated.")
+    g = caps("router.requesty.ai", {}, {"rpd": said}, day, lambda: [], now)
+    assert len(g) == 1 and g[0]["windows"][0] == {"label": "requests · day", "left": 99, "remaining": 198,
+                                                  "of": 200, "reset_at": _iso(mid + 86400)}, g
+    assert g[0]["label"] == "its replies · counted by the bench" and "not enforced" in g[0]["title"], g
+    g = caps("router.requesty.ai", {"router.requesty.ai": {"rpd": 10, "rpm": 5}}, {"rpd": said}, day, lambda: [], now)
+    assert g[0]["label"] == "your cap · counted by the bench" and g[0]["windows"] == [
+        {"label": "requests · day", "left": 80, "remaining": 8, "of": 10, "reset_at": _iso(mid + 86400)}], g
+    asked = []
+    g = caps("api.cohere.ai", {}, {"rpmo": (1000, "its replies", "")}, day, lambda: asked.append(1) or day, now)
+    assert asked and g[0]["windows"][0]["remaining"] == 999 and g[0]["windows"][0]["label"] == "requests · month", g
+    assert caps("nvidia-api", {}, {}, day, lambda: 1 / 0, now) == []                   # no cap: no ledger read
+    # NVIDIA's file: read, a change alerts (to dismiss), a move or a new shape alerts and keeps the last read.
+    nv = {"rateLimits": {"requestsPerMinute": "Up to 40 rpm", "requestsPerDay": "10,000 requests per day"}}
+    assert nvidia_figures(nv) == {"rpd": 10000, "rpm_text": "Up to 40 rpm"}
+    import tempfile
+    import alerts as al
+    g, seen = globals(), []
+    saved = (g["LIVE"], g["_get_json"], g["_nvidia_on_bench"], al.raise_alert, al.clear)
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            g["LIVE"], g["_nvidia_on_bench"] = pathlib.Path(tmp) / "live.json", lambda: True
+            al.raise_alert = lambda key, *a, **k: seen.append(("raise", key, k.get("dismiss")))
+            al.clear = lambda key: seen.append(("clear", key))
+            g["_get_json"] = lambda url, head: nv
+            assert watch_nvidia()["rpd"] == 10000 and not [s for s in seen if s[0] == "raise"], seen
+            nv = {"rateLimits": {"requestsPerMinute": "Up to 30 rpm", "requestsPerDay": "5,000 requests per day"}}
+            seen.clear()
+            assert watch_nvidia()["rpd"] == 5000 and ("raise", "limits:nvidia:changed", True) in seen, seen
+
+            def gone(url, head):
+                raise urllib.error.HTTPError(url, 404, "Not Found", {}, None)
+            g["_get_json"] = gone
+            seen.clear()
+            r = watch_nvidia()
+            assert r["rpd"] == 5000 and "404" in r["broken"] and ("raise", "limits:nvidia:unread", None) in seen, r
+            assert locked.read_json(g["LIVE"])["_nvidia"]["rpd"] == 5000                 # the last good read stays
+            g["_get_json"] = lambda url, head: {"somethingElse": {}}
+            assert "rateLimits" in watch_nvidia()["broken"]
+            g["_get_json"] = lambda url, head: nv
+            seen.clear()
+            assert "broken" not in watch_nvidia() and ("clear", "limits:nvidia:unread") in seen, seen
+    finally:
+        g["LIVE"], g["_get_json"], g["_nvidia_on_bench"], al.raise_alert, al.clear = saved
     assert _seconds_name(18000) == "5 hours" and _seconds_name(7200) == "2 hours"
     assert _utc_midnight(1793306000) == dt.datetime(2026, 10, 29, tzinfo=UTC).timestamp()
     assert _limit_text({"rpm": 20, "tpd": 200000}) == "20 requests a minute, 200,000 tokens a day"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/auto_retire.py

@@ -19,6 +19,14 @@ THE RULES (decided 2026-10-15)
   reason -- a rate limit that never lifts is a quota or a dead model.
   `mistral-medium-3-5` answered 429 to 11 calls out of 11.
 - NEVER on a rate limit or timeout alone inside those 7 days.
+- OUT OF QUOTA IS NOT FAILING (2026-11-02): a call the bench held back because
+  a limit is spent until a stated reset, or an account-wide quota, says
+  nothing about the model. Its days pause the clock, and while it is the
+  latest word there is no countdown. `gpt-5.6-luna` sat on "retires in 3
+  days" while CLI Proxy API's codex sign-in was out until 26 November and the
+  bench had not sent it a call since 28 October. A refusal that names a short
+  wait and is refused again after it still counts: that is a model that keeps
+  failing, not one that is tapped out.
 
 "No answer" means from ANY source: the perf probe, a useful offload, or a
 suite output that was a real answer. A model that answers in the suite resets
@@ -68,6 +76,16 @@ def account_quota(detail: str) -> bool:
     return any(marker in detail for marker in ACCOUNT_QUOTAS)
 
 
+HELD = "not sent; paced by the bench"                  # metrics/pacing.py's refusal
+
+
+def tapped_out(detail: str) -> bool:
+    """Out of quota right now, so the model could not be tried: the bench held
+    the call back (a limit spent until its reset, the provider's or a cap you
+    set), or the account's shared quota is spent (account_quota)."""
+    return HELD in detail or account_quota(detail)
+
+
 def plan(today: dt.date, models: dict[str, str], live: set[str],
          first_missing: dict[str, str], rows: list[dict],
          answers: dict[str, dt.date], restored: dict[str, dt.date],
@@ -87,15 +105,18 @@ def plan(today: dt.date, models: dict[str, str], live: set[str],
     down: dict[str, bool] = {}
     candidates: list[tuple[str, str, str]] = []      # (alias, bucket, why)
     for alias, served in models.items():
-        mine = [r for r in rows if r["alias"] == alias]
+        mine = sorted((r for r in rows if r["alias"] == alias), key=lambda r: r["date"])   # [-1] is the latest word
         last_ok = max([r["date"] for r in mine if r["answered"]]
                       + ([answers[alias]] if alias in answers else []),
                       default=None)
         floor = max(d for d in (last_ok, restored.get(alias), dt.date.min) if d)
         # A refused key (`signin`) is the endpoint's, fixed by signing in again.
         fails = sorted((r for r in mine if not r["answered"] and r["failure"] and r["failure"] != "signin"
-                        and r["date"] > floor and not account_quota(r["detail"])),
+                        and r["date"] > floor and not tapped_out(r["detail"])),
                        key=lambda r: r["date"])
+        quiet = {r["date"] for r in mine if not r["answered"] and r["date"] > floor
+                 and tapped_out(r["detail"])}
+        resting = bool(mine) and not mine[-1]["answered"] and tapped_out(mine[-1]["detail"])
         missing_since = None
         if served not in live and served in first_missing:
             missing_since = max(dt.date.fromisoformat(first_missing[served]),
@@ -107,8 +128,10 @@ def plan(today: dt.date, models: dict[str, str], live: set[str],
         why = (f"{latest['failure']}: {latest['detail']}".rstrip(": ")
                if latest else "")
         other_days = {r["date"] for r in fails if r["failure"] == "other"}
-        span = (today - fails[0]["date"]).days if fails else 0
         fail_days = {r["date"] for r in fails}
+        # Days it was out of quota, and nothing else, do not count: the clock pauses.
+        span = ((today - fails[0]["date"]).days
+                - len({q for q in quiet - fail_days if q > fails[0]["date"]})) if fails else 0
 
         # A CONNECTION PREFIX ADDED IN OPEN WEBUI renames every model on it
         # (`codestral-latest` -> `llm7.codestral-latest`), and the old names
@@ -146,7 +169,7 @@ def plan(today: dt.date, models: dict[str, str], live: set[str],
                 left.append((OTHER_DAYS - len(other_days), why))
             if fails:
                 left.append((BACKSTOP_DAYS - span, why))
-            if left:
+            if left and not resting:                   # resting: the Quota page says when it is back
                 days, reason = min(left)
                 out["pending"].append((alias, f"{max(days, 1)}|{reason}"))
 
@@ -351,6 +374,23 @@ def _selfcheck() -> None:
                    detail="429 Rate limit exceeded: free-models-per-day") for n in (8, 5, 0)]
     p = plan(d, {"quota": "v/quota"}, {"v/quota"}, {}, shared, {}, {}, {"v/quota": "v"})
     assert not p["retire"] and not p["pending"], p
+    # held back by the bench for days (its codex sign-in spent until a reset): no countdown
+    held = [dict(row("luna", n, "capacity"), detail=f"rate limit ({HELD}): 0 left") for n in (9, 5, 0)]
+    p = plan(d, {"luna": "c/luna"}, {"c/luna"}, {}, held, {}, {}, {"c/luna": "c"})
+    assert not p["retire"] and not p["pending"], p
+    # real failures, then out of quota for a week: those days pause the clock
+    paused = ([row("p", n, "capacity") for n in (10, 9, 8, 0)]
+              + [dict(row("p", n, "capacity"), detail=HELD) for n in range(1, 8)])
+    p = plan(d, {"p": "c/p"}, {"c/p"}, {}, paused, {}, {}, {"c/p": "c"})
+    assert not p["retire"] and dict(p["pending"])["p"].startswith("4|"), p
+    # ...and while it is still out of quota, no countdown is shown at all
+    p = plan(d, {"p": "c/p"}, {"c/p"}, {}, paused[:3] + paused[4:], {}, {}, {"c/p": "c"})
+    assert not p["retire"] and not p["pending"], p
+    # a short stated wait, refused again day after day, is a model that keeps failing
+    busy = [dict(row("u", n, "capacity"), detail="429 allows 1 request(s) every 1 min, retry in 60s")
+            for n in (8, 5, 0)]
+    p = plan(d, {"u": "v/u"}, {"v/u"}, {}, busy, {}, {}, {"v/u": "v"})
+    assert {a for a, _ in p["retire"]} == {"u"}, p
     # A refused key is the endpoint's, never the model's (suite/run.py SIGNIN)
     from run import failure_class
     assert failure_class("Error: Error code: 401 - {'error': 'invalid key'}") == "signin"