@wendellresearch-ai/benchwarmers 0.4.0 → 0.5.1

package/metrics/learn.py

@@ -60,9 +60,17 @@ RETRY = [
     r"\"retry_?delay\"\s*:\s*\"(\d+(?:\.\d+)?)s",
     r"\"retry_after\"\s*:\s*(\d+(?:\.\d+)?)",
     r"(?:try|retry) again in\s*(\d+(?:\.\d+)?)\s*(s|sec|seconds?|m|min|minutes?|h|hours?)\b",
-    r"please retry in\s*(\d+(?:\.\d+)?)\s*(s|m|h)",
+    r"\bretry in\s*(\d+(?:\.\d+)?)\s*(s|m|h)",         # "please retry in 2s", UnoRouter's "retry in 60s"
 ]
 RATE_LIMITED = re.compile(r"rate.?limit|too many requests|\b429\b|quota exceeded|resource_exhausted")
+# The number and its window in one phrase: UnoRouter's free tier, "allows 5
+# request(s) every 1 min per account on <model>:free". "every" only: the
+# bench's own refusal says "allows 3 requests per minute" (pacing.py), and a
+# limit the bench made up must never be learned back.
+ALLOWS = re.compile(r"allows?\s+([\d,]+)\s+requests?(?:\(s\))?\s+every\s+(?:1\s+)?(min(?:ute)?|day)\b")
+# new-api gateways (Agnes): "maximum 20 requests in 1 minutes" (its
+# middleware/model-rate-limit.go); a window of 1 minute or 1,440 only.
+MAXIMUM = re.compile(r"maximum\s+([\d,]+)\s+requests?\s+in\s+(1|1440)\s+minutes?\b")
 
 
 def _int(s: str) -> int:
@@ -83,6 +91,10 @@ def parse(text: str) -> dict:
     named = [axis for axis, pat in AXES.items() if re.search(pat, low)]
     if len(named) == 1 and (m := re.search(r"\blimit\b[\s:=]*([\d,]+)", low)):
         out["limits"] = {named[0]: _int(m.group(1))}
+    elif m := ALLOWS.search(low):
+        out["limits"] = {"rpd" if m.group(2) == "day" else "rpm": _int(m.group(1))}
+    elif m := MAXIMUM.search(low):
+        out["limits"] = {"rpd" if m.group(2) == "1440" else "rpm": _int(m.group(1))}
     for pat in RETRY:
         if m := re.search(pat, low):
             n = float(m.group(1))
@@ -205,6 +217,18 @@ def _selftest() -> None:
     assert p["retry_after_s"] == 180 and "limits" not in p, p      # no number under a named axis
     # Two axes named: the number is not filed under either.
     assert "limits" not in parse("limit 100 (tokens per minute) and requests per minute")
+    # UnoRouter's free tier says the number and its window together.
+    p = parse("Error code: 429 - {'error': {'message': 'Too many requests. The free tier allows 5 request(s) every"
+              " 1 min per account on llama-3.2-11b-vision:free - nothing is used up, retry in 60s.")
+    assert p["limits"] == {"rpm": 5} and p["retry_after_s"] == 60 and p["rate_limited"], p
+    assert parse("The free tier allows 200 requests every day")["limits"] == {"rpd": 200}
+    assert "limits" not in parse("allows 3 requests every 5 min")             # a window no axis holds
+    assert "limits" not in parse("rate limit (not sent; paced by the bench): x allows 3 requests per minute")
+    # A new-api gateway's 429 (Agnes).
+    p = parse("Error code: 429 - {'error': {'message': 'You have reached the request limit: maximum 20 requests"
+              " in 1 minutes', 'type': 'AgnesAI_error'}}")
+    assert p["limits"] == {"rpm": 20} and p["rate_limited"], p
+    assert "limits" not in parse("maximum 100 requests in 5 minutes")         # a window no axis holds
     assert parse("model not found") == {"rate_limited": False}
     assert parse("context length of 8k exceeded").get("context_window") is None   # "8" < 512
     print("ok")

package/metrics/pacing.py

@@ -177,6 +177,28 @@ def _ago(s: float) -> str:
     return "just now" if s < 60 else f"{s / 60:.0f} min ago" if s < 7200 else f"{s / 3600:.0f} h ago"
 
 
+def _since(rs: list[dict], records: list[dict]) -> float:
+    """The oldest ledger line any of these windows or quota records counts."""
+    spans = [AXES[r["axis"]][1] for r in rs]
+    return min([ledger.month_start(time.time()) if s == "month" else time.time() - s for s in spans]
+               + [_ts(q["observed_at"]) for q in records if q.get("observed_at")])
+
+
+def held(alias: str, connection: str, est_tokens: int = 1) -> tuple[float, str] | None:
+    """(seconds until there is room, why) when a call to `alias` would have to
+    wait or be refused now, else None. Read only: no slot is taken. For
+    picking seats, so a model the bench knows will be refused is not seated
+    (gateway.refused_now); `admit` still decides each call."""
+    import quota
+    rs, records = rules(alias, connection), quota.live(connection, alias)
+    if not rs and not records:
+        return None
+    now = time.time()
+    block = _blocked(rs, records, alias, connection, est_tokens, now,
+                     list(ledger.rows(_since(rs, records))), _inflight(now))
+    return None if block is None else (max(block[0] - now, 0.0), block[1])
+
+
 def admit(alias: str, connection: str, est_tokens: int, left: float, sleep=time.sleep) -> tuple[str | None, str]:
     """Wait for room, or refuse. Returns (ticket, refusal): a ticket to pass
     to `release` once the call's ledger line is written (None when no limit
@@ -189,9 +211,7 @@ def admit(alias: str, connection: str, est_tokens: int, left: float, sleep=time.
         records = quota.live(connection, alias)
         if not rs and not records:
             return None, ""
-        spans = [AXES[r["axis"]][1] for r in rs]
-        since = min([ledger.month_start(time.time()) if s == "month" else time.time() - s for s in spans]
-                    + [_ts(q["observed_at"]) for q in records if q.get("observed_at")])
+        since = _since(rs, records)
         with locked.lock(LOCK):
             now = time.time()
             rows = list(ledger.rows(since))

package/metrics/quota.py

@@ -21,7 +21,11 @@ its reset, so a connection is never held for want of traffic to observe one.
 ONE HEADER NAME, DIFFERENT WINDOWS
 ----------------------------------
 `x-ratelimit-limit-requests` is per minute at OpenAI and per day at Groq;
-Cerebras names its windows (`-requests-day`, `-tokens-minute`); OpenRouter
+Cerebras names its windows (`-requests-day`, `-tokens-minute`), Routeway only
+its windows (`X-RateLimit-Remaining-Day`, requests, resets in seconds), and a
+Cohere trial key its minute (`x-trial-endpoint-call-remaining`, no reset) and
+its month's limit with nothing left said (`x-endpoint-monthly-call-limit`:
+kept, never enforced, and quotaview counts the month from the ledger); OpenRouter
 sends bare `X-RateLimit-*` only on a 429, inside the body's metadata as well,
 and its message says which window ("free-models-per-day"); Anthropic's
 `anthropic-ratelimit-*` are per minute; Gemini sends no headers but its 429
@@ -76,16 +80,25 @@ PROFILES = {
 WINDOWS = {"minute": 60, "hour": 3600, "day": 86400}
 OPENAI_STYLE = re.compile(r"^x-ratelimit-(limit|remaining|reset)-(requests|tokens)(?:-(minute|hour|day))?$")
 BARE = re.compile(r"^x-ratelimit-(limit|remaining|reset)$")
+WINDOWED = re.compile(r"^x-ratelimit-(limit|remaining|reset)-(minute|hour|day)$")      # Routeway: requests
+COHERE = re.compile(r"^x-trial-endpoint-call-(limit|remaining)$")                      # a trial key's minute
+COHERE_MONTH = "x-endpoint-monthly-call-limit"          # a limit with no remaining: quotaview counts it
 ANTHROPIC = re.compile(r"^anthropic-ratelimit-(requests|tokens|input-tokens|output-tokens)-(limit|remaining|reset)$")
 DURATION = re.compile(r"^(?:(\d+(?:\.\d+)?)h)?(?:(\d+(?:\.\d+)?)m(?!s))?(?:(\d+(?:\.\d+)?)s)?(?:(\d+(?:\.\d+)?)ms)?$")
 
 
-def profile_for(base: str) -> tuple[str, dict]:
-    """The profile for an endpoint's base URL; ("generic", {}) for any other host."""
+def profile_for(base: str, connection: str = "") -> tuple[str, dict]:
+    """The profile for an endpoint's base URL, else for the connection (a hub
+    such as Open WebUI relays the provider's headers: `groq`, `api.groq.com`);
+    ("generic", {}) for any other."""
     host = (urllib.parse.urlsplit(base or "").hostname or "").lower()
+    conn = (connection or "").lower()
     for known, prof in PROFILES.items():
         if host == known or host.endswith("." + known):
             return prof
+    for known, prof in PROFILES.items():
+        if conn in (known, prof[0]) or conn.endswith("." + known):
+            return prof
     return "generic", {}
 
 
@@ -148,8 +161,15 @@ def from_headers(headers, profile: tuple[str, dict], now: float, status: int = 2
             what, metric, window = m.group(1), m.group(2), m.group(3) or defaults.get(m.group(2), "unknown")
         elif m := BARE.match(key):
             what, metric, window = m.group(1), "requests", _window_hint(hint)
+        elif m := WINDOWED.match(key):
+            what, metric, window = m.group(1), "requests", m.group(2)
         elif m := ANTHROPIC.match(key):
             metric, what, window = m.group(1), m.group(2), "minute"
+        elif m := COHERE.match(key):                   # no reset sent: the minute is the wait
+            what, metric, window = m.group(1), "requests", "minute"
+            fields.setdefault((metric, window), {}).setdefault("reset", "60")
+        elif key == COHERE_MONTH:                      # no remaining, no reset: never enforced (live())
+            what, metric, window = "limit", "requests", "month"
         else:
             continue
         fields.setdefault((metric, window), {})[what] = value
@@ -247,7 +267,7 @@ def store(connection: str, alias: str, records: list[dict], now: float | None =
 def observe(alias: str, connection: str, base: str, status: int, headers, body=None) -> list[dict]:
     """Record what one response states; returns the records. Called by the
     built-in client on every reply and every HTTP error."""
-    now, profile = time.time(), profile_for(base)
+    now, profile = time.time(), profile_for(base, connection)
     hint = " ".join(str(e.get("message") or "") for e in _error_objects(body)) if status == 429 else ""
     records = from_headers(headers, profile, now, status, hint)
     if status == 429 and body is not None:
@@ -318,12 +338,29 @@ def _selftest() -> None:
                       "X-Ratelimit-Reset-Requests": "2m59.56s"}, profile_for("https://api.groq.com/openai/v1"), now)
     g = one(r, "requests", "day")
     assert g["remaining"] == 0 and g["reset_at"] == _iso(now + 179.56) and g["source"] == "headers (groq)", g
+    # Through a hub, the connection names the provider.
+    assert profile_for("http://owui.lan/api", "groq")[0] == "groq" == profile_for("http://owui.lan/api", "api.groq.com")[0]
+    assert profile_for("http://owui.lan/api", "api.cohere.ai") == ("generic", {})
     # Cerebras names its windows.
     r = from_headers({"x-ratelimit-limit-requests-day": "14400", "x-ratelimit-remaining-requests-day": "14399",
                       "x-ratelimit-reset-requests-day": "33011.38", "x-ratelimit-limit-tokens-minute": "60000",
                       "x-ratelimit-remaining-tokens-minute": "59000", "x-ratelimit-reset-tokens-minute": "11.4"},
                      profile_for("https://api.cerebras.ai/v1"), now)
     assert one(r, "requests", "day")["limit"] == 14400 and one(r, "tokens", "minute")["remaining"] == 59000
+    # Routeway names only the window; requests, resets in seconds.
+    r = from_headers({"X-RateLimit-Limit-Day": "200", "X-RateLimit-Remaining-Day": "187", "X-RateLimit-Reset-Day": "3600",
+                      "X-RateLimit-Limit-Minute": "5", "X-RateLimit-Remaining-Minute": "4", "X-RateLimit-Reset-Minute": "42"},
+                     profile_for("http://owui.lan/api"), now)
+    d = one(r, "requests", "day")
+    assert d["limit"] == 200 and d["remaining"] == 187 and d["reset_at"] == _iso(now + 3600), d
+    assert one(r, "requests", "minute")["remaining"] == 4
+    # A Cohere trial key: its minute, with no reset (the minute is the wait); the month has no remaining.
+    r = from_headers({"x-trial-endpoint-call-limit": "20", "x-trial-endpoint-call-remaining": "19",
+                      "x-endpoint-monthly-call-limit": "1000"}, profile_for("http://owui.lan/api"), now)
+    c = one(r, "requests", "minute")
+    assert c["limit"] == 20 and c["remaining"] == 19 and c["reset_at"] == _iso(now + 60) and len(r) == 2, r
+    mo = one(r, "requests", "month")
+    assert mo["limit"] == 1000 and mo["remaining"] is None and mo["reset_at"] is None, mo
     # Anthropic: RFC 3339 resets, per minute.
     r = from_headers({"anthropic-ratelimit-requests-limit": "50", "anthropic-ratelimit-requests-remaining": "0",
                       "anthropic-ratelimit-requests-reset": "2025-10-24T00:00:45Z"},