@wendellresearch-ai/benchwarmers 0.2.1 → 0.3.0

package/metrics/guard.py

@@ -31,8 +31,10 @@ Decided by ROUTE, never by what a request happens to carry, so an API call
 with a bearer never meets the login cookie and a browser's cookie never opens
 the gateway:
 
-- the gateway's routes (`/v1/...`, `/bench/...`, on either port) take a
-  GATEWAY KEY as `Authorization: Bearer <key>`: named, revocable, kept only
+- the gateway's routes (`/v1/...`, `/v1beta/...`, `/bench/...` and Ollama's
+  five `/api/` names, on either port) take a GATEWAY KEY as `Authorization:
+  Bearer <key>` (or `x-api-key`, `x-goog-api-key`, a Gemini route's `?key=`,
+  as those SDKs send it): named, revocable, kept only
   as a SHA-256 in `config/gateway-keys.json`, shown once when made. A key
   never opens the dashboard, so a chat UI or agent given one cannot change
   settings, read keys or spend through it.
@@ -81,7 +83,15 @@ PREFIX = "bwk_"                    # every gateway key starts so: a secret scann
 COOKIE = "bw_session"
 SESSION_S = 7 * 24 * 3600
 MIN_TOKEN = 16
-GATEWAY_ROUTES = ("/v1/", "/bench/")
+GATEWAY_ROUTES = ("/v1/", "/v1beta/", "/bench/")
+# Ollama's routes, which the gateway answers too (metrics/formats.py). They sit
+# under the dashboard's /api/, so these five names are the gateway's, on either
+# port, and the dashboard must never use them.
+OLLAMA_ROUTES = ("/api/chat", "/api/generate", "/api/tags", "/api/version", "/api/show")
+# Where a client may send its key other than `Authorization: Bearer`: the
+# Anthropic SDK's header, and Google's (its SDKs send the header; REST
+# examples put the key in ?key=, which is taken on Gemini routes only).
+KEY_HEADERS = ("x-api-key", "x-goog-api-key")
 
 
 class Refused(Exception):
@@ -210,7 +220,7 @@ def startup(address: str) -> None:
 
 def gateway_route(path: str) -> bool:
     """A route the gateway answers, on its own port or the dashboard's."""
-    return path.startswith(GATEWAY_ROUTES)
+    return path.startswith(GATEWAY_ROUTES) or path.rstrip("/") in OLLAMA_ROUTES
 
 
 def _sign(expires: int) -> bytes:
@@ -267,17 +277,24 @@ def admin_caller(headers) -> str:
     return "admin"
 
 
-def gateway_caller(headers) -> str:
+def gateway_caller(headers, query_key: str | None = None) -> str:
     """Who is asking for a gateway route: the gateway key's name, or "local"
     when keys are not required (any bearer is ignored then: SDKs send a dummy
     one). Raises Refused(401), saying which of missing, malformed or unknown
-    it was. The cookie is never looked at."""
+    it was. The cookie is never looked at.
+
+    The key is the first of `Authorization: Bearer`, `x-api-key`,
+    `x-goog-api-key` and `query_key` (a Gemini route's ?key=) that is sent:
+    one credential is judged, never a second tried after a bad first."""
     if not keys_required():
         return "local"
     challenge = {"WWW-Authenticate": 'Bearer realm="benchwarmers"'}     # RFC 6750: every 401 says how
     raw = headers.get("Authorization")
+    other = next((headers.get(h) for h in KEY_HEADERS if headers.get(h) is not None), query_key)
+    if raw is None and other is not None:
+        raw = f"Bearer {other}"
     if raw is None:
-        raise Refused(401, "this route needs a gateway key (Authorization: Bearer <key>);"
+        raise Refused(401, "this route needs a gateway key (Authorization: Bearer <key>, or x-api-key);"
                            " make one on the dashboard's Settings page", challenge)
     scheme, _, value = raw.strip().partition(" ")
     value = value.strip()
@@ -540,6 +557,17 @@ def _selftest() -> None:
                 assert key_name("") is None and key_name(key + "x") is None and key_name("bwk_é") is None
                 assert gateway_caller({"Authorization": f"Bearer {key}"}) == "chat-ui"
                 assert gateway_caller({"Authorization": f"bearer  {key} "}) == "chat-ui"
+                # Other APIs' places for a key: Anthropic's header, Google's header and ?key=.
+                assert gateway_caller({"x-api-key": key}) == "chat-ui"
+                assert gateway_caller({"x-goog-api-key": key}) == "chat-ui"
+                assert gateway_caller({}, query_key=key) == "chat-ui"
+                for headers, q in (({"Authorization": "Bearer sk-dummy", "x-api-key": key}, None),   # the first sent is judged
+                                   ({"x-api-key": "nope"}, key), ({"x-api-key": ""}, None), ({}, "")):
+                    try:
+                        gateway_caller(headers, q)
+                        raise AssertionError(f"gateway let in {headers} {q!r}")
+                    except Refused as r:
+                        assert r.status == 401, r
                 entry = gateway_keys()["chat-ui"]
                 assert entry["hint"] == key[-4:] and entry["last_used"] and "rpm" not in entry, entry
                 # Each refusal says which it was, and how to authenticate (RFC 6750).
@@ -605,6 +633,8 @@ def _selftest() -> None:
                 KEYS = real
         assert gateway_route("/v1/models") and gateway_route("/bench/verdict")
         assert not gateway_route("/api/endpoints") and not gateway_route("/v1") and not gateway_route("//v1/x")
+        assert gateway_route("/api/tags") and gateway_route("/api/chat/") and gateway_route("/v1beta/models")
+        assert not gateway_route("/api/tagsx") and not gateway_route("/api/stamp")
     finally:
         for k, v in saved.items():
             os.environ.pop(k, None)

package/metrics/learn.py

@@ -114,12 +114,8 @@ def _known(alias: str, field: str, axis: str | None = None):
     return min(vals) if vals else None
 
 
-def record(alias: str, text: str, where: str) -> dict:
-    """Learn from one failed call. Returns what was stored or set, for logging."""
-    found = parse(text)
-    now = dt.datetime.now().astimezone()
-    stamp = now.isoformat(timespec="seconds")
-    evidence = " ".join((text or "").split())[:200]
+def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) -> dict:
+    """Store a context window or limits that are lower than what is known."""
     stored: dict = {}
 
     def tighten(data: dict) -> None:
@@ -140,6 +136,23 @@ def record(alias: str, text: str, where: str) -> dict:
 
     if alias and (found.get("context_window") or found.get("limits")):
         locked.update(LEARNED, tighten, ensure_ascii=False)
+    return stored
+
+
+def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
+    """Limits a provider stated outside its message text: a 429 body's quota
+    (metrics/quota.py, Gemini's "50 a day"). Kept the same way, only tighter."""
+    stamp = dt.datetime.now().astimezone().isoformat(timespec="seconds")
+    return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
+
+
+def record(alias: str, text: str, where: str) -> dict:
+    """Learn from one failed call. Returns what was stored or set, for logging."""
+    found = parse(text)
+    now = dt.datetime.now().astimezone()
+    stamp = now.isoformat(timespec="seconds")
+    evidence = " ".join((text or "").split())[:200]
+    stored = _tighten(alias, found, where, evidence, stamp)
     if alias and found.get("rate_limited"):
         wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
         cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),

package/metrics/ledger.py

@@ -0,0 +1,148 @@
+"""The usage ledger: one JSON line for every call the built-in client makes.
+
+    python metrics/ledger.py      # calls and tokens per connection: last hour, last day, this month
+
+RELEASE-PLAN Phase 2b, "evidence by default". A line:
+
+    {"t": 1761264000.123, "at": "2026-10-23T19:00:00-05:00", "alias": "...",
+     "connection": "groq", "endpoint": "openwebui", "served": "llama-...",
+     "status": "ok", "attempts": 1, "tokens_in": 812, "tokens_out": 96,
+     "reasoning": 0, "seconds": 2.4, "caller": "gateway", "purpose": ""}
+
+`t` is when the call STARTED (epoch seconds): that is the moment a provider's
+per-minute window counts, and pacing counts the same way. `at` is the same
+moment for people. `attempts` is how many HTTP requests the call sent,
+retries included, and 0 for a call the bench refused to send, since requests
+are what a provider's request limits count. `status` is ok, empty, error,
+timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
+provider's own figures from the reply's usage, absent when it gave none.
+`caller` is the script that made the call (gateway, perf/run, suite/run, ...)
+and `purpose` what the caller said it was for, when it said.
+
+A FILE A MONTH
+--------------
+metrics/usage/YYYY-MM.jsonl, by UTC month. Pacing (metrics/pacing.py) counts
+a window before every call on a limited connection, and its longest window
+is a calendar month, so a count reads this month's file and at most the one
+before, never the whole history.
+
+Lines are appended under locked.lock, so the gateway's seat threads and the
+suite's processes never interleave half-lines. A reader takes no lock and
+skips a line it cannot parse (one still being written).
+
+It replaces metrics/cost.py's read of the llm CLI's logs.db. Calls made
+through the llm backend are still only in logs.db, so cost.py reads both;
+nothing is in both.
+"""
+
+from __future__ import annotations
+
+import calendar
+import collections
+import datetime as dt
+import json
+import pathlib
+import sys
+import time
+
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+DIR = DATA / "metrics" / "usage"
+
+
+def _file(t: float) -> pathlib.Path:
+    return DIR / (time.strftime("%Y-%m", time.gmtime(t)) + ".jsonl")
+
+
+def append(row: dict) -> None:
+    """Add one call's line. `row["t"]` (its start) picks the month's file."""
+    line = json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n"
+    path = _file(row["t"])
+    with locked.lock(DIR / "usage"):
+        with open(path, "a", encoding="utf-8", newline="\n") as fh:
+            fh.write(line)
+
+
+def month_start(t: float) -> float:
+    """The first moment of t's calendar month, UTC."""
+    g = time.gmtime(t)
+    return float(calendar.timegm((g.tm_year, g.tm_mon, 1, 0, 0, 0)))
+
+
+def next_month(t: float) -> float:
+    """The first moment of the calendar month after t's, UTC."""
+    g = time.gmtime(t)
+    y, m = (g.tm_year + 1, 1) if g.tm_mon == 12 else (g.tm_year, g.tm_mon + 1)
+    return float(calendar.timegm((y, m, 1, 0, 0, 0)))
+
+
+def rows(since: float = 0.0, until: float | None = None):
+    """Every line that started at or after `since` (and before `until`), oldest
+    file first. since=0 reads every month there is."""
+    if since <= 0:
+        files = sorted(DIR.glob("*.jsonl"))
+    else:
+        files, t = [], month_start(since)
+        end = until or time.time()
+        while t <= end:
+            files.append(_file(t))
+            t = next_month(t)
+    for path in files:
+        try:
+            fh = open(path, encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        with fh:
+            for line in fh:
+                try:
+                    row = json.loads(line)
+                except ValueError:
+                    continue                           # a line still being written
+                if not isinstance(row, dict) or not isinstance(row.get("t"), (int, float)):
+                    continue
+                if row["t"] >= since and (until is None or row["t"] < until):
+                    yield row
+
+
+def tokens(usage: dict) -> dict:
+    """{tokens_in, tokens_out, reasoning} from a reply's usage, in either
+    spelling: chat completions' prompt/completion tokens, or the Responses
+    API's input/output tokens. Absent keys were not reported."""
+    if not isinstance(usage, dict):
+        return {}
+    out = {}
+    for key, names in (("tokens_in", ("prompt_tokens", "input_tokens")),
+                       ("tokens_out", ("completion_tokens", "output_tokens"))):
+        val = next((usage[n] for n in names if isinstance(usage.get(n), int)), None)
+        if val is not None:
+            out[key] = val
+    details = usage.get("completion_tokens_details") or usage.get("output_tokens_details") or {}
+    if isinstance(details, dict) and isinstance(details.get("reasoning_tokens"), int):
+        out["reasoning"] = details["reasoning_tokens"]
+    return out
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    now = time.time()
+    spans = (("hour", now - 3600), ("day", now - 86400), ("month", month_start(now)))
+    agg: dict = collections.defaultdict(lambda: {s: [0, 0] for s, _ in spans})
+    for row in rows(min(t for _, t in spans)):
+        for span, since in spans:
+            if row["t"] >= since:
+                a = agg[row.get("connection") or "?"][span]
+                a[0] += row.get("attempts") or 0
+                a[1] += (row.get("tokens_in") or 0) + (row.get("tokens_out") or 0)
+    if not agg:
+        print(f"no calls in the ledger this month ({DIR})")
+        return
+    print(f"{'connection':<24}" + "".join(f"{s + ' (req/tok)':>22}" for s, _ in spans))
+    for conn, a in sorted(agg.items()):
+        print(f"{conn:<24}" + "".join(f"{a[s][0]:>10,} / {a[s][1]:>9,}" for s, _ in spans))
+    print(f"\nmonth from {dt.datetime.fromtimestamp(month_start(now), dt.timezone.utc):%Y-%m-%d} UTC; {DIR}")
+
+
+if __name__ == "__main__":
+    main()

package/metrics/model_roles.py

@@ -16,6 +16,11 @@ roles:   general  reasoning / do-it-all: eligible for every shape
                   bounded tasks (classify, extract, summarise, enumerate over a
                   small file); not review or design over a large packet
          image    image generation; never a reviewer
+         vision   a specialist at reading images (OCR, documents, screenshots,
+                  charts), weak at text-only work: admitted by the vision shape
+                  alone, and a collection member with this role sits only when
+                  the request carries images (gateway.py). A model that reads
+                  images AND does everything else stays general.
          decision typed answers with probabilities through a Decisions API, not
                   chat; no shape admits them (config/decision-models.json)
 
@@ -30,8 +35,11 @@ shapes:  today's task, `route.py --shape <name>`; general when none is given.
 ad_hoc:  aliases NEVER chosen automatically: not by the router's seats or
          reserve, not by a benchmark sweep or `--next` rotation, not by the
          daily perf run. Orthogonal to role: a role says what a model is FOR,
-         this says what it COSTS. They answer only when named (`--with`,
-         `llm -m`, a benchmark's `--model`).
+         this says what it COSTS. They answer only when named: `--with` or
+         the gateway's `with`, a collection that lists them as a member or
+         merger (every request through it spends their quota), `llm -m`, a
+         benchmark's `--model`. `bench onboard --quick --ad-hoc` (or Quick + ad
+         hoc in the picker) onboards one this way from the start.
 
 A role never lowers a score: rank is decided by evidence when seats are
 picked; a role only decides which shapes a model is eligible for.
@@ -51,6 +59,8 @@ ROLES = {
     "light": "small, fast instruct model — eligible for quick and enumeration;"
              " not for review or design over a large packet",
     "image": "image generation — never a reviewer",
+    "vision": "reads images (OCR, documents, screenshots) — sits only for requests that carry"
+              " images; never a text reviewer",
     "decision": "typed choice / score / yes-no with probabilities, through a Decisions API"
                 " — no chat, never a seat; see config/decision-models.json",
 }
@@ -62,7 +72,7 @@ SHAPES = {
     "code": ["general", "code"],
     "enumeration": ["general", "code", "light"],
     "quick": ["general", "light"],
-    "vision": ["general", "light"],
+    "vision": ["general", "light", "vision"],
     "image": ["image"],
 }
 

package/metrics/pacing.py

@@ -0,0 +1,248 @@
+"""Pacing: the stored rate limits enforced before a call, not only shown.
+
+    python metrics/pacing.py <alias>     # the limits that apply, and how much of each is used
+
+RELEASE-PLAN Phase 2b ("limits enforced, not just shown"). Until now the
+limits in config/connection-limits.json (the Endpoints dialog), a model's own
+`limits` in config/model-overrides.json (its popup) and the limits
+metrics/learn.py read out of providers' errors were display only. The
+built-in client now asks `admit` before every call. When a window is used
+up it waits, if the wait ends before the call's deadline, and otherwise
+refuses with an error that names the limit, so nothing is sent that the
+provider would refuse, and nothing is spent of a cap that does not come back
+(the motivating case: a trial key at 20 a minute and 1,000 a month).
+
+WHAT IS COUNTED
+---------------
+- rpm, tpm: the last 60 s.  rpd, tpd: the last 24 h, which never allows more
+  than a provider's calendar day would, however it counts its day.
+  rpmo (connection limits only): the calendar month in UTC, how trial keys
+  are reset.
+- Requests are the ledger's `attempts` (metrics/ledger.py), retries included:
+  what the provider saw. Tokens are its in plus out.
+- Calls in flight count too. A panel fires its seats at once, long before
+  any of them is in the ledger, so each admitted call is written to
+  runs/inflight.json until its ledger line exists. The check and the write
+  happen under one lock (runs/pacing.lock), which the gateway's seat threads
+  and the suite's processes share; an entry whose call outlived its own
+  deadline is dropped, so a killed process cannot hold a slot.
+- A connection's limits count every model on it (a key's limits are the
+  key's); a model's own or learned limits count that model alone, its own
+  beating the learned (as everywhere: your values win).
+- Quota records (metrics/quota.py): what a provider last said was left, less
+  the calls sent since, until its reset.
+
+The catalog refresh's figures (FreeLLMAPI's listing) are not enforced: they
+describe a typical free tier, not this key's.
+
+THE REFUSAL IS A RATE LIMIT
+---------------------------
+`Error: client failed: rate limit (not sent; ...): ...; try again in 41s`.
+"client failed" and "rate limit" are markers (suite/run.py ERRORS and
+CAPACITY), so it counts as a retryable capacity failure, never as the model
+answering badly or going away; and learn.py reads "try again in 41s" as the
+cooldown, so the gateway's explorer seats pass the model by meanwhile. The
+text names no "limit N" figure, which learn.py would file as a new limit.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import os
+import pathlib
+import sys
+import time
+import uuid
+
+import ledger
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+LOCK = DATA / "runs" / "pacing"
+INFLIGHT = DATA / "runs" / "inflight.json"
+# axis -> (what it counts, the window in seconds or "month", how it reads)
+AXES = {"rpm": ("requests", 60, "requests per minute"), "rpd": ("requests", 86400, "requests per day"),
+        "tpm": ("tokens", 60, "tokens per minute"), "tpd": ("tokens", 86400, "tokens per day"),
+        "rpmo": ("requests", "month", "requests a calendar month (UTC)")}
+MARGIN_S = 2.0                         # a wait must leave this long for the call itself
+POLL_S = 5.0                           # re-check at least this often while waiting
+
+
+def rules(alias: str, connection: str) -> list[dict]:
+    """The limits that apply: {scope, axis, limit, source}."""
+    import catalog
+    import learn
+    conn = catalog.connection_limits().get(connection) or {}
+    mine = (catalog.overrides().get(alias) or {}).get("limits") or {}
+    seen = (learn.learned().get(alias) or {}).get("limits") or {}
+    out = []
+
+    def ok(n) -> bool:
+        return isinstance(n, int) and not isinstance(n, bool) and n > 0
+    for axis in AXES:
+        if ok(n := conn.get(axis)):
+            out.append({"scope": "connection", "axis": axis, "limit": n,
+                        "source": f"you, for {connection} (config/connection-limits.json)"})
+        if axis == "rpmo":
+            continue
+        if ok(n := mine.get(axis)):
+            out.append({"scope": "alias", "axis": axis, "limit": n,
+                        "source": "you, for this model (config/model-overrides.json)"})
+        elif ok(n := (seen.get(axis) or {}).get("value")):
+            out.append({"scope": "alias", "axis": axis, "limit": n,
+                        "source": f"learned {str(seen[axis].get('at', ''))[:10]} from the provider"})
+    return out
+
+
+def _wait_text(s: float) -> str:
+    """learn.py's spelling (s, m or h), so it reads the cooldown back."""
+    return f"{s:.0f}s" if s < 120 else f"{s / 60:.0f}m" if s < 7200 else f"{s / 3600:.0f}h"
+
+
+def _inflight(now: float) -> dict:
+    return {k: v for k, v in locked.read_json(INFLIGHT).items()
+            if isinstance(v, dict) and (v.get("until") or 0) > now}
+
+
+def _blocked(rs: list[dict], records: list[dict], alias: str, connection: str, est: int,
+             now: float, rows: list[dict], flying: dict) -> tuple[float, str] | None:
+    """(when a slot frees, why) for the window that frees last, or None when
+    every window has room for this call."""
+    worst = None
+
+    def mine(e: dict, scope: str) -> bool:
+        return e.get("connection") == connection if scope == "connection" else e.get("alias") == alias
+    for r in rs:
+        what, span, reads = AXES[r["axis"]]
+        since = ledger.month_start(now) if span == "month" else now - span
+        events = sorted(
+            [(e["t"], (e.get("attempts") or 0) if what == "requests"
+              else (e.get("tokens_in") or 0) + (e.get("tokens_out") or 0))
+             for e in rows if e["t"] >= since and mine(e, r["scope"])]
+            + [(f["t"], 1 if what == "requests" else f.get("tokens") or 0)
+               for f in flying.values() if mine(f, r["scope"])])
+        used = sum(w for _, w in events)
+        # A prompt larger than the whole window waits for an empty one, then goes:
+        # the estimate is rough, and the provider has the last word.
+        need = 1 if what == "requests" else min(max(est, 1), r["limit"])
+        if used + need <= r["limit"]:
+            continue
+        if span == "month":
+            frees = ledger.next_month(now)
+        else:
+            frees, gone = now, 0
+            for t, w in events:                        # oldest first, until enough have aged out
+                gone += w
+                frees = t + span
+                if used - gone + need <= r["limit"]:
+                    break
+        whose = connection if r["scope"] == "connection" else alias
+        why = (f"{whose} allows {r['limit']:,} {reads} ({r['source']}) and {used:,} "
+               f"{'were' if what == 'requests' else 'tokens were'} used"
+               f"{' this month' if span == 'month' else ''}")
+        if worst is None or frees > worst[0]:
+            worst = (frees, why)
+    for q in records:
+        what = "tokens" if "tokens" in q.get("metric", "") else "requests"
+        if not isinstance(q.get("remaining"), int):
+            continue
+        try:
+            seen = _ts(q["observed_at"])
+            reset = _ts(q["reset_at"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        used = sum((e.get("attempts") or 0) if what == "requests"
+                   else (e.get("tokens_in") or 0) + (e.get("tokens_out") or 0)
+                   for e in rows if e["t"] >= seen and e.get("alias") == alias)
+        used += sum(1 if what == "requests" else f.get("tokens") or 0
+                    for f in flying.values() if f.get("alias") == alias)
+        if q["remaining"] - used > 0:
+            continue
+        limit = f" of {q['limit']:,}" if isinstance(q.get("limit"), int) else ""
+        window = "" if q.get("window") in ("unknown", "retry-after") else f" a {q['window']}"
+        why = (f"{connection} said {max(q['remaining'], 0):,}{limit} {q['metric']}{window} were left for {alias}"
+               f" ({q.get('source')}, {_ago(now - seen)})")
+        if worst is None or reset > worst[0]:
+            worst = (reset, why)
+    return worst
+
+
+def _ts(iso: str) -> float:
+    return dt.datetime.fromisoformat(iso).timestamp()
+
+
+def _ago(s: float) -> str:
+    return "just now" if s < 60 else f"{s / 60:.0f} min ago" if s < 7200 else f"{s / 3600:.0f} h ago"
+
+
+def admit(alias: str, connection: str, est_tokens: int, left: float, sleep=time.sleep) -> tuple[str | None, str]:
+    """Wait for room, or refuse. Returns (ticket, refusal): a ticket to pass
+    to `release` once the call's ledger line is written (None when no limit
+    applies, so there is nothing to release), or the refusal text."""
+    import quota
+    rs = rules(alias, connection)
+    end = time.time() + left                           # wall clock: the in-flight entry's expiry, read by other processes
+    stop = time.monotonic() + left                     # the caller's deadline, which a clock change cannot move
+    while True:
+        records = quota.live(connection, alias)
+        if not rs and not records:
+            return None, ""
+        spans = [AXES[r["axis"]][1] for r in rs]
+        since = min([ledger.month_start(time.time()) if s == "month" else time.time() - s for s in spans]
+                    + [_ts(q["observed_at"]) for q in records if q.get("observed_at")])
+        with locked.lock(LOCK):
+            now = time.time()
+            rows = list(ledger.rows(since))
+            flying = _inflight(now)
+            block = _blocked(rs, records, alias, connection, est_tokens, now, rows, flying)
+            if block is None:
+                ticket = uuid.uuid4().hex
+                flying[ticket] = {"alias": alias, "connection": connection, "t": now, "tokens": est_tokens,
+                                  "until": end + 60, "pid": os.getpid()}
+                locked.write_atomic(INFLIGHT, json.dumps(flying, indent=1) + "\n")
+                return ticket, ""
+        frees, why = block
+        wait = max(frees - time.time(), 0.05)
+        if time.monotonic() + wait > stop - MARGIN_S:
+            return None, (f"Error: client failed: rate limit (not sent; paced by the bench): {why};"
+                          f" try again in {_wait_text(wait)}")
+        sleep(min(wait, POLL_S))
+
+
+def release(ticket: str | None) -> None:
+    """The call's ledger line is written: it no longer counts as in flight."""
+    if not ticket:
+        return
+    with locked.lock(LOCK):
+        flying = _inflight(time.time())               # stale entries go too
+        flying.pop(ticket, None)
+        locked.write_atomic(INFLIGHT, json.dumps(flying, indent=1) + "\n")
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    if len(sys.argv) != 2:
+        print(__doc__.split("\n\n")[0])
+        return
+    sys.path.insert(0, str(ROOT / "suite"))
+    import route
+    alias = sys.argv[1]
+    conn = route.connection_of(alias)
+    now = time.time()
+    rows = list(ledger.rows(ledger.month_start(now) - 86400))
+    rs = rules(alias, conn)
+    if not rs:
+        print(f"{alias} ({conn}): no limit is set or learned, so nothing paces it")
+    for r in rs:
+        what, span, reads = AXES[r["axis"]]
+        since = ledger.month_start(now) if span == "month" else now - span
+        used = sum((e.get("attempts") or 0) if what == "requests" else
+                   (e.get("tokens_in") or 0) + (e.get("tokens_out") or 0) for e in rows if e["t"] >= since and
+                   (e.get("connection") == conn if r["scope"] == "connection" else e.get("alias") == alias))
+        print(f"  {r['axis']:<5} {used:>9,} of {r['limit']:<9,} {reads}  ({r['source']})")
+
+
+if __name__ == "__main__":
+    main()