@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2

package/metrics/learn.py

@@ -1,221 +1,234 @@
-"""Learn a model's real limits from its own error messages.
-
-    python metrics/learn.py              # what has been learned, and active cooldowns
-    python metrics/learn.py --selftest   # canned error strings, no calls
-
-THE CATALOG IS CURATED; THE ERRORS ARE THE TRUTH
-------------------------------------------------
-A provider that says "maximum context length is 131072 tokens" has told us its
-real window, whatever a catalog claims; one that says "Limit 30, Used 30 ...
-requests per minute" has told us its rate limit; one that says "try again in
-17s" has told us when to come back. FreeLLMAPI's router (MIT, read 2026-10-17)
-learns exactly these at runtime, and so does this: every failed call from the
-perf probe, the suite (and so suite/pace.py) and the gateway passes its error
-text through `record()`.
-
-ONLY EVER TIGHTER
------------------
-A learned context window or rate limit is kept only if it is LOWER than what is
-known, and replaces an older learned value only when lower still. An error can
-prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
-says. Values you entered in the model popup (config/model-overrides.json)
-still win: you can always correct a bad parse.
-
-    config/model-learned.json   alias -> {context_window, limits, capabilities, served_as}
-                                with the date and the evidence for each
-    runs/cooldowns.json         alias -> {until, why}: rate-limited, come back later.
-                                Not config: it expires, and is nobody's decision.
-"""
-
-from __future__ import annotations
-
-import datetime as dt
-import json
-import pathlib
-import re
-import sys
-
-import locked
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-LEARNED = DATA / "config" / "model-learned.json"
-COOLDOWNS = DATA / "runs" / "cooldowns.json"
-DEFAULT_COOLDOWN_S = 1800          # a 429 with no stated wait
-MAX_COOLDOWN_S = 24 * 3600
-
-CONTEXT = [
-    r"maximum context length is\s*([\d,]+)",
-    r"context length(?: of| is)?\s*([\d,]+)",
-    r"context window(?: of| is)?\s*([\d,]+)",
-    r"max_model_len[^\d]{0,20}([\d,]+)",
-    r"maximum (?:input|prompt) (?:length|tokens?)(?: is)?\s*([\d,]+)",
-]
-AXES = {"rpm": r"requests? per minute|\brpm\b",
-        "rpd": r"requests? per day|\brpd\b",
-        "tpm": r"tokens? per minute|\btpm\b",
-        "tpd": r"tokens? per day|\btpd\b"}
-RETRY = [
-    r"retry-after[\"':\s]+(\d+(?:\.\d+)?)",
-    r"\"retry_?delay\"\s*:\s*\"(\d+(?:\.\d+)?)s",
-    r"\"retry_after\"\s*:\s*(\d+(?:\.\d+)?)",
-    r"(?:try|retry) again in\s*(\d+(?:\.\d+)?)\s*(s|sec|seconds?|m|min|minutes?|h|hours?)\b",
-    r"please retry in\s*(\d+(?:\.\d+)?)\s*(s|m|h)",
-]
-RATE_LIMITED = re.compile(r"rate.?limit|too many requests|\b429\b|quota exceeded|resource_exhausted")
-
-
-def _int(s: str) -> int:
-    return int(s.replace(",", ""))
-
-
-def parse(text: str) -> dict:
-    """What an error message proves: {context_window, limits{axis: n},
-    retry_after_s, rate_limited}. Absent keys were not stated."""
-    low = (text or "").lower()
-    out: dict = {}
-    for pat in CONTEXT:
-        if (m := re.search(pat, low)) and _int(m.group(1)) >= 512:
-            out["context_window"] = _int(m.group(1))
-            break
-    # "Limit 30" (or "limit: 30") only counts when one axis is named, so a
-    # number is never filed under the wrong limit.
-    named = [axis for axis, pat in AXES.items() if re.search(pat, low)]
-    if len(named) == 1 and (m := re.search(r"\blimit\b[\s:=]*([\d,]+)", low)):
-        out["limits"] = {named[0]: _int(m.group(1))}
-    for pat in RETRY:
-        if m := re.search(pat, low):
-            n = float(m.group(1))
-            unit = (m.group(2) if m.lastindex and m.lastindex >= 2 else "s")[0]
-            out["retry_after_s"] = min(n * {"s": 1, "m": 60, "h": 3600}[unit], MAX_COOLDOWN_S)
-            break
-    out["rate_limited"] = bool(RATE_LIMITED.search(low)) or "retry_after_s" in out
-    return out
-
-
-_read = locked.read_json
-
-
-def learned() -> dict:
-    return {k: v for k, v in _read(LEARNED).items() if not k.startswith("_")}
-
-
-def _known(alias: str, field: str, axis: str | None = None):
-    """The value a learned figure has to beat: the catalog's, else the last learned."""
-    import catalog
-    auto = catalog.catalog().get(alias, {})
-    mine = learned().get(alias, {})
-    if axis:
-        vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
-                catalog.connection_limits().get(catalog.connection_of(alias), {}).get(axis)]
-    else:
-        vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
-    vals = [v for v in vals if v]
-    return min(vals) if vals else None
-
-
-def record(alias: str, text: str, where: str) -> dict:
-    """Learn from one failed call. Returns what was stored or set, for logging."""
-    found = parse(text)
-    now = dt.datetime.now().astimezone()
-    stamp = now.isoformat(timespec="seconds")
-    evidence = " ".join((text or "").split())[:200]
-    stored: dict = {}
-
-    def tighten(data: dict) -> None:
-        # Compared inside the lock, so two seats learning at once both count.
-        entry = data.get(alias, {})
-        if (n := found.get("context_window")) and n < (_known(alias, "context_window") or 1 << 62):
-            entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
-            stored["context_window"] = n
-        for axis, n in (found.get("limits") or {}).items():
-            if n < (_known(alias, "", axis) or 1 << 62):
-                entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
-                                                        "evidence": evidence}
-                stored[axis] = n
-        if stored:
-            data[alias] = entry
-            data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
-                                    " tightened. Your values in config/model-overrides.json win.")
-
-    if alias and (found.get("context_window") or found.get("limits")):
-        locked.update(LEARNED, tighten, ensure_ascii=False)
-    if alias and found.get("rate_limited"):
-        wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
-        cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
-                "why": evidence, "where": where, "stated": "retry_after_s" in found}
-        locked.update(COOLDOWNS, lambda cd: cd.__setitem__(alias, cool), ensure_ascii=False)
-        stored["cooldown_s"] = round(wait)
-    return stored
-
-
-def cooldowns() -> dict:
-    """Active cooldowns only: alias -> {until, why}."""
-    now = dt.datetime.now().astimezone()
-    out = {}
-    for alias, c in _read(COOLDOWNS).items():
-        try:
-            if dt.datetime.fromisoformat(c["until"]) > now:
-                out[alias] = c
-        except (KeyError, ValueError, TypeError):
-            continue
-    return out
-
-
-def note_capability(alias: str, cap: str, evidence: str) -> None:
-    """A capability SEEN working (a tool call returned, reasoning tokens
-    present). Only positives are stored: a probe that fails proves nothing."""
-    seen = {"value": True, "at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
-            "evidence": evidence[:200]}
-    locked.update(LEARNED, lambda data: data.setdefault(alias, {}).setdefault("capabilities", {})
-                  .__setitem__(cap, seen), ensure_ascii=False)
-
-
-def note_served_as(alias: str, requested: str, served: str) -> None:
-    """The response named a different model than the one asked for."""
-    swap = {"requested": requested, "served": served,
-            "at": dt.datetime.now().astimezone().isoformat(timespec="seconds")}
-    locked.update(LEARNED, lambda data: data.setdefault(alias, {}).__setitem__("served_as", swap),
-                  ensure_ascii=False)
-
-
-def _selftest() -> None:
-    p = parse("Error code: 400 - This model's maximum context length is 131,072 tokens. However,"
-              " you requested 140000 tokens")
-    assert p["context_window"] == 131072 and not p["rate_limited"], p
-    p = parse("429 Too Many Requests: Rate limit reached for model in organization: Limit 30,"
-              " Used 30, Requested 1. Please try again in 2.5s. (requests per minute)")
-    assert p["limits"] == {"rpm": 30} and p["retry_after_s"] == 2.5 and p["rate_limited"], p
-    p = parse('{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"retryDelay":"17s"}]}}')
-    assert p["retry_after_s"] == 17 and p["rate_limited"], p
-    p = parse("Rate limit exceeded: free-models-per-day. Try again in 3 minutes")
-    assert p["retry_after_s"] == 180 and "limits" not in p, p      # no number under a named axis
-    # Two axes named: the number is not filed under either.
-    assert "limits" not in parse("limit 100 (tokens per minute) and requests per minute")
-    assert parse("model not found") == {"rate_limited": False}
-    assert parse("context length of 8k exceeded").get("context_window") is None   # "8" < 512
-    print("ok")
-
-
-def main() -> None:
-    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-    if "--selftest" in sys.argv:
-        _selftest()
-        return
-    for alias, e in sorted(learned().items()):
-        bits = []
-        if e.get("context_window"):
-            bits.append(f"context {e['context_window']['value']:,}")
-        bits += [f"{k} {v['value']}" for k, v in (e.get("limits") or {}).items()]
-        bits += [f"can {k}" for k in (e.get("capabilities") or {})]
-        if e.get("served_as"):
-            bits.append(f"served as {e['served_as']['served']}")
-        print(f"  {alias:<34} {', '.join(bits)}")
-    for alias, c in sorted(cooldowns().items()):
-        print(f"  cooling  {alias:<26} until {c['until'][11:16]}  ({c['why'][:70]})")
-
-
-if __name__ == "__main__":
-    sys.path.insert(0, str(ROOT / "metrics"))
-    sys.path.insert(0, str(ROOT / "suite"))
-    main()
+"""Learn a model's real limits from its own error messages.
+
+    python metrics/learn.py              # what has been learned, and active cooldowns
+    python metrics/learn.py --selftest   # canned error strings, no calls
+
+THE CATALOG IS CURATED; THE ERRORS ARE THE TRUTH
+------------------------------------------------
+A provider that says "maximum context length is 131072 tokens" has told us its
+real window, whatever a catalog claims; one that says "Limit 30, Used 30 ...
+requests per minute" has told us its rate limit; one that says "try again in
+17s" has told us when to come back. FreeLLMAPI's router (MIT, read 2026-10-17)
+learns exactly these at runtime, and so does this: every failed call from the
+perf probe, the suite (and so suite/pace.py) and the gateway passes its error
+text through `record()`.
+
+ONLY EVER TIGHTER
+-----------------
+A learned context window or rate limit is kept only if it is LOWER than what is
+known, and replaces an older learned value only when lower still. An error can
+prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
+says. Values you entered in the model popup (config/model-overrides.json)
+still win: you can always correct a bad parse.
+
+    config/model-learned.json   alias -> {context_window, limits, capabilities, served_as}
+                                with the date and the evidence for each
+    runs/cooldowns.json         alias -> {until, why}: rate-limited, come back later.
+                                Not config: it expires, and is nobody's decision.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import pathlib
+import re
+import sys
+
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+LEARNED = DATA / "config" / "model-learned.json"
+COOLDOWNS = DATA / "runs" / "cooldowns.json"
+DEFAULT_COOLDOWN_S = 1800          # a 429 with no stated wait
+MAX_COOLDOWN_S = 24 * 3600
+
+CONTEXT = [
+    r"maximum context length is\s*([\d,]+)",
+    r"context length(?: of| is)?\s*([\d,]+)",
+    r"context window(?: of| is)?\s*([\d,]+)",
+    r"max_model_len[^\d]{0,20}([\d,]+)",
+    r"maximum (?:input|prompt) (?:length|tokens?)(?: is)?\s*([\d,]+)",
+]
+AXES = {"rpm": r"requests? per minute|\brpm\b",
+        "rpd": r"requests? per day|\brpd\b",
+        "tpm": r"tokens? per minute|\btpm\b",
+        "tpd": r"tokens? per day|\btpd\b"}
+RETRY = [
+    r"retry-after[\"':\s]+(\d+(?:\.\d+)?)",
+    r"\"retry_?delay\"\s*:\s*\"(\d+(?:\.\d+)?)s",
+    r"\"retry_after\"\s*:\s*(\d+(?:\.\d+)?)",
+    r"(?:try|retry) again in\s*(\d+(?:\.\d+)?)\s*(s|sec|seconds?|m|min|minutes?|h|hours?)\b",
+    r"please retry in\s*(\d+(?:\.\d+)?)\s*(s|m|h)",
+]
+RATE_LIMITED = re.compile(r"rate.?limit|too many requests|\b429\b|quota exceeded|resource_exhausted")
+
+
+def _int(s: str) -> int:
+    return int(s.replace(",", ""))
+
+
+def parse(text: str) -> dict:
+    """What an error message proves: {context_window, limits{axis: n},
+    retry_after_s, rate_limited}. Absent keys were not stated."""
+    low = (text or "").lower()
+    out: dict = {}
+    for pat in CONTEXT:
+        if (m := re.search(pat, low)) and _int(m.group(1)) >= 512:
+            out["context_window"] = _int(m.group(1))
+            break
+    # "Limit 30" (or "limit: 30") only counts when one axis is named, so a
+    # number is never filed under the wrong limit.
+    named = [axis for axis, pat in AXES.items() if re.search(pat, low)]
+    if len(named) == 1 and (m := re.search(r"\blimit\b[\s:=]*([\d,]+)", low)):
+        out["limits"] = {named[0]: _int(m.group(1))}
+    for pat in RETRY:
+        if m := re.search(pat, low):
+            n = float(m.group(1))
+            unit = (m.group(2) if m.lastindex and m.lastindex >= 2 else "s")[0]
+            out["retry_after_s"] = min(n * {"s": 1, "m": 60, "h": 3600}[unit], MAX_COOLDOWN_S)
+            break
+    out["rate_limited"] = bool(RATE_LIMITED.search(low)) or "retry_after_s" in out
+    return out
+
+
+_read = locked.read_json
+
+
+def learned() -> dict:
+    return {k: v for k, v in _read(LEARNED).items() if not k.startswith("_")}
+
+
+def _known(alias: str, field: str, axis: str | None = None):
+    """The value a learned figure has to beat: the catalog's, else the last learned."""
+    import catalog
+    auto = catalog.catalog().get(alias, {})
+    mine = learned().get(alias, {})
+    if axis:
+        vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
+                catalog.connection_limits().get(catalog.connection_of(alias), {}).get(axis)]
+    else:
+        vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
+    vals = [v for v in vals if v]
+    return min(vals) if vals else None
+
+
+def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) -> dict:
+    """Store a context window or limits that are lower than what is known."""
+    stored: dict = {}
+
+    def tighten(data: dict) -> None:
+        # Compared inside the lock, so two seats learning at once both count.
+        entry = data.get(alias, {})
+        if (n := found.get("context_window")) and n < (_known(alias, "context_window") or 1 << 62):
+            entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
+            stored["context_window"] = n
+        for axis, n in (found.get("limits") or {}).items():
+            if n < (_known(alias, "", axis) or 1 << 62):
+                entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
+                                                        "evidence": evidence}
+                stored[axis] = n
+        if stored:
+            data[alias] = entry
+            data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
+                                    " tightened. Your values in config/model-overrides.json win.")
+
+    if alias and (found.get("context_window") or found.get("limits")):
+        locked.update(LEARNED, tighten, ensure_ascii=False)
+    return stored
+
+
+def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
+    """Limits a provider stated outside its message text: a 429 body's quota
+    (metrics/quota.py, Gemini's "50 a day"). Kept the same way, only tighter."""
+    stamp = dt.datetime.now().astimezone().isoformat(timespec="seconds")
+    return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
+
+
+def record(alias: str, text: str, where: str) -> dict:
+    """Learn from one failed call. Returns what was stored or set, for logging."""
+    found = parse(text)
+    now = dt.datetime.now().astimezone()
+    stamp = now.isoformat(timespec="seconds")
+    evidence = " ".join((text or "").split())[:200]
+    stored = _tighten(alias, found, where, evidence, stamp)
+    if alias and found.get("rate_limited"):
+        wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
+        cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
+                "why": evidence, "where": where, "stated": "retry_after_s" in found}
+        locked.update(COOLDOWNS, lambda cd: cd.__setitem__(alias, cool), ensure_ascii=False)
+        stored["cooldown_s"] = round(wait)
+    return stored
+
+
+def cooldowns() -> dict:
+    """Active cooldowns only: alias -> {until, why}."""
+    now = dt.datetime.now().astimezone()
+    out = {}
+    for alias, c in _read(COOLDOWNS).items():
+        try:
+            if dt.datetime.fromisoformat(c["until"]) > now:
+                out[alias] = c
+        except (KeyError, ValueError, TypeError):
+            continue
+    return out
+
+
+def note_capability(alias: str, cap: str, evidence: str) -> None:
+    """A capability SEEN working (a tool call returned, reasoning tokens
+    present). Only positives are stored: a probe that fails proves nothing."""
+    seen = {"value": True, "at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
+            "evidence": evidence[:200]}
+    locked.update(LEARNED, lambda data: data.setdefault(alias, {}).setdefault("capabilities", {})
+                  .__setitem__(cap, seen), ensure_ascii=False)
+
+
+def note_served_as(alias: str, requested: str, served: str) -> None:
+    """The response named a different model than the one asked for."""
+    swap = {"requested": requested, "served": served,
+            "at": dt.datetime.now().astimezone().isoformat(timespec="seconds")}
+    locked.update(LEARNED, lambda data: data.setdefault(alias, {}).__setitem__("served_as", swap),
+                  ensure_ascii=False)
+
+
+def _selftest() -> None:
+    p = parse("Error code: 400 - This model's maximum context length is 131,072 tokens. However,"
+              " you requested 140000 tokens")
+    assert p["context_window"] == 131072 and not p["rate_limited"], p
+    p = parse("429 Too Many Requests: Rate limit reached for model in organization: Limit 30,"
+              " Used 30, Requested 1. Please try again in 2.5s. (requests per minute)")
+    assert p["limits"] == {"rpm": 30} and p["retry_after_s"] == 2.5 and p["rate_limited"], p
+    p = parse('{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"retryDelay":"17s"}]}}')
+    assert p["retry_after_s"] == 17 and p["rate_limited"], p
+    p = parse("Rate limit exceeded: free-models-per-day. Try again in 3 minutes")
+    assert p["retry_after_s"] == 180 and "limits" not in p, p      # no number under a named axis
+    # Two axes named: the number is not filed under either.
+    assert "limits" not in parse("limit 100 (tokens per minute) and requests per minute")
+    assert parse("model not found") == {"rate_limited": False}
+    assert parse("context length of 8k exceeded").get("context_window") is None   # "8" < 512
+    print("ok")
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    if "--selftest" in sys.argv:
+        _selftest()
+        return
+    for alias, e in sorted(learned().items()):
+        bits = []
+        if e.get("context_window"):
+            bits.append(f"context {e['context_window']['value']:,}")
+        bits += [f"{k} {v['value']}" for k, v in (e.get("limits") or {}).items()]
+        bits += [f"can {k}" for k in (e.get("capabilities") or {})]
+        if e.get("served_as"):
+            bits.append(f"served as {e['served_as']['served']}")
+        print(f"  {alias:<34} {', '.join(bits)}")
+    for alias, c in sorted(cooldowns().items()):
+        print(f"  cooling  {alias:<26} until {c['until'][11:16]}  ({c['why'][:70]})")
+
+
+if __name__ == "__main__":
+    sys.path.insert(0, str(ROOT / "metrics"))
+    sys.path.insert(0, str(ROOT / "suite"))
+    main()

package/metrics/ledger.py

@@ -0,0 +1,148 @@
+"""The usage ledger: one JSON line for every call the built-in client makes.
+
+    python metrics/ledger.py      # calls and tokens per connection: last hour, last day, this month
+
+RELEASE-PLAN Phase 2b, "evidence by default". A line:
+
+    {"t": 1761264000.123, "at": "2026-10-23T19:00:00-05:00", "alias": "...",
+     "connection": "groq", "endpoint": "openwebui", "served": "llama-...",
+     "status": "ok", "attempts": 1, "tokens_in": 812, "tokens_out": 96,
+     "reasoning": 0, "seconds": 2.4, "caller": "gateway", "purpose": ""}
+
+`t` is when the call STARTED (epoch seconds): that is the moment a provider's
+per-minute window counts, and pacing counts the same way. `at` is the same
+moment for people. `attempts` is how many HTTP requests the call sent,
+retries included, and 0 for a call the bench refused to send, since requests
+are what a provider's request limits count. `status` is ok, empty, error,
+timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
+provider's own figures from the reply's usage, absent when it gave none.
+`caller` is the script that made the call (gateway, perf/run, suite/run, ...)
+and `purpose` what the caller said it was for, when it said.
+
+A FILE A MONTH
+--------------
+metrics/usage/YYYY-MM.jsonl, by UTC month. Pacing (metrics/pacing.py) counts
+a window before every call on a limited connection, and its longest window
+is a calendar month, so a count reads this month's file and at most the one
+before, never the whole history.
+
+Lines are appended under locked.lock, so the gateway's seat threads and the
+suite's processes never interleave half-lines. A reader takes no lock and
+skips a line it cannot parse (one still being written).
+
+It replaces metrics/cost.py's read of the llm CLI's logs.db. Calls made
+through the llm backend are still only in logs.db, so cost.py reads both;
+nothing is in both.
+"""
+
+from __future__ import annotations
+
+import calendar
+import collections
+import datetime as dt
+import json
+import pathlib
+import sys
+import time
+
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+DIR = DATA / "metrics" / "usage"
+
+
+def _file(t: float) -> pathlib.Path:
+    return DIR / (time.strftime("%Y-%m", time.gmtime(t)) + ".jsonl")
+
+
+def append(row: dict) -> None:
+    """Add one call's line. `row["t"]` (its start) picks the month's file."""
+    line = json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n"
+    path = _file(row["t"])
+    with locked.lock(DIR / "usage"):
+        with open(path, "a", encoding="utf-8", newline="\n") as fh:
+            fh.write(line)
+
+
+def month_start(t: float) -> float:
+    """The first moment of t's calendar month, UTC."""
+    g = time.gmtime(t)
+    return float(calendar.timegm((g.tm_year, g.tm_mon, 1, 0, 0, 0)))
+
+
+def next_month(t: float) -> float:
+    """The first moment of the calendar month after t's, UTC."""
+    g = time.gmtime(t)
+    y, m = (g.tm_year + 1, 1) if g.tm_mon == 12 else (g.tm_year, g.tm_mon + 1)
+    return float(calendar.timegm((y, m, 1, 0, 0, 0)))
+
+
+def rows(since: float = 0.0, until: float | None = None):
+    """Every line that started at or after `since` (and before `until`), oldest
+    file first. since=0 reads every month there is."""
+    if since <= 0:
+        files = sorted(DIR.glob("*.jsonl"))
+    else:
+        files, t = [], month_start(since)
+        end = until or time.time()
+        while t <= end:
+            files.append(_file(t))
+            t = next_month(t)
+    for path in files:
+        try:
+            fh = open(path, encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        with fh:
+            for line in fh:
+                try:
+                    row = json.loads(line)
+                except ValueError:
+                    continue                           # a line still being written
+                if not isinstance(row, dict) or not isinstance(row.get("t"), (int, float)):
+                    continue
+                if row["t"] >= since and (until is None or row["t"] < until):
+                    yield row
+
+
+def tokens(usage: dict) -> dict:
+    """{tokens_in, tokens_out, reasoning} from a reply's usage, in either
+    spelling: chat completions' prompt/completion tokens, or the Responses
+    API's input/output tokens. Absent keys were not reported."""
+    if not isinstance(usage, dict):
+        return {}
+    out = {}
+    for key, names in (("tokens_in", ("prompt_tokens", "input_tokens")),
+                       ("tokens_out", ("completion_tokens", "output_tokens"))):
+        val = next((usage[n] for n in names if isinstance(usage.get(n), int)), None)
+        if val is not None:
+            out[key] = val
+    details = usage.get("completion_tokens_details") or usage.get("output_tokens_details") or {}
+    if isinstance(details, dict) and isinstance(details.get("reasoning_tokens"), int):
+        out["reasoning"] = details["reasoning_tokens"]
+    return out
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    now = time.time()
+    spans = (("hour", now - 3600), ("day", now - 86400), ("month", month_start(now)))
+    agg: dict = collections.defaultdict(lambda: {s: [0, 0] for s, _ in spans})
+    for row in rows(min(t for _, t in spans)):
+        for span, since in spans:
+            if row["t"] >= since:
+                a = agg[row.get("connection") or "?"][span]
+                a[0] += row.get("attempts") or 0
+                a[1] += (row.get("tokens_in") or 0) + (row.get("tokens_out") or 0)
+    if not agg:
+        print(f"no calls in the ledger this month ({DIR})")
+        return
+    print(f"{'connection':<24}" + "".join(f"{s + ' (req/tok)':>22}" for s, _ in spans))
+    for conn, a in sorted(agg.items()):
+        print(f"{conn:<24}" + "".join(f"{a[s][0]:>10,} / {a[s][1]:>9,}" for s, _ in spans))
+    print(f"\nmonth from {dt.datetime.fromtimestamp(month_start(now), dt.timezone.utc):%Y-%m-%d} UTC; {DIR}")
+
+
+if __name__ == "__main__":
+    main()