@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/logo-dark.svg +49 -0
- package/.github/logo.svg +49 -0
- package/BENCHMARKS.md +160 -0
- package/GATEWAY.md +33 -11
- package/PRIVACY.md +20 -6
- package/README.md +46 -11
- package/SECURITY.md +20 -10
- package/USAGE.md +81 -11
- package/VERSION +1 -1
- package/gateway.py +93 -82
- package/metrics/bench_collections.py +27 -3
- package/metrics/catalog.py +52 -4
- package/metrics/client.py +51 -1
- package/metrics/connect.py +225 -0
- package/metrics/decide.py +720 -31
- package/metrics/endpoints.py +9 -7
- package/metrics/guard.py +6 -4
- package/metrics/learn.py +264 -10
- package/metrics/ledger.py +3 -1
- package/metrics/locked.py +102 -1
- package/metrics/offload.py +36 -19
- package/metrics/pacing.py +8 -1
- package/metrics/quotaview.py +19 -5
- package/metrics/route.py +9 -1
- package/metrics/scheduler.py +11 -1
- package/metrics/test_decide.py +299 -7
- package/metrics/wrong.py +21 -9
- package/package.json +3 -1
- package/scripts/auto_retire.py +96 -21
- package/scripts/onboard.py +51 -4
- package/scripts/test_connect.py +152 -3
- package/scripts/test_first_run.py +4 -2
- package/scripts/test_formats.py +39 -0
- package/scripts/test_quota.py +27 -0
- package/scripts/test_serve_edit.py +308 -4
- package/serve.py +1721 -145
- package/suite/README.md +5 -2
- package/suite/run.py +226 -6
- package/suite/test_pick.py +15 -0
- package/suite/test_score.py +82 -1
- package/ui/bench.css +44 -0
- package/ui/bench.js +933 -203
package/metrics/endpoints.py
CHANGED
|
@@ -131,8 +131,9 @@ def load() -> dict[str, dict]:
|
|
|
131
131
|
else's machine is a timeout with no explanation. Whatever needs an
|
|
132
132
|
endpoint asks default() (base, key, models and yaml_block all do), and
|
|
133
133
|
that refuses, so a job run before any endpoint exists says what to do."""
|
|
134
|
+
import locked
|
|
134
135
|
try:
|
|
135
|
-
data = json.loads(
|
|
136
|
+
data = json.loads(locked.read_text(FILE)) # inside a render, from its snapshot
|
|
136
137
|
except FileNotFoundError:
|
|
137
138
|
return {}
|
|
138
139
|
except (OSError, ValueError) as exc:
|
|
@@ -147,11 +148,8 @@ def connections() -> dict:
|
|
|
147
148
|
ties, names for bare-IP hosts, models.dev ids, providers whose free tier
|
|
148
149
|
works only in their own client. {} when absent: a fresh
|
|
149
150
|
install has no such facts, and none are assumed."""
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
except (OSError, ValueError):
|
|
153
|
-
return {}
|
|
154
|
-
return data if isinstance(data, dict) else {}
|
|
151
|
+
import locked
|
|
152
|
+
return locked.read_json(CONNECTIONS) # inside a render, from its snapshot
|
|
155
153
|
|
|
156
154
|
|
|
157
155
|
_default: tuple = (None, "")
|
|
@@ -297,7 +295,8 @@ def models(name: str | None = None, timeout: int = 30) -> dict[str, dict]:
|
|
|
297
295
|
def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, dict]:
|
|
298
296
|
"""What an endpoint entry lists, saved or not yet (the dashboard tests one
|
|
299
297
|
before saving it). `models_url` when the list lives elsewhere (GitHub
|
|
300
|
-
Models: its catalog, a bare JSON list
|
|
298
|
+
Models: its catalog, a bare JSON list; Cloudflare: its catalog search,
|
|
299
|
+
{"result": [...]} by `name`); entries marked `visibility` other
|
|
301
300
|
than "list" are left out (ChatGPT's plan lists ones it will not serve)."""
|
|
302
301
|
api = entry.get("api") or "chat"
|
|
303
302
|
headers = dict(entry.get("headers") or {})
|
|
@@ -315,6 +314,9 @@ def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, d
|
|
|
315
314
|
# what generateContent serves, by its bare name.
|
|
316
315
|
rows = [{**m, "id": str(m.get("name", "")).removeprefix("models/")} for m in data.get("models", [])
|
|
317
316
|
if isinstance(m, dict) and "generateContent" in (m.get("supportedGenerationMethods") or ())]
|
|
317
|
+
elif isinstance(data, dict) and isinstance(data.get("result"), list):
|
|
318
|
+
# Cloudflare's catalog search: {"result": [{"name": "@cf/...", ...}]}.
|
|
319
|
+
rows = [{**m, "id": m.get("name")} for m in data["result"] if isinstance(m, dict)]
|
|
318
320
|
else:
|
|
319
321
|
rows = data.get("data", []) if isinstance(data, dict) else data
|
|
320
322
|
return {e["id"]: e for e in rows
|
package/metrics/guard.py
CHANGED
|
@@ -146,9 +146,11 @@ def cors_origin(headers, port: int) -> str:
|
|
|
146
146
|
return origin if origin.lower() in origins(port) else ""
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
def check(headers, port: int, post: bool, max_body: int, cors: bool = False
|
|
149
|
+
def check(headers, port: int, post: bool, max_body: int, cors: bool = False,
|
|
150
|
+
kind: str = "application/json") -> int:
|
|
150
151
|
"""Raise Refused, or return how many body bytes to read (0 for a GET).
|
|
151
|
-
`cors`: a gateway route, where an allowed origin may be cross-site.
|
|
152
|
+
`cors`: a gateway route, where an allowed origin may be cross-site.
|
|
153
|
+
`kind`: the body's type; never one a plain form can send (an import's zip)."""
|
|
152
154
|
host = (headers.get("Host") or "").strip().lower()
|
|
153
155
|
if host not in hosts(port):
|
|
154
156
|
raise Refused(403, f"unknown Host {host or '(none)'!r}; set BENCHWARMERS_PUBLIC_URL"
|
|
@@ -161,8 +163,8 @@ def check(headers, port: int, post: bool, max_body: int, cors: bool = False) ->
|
|
|
161
163
|
raise Refused(403, "cross-site requests are refused")
|
|
162
164
|
if origin and origin not in allowed:
|
|
163
165
|
raise Refused(403, f"Origin {origin!r} is not allowed; set BENCH_ALLOWED_ORIGINS to add one")
|
|
164
|
-
if not (headers.get("Content-Type") or "").lower().startswith(
|
|
165
|
-
raise Refused(415, "send Content-Type:
|
|
166
|
+
if not (headers.get("Content-Type") or "").lower().startswith(kind):
|
|
167
|
+
raise Refused(415, f"send Content-Type: {kind}")
|
|
166
168
|
try:
|
|
167
169
|
length = int(headers.get("Content-Length") or 0)
|
|
168
170
|
except ValueError:
|
package/metrics/learn.py
CHANGED
|
@@ -13,18 +13,40 @@ learns exactly these at runtime, and so does this: every failed call from the
|
|
|
13
13
|
perf probe, the suite (and so suite/pace.py) and the gateway passes its error
|
|
14
14
|
text through `record()`.
|
|
15
15
|
|
|
16
|
-
ONLY EVER TIGHTER
|
|
17
|
-
|
|
16
|
+
ONLY EVER TIGHTER, UNTIL IT EXPIRES
|
|
17
|
+
-----------------------------------
|
|
18
18
|
A learned context window or rate limit is kept only if it is LOWER than what is
|
|
19
19
|
known, and replaces an older learned value only when lower still. An error can
|
|
20
20
|
prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
|
|
21
21
|
says. Values you entered in the model popup (config/model-overrides.json)
|
|
22
22
|
still win: you can always correct a bad parse.
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
A rate limit can be raised, though (a plan upgrade, a provider's change), and
|
|
25
|
+
no error ever says so. So a learned RATE limit that has not been seen again for
|
|
26
|
+
LEARNED_TTL_S (14 days) is left out wherever it is read: the bench goes back to
|
|
27
|
+
the higher rate, and the provider's next refusal teaches the limit again. The
|
|
28
|
+
same limit seen again refreshes its `confirmed_at`, so one that still holds
|
|
29
|
+
does not expire. A context window does not: a model's window does not grow.
|
|
30
|
+
|
|
31
|
+
A 429 THAT NAMES NO NUMBER
|
|
32
|
+
--------------------------
|
|
33
|
+
Cloudflare Workers AI says only that the day's free Neurons are used up. The
|
|
34
|
+
bench knows how many requests it sent on that connection in the window before
|
|
35
|
+
the refusal (metrics/ledger.py), and that count is the limit, near enough. It
|
|
36
|
+
is a connection's limit, not a model's: the key's allowance is shared. One
|
|
37
|
+
refusal is a candidate; a second at a count within 20% of it, a window or more
|
|
38
|
+
later (sooner is the same refusal, met by another call), is adopted (the lower
|
|
39
|
+
of the two) and enforced by metrics/pacing.py unless you set your own for that
|
|
40
|
+
connection. Seen again near the same count it is confirmed; a refusal at a
|
|
41
|
+
clearly lower count lowers it at once. Only rpm and rpd, and never from the
|
|
42
|
+
bench's own refusal.
|
|
43
|
+
|
|
44
|
+
config/model-learned.json alias -> {context_window, limits, capabilities, served_as}
|
|
45
|
+
with the date and the evidence for each
|
|
46
|
+
config/connection-learned.json connection -> {rpm, rpd}: {value, at, confirmed_at, where,
|
|
47
|
+
evidence}, or the candidates until one is adopted
|
|
48
|
+
runs/cooldowns.json alias -> {until, why}: rate-limited, come back later.
|
|
49
|
+
Not config: it expires, and is nobody's decision.
|
|
28
50
|
"""
|
|
29
51
|
|
|
30
52
|
from __future__ import annotations
|
|
@@ -40,9 +62,13 @@ import locked
|
|
|
40
62
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
41
63
|
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
42
64
|
LEARNED = DATA / "config" / "model-learned.json"
|
|
65
|
+
CONN_LEARNED = DATA / "config" / "connection-learned.json"
|
|
43
66
|
COOLDOWNS = DATA / "runs" / "cooldowns.json"
|
|
44
67
|
DEFAULT_COOLDOWN_S = 1800 # a 429 with no stated wait
|
|
45
68
|
MAX_COOLDOWN_S = 24 * 3600
|
|
69
|
+
LEARNED_TTL_S = 14 * 86400 # a learned rate limit not seen again for this long is forgotten
|
|
70
|
+
CONN_AXES = {"rpm": 60, "rpd": 86400} # what a connection's limit is learned for: its window, seconds
|
|
71
|
+
MIN_COUNT = 3 # fewer requests in the window than this prove nothing
|
|
46
72
|
|
|
47
73
|
CONTEXT = [
|
|
48
74
|
r"maximum context length is\s*([\d,]+)",
|
|
@@ -71,6 +97,10 @@ ALLOWS = re.compile(r"allows?\s+([\d,]+)\s+requests?(?:\(s\))?\s+every\s+(?:1\s+
|
|
|
71
97
|
# new-api gateways (Agnes): "maximum 20 requests in 1 minutes" (its
|
|
72
98
|
# middleware/model-rate-limit.go); a window of 1 minute or 1,440 only.
|
|
73
99
|
MAXIMUM = re.compile(r"maximum\s+([\d,]+)\s+requests?\s+in\s+(1|1440)\s+minutes?\b")
|
|
100
|
+
# Which window a 429 with no number is about. A window, not a wait: "try
|
|
101
|
+
# again in 3 minutes" names no per-minute limit.
|
|
102
|
+
DAY = re.compile(r"\bday\b|daily|free allocation|neurons")
|
|
103
|
+
MINUTE = re.compile(r"per[- ]?min(?:ute)?\b|a minute|\brpm\b")
|
|
74
104
|
|
|
75
105
|
|
|
76
106
|
def _int(s: str) -> int:
|
|
@@ -108,18 +138,64 @@ def parse(text: str) -> dict:
|
|
|
108
138
|
_read = locked.read_json
|
|
109
139
|
|
|
110
140
|
|
|
141
|
+
def _age(stamp, now: dt.datetime) -> float:
|
|
142
|
+
"""Seconds since an ISO stamp; forever when it is missing or unreadable."""
|
|
143
|
+
try:
|
|
144
|
+
return (now - dt.datetime.fromisoformat(stamp)).total_seconds()
|
|
145
|
+
except (TypeError, ValueError):
|
|
146
|
+
return float("inf")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _live(ev: dict, now: dt.datetime) -> bool:
|
|
150
|
+
"""Seen within LEARNED_TTL_S: an older rate limit may have been raised since."""
|
|
151
|
+
return _age(ev.get("confirmed_at") or ev.get("at"), now) <= LEARNED_TTL_S
|
|
152
|
+
|
|
153
|
+
|
|
111
154
|
def learned() -> dict:
|
|
112
|
-
|
|
155
|
+
"""alias -> what was learned, without the rate limits that have expired."""
|
|
156
|
+
now = dt.datetime.now().astimezone()
|
|
157
|
+
out = {}
|
|
158
|
+
for alias, e in _read(LEARNED).items():
|
|
159
|
+
if alias.startswith("_"):
|
|
160
|
+
continue
|
|
161
|
+
if isinstance(e, dict) and e.get("limits"):
|
|
162
|
+
e = {**e, "limits": {k: ev for k, ev in e["limits"].items() if _live(ev, now)}}
|
|
163
|
+
out[alias] = e
|
|
164
|
+
return out
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def connection_learned(pending: bool = False) -> dict:
|
|
168
|
+
"""connection -> {axis: {value, at, confirmed_at, where, evidence}}: the
|
|
169
|
+
adopted limits that have not expired. pending=True: for the axes with none,
|
|
170
|
+
the latest candidate instead ({n, at}), seen once and not enforced."""
|
|
171
|
+
now = dt.datetime.now().astimezone()
|
|
172
|
+
out: dict = {}
|
|
173
|
+
for conn, axes in _read(CONN_LEARNED).items():
|
|
174
|
+
if conn.startswith("_") or not isinstance(axes, dict):
|
|
175
|
+
continue
|
|
176
|
+
for axis, e in axes.items():
|
|
177
|
+
if axis not in CONN_AXES or not isinstance(e, dict):
|
|
178
|
+
continue
|
|
179
|
+
adopted = bool(e.get("value")) and _live(e, now)
|
|
180
|
+
waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
|
|
181
|
+
if adopted and not pending:
|
|
182
|
+
out.setdefault(conn, {})[axis] = {k: v for k, v in e.items() if k != "candidates"}
|
|
183
|
+
elif pending and not adopted and waiting:
|
|
184
|
+
out.setdefault(conn, {})[axis] = waiting[-1]
|
|
185
|
+
return out
|
|
113
186
|
|
|
114
187
|
|
|
115
188
|
def _known(alias: str, field: str, axis: str | None = None):
|
|
116
|
-
"""The value a learned figure has to beat: the catalog's, else the last
|
|
189
|
+
"""The value a learned figure has to beat: the catalog's, else the last
|
|
190
|
+
learned; for a rate limit, its connection's too, yours and the learned."""
|
|
117
191
|
import catalog
|
|
118
192
|
auto = catalog.catalog().get(alias, {})
|
|
119
193
|
mine = learned().get(alias, {})
|
|
120
194
|
if axis:
|
|
195
|
+
conn = catalog.connection_of(alias)
|
|
121
196
|
vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
|
|
122
|
-
catalog.connection_limits().get(
|
|
197
|
+
catalog.connection_limits().get(conn, {}).get(axis),
|
|
198
|
+
connection_learned().get(conn, {}).get(axis, {}).get("value")]
|
|
123
199
|
else:
|
|
124
200
|
vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
|
|
125
201
|
vals = [v for v in vals if v]
|
|
@@ -137,10 +213,13 @@ def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) ->
|
|
|
137
213
|
entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
|
|
138
214
|
stored["context_window"] = n
|
|
139
215
|
for axis, n in (found.get("limits") or {}).items():
|
|
140
|
-
|
|
216
|
+
had = (entry.get("limits") or {}).get(axis) or {}
|
|
217
|
+
if n < (_known(alias, "", axis) or 1 << 62): # an expired one is not known: a higher value may replace it
|
|
141
218
|
entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
|
|
142
219
|
"evidence": evidence}
|
|
143
220
|
stored[axis] = n
|
|
221
|
+
elif had.get("value") == n:
|
|
222
|
+
had["confirmed_at"] = stamp # still true, so it does not expire
|
|
144
223
|
if stored:
|
|
145
224
|
data[alias] = entry
|
|
146
225
|
data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
|
|
@@ -158,6 +237,75 @@ def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
|
|
|
158
237
|
return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
|
|
159
238
|
|
|
160
239
|
|
|
240
|
+
def _axis(text: str, found: dict) -> str | None:
|
|
241
|
+
"""The window a 429 that names no number is about, or None: nothing to
|
|
242
|
+
learn. Never the bench's own refusal ("not sent", pacing.py): a limit the
|
|
243
|
+
bench enforced must not be learned back from itself. Nor a token limit: the
|
|
244
|
+
count is of requests."""
|
|
245
|
+
low = (text or "").lower()
|
|
246
|
+
if not found.get("rate_limited") or found.get("limits") or "not sent" in low \
|
|
247
|
+
or re.search(AXES["tpm"] + "|" + AXES["tpd"], low):
|
|
248
|
+
return None
|
|
249
|
+
wait = found.get("retry_after_s")
|
|
250
|
+
if DAY.search(low) or (wait or 0) >= 3600:
|
|
251
|
+
return "rpd"
|
|
252
|
+
# A short wait alone is not enough for rpm: calls in flight are not in the
|
|
253
|
+
# ledger yet, so a minute's count undercounts, and two passing 429s could
|
|
254
|
+
# pin a connection at rpm 3 for 14 days. Only a named minute window counts.
|
|
255
|
+
if MINUTE.search(low):
|
|
256
|
+
return "rpm"
|
|
257
|
+
return None
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _sent(connection: str, since: float) -> int:
|
|
261
|
+
"""Requests the provider took on a connection since then: the ledger's
|
|
262
|
+
attempts, without the calls that failed, whose own retries would count the
|
|
263
|
+
refusal as use."""
|
|
264
|
+
import ledger
|
|
265
|
+
return sum(r.get("attempts") or 0 for r in ledger.rows(since)
|
|
266
|
+
if r.get("connection") == connection and r.get("status") != "error")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _near(n: int, value: int) -> bool:
|
|
270
|
+
return 0.8 * value <= n <= 1.2 * value
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _learn_connection(connection: str, axis: str, n: int, where: str, evidence: str, now: dt.datetime) -> dict:
|
|
274
|
+
"""A connection refused after `n` requests in the axis's window. Returns
|
|
275
|
+
{axis: value} when that adopted a limit or lowered one."""
|
|
276
|
+
stamp, span = now.isoformat(timespec="seconds"), CONN_AXES[axis]
|
|
277
|
+
stored: dict = {}
|
|
278
|
+
|
|
279
|
+
def change(data: dict) -> None:
|
|
280
|
+
axes = data.setdefault(connection, {})
|
|
281
|
+
e = axes.get(axis) or {}
|
|
282
|
+
fresh = {"value": n, "at": stamp, "confirmed_at": stamp, "where": where, "evidence": evidence}
|
|
283
|
+
waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
|
|
284
|
+
if e.get("value") and _live(e, now):
|
|
285
|
+
# One lesson a window: while a provider keeps refusing, the count
|
|
286
|
+
# of what it took only falls, and would drag the limit down with it.
|
|
287
|
+
if _age(e.get("confirmed_at") or e.get("at"), now) < span:
|
|
288
|
+
return
|
|
289
|
+
if n < 0.8 * e["value"]: # refused well below it: that is proof
|
|
290
|
+
axes[axis] = fresh
|
|
291
|
+
stored[axis] = n
|
|
292
|
+
elif _near(n, e["value"]):
|
|
293
|
+
e["confirmed_at"] = stamp
|
|
294
|
+
elif waiting and _age(waiting[-1]["at"], now) < span:
|
|
295
|
+
return # the same refusal, met by another call
|
|
296
|
+
elif agree := [c["n"] for c in waiting if _near(n, c["n"])]:
|
|
297
|
+
axes[axis] = fresh | {"value": min(agree + [n])}
|
|
298
|
+
stored[axis] = axes[axis]["value"]
|
|
299
|
+
else:
|
|
300
|
+
axes[axis] = {"candidates": (waiting + [{"n": n, "at": stamp}])[-5:]}
|
|
301
|
+
data.setdefault("_why", "Learned by metrics/learn.py from 429s that named no number: the requests"
|
|
302
|
+
" the bench had sent in the window. Enforced unless you set your own"
|
|
303
|
+
" (config/connection-limits.json); forgotten 14 days after last seen.")
|
|
304
|
+
|
|
305
|
+
locked.update(CONN_LEARNED, change, ensure_ascii=False)
|
|
306
|
+
return stored
|
|
307
|
+
|
|
308
|
+
|
|
161
309
|
def record(alias: str, text: str, where: str) -> dict:
|
|
162
310
|
"""Learn from one failed call. Returns what was stored or set, for logging."""
|
|
163
311
|
found = parse(text)
|
|
@@ -165,6 +313,14 @@ def record(alias: str, text: str, where: str) -> dict:
|
|
|
165
313
|
stamp = now.isoformat(timespec="seconds")
|
|
166
314
|
evidence = " ".join((text or "").split())[:200]
|
|
167
315
|
stored = _tighten(alias, found, where, evidence, stamp)
|
|
316
|
+
if alias and (axis := _axis(text, found)):
|
|
317
|
+
try:
|
|
318
|
+
import catalog
|
|
319
|
+
conn = catalog.connection_of(alias)
|
|
320
|
+
if conn and (n := _sent(conn, now.timestamp() - CONN_AXES[axis])) >= MIN_COUNT:
|
|
321
|
+
stored.update(_learn_connection(conn, axis, n, where, evidence, now))
|
|
322
|
+
except Exception: # noqa: BLE001 -- learning never breaks a call
|
|
323
|
+
pass
|
|
168
324
|
if alias and found.get("rate_limited"):
|
|
169
325
|
wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
|
|
170
326
|
cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
|
|
@@ -231,9 +387,104 @@ def _selftest() -> None:
|
|
|
231
387
|
assert "limits" not in parse("maximum 100 requests in 5 minutes") # a window no axis holds
|
|
232
388
|
assert parse("model not found") == {"rate_limited": False}
|
|
233
389
|
assert parse("context length of 8k exceeded").get("context_window") is None # "8" < 512
|
|
390
|
+
|
|
391
|
+
# Which window a 429 with no number is about.
|
|
392
|
+
def axis(text: str):
|
|
393
|
+
return _axis(text, parse(text))
|
|
394
|
+
neurons = "Error code: 429 - you have used up your daily free allocation of 10,000 neurons"
|
|
395
|
+
assert axis(neurons) == "rpd" and "limits" not in parse(neurons)
|
|
396
|
+
assert axis("Rate limit exceeded: free-models-per-day. Try again in 3 minutes") == "rpd"
|
|
397
|
+
assert axis("429 Too Many Requests, retry-after: 7200") == "rpd"
|
|
398
|
+
assert axis("429 Too Many Requests, retry-after: 30") is None # a short wait alone: no window
|
|
399
|
+
assert axis("429: too many requests per minute") == "rpm"
|
|
400
|
+
assert axis("429 Too Many Requests") is None # no window named
|
|
401
|
+
assert axis("Rate limit exceeded. Try again in 3 minutes") is None # a wait, not a window
|
|
402
|
+
assert axis("429: rate limit on tokens per minute") is None # requests are what is counted
|
|
403
|
+
assert axis("Limit 30, Used 30 (requests per minute) 429") is None # it gave the number
|
|
404
|
+
assert axis("model not found") is None
|
|
405
|
+
# The bench's own refusal is never learned from.
|
|
406
|
+
refusal = ("Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests"
|
|
407
|
+
" per day (learned) and 50 were used; try again in 2h")
|
|
408
|
+
assert axis(refusal) is None and axis(refusal.replace("not sent; ", "")) == "rpd"
|
|
409
|
+
_selftest_stores(neurons, refusal)
|
|
234
410
|
print("ok")
|
|
235
411
|
|
|
236
412
|
|
|
413
|
+
def _selftest_stores(neurons: str, refusal: str) -> None:
|
|
414
|
+
"""The connection store and the expiry, in a temp folder with a stub
|
|
415
|
+
catalog: nothing of the install's is read or written."""
|
|
416
|
+
import tempfile
|
|
417
|
+
|
|
418
|
+
import catalog
|
|
419
|
+
import ledger
|
|
420
|
+
with tempfile.TemporaryDirectory() as folder:
|
|
421
|
+
tmp = pathlib.Path(folder)
|
|
422
|
+
globals().update(LEARNED=tmp / "model-learned.json", CONN_LEARNED=tmp / "connection-learned.json",
|
|
423
|
+
COOLDOWNS=tmp / "cooldowns.json")
|
|
424
|
+
ledger.DIR = tmp / "usage"
|
|
425
|
+
catalog.catalog = catalog.connection_limits = lambda: {}
|
|
426
|
+
catalog.connection_of = lambda alias: "conn-" + alias
|
|
427
|
+
now = dt.datetime.now().astimezone()
|
|
428
|
+
|
|
429
|
+
def ago(**kw) -> dt.datetime:
|
|
430
|
+
return now - dt.timedelta(**kw)
|
|
431
|
+
|
|
432
|
+
def refused(n: int, when: dt.datetime, conn: str = "prov", ax: str = "rpd") -> dict:
|
|
433
|
+
return _learn_connection(conn, ax, n, "test", "429", when)
|
|
434
|
+
# One refusal is a candidate; the same refusal met again changes nothing.
|
|
435
|
+
assert refused(100, ago(days=5)) == {} and connection_learned() == {}
|
|
436
|
+
assert connection_learned(pending=True)["prov"]["rpd"]["n"] == 100
|
|
437
|
+
assert refused(104, ago(days=5, hours=-1)) == {} and connection_learned() == {}
|
|
438
|
+
# A second count far from the first does not adopt; one within 20% does, the lower of the two.
|
|
439
|
+
assert refused(300, ago(days=4)) == {} and connection_learned() == {}
|
|
440
|
+
assert refused(110, ago(days=3)) == {"rpd": 100}
|
|
441
|
+
got = connection_learned()["prov"]["rpd"]
|
|
442
|
+
assert got["value"] == 100 and "candidates" not in got and connection_learned(pending=True) == {}, got
|
|
443
|
+
# Seen again near it: confirmed. Clearly lower: lowered at once. Higher: left alone.
|
|
444
|
+
assert refused(95, ago(days=2)) == {}
|
|
445
|
+
got = connection_learned()["prov"]["rpd"]
|
|
446
|
+
assert got["value"] == 100 and got["confirmed_at"] == ago(days=2).isoformat(timespec="seconds") != got["at"]
|
|
447
|
+
assert refused(40, ago(days=2, hours=-1)) == {} # within the window: the same refusal
|
|
448
|
+
assert refused(500, ago(days=1)) == {} and connection_learned()["prov"]["rpd"]["value"] == 100
|
|
449
|
+
assert refused(60, now) == {"rpd": 60} and connection_learned()["prov"]["rpd"]["value"] == 60
|
|
450
|
+
# Not seen for 15 days: forgotten on read, and a higher limit can be learned.
|
|
451
|
+
assert refused(20, ago(days=16), "old", "rpm") == {} and refused(20, ago(days=15), "old", "rpm") == {"rpm": 20}
|
|
452
|
+
assert "old" not in connection_learned()
|
|
453
|
+
assert refused(50, ago(minutes=5), "old", "rpm") == {} and refused(50, now, "old", "rpm") == {"rpm": 50}
|
|
454
|
+
|
|
455
|
+
# A model's own learned limit: the same expiry, and the same limit seen again is confirmed.
|
|
456
|
+
old = {"value": 5, "at": ago(days=15).isoformat(timespec="seconds"), "where": "test", "evidence": ""}
|
|
457
|
+
LEARNED.write_text(json.dumps({"m": {"limits": {"rpm": dict(old), "rpd": dict(old)},
|
|
458
|
+
"context_window": dict(old, value=8192)}}), encoding="utf-8")
|
|
459
|
+
assert learned()["m"]["limits"] == {} and learned()["m"]["context_window"]["value"] == 8192
|
|
460
|
+
stamp = now.isoformat(timespec="seconds")
|
|
461
|
+
assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", stamp) == {"rpm": 20}
|
|
462
|
+
assert _tighten("m", {"limits": {"rpm": 30}}, "test", "", stamp) == {} # live: only ever tighter
|
|
463
|
+
assert "confirmed_at" not in learned()["m"]["limits"]["rpm"]
|
|
464
|
+
again = ago(hours=-1).isoformat(timespec="seconds")
|
|
465
|
+
assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", again) == {}
|
|
466
|
+
got = learned()["m"]["limits"]["rpm"]
|
|
467
|
+
assert got["value"] == 20 and got["at"] == stamp and got["confirmed_at"] == again, got
|
|
468
|
+
# A model's limit must beat its connection's learned one too.
|
|
469
|
+
assert refused(10, ago(minutes=5), "conn-m2", "rpm") == {} and refused(10, now, "conn-m2", "rpm") == {"rpm": 10}
|
|
470
|
+
assert _tighten("m2", {"limits": {"rpm": 60}}, "test", "", stamp) == {}
|
|
471
|
+
assert _tighten("m2", {"limits": {"rpm": 6}}, "test", "", stamp) == {"rpm": 6}
|
|
472
|
+
|
|
473
|
+
# record(): the count is the ledger's, of what the provider took on that connection.
|
|
474
|
+
def sent(conn: str, n: int, status: str = "ok", attempts: int = 1) -> None:
|
|
475
|
+
for i in range(n):
|
|
476
|
+
ledger.append({"t": now.timestamp() - 30 - i, "alias": "x", "connection": conn, "status": status,
|
|
477
|
+
"attempts": attempts})
|
|
478
|
+
sent("conn-r", 4), sent("conn-r", 1, "error", 3), sent("elsewhere", 9), sent("conn-p", 4), sent("conn-few", 2)
|
|
479
|
+
assert record("r", neurons, "test") == {"cooldown_s": DEFAULT_COOLDOWN_S} # a candidate: nothing stored yet
|
|
480
|
+
assert connection_learned(pending=True)["conn-r"]["rpd"]["n"] == 4
|
|
481
|
+
assert record("few", neurons, "test") and "conn-few" not in connection_learned(pending=True) # too few to say
|
|
482
|
+
# The bench's own refusal cools the model as before, and teaches nothing.
|
|
483
|
+
assert record("p", refusal, "test") == {"cooldown_s": 7200} and "p" in cooldowns()
|
|
484
|
+
assert "conn-p" not in connection_learned(pending=True)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
|
|
237
488
|
def main() -> None:
|
|
238
489
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
239
490
|
if "--selftest" in sys.argv:
|
|
@@ -248,6 +499,9 @@ def main() -> None:
|
|
|
248
499
|
if e.get("served_as"):
|
|
249
500
|
bits.append(f"served as {e['served_as']['served']}")
|
|
250
501
|
print(f" {alias:<34} {', '.join(bits)}")
|
|
502
|
+
for conn, axes in sorted(connection_learned().items()):
|
|
503
|
+
bits = [f"{k} {v['value']}" for k, v in axes.items()]
|
|
504
|
+
print(f" {conn:<34} {', '.join(bits)} (the connection's, from 429s with no number)")
|
|
251
505
|
for alias, c in sorted(cooldowns().items()):
|
|
252
506
|
print(f" cooling {alias:<26} until {c['until'][11:16]} ({c['why'][:70]})")
|
|
253
507
|
|
package/metrics/ledger.py
CHANGED
|
@@ -17,7 +17,9 @@ are what a provider's request limits count. `status` is ok, empty, error,
|
|
|
17
17
|
timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
|
|
18
18
|
provider's own figures from the reply's usage, absent when it gave none.
|
|
19
19
|
`caller` is the script that made the call (gateway, perf/run, suite/run, ...)
|
|
20
|
-
and `purpose` what the caller said it was for, when it said.
|
|
20
|
+
and `purpose` what the caller said it was for, when it said. A failed call
|
|
21
|
+
(status error) carries its `error`, start and end (client.clip), from
|
|
22
|
+
2026-11-08: the uptime report classifies it (suite/run.uptime_class).
|
|
21
23
|
|
|
22
24
|
A FILE A MONTH
|
|
23
25
|
--------------
|
package/metrics/locked.py
CHANGED
|
@@ -26,6 +26,7 @@ import json
|
|
|
26
26
|
import os
|
|
27
27
|
import pathlib
|
|
28
28
|
import tempfile
|
|
29
|
+
import threading
|
|
29
30
|
import time
|
|
30
31
|
|
|
31
32
|
if os.name == "nt":
|
|
@@ -71,6 +72,7 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
|
|
|
71
72
|
writers never share it (a fixed `.part` name let one writer move another's
|
|
72
73
|
half-written file into place)."""
|
|
73
74
|
path = pathlib.Path(path)
|
|
75
|
+
_forget(path)
|
|
74
76
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
75
77
|
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
|
|
76
78
|
try:
|
|
@@ -90,10 +92,85 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
|
|
|
90
92
|
os.remove(tmp)
|
|
91
93
|
|
|
92
94
|
|
|
95
|
+
_seen = threading.local()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@contextlib.contextmanager
|
|
99
|
+
def snapshot():
|
|
100
|
+
"""Inside it, this thread reads each file from disk once and reuses the
|
|
101
|
+
text: one dashboard render asked for the same dozen config files 11,000
|
|
102
|
+
times, seconds of opens on Windows. Text, not the parsed object, so a
|
|
103
|
+
caller changing what it parsed changes nothing for the next one. Other
|
|
104
|
+
threads (the gateway, a POST) read the disk as before; a write from this
|
|
105
|
+
thread drops its copy."""
|
|
106
|
+
if getattr(_seen, "texts", None) is not None: # already inside one
|
|
107
|
+
yield
|
|
108
|
+
return
|
|
109
|
+
_seen.texts = {}
|
|
110
|
+
try:
|
|
111
|
+
yield
|
|
112
|
+
finally:
|
|
113
|
+
_seen.texts = None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def once(key, compute):
|
|
117
|
+
"""compute(); inside snapshot(), its first answer for `key` (an OSError as
|
|
118
|
+
first raised). Only for answers no caller can change, such as a string."""
|
|
119
|
+
texts = getattr(_seen, "texts", None)
|
|
120
|
+
if texts is None:
|
|
121
|
+
return compute()
|
|
122
|
+
if key not in texts:
|
|
123
|
+
try:
|
|
124
|
+
texts[key] = compute()
|
|
125
|
+
except OSError as exc:
|
|
126
|
+
texts[key] = exc # missing stays missing, as the same error
|
|
127
|
+
if isinstance(texts[key], OSError):
|
|
128
|
+
raise texts[key]
|
|
129
|
+
return texts[key]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def read_text(path) -> str:
|
|
133
|
+
"""The file's text (OSError if it cannot be read); inside snapshot(), as
|
|
134
|
+
first read."""
|
|
135
|
+
return once(str(path), lambda: pathlib.Path(path).read_text(encoding="utf-8"))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def appended(path, mark: dict) -> tuple[bool, list[str]]:
|
|
139
|
+
"""(start over?, lines) for a log that is only ever appended to: the
|
|
140
|
+
whole lines added since `mark` last saw it. The first time, or when the
|
|
141
|
+
file was replaced or rewritten rather than appended to (shorter, or the
|
|
142
|
+
bytes before where it stopped differ), it is True and every line. A last
|
|
143
|
+
line still missing its newline waits for the next call. `mark` is the
|
|
144
|
+
caller's own dict, kept between calls; hold one lock around this call
|
|
145
|
+
and whatever folds the lines in."""
|
|
146
|
+
path = str(path)
|
|
147
|
+
at, before = (mark.get("at", 0), mark.get("before", b"")) if mark.get("path") == path else (0, b"")
|
|
148
|
+
try:
|
|
149
|
+
with open(path, "rb") as fh:
|
|
150
|
+
size = fh.seek(0, os.SEEK_END)
|
|
151
|
+
if at:
|
|
152
|
+
fh.seek(max(at - len(before), 0))
|
|
153
|
+
if size < at or fh.read(len(before)) != before:
|
|
154
|
+
at, before = 0, b""
|
|
155
|
+
fh.seek(at)
|
|
156
|
+
data = fh.read()
|
|
157
|
+
except OSError:
|
|
158
|
+
mark.clear()
|
|
159
|
+
return True, []
|
|
160
|
+
end = data.rfind(b"\n") + 1
|
|
161
|
+
mark.update(path=path, at=at + end, before=(before + data[:end])[-64:])
|
|
162
|
+
return at == 0, data[:end].decode("utf-8", "replace").splitlines()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _forget(path) -> None:
|
|
166
|
+
if (texts := getattr(_seen, "texts", None)) is not None:
|
|
167
|
+
texts.pop(str(path), None)
|
|
168
|
+
|
|
169
|
+
|
|
93
170
|
def read_json(path: pathlib.Path) -> dict:
|
|
94
171
|
"""The file's object, or {} when it is missing, unreadable or not an object."""
|
|
95
172
|
try:
|
|
96
|
-
data = json.loads(
|
|
173
|
+
data = json.loads(read_text(path))
|
|
97
174
|
except (OSError, ValueError):
|
|
98
175
|
return {}
|
|
99
176
|
return data if isinstance(data, dict) else {}
|
|
@@ -105,6 +182,7 @@ def update(path: pathlib.Path, change, **dump):
|
|
|
105
182
|
json.dumps (indent defaults to 2)."""
|
|
106
183
|
dump.setdefault("indent", 2)
|
|
107
184
|
with lock(path):
|
|
185
|
+
_forget(path) # under the lock, the disk's copy
|
|
108
186
|
data = read_json(path)
|
|
109
187
|
before = json.dumps(data, sort_keys=True)
|
|
110
188
|
result = change(data)
|
|
@@ -135,6 +213,29 @@ def _selftest() -> None:
|
|
|
135
213
|
assert data == {"t": 200, "p": 150}, data
|
|
136
214
|
assert update(f, lambda d: "same") == "same" # no change, no write, result passed back
|
|
137
215
|
assert not [p for p in f.parent.iterdir() if p.suffix == ".part"], "temp file left behind"
|
|
216
|
+
# A snapshot reads each file once; its own writes, and leaving it, read the disk again.
|
|
217
|
+
g = pathlib.Path(tmp) / "g.json"
|
|
218
|
+
with snapshot():
|
|
219
|
+
assert read_json(g) == {} # missing stays missing
|
|
220
|
+
g.write_text('{"a": 1}', encoding="utf-8")
|
|
221
|
+
assert read_json(g) == {}
|
|
222
|
+
write_atomic(g, '{"a": 2}')
|
|
223
|
+
assert read_json(g) == {"a": 2}
|
|
224
|
+
g.write_text('{"a": 3}', encoding="utf-8")
|
|
225
|
+
read_json(g)["a"] = 9 # changing what it parsed changes nothing
|
|
226
|
+
assert read_json(g) == {"a": 2}
|
|
227
|
+
assert read_json(g) == {"a": 3}
|
|
228
|
+
# A log read in pieces: only what was added, a half-written line waits,
|
|
229
|
+
# and a rewrite starts over.
|
|
230
|
+
log, mark = pathlib.Path(tmp) / "l.jsonl", {}
|
|
231
|
+
assert appended(log, mark) == (True, [])
|
|
232
|
+
log.write_bytes(b"a\nb\nc")
|
|
233
|
+
assert appended(log, mark) == (True, ["a", "b"])
|
|
234
|
+
with log.open("ab") as fh:
|
|
235
|
+
fh.write(b"\nd\n")
|
|
236
|
+
assert appended(log, mark) == (False, ["c", "d"]) and appended(log, mark) == (False, [])
|
|
237
|
+
log.write_bytes(b"x\ny\nz\nw\n") # longer, but not what was there
|
|
238
|
+
assert appended(log, mark) == (True, ["x", "y", "z", "w"])
|
|
138
239
|
print("ok")
|
|
139
240
|
|
|
140
241
|
|
package/metrics/offload.py
CHANGED
|
@@ -29,34 +29,51 @@ import collections
|
|
|
29
29
|
import datetime as dt
|
|
30
30
|
import json
|
|
31
31
|
import pathlib
|
|
32
|
+
import threading
|
|
32
33
|
|
|
33
34
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
34
35
|
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
35
36
|
LOG = DATA / "metrics" / "offloads.jsonl"
|
|
36
37
|
|
|
37
38
|
|
|
39
|
+
_lock = threading.Lock()
|
|
40
|
+
_seen: dict = {"mark": {}, "renames": None, "out": [], "by_call": {}}
|
|
41
|
+
|
|
42
|
+
|
|
38
43
|
def entries() -> list[dict]:
|
|
39
44
|
"""Every offload, oldest first. An entry with a `call` id (written by the
|
|
40
45
|
bench gateway) is superseded by a later one with the same id: the gateway
|
|
41
|
-
records each seat unjudged as it answers, and the verdict comes after.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
46
|
+
records each seat unjudged as it answers, and the verdict comes after.
|
|
47
|
+
|
|
48
|
+
Only the lines added since the last call are read (all of them again
|
|
49
|
+
after a rename): one dashboard render asked seven times, a log of 44,000
|
|
50
|
+
lines took most of a second each, and the gateway adds to it on every
|
|
51
|
+
call. A new list each call; the entries in it are shared, so never change
|
|
52
|
+
one."""
|
|
53
|
+
import aliases
|
|
54
|
+
import locked
|
|
55
|
+
with _lock:
|
|
56
|
+
s = _seen
|
|
57
|
+
if s["renames"] is not aliases.history(): # a rename re-reads the history as a new dict
|
|
58
|
+
s.update(mark={}, renames=aliases.history())
|
|
59
|
+
fresh, lines = locked.appended(LOG, s["mark"])
|
|
60
|
+
if fresh:
|
|
61
|
+
s.update(out=[], by_call={})
|
|
62
|
+
out, by_call = s["out"], s["by_call"]
|
|
63
|
+
for line in lines:
|
|
64
|
+
if line.strip():
|
|
65
|
+
try:
|
|
66
|
+
e = json.loads(line)
|
|
67
|
+
except ValueError:
|
|
68
|
+
continue
|
|
69
|
+
e["model"] = aliases.current(e.get("model", ""), s["renames"]) # a renamed model keeps its record
|
|
70
|
+
if e.get("call") in by_call:
|
|
71
|
+
out[by_call[e["call"]]] = e
|
|
72
|
+
continue
|
|
73
|
+
if e.get("call"):
|
|
74
|
+
by_call[e["call"]] = len(out)
|
|
75
|
+
out.append(e)
|
|
76
|
+
return list(out)
|
|
60
77
|
|
|
61
78
|
|
|
62
79
|
def add(model: str, task: str, useful: bool | None, note: str, call: str | None = None,
|