@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -131,8 +131,9 @@ def load() -> dict[str, dict]:
131
131
  else's machine is a timeout with no explanation. Whatever needs an
132
132
  endpoint asks default() (base, key, models and yaml_block all do), and
133
133
  that refuses, so a job run before any endpoint exists says what to do."""
134
+ import locked
134
135
  try:
135
- data = json.loads(FILE.read_text(encoding="utf-8"))
136
+ data = json.loads(locked.read_text(FILE)) # inside a render, from its snapshot
136
137
  except FileNotFoundError:
137
138
  return {}
138
139
  except (OSError, ValueError) as exc:
@@ -147,11 +148,8 @@ def connections() -> dict:
147
148
  ties, names for bare-IP hosts, models.dev ids, providers whose free tier
148
149
  works only in their own client. {} when absent: a fresh
149
150
  install has no such facts, and none are assumed."""
150
- try:
151
- data = json.loads(CONNECTIONS.read_text(encoding="utf-8"))
152
- except (OSError, ValueError):
153
- return {}
154
- return data if isinstance(data, dict) else {}
151
+ import locked
152
+ return locked.read_json(CONNECTIONS) # inside a render, from its snapshot
155
153
 
156
154
 
157
155
  _default: tuple = (None, "")
@@ -297,7 +295,8 @@ def models(name: str | None = None, timeout: int = 30) -> dict[str, dict]:
297
295
  def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, dict]:
298
296
  """What an endpoint entry lists, saved or not yet (the dashboard tests one
299
297
  before saving it). `models_url` when the list lives elsewhere (GitHub
300
- Models: its catalog, a bare JSON list); entries marked `visibility` other
298
+ Models: its catalog, a bare JSON list; Cloudflare: its catalog search,
299
+ {"result": [...]} by `name`); entries marked `visibility` other
301
300
  than "list" are left out (ChatGPT's plan lists ones it will not serve)."""
302
301
  api = entry.get("api") or "chat"
303
302
  headers = dict(entry.get("headers") or {})
@@ -315,6 +314,9 @@ def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, d
315
314
  # what generateContent serves, by its bare name.
316
315
  rows = [{**m, "id": str(m.get("name", "")).removeprefix("models/")} for m in data.get("models", [])
317
316
  if isinstance(m, dict) and "generateContent" in (m.get("supportedGenerationMethods") or ())]
317
+ elif isinstance(data, dict) and isinstance(data.get("result"), list):
318
+ # Cloudflare's catalog search: {"result": [{"name": "@cf/...", ...}]}.
319
+ rows = [{**m, "id": m.get("name")} for m in data["result"] if isinstance(m, dict)]
318
320
  else:
319
321
  rows = data.get("data", []) if isinstance(data, dict) else data
320
322
  return {e["id"]: e for e in rows
package/metrics/guard.py CHANGED
@@ -146,9 +146,11 @@ def cors_origin(headers, port: int) -> str:
146
146
  return origin if origin.lower() in origins(port) else ""
147
147
 
148
148
 
149
- def check(headers, port: int, post: bool, max_body: int, cors: bool = False) -> int:
149
+ def check(headers, port: int, post: bool, max_body: int, cors: bool = False,
150
+ kind: str = "application/json") -> int:
150
151
  """Raise Refused, or return how many body bytes to read (0 for a GET).
151
- `cors`: a gateway route, where an allowed origin may be cross-site."""
152
+ `cors`: a gateway route, where an allowed origin may be cross-site.
153
+ `kind`: the body's type; never one a plain form can send (an import's zip)."""
152
154
  host = (headers.get("Host") or "").strip().lower()
153
155
  if host not in hosts(port):
154
156
  raise Refused(403, f"unknown Host {host or '(none)'!r}; set BENCHWARMERS_PUBLIC_URL"
@@ -161,8 +163,8 @@ def check(headers, port: int, post: bool, max_body: int, cors: bool = False) ->
161
163
  raise Refused(403, "cross-site requests are refused")
162
164
  if origin and origin not in allowed:
163
165
  raise Refused(403, f"Origin {origin!r} is not allowed; set BENCH_ALLOWED_ORIGINS to add one")
164
- if not (headers.get("Content-Type") or "").lower().startswith("application/json"):
165
- raise Refused(415, "send Content-Type: application/json")
166
+ if not (headers.get("Content-Type") or "").lower().startswith(kind):
167
+ raise Refused(415, f"send Content-Type: {kind}")
166
168
  try:
167
169
  length = int(headers.get("Content-Length") or 0)
168
170
  except ValueError:
package/metrics/learn.py CHANGED
@@ -13,18 +13,40 @@ learns exactly these at runtime, and so does this: every failed call from the
13
13
  perf probe, the suite (and so suite/pace.py) and the gateway passes its error
14
14
  text through `record()`.
15
15
 
16
- ONLY EVER TIGHTER
17
- -----------------
16
+ ONLY EVER TIGHTER, UNTIL IT EXPIRES
17
+ -----------------------------------
18
18
  A learned context window or rate limit is kept only if it is LOWER than what is
19
19
  known, and replaces an older learned value only when lower still. An error can
20
20
  prove a ceiling exists; it cannot prove the ceiling is higher than a catalog
21
21
  says. Values you entered in the model popup (config/model-overrides.json)
22
22
  still win: you can always correct a bad parse.
23
23
 
24
- config/model-learned.json alias -> {context_window, limits, capabilities, served_as}
25
- with the date and the evidence for each
26
- runs/cooldowns.json alias -> {until, why}: rate-limited, come back later.
27
- Not config: it expires, and is nobody's decision.
24
+ A rate limit can be raised, though (a plan upgrade, a provider's change), and
25
+ no error ever says so. So a learned RATE limit that has not been seen again for
26
+ LEARNED_TTL_S (14 days) is left out wherever it is read: the bench goes back to
27
+ the higher rate, and the provider's next refusal teaches the limit again. The
28
+ same limit seen again refreshes its `confirmed_at`, so one that still holds
29
+ does not expire. A context window does not: a model's window does not grow.
30
+
31
+ A 429 THAT NAMES NO NUMBER
32
+ --------------------------
33
+ Cloudflare Workers AI says only that the day's free Neurons are used up. The
34
+ bench knows how many requests it sent on that connection in the window before
35
+ the refusal (metrics/ledger.py), and that count is the limit, near enough. It
36
+ is a connection's limit, not a model's: the key's allowance is shared. One
37
+ refusal is a candidate; a second at a count within 20% of it, a window or more
38
+ later (sooner is the same refusal, met by another call), is adopted (the lower
39
+ of the two) and enforced by metrics/pacing.py unless you set your own for that
40
+ connection. Seen again near the same count it is confirmed; a refusal at a
41
+ clearly lower count lowers it at once. Only rpm and rpd, and never from the
42
+ bench's own refusal.
43
+
44
+ config/model-learned.json alias -> {context_window, limits, capabilities, served_as}
45
+ with the date and the evidence for each
46
+ config/connection-learned.json connection -> {rpm, rpd}: {value, at, confirmed_at, where,
47
+ evidence}, or the candidates until one is adopted
48
+ runs/cooldowns.json alias -> {until, why}: rate-limited, come back later.
49
+ Not config: it expires, and is nobody's decision.
28
50
  """
29
51
 
30
52
  from __future__ import annotations
@@ -40,9 +62,13 @@ import locked
40
62
  ROOT = pathlib.Path(__file__).resolve().parent.parent
41
63
  from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
42
64
  LEARNED = DATA / "config" / "model-learned.json"
65
+ CONN_LEARNED = DATA / "config" / "connection-learned.json"
43
66
  COOLDOWNS = DATA / "runs" / "cooldowns.json"
44
67
  DEFAULT_COOLDOWN_S = 1800 # a 429 with no stated wait
45
68
  MAX_COOLDOWN_S = 24 * 3600
69
+ LEARNED_TTL_S = 14 * 86400 # a learned rate limit not seen again for this long is forgotten
70
+ CONN_AXES = {"rpm": 60, "rpd": 86400} # what a connection's limit is learned for: its window, seconds
71
+ MIN_COUNT = 3 # fewer requests in the window than this prove nothing
46
72
 
47
73
  CONTEXT = [
48
74
  r"maximum context length is\s*([\d,]+)",
@@ -71,6 +97,10 @@ ALLOWS = re.compile(r"allows?\s+([\d,]+)\s+requests?(?:\(s\))?\s+every\s+(?:1\s+
71
97
  # new-api gateways (Agnes): "maximum 20 requests in 1 minutes" (its
72
98
  # middleware/model-rate-limit.go); a window of 1 minute or 1,440 only.
73
99
  MAXIMUM = re.compile(r"maximum\s+([\d,]+)\s+requests?\s+in\s+(1|1440)\s+minutes?\b")
100
+ # Which window a 429 with no number is about. A window, not a wait: "try
101
+ # again in 3 minutes" names no per-minute limit.
102
+ DAY = re.compile(r"\bday\b|daily|free allocation|neurons")
103
+ MINUTE = re.compile(r"per[- ]?min(?:ute)?\b|a minute|\brpm\b")
74
104
 
75
105
 
76
106
  def _int(s: str) -> int:
@@ -108,18 +138,64 @@ def parse(text: str) -> dict:
108
138
  _read = locked.read_json
109
139
 
110
140
 
141
+ def _age(stamp, now: dt.datetime) -> float:
142
+ """Seconds since an ISO stamp; forever when it is missing or unreadable."""
143
+ try:
144
+ return (now - dt.datetime.fromisoformat(stamp)).total_seconds()
145
+ except (TypeError, ValueError):
146
+ return float("inf")
147
+
148
+
149
+ def _live(ev: dict, now: dt.datetime) -> bool:
150
+ """Seen within LEARNED_TTL_S: an older rate limit may have been raised since."""
151
+ return _age(ev.get("confirmed_at") or ev.get("at"), now) <= LEARNED_TTL_S
152
+
153
+
111
154
  def learned() -> dict:
112
- return {k: v for k, v in _read(LEARNED).items() if not k.startswith("_")}
155
+ """alias -> what was learned, without the rate limits that have expired."""
156
+ now = dt.datetime.now().astimezone()
157
+ out = {}
158
+ for alias, e in _read(LEARNED).items():
159
+ if alias.startswith("_"):
160
+ continue
161
+ if isinstance(e, dict) and e.get("limits"):
162
+ e = {**e, "limits": {k: ev for k, ev in e["limits"].items() if _live(ev, now)}}
163
+ out[alias] = e
164
+ return out
165
+
166
+
167
+ def connection_learned(pending: bool = False) -> dict:
168
+ """connection -> {axis: {value, at, confirmed_at, where, evidence}}: the
169
+ adopted limits that have not expired. pending=True: for the axes with none,
170
+ the latest candidate instead ({n, at}), seen once and not enforced."""
171
+ now = dt.datetime.now().astimezone()
172
+ out: dict = {}
173
+ for conn, axes in _read(CONN_LEARNED).items():
174
+ if conn.startswith("_") or not isinstance(axes, dict):
175
+ continue
176
+ for axis, e in axes.items():
177
+ if axis not in CONN_AXES or not isinstance(e, dict):
178
+ continue
179
+ adopted = bool(e.get("value")) and _live(e, now)
180
+ waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
181
+ if adopted and not pending:
182
+ out.setdefault(conn, {})[axis] = {k: v for k, v in e.items() if k != "candidates"}
183
+ elif pending and not adopted and waiting:
184
+ out.setdefault(conn, {})[axis] = waiting[-1]
185
+ return out
113
186
 
114
187
 
115
188
  def _known(alias: str, field: str, axis: str | None = None):
116
- """The value a learned figure has to beat: the catalog's, else the last learned."""
189
+ """The value a learned figure has to beat: the catalog's, else the last
190
+ learned; for a rate limit, its connection's too, yours and the learned."""
117
191
  import catalog
118
192
  auto = catalog.catalog().get(alias, {})
119
193
  mine = learned().get(alias, {})
120
194
  if axis:
195
+ conn = catalog.connection_of(alias)
121
196
  vals = [(mine.get("limits") or {}).get(axis, {}).get("value"),
122
- catalog.connection_limits().get(catalog.connection_of(alias), {}).get(axis)]
197
+ catalog.connection_limits().get(conn, {}).get(axis),
198
+ connection_learned().get(conn, {}).get(axis, {}).get("value")]
123
199
  else:
124
200
  vals = [(mine.get(field) or {}).get("value"), auto.get(field)]
125
201
  vals = [v for v in vals if v]
@@ -137,10 +213,13 @@ def _tighten(alias: str, found: dict, where: str, evidence: str, stamp: str) ->
137
213
  entry["context_window"] = {"value": n, "at": stamp, "where": where, "evidence": evidence}
138
214
  stored["context_window"] = n
139
215
  for axis, n in (found.get("limits") or {}).items():
140
- if n < (_known(alias, "", axis) or 1 << 62):
216
+ had = (entry.get("limits") or {}).get(axis) or {}
217
+ if n < (_known(alias, "", axis) or 1 << 62): # an expired one is not known: a higher value may replace it
141
218
  entry.setdefault("limits", {})[axis] = {"value": n, "at": stamp, "where": where,
142
219
  "evidence": evidence}
143
220
  stored[axis] = n
221
+ elif had.get("value") == n:
222
+ had["confirmed_at"] = stamp # still true, so it does not expire
144
223
  if stored:
145
224
  data[alias] = entry
146
225
  data.setdefault("_why", "Learned from error messages by metrics/learn.py; only ever"
@@ -158,6 +237,75 @@ def note_limits(alias: str, limits: dict, where: str, evidence: str) -> dict:
158
237
  return _tighten(alias, {"limits": {k: v for k, v in limits.items() if k in AXES}}, where, evidence, stamp)
159
238
 
160
239
 
240
+ def _axis(text: str, found: dict) -> str | None:
241
+ """The window a 429 that names no number is about, or None: nothing to
242
+ learn. Never the bench's own refusal ("not sent", pacing.py): a limit the
243
+ bench enforced must not be learned back from itself. Nor a token limit: the
244
+ count is of requests."""
245
+ low = (text or "").lower()
246
+ if not found.get("rate_limited") or found.get("limits") or "not sent" in low \
247
+ or re.search(AXES["tpm"] + "|" + AXES["tpd"], low):
248
+ return None
249
+ wait = found.get("retry_after_s")
250
+ if DAY.search(low) or (wait or 0) >= 3600:
251
+ return "rpd"
252
+ # A short wait alone is not enough for rpm: calls in flight are not in the
253
+ # ledger yet, so a minute's count undercounts, and two passing 429s could
254
+ # pin a connection at rpm 3 for 14 days. Only a named minute window counts.
255
+ if MINUTE.search(low):
256
+ return "rpm"
257
+ return None
258
+
259
+
260
+ def _sent(connection: str, since: float) -> int:
261
+ """Requests the provider took on a connection since then: the ledger's
262
+ attempts, without the calls that failed, whose own retries would count the
263
+ refusal as use."""
264
+ import ledger
265
+ return sum(r.get("attempts") or 0 for r in ledger.rows(since)
266
+ if r.get("connection") == connection and r.get("status") != "error")
267
+
268
+
269
+ def _near(n: int, value: int) -> bool:
270
+ return 0.8 * value <= n <= 1.2 * value
271
+
272
+
273
+ def _learn_connection(connection: str, axis: str, n: int, where: str, evidence: str, now: dt.datetime) -> dict:
274
+ """A connection refused after `n` requests in the axis's window. Returns
275
+ {axis: value} when that adopted a limit or lowered one."""
276
+ stamp, span = now.isoformat(timespec="seconds"), CONN_AXES[axis]
277
+ stored: dict = {}
278
+
279
+ def change(data: dict) -> None:
280
+ axes = data.setdefault(connection, {})
281
+ e = axes.get(axis) or {}
282
+ fresh = {"value": n, "at": stamp, "confirmed_at": stamp, "where": where, "evidence": evidence}
283
+ waiting = [c for c in e.get("candidates") or [] if _live(c, now)]
284
+ if e.get("value") and _live(e, now):
285
+ # One lesson a window: while a provider keeps refusing, the count
286
+ # of what it took only falls, and would drag the limit down with it.
287
+ if _age(e.get("confirmed_at") or e.get("at"), now) < span:
288
+ return
289
+ if n < 0.8 * e["value"]: # refused well below it: that is proof
290
+ axes[axis] = fresh
291
+ stored[axis] = n
292
+ elif _near(n, e["value"]):
293
+ e["confirmed_at"] = stamp
294
+ elif waiting and _age(waiting[-1]["at"], now) < span:
295
+ return # the same refusal, met by another call
296
+ elif agree := [c["n"] for c in waiting if _near(n, c["n"])]:
297
+ axes[axis] = fresh | {"value": min(agree + [n])}
298
+ stored[axis] = axes[axis]["value"]
299
+ else:
300
+ axes[axis] = {"candidates": (waiting + [{"n": n, "at": stamp}])[-5:]}
301
+ data.setdefault("_why", "Learned by metrics/learn.py from 429s that named no number: the requests"
302
+ " the bench had sent in the window. Enforced unless you set your own"
303
+ " (config/connection-limits.json); forgotten 14 days after last seen.")
304
+
305
+ locked.update(CONN_LEARNED, change, ensure_ascii=False)
306
+ return stored
307
+
308
+
161
309
  def record(alias: str, text: str, where: str) -> dict:
162
310
  """Learn from one failed call. Returns what was stored or set, for logging."""
163
311
  found = parse(text)
@@ -165,6 +313,14 @@ def record(alias: str, text: str, where: str) -> dict:
165
313
  stamp = now.isoformat(timespec="seconds")
166
314
  evidence = " ".join((text or "").split())[:200]
167
315
  stored = _tighten(alias, found, where, evidence, stamp)
316
+ if alias and (axis := _axis(text, found)):
317
+ try:
318
+ import catalog
319
+ conn = catalog.connection_of(alias)
320
+ if conn and (n := _sent(conn, now.timestamp() - CONN_AXES[axis])) >= MIN_COUNT:
321
+ stored.update(_learn_connection(conn, axis, n, where, evidence, now))
322
+ except Exception: # noqa: BLE001 -- learning never breaks a call
323
+ pass
168
324
  if alias and found.get("rate_limited"):
169
325
  wait = found.get("retry_after_s") or DEFAULT_COOLDOWN_S
170
326
  cool = {"until": (now + dt.timedelta(seconds=wait)).isoformat(timespec="seconds"),
@@ -231,9 +387,104 @@ def _selftest() -> None:
231
387
  assert "limits" not in parse("maximum 100 requests in 5 minutes") # a window no axis holds
232
388
  assert parse("model not found") == {"rate_limited": False}
233
389
  assert parse("context length of 8k exceeded").get("context_window") is None # "8" < 512
390
+
391
+ # Which window a 429 with no number is about.
392
+ def axis(text: str):
393
+ return _axis(text, parse(text))
394
+ neurons = "Error code: 429 - you have used up your daily free allocation of 10,000 neurons"
395
+ assert axis(neurons) == "rpd" and "limits" not in parse(neurons)
396
+ assert axis("Rate limit exceeded: free-models-per-day. Try again in 3 minutes") == "rpd"
397
+ assert axis("429 Too Many Requests, retry-after: 7200") == "rpd"
398
+ assert axis("429 Too Many Requests, retry-after: 30") is None # a short wait alone: no window
399
+ assert axis("429: too many requests per minute") == "rpm"
400
+ assert axis("429 Too Many Requests") is None # no window named
401
+ assert axis("Rate limit exceeded. Try again in 3 minutes") is None # a wait, not a window
402
+ assert axis("429: rate limit on tokens per minute") is None # requests are what is counted
403
+ assert axis("Limit 30, Used 30 (requests per minute) 429") is None # it gave the number
404
+ assert axis("model not found") is None
405
+ # The bench's own refusal is never learned from.
406
+ refusal = ("Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests"
407
+ " per day (learned) and 50 were used; try again in 2h")
408
+ assert axis(refusal) is None and axis(refusal.replace("not sent; ", "")) == "rpd"
409
+ _selftest_stores(neurons, refusal)
234
410
  print("ok")
235
411
 
236
412
 
413
+ def _selftest_stores(neurons: str, refusal: str) -> None:
414
+ """The connection store and the expiry, in a temp folder with a stub
415
+ catalog: nothing of the install's is read or written."""
416
+ import tempfile
417
+
418
+ import catalog
419
+ import ledger
420
+ with tempfile.TemporaryDirectory() as folder:
421
+ tmp = pathlib.Path(folder)
422
+ globals().update(LEARNED=tmp / "model-learned.json", CONN_LEARNED=tmp / "connection-learned.json",
423
+ COOLDOWNS=tmp / "cooldowns.json")
424
+ ledger.DIR = tmp / "usage"
425
+ catalog.catalog = catalog.connection_limits = lambda: {}
426
+ catalog.connection_of = lambda alias: "conn-" + alias
427
+ now = dt.datetime.now().astimezone()
428
+
429
+ def ago(**kw) -> dt.datetime:
430
+ return now - dt.timedelta(**kw)
431
+
432
+ def refused(n: int, when: dt.datetime, conn: str = "prov", ax: str = "rpd") -> dict:
433
+ return _learn_connection(conn, ax, n, "test", "429", when)
434
+ # One refusal is a candidate; the same refusal met again changes nothing.
435
+ assert refused(100, ago(days=5)) == {} and connection_learned() == {}
436
+ assert connection_learned(pending=True)["prov"]["rpd"]["n"] == 100
437
+ assert refused(104, ago(days=5, hours=-1)) == {} and connection_learned() == {}
438
+ # A second count far from the first does not adopt; one within 20% does, the lower of the two.
439
+ assert refused(300, ago(days=4)) == {} and connection_learned() == {}
440
+ assert refused(110, ago(days=3)) == {"rpd": 100}
441
+ got = connection_learned()["prov"]["rpd"]
442
+ assert got["value"] == 100 and "candidates" not in got and connection_learned(pending=True) == {}, got
443
+ # Seen again near it: confirmed. Clearly lower: lowered at once. Higher: left alone.
444
+ assert refused(95, ago(days=2)) == {}
445
+ got = connection_learned()["prov"]["rpd"]
446
+ assert got["value"] == 100 and got["confirmed_at"] == ago(days=2).isoformat(timespec="seconds") != got["at"]
447
+ assert refused(40, ago(days=2, hours=-1)) == {} # within the window: the same refusal
448
+ assert refused(500, ago(days=1)) == {} and connection_learned()["prov"]["rpd"]["value"] == 100
449
+ assert refused(60, now) == {"rpd": 60} and connection_learned()["prov"]["rpd"]["value"] == 60
450
+ # Not seen for 15 days: forgotten on read, and a higher limit can be learned.
451
+ assert refused(20, ago(days=16), "old", "rpm") == {} and refused(20, ago(days=15), "old", "rpm") == {"rpm": 20}
452
+ assert "old" not in connection_learned()
453
+ assert refused(50, ago(minutes=5), "old", "rpm") == {} and refused(50, now, "old", "rpm") == {"rpm": 50}
454
+
455
+ # A model's own learned limit: the same expiry, and the same limit seen again is confirmed.
456
+ old = {"value": 5, "at": ago(days=15).isoformat(timespec="seconds"), "where": "test", "evidence": ""}
457
+ LEARNED.write_text(json.dumps({"m": {"limits": {"rpm": dict(old), "rpd": dict(old)},
458
+ "context_window": dict(old, value=8192)}}), encoding="utf-8")
459
+ assert learned()["m"]["limits"] == {} and learned()["m"]["context_window"]["value"] == 8192
460
+ stamp = now.isoformat(timespec="seconds")
461
+ assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", stamp) == {"rpm": 20}
462
+ assert _tighten("m", {"limits": {"rpm": 30}}, "test", "", stamp) == {} # live: only ever tighter
463
+ assert "confirmed_at" not in learned()["m"]["limits"]["rpm"]
464
+ again = ago(hours=-1).isoformat(timespec="seconds")
465
+ assert _tighten("m", {"limits": {"rpm": 20}}, "test", "", again) == {}
466
+ got = learned()["m"]["limits"]["rpm"]
467
+ assert got["value"] == 20 and got["at"] == stamp and got["confirmed_at"] == again, got
468
+ # A model's limit must beat its connection's learned one too.
469
+ assert refused(10, ago(minutes=5), "conn-m2", "rpm") == {} and refused(10, now, "conn-m2", "rpm") == {"rpm": 10}
470
+ assert _tighten("m2", {"limits": {"rpm": 60}}, "test", "", stamp) == {}
471
+ assert _tighten("m2", {"limits": {"rpm": 6}}, "test", "", stamp) == {"rpm": 6}
472
+
473
+ # record(): the count is the ledger's, of what the provider took on that connection.
474
+ def sent(conn: str, n: int, status: str = "ok", attempts: int = 1) -> None:
475
+ for i in range(n):
476
+ ledger.append({"t": now.timestamp() - 30 - i, "alias": "x", "connection": conn, "status": status,
477
+ "attempts": attempts})
478
+ sent("conn-r", 4), sent("conn-r", 1, "error", 3), sent("elsewhere", 9), sent("conn-p", 4), sent("conn-few", 2)
479
+ assert record("r", neurons, "test") == {"cooldown_s": DEFAULT_COOLDOWN_S} # a candidate: nothing stored yet
480
+ assert connection_learned(pending=True)["conn-r"]["rpd"]["n"] == 4
481
+ assert record("few", neurons, "test") and "conn-few" not in connection_learned(pending=True) # too few to say
482
+ # The bench's own refusal cools the model as before, and teaches nothing.
483
+ assert record("p", refusal, "test") == {"cooldown_s": 7200} and "p" in cooldowns()
484
+ assert "conn-p" not in connection_learned(pending=True)
485
+
486
+
487
+
237
488
  def main() -> None:
238
489
  sys.stdout.reconfigure(encoding="utf-8", errors="replace")
239
490
  if "--selftest" in sys.argv:
@@ -248,6 +499,9 @@ def main() -> None:
248
499
  if e.get("served_as"):
249
500
  bits.append(f"served as {e['served_as']['served']}")
250
501
  print(f" {alias:<34} {', '.join(bits)}")
502
+ for conn, axes in sorted(connection_learned().items()):
503
+ bits = [f"{k} {v['value']}" for k, v in axes.items()]
504
+ print(f" {conn:<34} {', '.join(bits)} (the connection's, from 429s with no number)")
251
505
  for alias, c in sorted(cooldowns().items()):
252
506
  print(f" cooling {alias:<26} until {c['until'][11:16]} ({c['why'][:70]})")
253
507
 
package/metrics/ledger.py CHANGED
@@ -17,7 +17,9 @@ are what a provider's request limits count. `status` is ok, empty, error,
17
17
  timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
18
18
  provider's own figures from the reply's usage, absent when it gave none.
19
19
  `caller` is the script that made the call (gateway, perf/run, suite/run, ...)
20
- and `purpose` what the caller said it was for, when it said.
20
+ and `purpose` what the caller said it was for, when it said. A failed call
21
+ (status error) carries its `error`, start and end (client.clip), from
22
+ 2026-11-08: the uptime report classifies it (suite/run.uptime_class).
21
23
 
22
24
  A FILE A MONTH
23
25
  --------------
package/metrics/locked.py CHANGED
@@ -26,6 +26,7 @@ import json
26
26
  import os
27
27
  import pathlib
28
28
  import tempfile
29
+ import threading
29
30
  import time
30
31
 
31
32
  if os.name == "nt":
@@ -71,6 +72,7 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
71
72
  writers never share it (a fixed `.part` name let one writer move another's
72
73
  half-written file into place)."""
73
74
  path = pathlib.Path(path)
75
+ _forget(path)
74
76
  path.parent.mkdir(parents=True, exist_ok=True)
75
77
  fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
76
78
  try:
@@ -90,10 +92,85 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
90
92
  os.remove(tmp)
91
93
 
92
94
 
95
+ _seen = threading.local()
96
+
97
+
98
+ @contextlib.contextmanager
99
+ def snapshot():
100
+ """Inside it, this thread reads each file from disk once and reuses the
101
+ text: one dashboard render asked for the same dozen config files 11,000
102
+ times, seconds of opens on Windows. Text, not the parsed object, so a
103
+ caller changing what it parsed changes nothing for the next one. Other
104
+ threads (the gateway, a POST) read the disk as before; a write from this
105
+ thread drops its copy."""
106
+ if getattr(_seen, "texts", None) is not None: # already inside one
107
+ yield
108
+ return
109
+ _seen.texts = {}
110
+ try:
111
+ yield
112
+ finally:
113
+ _seen.texts = None
114
+
115
+
116
+ def once(key, compute):
117
+ """compute(); inside snapshot(), its first answer for `key` (an OSError as
118
+ first raised). Only for answers no caller can change, such as a string."""
119
+ texts = getattr(_seen, "texts", None)
120
+ if texts is None:
121
+ return compute()
122
+ if key not in texts:
123
+ try:
124
+ texts[key] = compute()
125
+ except OSError as exc:
126
+ texts[key] = exc # missing stays missing, as the same error
127
+ if isinstance(texts[key], OSError):
128
+ raise texts[key]
129
+ return texts[key]
130
+
131
+
132
+ def read_text(path) -> str:
133
+ """The file's text (OSError if it cannot be read); inside snapshot(), as
134
+ first read."""
135
+ return once(str(path), lambda: pathlib.Path(path).read_text(encoding="utf-8"))
136
+
137
+
138
+ def appended(path, mark: dict) -> tuple[bool, list[str]]:
139
+ """(start over?, lines) for a log that is only ever appended to: the
140
+ whole lines added since `mark` last saw it. The first time, or when the
141
+ file was replaced or rewritten rather than appended to (shorter, or the
142
+ bytes before where it stopped differ), it is True and every line. A last
143
+ line still missing its newline waits for the next call. `mark` is the
144
+ caller's own dict, kept between calls; hold one lock around this call
145
+ and whatever folds the lines in."""
146
+ path = str(path)
147
+ at, before = (mark.get("at", 0), mark.get("before", b"")) if mark.get("path") == path else (0, b"")
148
+ try:
149
+ with open(path, "rb") as fh:
150
+ size = fh.seek(0, os.SEEK_END)
151
+ if at:
152
+ fh.seek(max(at - len(before), 0))
153
+ if size < at or fh.read(len(before)) != before:
154
+ at, before = 0, b""
155
+ fh.seek(at)
156
+ data = fh.read()
157
+ except OSError:
158
+ mark.clear()
159
+ return True, []
160
+ end = data.rfind(b"\n") + 1
161
+ mark.update(path=path, at=at + end, before=(before + data[:end])[-64:])
162
+ return at == 0, data[:end].decode("utf-8", "replace").splitlines()
163
+
164
+
165
+ def _forget(path) -> None:
166
+ if (texts := getattr(_seen, "texts", None)) is not None:
167
+ texts.pop(str(path), None)
168
+
169
+
93
170
  def read_json(path: pathlib.Path) -> dict:
94
171
  """The file's object, or {} when it is missing, unreadable or not an object."""
95
172
  try:
96
- data = json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
173
+ data = json.loads(read_text(path))
97
174
  except (OSError, ValueError):
98
175
  return {}
99
176
  return data if isinstance(data, dict) else {}
@@ -105,6 +182,7 @@ def update(path: pathlib.Path, change, **dump):
105
182
  json.dumps (indent defaults to 2)."""
106
183
  dump.setdefault("indent", 2)
107
184
  with lock(path):
185
+ _forget(path) # under the lock, the disk's copy
108
186
  data = read_json(path)
109
187
  before = json.dumps(data, sort_keys=True)
110
188
  result = change(data)
@@ -135,6 +213,29 @@ def _selftest() -> None:
135
213
  assert data == {"t": 200, "p": 150}, data
136
214
  assert update(f, lambda d: "same") == "same" # no change, no write, result passed back
137
215
  assert not [p for p in f.parent.iterdir() if p.suffix == ".part"], "temp file left behind"
216
+ # A snapshot reads each file once; its own writes, and leaving it, read the disk again.
217
+ g = pathlib.Path(tmp) / "g.json"
218
+ with snapshot():
219
+ assert read_json(g) == {} # missing stays missing
220
+ g.write_text('{"a": 1}', encoding="utf-8")
221
+ assert read_json(g) == {}
222
+ write_atomic(g, '{"a": 2}')
223
+ assert read_json(g) == {"a": 2}
224
+ g.write_text('{"a": 3}', encoding="utf-8")
225
+ read_json(g)["a"] = 9 # changing what it parsed changes nothing
226
+ assert read_json(g) == {"a": 2}
227
+ assert read_json(g) == {"a": 3}
228
+ # A log read in pieces: only what was added, a half-written line waits,
229
+ # and a rewrite starts over.
230
+ log, mark = pathlib.Path(tmp) / "l.jsonl", {}
231
+ assert appended(log, mark) == (True, [])
232
+ log.write_bytes(b"a\nb\nc")
233
+ assert appended(log, mark) == (True, ["a", "b"])
234
+ with log.open("ab") as fh:
235
+ fh.write(b"\nd\n")
236
+ assert appended(log, mark) == (False, ["c", "d"]) and appended(log, mark) == (False, [])
237
+ log.write_bytes(b"x\ny\nz\nw\n") # longer, but not what was there
238
+ assert appended(log, mark) == (True, ["x", "y", "z", "w"])
138
239
  print("ok")
139
240
 
140
241
 
@@ -29,34 +29,51 @@ import collections
29
29
  import datetime as dt
30
30
  import json
31
31
  import pathlib
32
+ import threading
32
33
 
33
34
  ROOT = pathlib.Path(__file__).resolve().parent.parent
34
35
  from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
35
36
  LOG = DATA / "metrics" / "offloads.jsonl"
36
37
 
37
38
 
39
+ _lock = threading.Lock()
40
+ _seen: dict = {"mark": {}, "renames": None, "out": [], "by_call": {}}
41
+
42
+
38
43
  def entries() -> list[dict]:
39
44
  """Every offload, oldest first. An entry with a `call` id (written by the
40
45
  bench gateway) is superseded by a later one with the same id: the gateway
41
- records each seat unjudged as it answers, and the verdict comes after."""
42
- if not LOG.exists():
43
- return []
44
- out, by_call = [], {}
45
- from aliases import current
46
- for line in LOG.read_text(encoding="utf-8").splitlines():
47
- if line.strip():
48
- try:
49
- e = json.loads(line)
50
- except ValueError:
51
- continue
52
- e["model"] = current(e.get("model", "")) # a renamed model keeps its record
53
- if e.get("call") in by_call:
54
- out[by_call[e["call"]]] = e
55
- continue
56
- if e.get("call"):
57
- by_call[e["call"]] = len(out)
58
- out.append(e)
59
- return out
46
+ records each seat unjudged as it answers, and the verdict comes after.
47
+
48
+ Only the lines added since the last call are read (all of them again
49
+ after a rename): one dashboard render asked seven times, a log of 44,000
50
+ lines took most of a second each, and the gateway adds to it on every
51
+ call. A new list each call; the entries in it are shared, so never change
52
+ one."""
53
+ import aliases
54
+ import locked
55
+ with _lock:
56
+ s = _seen
57
+ if s["renames"] is not aliases.history(): # a rename re-reads the history as a new dict
58
+ s.update(mark={}, renames=aliases.history())
59
+ fresh, lines = locked.appended(LOG, s["mark"])
60
+ if fresh:
61
+ s.update(out=[], by_call={})
62
+ out, by_call = s["out"], s["by_call"]
63
+ for line in lines:
64
+ if line.strip():
65
+ try:
66
+ e = json.loads(line)
67
+ except ValueError:
68
+ continue
69
+ e["model"] = aliases.current(e.get("model", ""), s["renames"]) # a renamed model keeps its record
70
+ if e.get("call") in by_call:
71
+ out[by_call[e["call"]]] = e
72
+ continue
73
+ if e.get("call"):
74
+ by_call[e["call"]] = len(out)
75
+ out.append(e)
76
+ return list(out)
60
77
 
61
78
 
62
79
  def add(model: str, task: str, useful: bool | None, note: str, call: str | None = None,