@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/suite/README.md CHANGED
@@ -34,8 +34,11 @@ python suite/run.py --model gemini-flash-3.8
34
34
  python suite/run.py --dry-run # print the prompts, call nothing
35
35
  ```
36
36
 
37
- Results append to `runs/suite-<date>.md` and print a table. Nothing is
38
- overwritten — a run is evidence, and evidence accumulates.
37
+ Results append to `runs/suite-<date>.md` and print a table. Each answer is
38
+ also kept at `runs/answers/<task>/<model>/<run start>.md` (since 2026-11-08).
39
+ Nothing there is overwritten — a run is evidence, and evidence accumulates.
40
+ The Benchmarks page's Trend compares each model's scores with its own earlier
41
+ ones ([BENCHMARKS.md](../BENCHMARKS.md)).
39
42
 
40
43
  ## Claim verification, and tasks mined from real work
41
44
 
package/suite/run.py CHANGED
@@ -29,6 +29,9 @@ sys.path.insert(0, str(ROOT / "metrics"))
29
29
  from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
30
30
  TASKS = DATA / "suite" / "tasks"
31
31
  RUNS = DATA / "runs"
32
+ # Every answer, kept: output/<model>.md is replaced by the next run, so a
33
+ # score in an old run note had no answer left behind it (before 2026-11-08).
34
+ ANSWERS = RUNS / "answers"
32
35
 
33
36
  # THE CODEBASES THIS BENCHMARK IS DRAWN FROM.
34
37
  #
@@ -647,8 +650,9 @@ CAPACITY = (
647
650
  # than a wrong class: `check_deprecations` skips blank failures and keeps
648
651
  # whatever it saw last, so two retired models silently kept their old
649
652
  # classification. Imported rather than repeated, so the next class added
650
- # cannot reintroduce this.
651
- FAILURE_CLASSES = ("structural", "paywalled", "entitlement", "capacity",
653
+ # cannot reintroduce this. `signin` was missing until 2026-11-08 (the same
654
+ # blank failure); it leads, as it does in `failure_class`.
655
+ FAILURE_CLASSES = ("signin", "structural", "paywalled", "entitlement", "capacity",
652
656
  "timeout", "empty", "other")
653
657
 
654
658
 
@@ -710,12 +714,202 @@ PROVIDER_NOTICES = (
710
714
  )
711
715
 
712
716
 
713
- def provider_notice(output: str) -> str | None:
714
- """The notice phrase, if a short reply is a provider's quota/credit notice."""
717
+ # A short reply the phrases miss is put to a decision model only when it LOOKS
718
+ # like a notice: under NOTICE_LIKE_MAX characters (a notice is a sentence or
719
+ # two; the probe's one-word "ready" has none of these words) and speaking of
720
+ # the account: credit, quota, balance, billing, a plan or subscription, an
721
+ # upgrade, a top-up, a limit or allowance used up. A short answer that happens
722
+ # to use those words costs one question, once per shape (decide.notice keeps
723
+ # every answer); any other answer costs nothing, not even a file read.
724
+ NOTICE_LIKE = re.compile(r"\b(credits?|quota|balance|billing|subscription|plan|upgrade|top[ -]?up|limit|"
725
+ r"allowance|exhausted|exceeded|insufficient|payment)\b")
726
+ NOTICE_LIKE_MAX = 300
727
+
728
+
729
+ def provider_notice(output: str, ask: bool = False) -> str | None:
730
+ """The notice phrase, if a short reply is a provider's quota/credit notice.
731
+ When no phrase matches but the reply looks like one (NOTICE_LIKE): what a
732
+ decision model said of its shape ("decided by <model> at <p>"), read from
733
+ metrics/decided.json; asked now only with `ask` (the gateway's seats, as
734
+ the reply comes in), never while a page or a score is worked out."""
715
735
  text = output.strip().lower()
716
736
  if len(text) > 400:
717
737
  return None
718
- return next((m for m in PROVIDER_NOTICES if m in text), None)
738
+ if hit := next((m for m in PROVIDER_NOTICES if m in text), None):
739
+ return hit
740
+ if len(text) > NOTICE_LIKE_MAX or not NOTICE_LIKE.search(text):
741
+ return None
742
+ try:
743
+ import decide
744
+ return decide.notice(output.strip(), ask_now=ask)
745
+ except Exception: # noqa: BLE001 -- the fallback never fails a reply
746
+ return None
747
+
748
+
749
+ # WAS THE PROVIDER THERE? One call at a time, for the Uptime report (serve.py).
750
+ #
751
+ # `failure_class` asks what to do about a MODEL; this asks something narrower
752
+ # about one CALL: the provider failed us (down), it answered (up), or the call
753
+ # failed for a reason that is ours and says nothing about the provider (left
754
+ # out): our own quota or rate allowance, our pacer, our client, our LAN hub,
755
+ # or a setup problem (sign-in, no such model, no access, our own request).
756
+ # The owner's rules, 2026-11-08. Read from the stored error text every time,
757
+ # so a better rule reclassifies the past too. Generic first (status codes,
758
+ # standard codes and phrases), so a provider the bench does not use yet is
759
+ # classified; first match wins. Shapes and cases: suite/test_score.py.
760
+ UPTIME_UP = ("ok", "empty") # an empty reply is still an answer
761
+ UPTIME_DOWN = ("busy", "timeout", "unreachable", "unrecognised")
762
+ UPTIME_OUT = ("quota", "rate", "setup", "notsent", "client", "relay")
763
+
764
+
765
+ def _rx(*parts: str) -> re.Pattern:
766
+ return re.compile("|".join(parts))
767
+
768
+
769
+ _NOTSENT = _rx(r"\(not sent;", r"paced by the bench", r"free-tier guard")
770
+ _CLIENT = _rx(r"no model is registered", r"no key named", r"endpoint base must be", r"codec can't (en|de)code",
771
+ r"traceback \(most recent", r"database is locked", r"sqlite3?\.")
772
+ _TIMEOUT = _rx(r"<<timed out", r"no answer in \d+s", r"\btimed? ?out\b", r"read timeout", r"deadline[ _]exceeded",
773
+ r"gateway time-?out", r"request_timeout", r"timeout_error", r"stream_timeout",
774
+ r"error code: (408|504|524)\b")
775
+ _SIGNIN = _rx(r"error code: 401\b", r"['\"](code|status)['\"]: ?['\"]?401\b", r"invalid[ _]api[ _]key",
776
+ r"incorrect api key", r"unauthenticated", r"authentication[ _](error|failed)", r"sign in again",
777
+ r"token (has )?expired", r"invalid credentials", r"sharing_usage_unavailable", r"api key not valid",
778
+ r"api_key_invalid", r"invalid username or password", r"(didn'?t|did not) provide an api key",
779
+ r"missing api key")
780
+ _UPSTREAM_WRAP = _rx(r"provider returned error", r"upstream error", r"upstream provider")
781
+ _PAYWALLED = _rx(*map(re.escape, PAYWALLED))
782
+ _NOTFOUND = _rx(r"error code: 404\b", r"model_not_found", r"model not found", r"no endpoints found",
783
+ r"does not exist", r"unknown model", r"invalid model", r"no model found", r"no such model",
784
+ r"not_found_error", r"unavailable_model", r"decommissioned", r"has been deprecated",
785
+ r"model '[^']*' not found", r"try pulling it first", r"model identifier is invalid")
786
+ _ENTITLEMENT = _rx(r"error code: 40[23]\b", r"payment required", r"insufficient[ _]balance", r"insufficient[ _]credit",
787
+ r"not available in your subscription", r"permission[ _]denied", r"not authori[sz]ed",
788
+ r"unauthori[sz]ed", r"no access to model", r"only available on", r"consent_required",
789
+ r"needs your consent", r"paid model", r"credits required", r"zero balance",
790
+ r"failed_precondition", r"not available in your (country|region)",
791
+ r"user location is not supported", r"accessdeniedexception",
792
+ r"(don'?t|do not|does not) have (access|permission)", r"not allowed to access")
793
+ # A request the provider refused as OURS: wins over busy words ("capacity",
794
+ # "upstream") in the same text, so a wrapped "prompt is too long" is setup.
795
+ _REQUEST = _rx(r"request[ _]too[ _]large", r"context[ _]length", r"maximum context", r"too many tokens(?! (per|in the last))",
796
+ r"prompt is too long", r"payload too large", r"invalid_argument", r"invalid_request_error",
797
+ r"content_filter", r"content management policy", r"input validation error",
798
+ r"must have less than \d+ tokens", r"extra inputs are not permitted", r"validationexception",
799
+ r"malformed input", r"error code: 413\b")
800
+ _REQUEST_STATUS = _rx(r"error code: (400|422)\b", r"['\"]status['\"]: ?['\"]?(400|422)\b")
801
+ # Our own LONG-window allowance used up: per day, month or hour, credits.
802
+ _QUOTA = _rx(r"insufficient_quota", r"quota[ _]?exceeded", r"exceeded your (current|service) quota",
803
+ r"quota has been exhausted", r"free usage quota", r"usage_limit_(exceeded|reached)",
804
+ r"usage limit has been reached", r"free-models-per-day", r"per[ -](day|month|hour)\b",
805
+ r"(\d|calls?|requests?|tokens?) ?/ ?(day|month|hour)\b", r"\brpd\b", r"\btpd\b", r"(requests?|tokens?|calls?)[ _]?per[ _]?day",
806
+ r"allowance renews", r"(?<!nothing is )(?<!not )(?<!n't )\bused up\b", r"out of credits",
807
+ r"no credits", r"add (\d+ )?credits", r"credit balance", r"spend(ing)? limit", r"budget exceeded",
808
+ r"limit_source[\"': ]+\w*(daily|monthly)", r"for the hour")
809
+ # Words that mean quota only beside a limit word ("daily maintenance window" is not one).
810
+ _QUOTA_WEAK = _rx(r"\bdaily\b", r"\bmonthly\b", r"\bweekly\b", r"\bhourly\b", r"today'?s")
811
+ _LIMIT_WORD = _rx(r"quota", r"limit", r"allowance", r"(?<!connection )\breset", r"exceeded", r"reached", r"\bused\b")
812
+ # Our own SHORT-window allowance: per account, key or org, per minute, concurrency.
813
+ _RATE = _rx(r"per account", r"per key", r"per[ -]user\b", r"limit for (this|the) user",
814
+ r"(your|this) (account|organi[sz]ation).{0,80}(limit|\brate\b|concurrent|per[ -])",
815
+ r"in organi[sz]ation", r"account per-minute", r"per[ -](second|minute|min)\b", r"free-models-per-min",
816
+ r"\b(rpm|tpm|itpm|qpm|qps|rps)\b", r"(requests?|tokens?|queries|calls)[ _]?per[ _]?(minute|second)",
817
+ r"(calls|requests) ?/ ?(min|minute|sec|second)\b", r"limit:? *\d+, *used:? *\d+",
818
+ r"allows \d+ requests?(\(s\))? every", r"concurrent requests", r"rate limit for free users",
819
+ r"to unlock higher limits")
820
+ # A pool shared with other people: its 429 is the provider busy, not our allowance.
821
+ _SHARED = _rx(r"shared_capacity", r"rate-limited upstream", r"shared pool", r"all providers", r"no upstreams")
822
+ _BUSY = _rx(r"overloaded", r"high demand", r"\bbusy\b", r"capacity", r"temporarily", r"upstream", r"saturated",
823
+ r"try again (later|shortly|in a)", r"retry shortly", r"service[ _]unavailable", r"\bunavailable\b",
824
+ r"resource_?exhausted", r"request limit reached", r"queue is full", r"max connections",
825
+ r"rate[ _-]?limit", r"rate exceeded", r"too many requests", r"throttl", r"provider returned error",
826
+ r"internal server error", r"server[ _]error", r"internal_error", r"bad[ _]gateway",
827
+ r"failed to generate", r"provider_unavailable", r"get_channel_failed", r"model_cooldown",
828
+ r"modelnotreadyexception", r"not ready for inference", r"currently loading",
829
+ r"error code: (429|50\d|529)\b", r"['\"](code|status)['\"]: ?['\"]?(429|50\d|529)\b",
830
+ r"status code '?(429|50\d|529)\b")
831
+ _UNREACHABLE = _rx(r"connection error", r"connection refused", r"actively refused", r"no connection could be made",
832
+ r"connection reset",
833
+ r"forcibly closed", r"connection aborted", r"remote end closed", r"getaddrinfo",
834
+ r"name resolution", r"\bssl\b", r"certificate", r"eof occurred", r"broke off", r"had no choices",
835
+ r"was not json", r"stream failed", r"cloudflare", r"cf-footer", r"<html", r"<!doctype",
836
+ r"protocol violation", r"network (connection lost|is unreachable)", r"no route to host",
837
+ r"error code: 52[0-8]\b")
838
+ _RELAY_DOWN = _rx(r"actively refused", r"connection refused", r"no connection could be made", r"getaddrinfo")
839
+ _RETRY_AFTER = re.compile(r"retry-after:? *(\d+)|retry after (\d+) ?s")
840
+
841
+
842
+ def uptime_class(text: str, status: str = "error", private_endpoint: bool = False,
843
+ decided: bool = True) -> tuple[str, str]:
844
+ """(class, reason) for one call: a class from UPTIME_UP, UPTIME_DOWN or
845
+ UPTIME_OUT. `status` is the call's own (ok, empty, error, timeout, or
846
+ refused: not sent by the bench); `private_endpoint`: the call went to a
847
+ hub on our own network, so a refused connection is that hub, not the
848
+ provider. When no rule matches, the class a decision model gave the
849
+ error's shape, read from metrics/decided.json (decide.classify_errors
850
+ asks, a few new shapes a day; nothing is asked here), its reason
851
+ "decided by <model> at <confidence>"; `decided=False` is the rules alone,
852
+ which the decision benchmark labels with."""
853
+ if status in ("ok", "empty"):
854
+ return status, "answered"
855
+ if status == "refused":
856
+ return "notsent", "not sent by the bench"
857
+ if status == "timeout":
858
+ return "timeout", "no answer in time"
859
+ low = " ".join((text or "").lower().split())
860
+ if not low:
861
+ return "unrecognised", "no error text"
862
+ if _NOTSENT.search(low):
863
+ return "notsent", "the bench's own pacing"
864
+ # Not a provider's own crash sent back with its status (a 500 carrying a traceback).
865
+ if _CLIENT.search(low) and not re.search(r"error code: \d{3}\b", low):
866
+ return "client", "our client failed"
867
+ if _TIMEOUT.search(low):
868
+ return "timeout", "timed out"
869
+ if _SIGNIN.search(low):
870
+ # A router relaying ITS upstream's refused key: not our key, the provider's problem.
871
+ return ("busy", "upstream's own key refused") if _UPSTREAM_WRAP.search(low) else ("setup", "sign-in")
872
+ if _PAYWALLED.search(low) and not re.search(r"high demand|overloaded|temporarily", low):
873
+ return "setup", "paywalled"
874
+ if _NOTFOUND.search(low) and not re.search(r"temporarily|overloaded|high demand|capacity|saturated", low):
875
+ return "setup", "not found"
876
+ if _REQUEST.search(low):
877
+ return "setup", "our request"
878
+ if not _SHARED.search(low) or "free usage quota" in low:
879
+ # A weak word ("daily limit") beside the provider's own trouble is its advice, not our quota.
880
+ weak = (_QUOTA_WEAK.search(low) and _LIMIT_WORD.search(low)
881
+ and not re.search(r"high demand|overloaded|temporarily", low))
882
+ if _QUOTA.search(low) or weak:
883
+ if "credits required" in low:
884
+ return "setup", "no access"
885
+ return "quota", "our quota"
886
+ if _RATE.search(low):
887
+ return "rate", "our rate limit"
888
+ if _ENTITLEMENT.search(low):
889
+ # As with sign-in: a router relaying its upstream's refused key is the provider's problem.
890
+ if _UPSTREAM_WRAP.search(low) and re.search(r"unauthori[sz]ed|permission[ _]denied", low):
891
+ return "busy", "upstream's own key refused"
892
+ return "setup", "no access"
893
+ if _REQUEST_STATUS.search(low) and not _BUSY.search(low):
894
+ return "setup", "our request"
895
+ # An otherwise silent 429 that asks for an hour or more is a long window: ours.
896
+ if ((m := _RETRY_AFTER.search(low)) and int(m.group(1) or m.group(2)) >= 3600
897
+ and re.search(r"\b429\b", low) and not _SHARED.search(low)):
898
+ return "quota", "our quota (a wait of an hour or more)"
899
+ if _BUSY.search(low):
900
+ return "busy", "busy"
901
+ if _UNREACHABLE.search(low):
902
+ if private_endpoint and _RELAY_DOWN.search(low):
903
+ return "relay", "our LAN hub refused the connection"
904
+ return "unreachable", "unreachable"
905
+ if decided:
906
+ try:
907
+ import decide
908
+ if hit := decide.cached_error_class(text):
909
+ return hit
910
+ except Exception: # noqa: BLE001 -- a broken cache is no decision
911
+ pass
912
+ return "unrecognised", "nothing matched"
719
913
 
720
914
 
721
915
  def outcome(output: str, min_length: int = 40) -> tuple[str, str] | None:
@@ -1063,7 +1257,8 @@ def main() -> int:
1063
1257
  return 0
1064
1258
 
1065
1259
  RUNS.mkdir(exist_ok=True)
1066
- stamp = dt.datetime.now().strftime("%Y-%m-%d")
1260
+ started = dt.datetime.now()
1261
+ stamp = started.strftime("%Y-%m-%d")
1067
1262
  report = RUNS / f"suite-{stamp}.md"
1068
1263
  rows = []
1069
1264
 
@@ -1123,12 +1318,37 @@ def main() -> int:
1123
1318
  f" {seconds:6.1f}s{tries}{flag}")
1124
1319
  (spec["dir"] / "output").mkdir(exist_ok=True)
1125
1320
  (spec["dir"] / "output" / f"{model}.md").write_text(body, encoding="utf-8")
1321
+ keep_answer(spec["id"], model, started, body)
1126
1322
 
1127
1323
  _write_report(report, stamp, rows)
1128
1324
  print(f"\nwritten: {report}")
1129
1325
  return 0
1130
1326
 
1131
1327
 
1328
+ def keep_answer(task: str, model: str, started: dt.datetime, body: str) -> pathlib.Path | None:
1329
+ """A copy of the answer that the next run cannot replace:
1330
+ runs/answers/<task>/<model>/<run start>.md. The run's start ties it to its
1331
+ score, a row of that run's section in runs/suite-<its date>.md (the
1332
+ sections are in start order); the file's own time is when it was saved.
1333
+ No reader of the saved answers looks there (they read suite/tasks/*/output).
1334
+ A copy that cannot be written costs the copy, not the run's note."""
1335
+ folder = ANSWERS / task / model
1336
+ name = started.strftime("%Y-%m-%dT%H%M%S")
1337
+ try:
1338
+ folder.mkdir(parents=True, exist_ok=True)
1339
+ for n in range(1, 100): # two runs started in the same second ("_" sorts after ".")
1340
+ path = folder / (f"{name}.md" if n == 1 else f"{name}_{n}.md")
1341
+ try:
1342
+ with path.open("x", encoding="utf-8") as fh:
1343
+ fh.write(body)
1344
+ return path
1345
+ except FileExistsError:
1346
+ continue
1347
+ except OSError as exc:
1348
+ print(f" (answer not kept: {exc})")
1349
+ return None
1350
+
1351
+
1132
1352
  def _learn(model: str, body: str, where: str) -> None:
1133
1353
  """Pass a failed call's text to metrics/learn.py: a context window or rate
1134
1354
  limit the error states, and a cooldown if it was rate-limited. Never lets
@@ -44,6 +44,21 @@ def main() -> None:
44
44
  (tasks[3]["dir"] / "output" / "m.md").write_text("<<TIMED OUT after 300s>>", encoding="utf-8")
45
45
  assert [t["id"] for t in pick_tasks(tasks, "m")] == ["t2"], pick_tasks(tasks, "m")
46
46
  assert [t["id"] for t in pick_tasks(tasks, "other", n=2)] == ["t0", "t1"] # per model
47
+
48
+ # Every answer is also kept, by its run's start, and never replaced: two runs
49
+ # that started in the same second keep both.
50
+ import datetime as dt
51
+ import run
52
+ with tempfile.TemporaryDirectory() as tmp:
53
+ kept, run.ANSWERS = run.ANSWERS, pathlib.Path(tmp)
54
+ try:
55
+ at = dt.datetime(2026, 11, 8, 9, 30, 5)
56
+ a, b = run.keep_answer("t", "m", at, "first"), run.keep_answer("t", "m", at, "second")
57
+ assert (a.relative_to(tmp).as_posix(), b.name) == ("t/m/2026-11-08T093005.md", "2026-11-08T093005_2.md"), (a, b)
58
+ assert sorted(a.parent.iterdir()) == [a, b] # by name is in order
59
+ assert (a.read_text(encoding="utf-8"), b.read_text(encoding="utf-8")) == ("first", "second")
60
+ finally:
61
+ run.ANSWERS = kept
47
62
  print("ok")
48
63
 
49
64
 
@@ -1,4 +1,5 @@
1
- """python suite/test_score.py — the scorer's negation guard and first-line scoring, no calls."""
1
+ """python suite/test_score.py — the scorer's negation guard and first-line scoring, and the
2
+ uptime classifier's cases; no calls."""
2
3
  import pathlib
3
4
  import sys
4
5
 
@@ -27,4 +28,84 @@ seeded = {"must_find": [{"regex": r"^fault:\s*line\s*1\b", "label": "fault: line
27
28
  assert run.score(seeded, "FAULT: line 1\nthe comparison ...")["found"] == 1
28
29
  assert run.score(seeded, "FAULT: line 12\n...")["found"] == 0 # not a prefix match
29
30
  assert run.score(seeded, "NO FAULT")["invented"] == ["no fault"]
31
+
32
+ # Every class failure_class returns has a place in the worst-first order (perf/run.py
33
+ # wrote a blank failure for signin while it had none).
34
+ assert run.failure_class("Error: Error code: 401 - {'error': 'invalid key'}") in run.FAILURE_CLASSES
35
+
36
+ # run.uptime_class: shapes from the bench's own data first, then public providers'
37
+ # documented formats and the heavy-hitters panel's cases (2026-11-08).
38
+ UPTIME = {
39
+ "Error: Error code: 429 - {'error': {'message': 'Provider returned error', 'code': 429, 'metadata': {'raw': 'x/y:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits'}}}": "busy",
40
+ "Error: Error code: 429 - {'error': {'message': 'Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day', 'code': 429}}": "quota",
41
+ "s. Credits don't affect this cap. Add your own keys to get a boost.', 'code': 429, 'metadata': {'limit_source': 'openrouter_shared_capacity'}}": "busy",
42
+ "Error: Error code: 429 - {'error': {'message': 'Too many requests. The free tier allows 1 request(s) every 1 min per account on m:free - nothing is used up, retry in 60s.'}}": "rate",
43
+ "Error: Error code: 503 - {'error': {'message': 'The free usage quota for this model is used up and it is currently unavailable.'}}": "quota",
44
+ "Error: Error code: 503 - {'error': {'code': 'get_channel_failed', 'message': 'All providers for model m are busy right now (they hit their rate limit).'}}": "busy",
45
+ "Error: Error code: 500 - ResourceExhausted: Worker local total request limit reached (33/32)": "busy",
46
+ "Error: Error code: 404 - {'error': 'The requested model is currently unavailable.', 'error_type': 'unavailable_model'}": "setup",
47
+ "Error: Error code: 400 - {'error': {'code': 400, 'message': 'Request contains an invalid argument.', 'status': 'INVALID_ARGUMENT'}}": "setup",
48
+ "Error: Error code: 413 - Request too large for model in organization on input tokens per minute (ITPM): Limit 7000, Requested 9000": "setup",
49
+ "Error: Error code: 429 - Your organization reached its limit of concurrent requests. top up at <url>": "rate",
50
+ "Error: Error code: 402 - {'error': {'title': 'Paid Model - Credits Required', 'message': 'Add credits to continue'}}": "setup",
51
+ "Error: Error code: 429 - {'error': {'message': \"You've reached today's free-model token quota. Your plan's paid allowance is separate, or wait for the daily reset.\"}} Retry-After: 14652": "quota",
52
+ "Error: client failed: Connection error. ([WinError 10061] No connection could be made because the target machine actively refused it)": "relay",
53
+ "Error: client failed: the reply broke off part way ([WinError 10054] An existing connection was forcibly closed by the remote host)": "unreachable",
54
+ "Error: client failed: the reply had no choices: the stream sent none": "unreachable",
55
+ "<<TIMED OUT after 90s>>": "timeout",
56
+ "no answer in 420s": "timeout",
57
+ "Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests per day": "notsent",
58
+ "sqlite3.OperationalError: database is locked": "client",
59
+ "Error: Error code: 429 - {'error': {'code': 'model_cooldown', 'last_upstream_error': 'usage_limit_reached: The usage limit has been reached', 'message': 'All credentials for model m are cooling down via provider codex'}} Retry-After: 2528616": "quota",
60
+ "Error: Error code: 503 - {'error': {'message': 'The model is temporarily unavailable due to high demand. Please retry in a moment.', 'type': 'server_error'}}": "busy",
61
+ "Error: Error code: 418 - {'error': 'teapot'}": "unrecognised",
62
+ "Error: Error code: 529 - {'type': 'error', 'error': {'type': 'overloaded_error', 'message': 'Overloaded'}}": "busy",
63
+ "Error: Error code: 429 - {'error': {'code': 429, 'message': 'You exceeded your current quota', 'status': 'RESOURCE_EXHAUSTED'}}": "quota",
64
+ "Error: Error code: 429 - {'error': {'code': 429, 'message': 'Resource has been exhausted (e.g. check quota).', 'status': 'RESOURCE_EXHAUSTED'}}": "busy",
65
+ "Error: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of request tokens has exceeded your per-minute rate limit'}}": "rate",
66
+ "Error: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-x in organization org-x on requests per min (RPM): Limit 3, Used 3, Requested 1.', 'type': 'requests', 'code': 'rate_limit_exceeded'}}": "rate",
67
+ "Error: Error code: 429 - {'message': \"You are using a Trial key, which is limited to 1000 API calls / month.\"}": "quota",
68
+ "Error: Error code: 429 - {'error': {'message': 'Rate limit exceeded: free-models-per-min.', 'code': 429}}": "rate",
69
+ "Error: Error code: 400 - {'error': {'message': 'Provider returned error: prompt is too long: 210000 tokens > 200000 maximum', 'code': 400}}": "setup",
70
+ "Error: Error code: 400 - {'message': 'Prompt exceeds the capacity of this model: too many tokens'}": "setup",
71
+ "Error: Error code: 503 - {'error': 'Service temporarily unavailable during daily maintenance window'}": "busy",
72
+ "Error: Error code: 503 - Your account is temporarily overloaded.": "busy",
73
+ "Error: Error code: 500 - Your account failed to generate a reply.": "busy",
74
+ "403: Your account does not have access to this model.": "setup",
75
+ "An error occurred (ServiceQuotaExceededException) when calling the InvokeModel operation: Rate exceeded.": "quota",
76
+ "An error occurred (AccessDeniedException) when calling the InvokeModel operation: You don't have access to the model.": "setup",
77
+ "An error occurred (ValidationException) when calling the InvokeModel operation: The provided model identifier is invalid.": "setup",
78
+ "An error occurred (ModelNotReadyException) when calling the Converse operation: Model is not ready for inference.": "busy",
79
+ "An error occurred (ThrottlingException) when calling the InvokeModel operation: Too many requests, please wait.": "busy",
80
+ "{\"error\": \"model 'llama3:70b' not found, try pulling it first\"}": "setup",
81
+ "{\"success\": false, \"errors\": [{\"code\": 10000, \"message\": \"Authentication error\"}]}": "setup",
82
+ "... 'code': 'content_filter', 'status': 400}}": "setup",
83
+ "Error: Error code: 401 - {'error': {'message': 'Provider returned error', 'metadata': {'raw': 'invalid api key'}}}": "busy",
84
+ "Error: Error code: 429 - {'error': {'message': 'Too many requests'}} Retry-After: 86400": "quota",
85
+ "Error: Error code: 429 - {'error': {'message': 'Too many requests'}} Retry-After: 30": "busy",
86
+ "Error: Error code: 503 - model unavailable for free users due to high demand": "busy",
87
+ "provider notice in a success reply (out of credits): You are out of credits.": "quota",
88
+ # the heavy-hitters review of the build (p20261108-104415-db82)
89
+ "Error: Error code: 403 - {'error': {'message': 'Provider returned error', 'metadata': {'raw': 'Unauthorized'}}}": "busy",
90
+ "Error: Error code: 403 - {'error': {'message': 'Unauthorized'}}": "setup",
91
+ "Error: Error code: 429 - {'error': {'message': 'Too many tokens per minute'}}": "rate",
92
+ "Error: Error code: 502 - Bad Gateway at https://example.com/api/day/status": "busy",
93
+ "Error: client failed: the reply broke off part way (connection reset while fetching today's list)": "unreachable",
94
+ "Error: Error code: 429 - Model is experiencing high demand. Please retry shortly, or add your own key for a higher daily limit.": "busy",
95
+ "[WinError 10065] No connection could be made to the host": "relay",
96
+ "ne 12, in handler ... Traceback (most recent call last): ... 'status': 500, 'error code: 500'": "busy",
97
+ }
98
+ # The rules alone (decided=False): what this install's decision models said of a shape is not a rule.
99
+ bad = [(t[:80], run.uptime_class(t, private_endpoint=True, decided=False)[0], want) for t, want in UPTIME.items()
100
+ if run.uptime_class(t, private_endpoint=True, decided=False)[0] != want]
101
+ assert not bad, bad
102
+ # The call's own status decides first; an error with no text is never an answer.
103
+ assert run.uptime_class("", "ok")[0] == "ok" and run.uptime_class("", "empty")[0] == "empty"
104
+ assert run.uptime_class("", "refused")[0] == "notsent" and run.uptime_class("", "timeout")[0] == "timeout"
105
+ assert run.uptime_class("", "error")[0] == "unrecognised"
106
+ # A refused connection is our hub only when the call went to one.
107
+ assert run.uptime_class("Error: client failed: Connection error. (connection refused)")[0] == "unreachable"
108
+ assert set(run.UPTIME_UP + run.UPTIME_DOWN + run.UPTIME_OUT) >= {run.uptime_class(t, decided=False)[0] for t in UPTIME}
109
+ # A normal answer never reaches the notice fallback (metrics/test_decide.py tests the fallback itself).
110
+ assert not run.NOTICE_LIKE.search("ready") and run.NOTICE_LIKE.search("your credit balance is low")
30
111
  print("ok")
package/ui/bench.css CHANGED
@@ -185,6 +185,15 @@ table.a-table col.c-acts { width: 7.5rem; }
185
185
  padding: .05rem .4rem; border-radius: 999px; font: 700 .75rem/1.25 var(--font-mono);
186
186
  background: var(--err-ground); color: var(--err); }
187
187
  .tab { display: none; } .tab.active { display: block; }
188
+ /* A page not shown keeps its layout (content-visibility), so showing it again
189
+ is cheap: with display none, Discover's 1,200 rows were laid out again on
190
+ every switch to it (0.2–0.3 s), now 15 ms. Its contents stay out of the
191
+ accessibility tree and focus order, as with display none. */
192
+ @supports (content-visibility: hidden) {
193
+ .tab { display: block; content-visibility: hidden; } .tab.active { content-visibility: visible; }
194
+ }
195
+ /* A page not fetched yet (bench.js load), while the bar sweeps the top. */
196
+ .tab[data-lazy]:empty::before { content: "Loading…"; display: block; padding: 1.4rem 1.6rem; color: var(--muted); }
188
197
  /* One surface for the page's content: its panels are sections of it,
189
198
  divided by a hairline, rather than separate boxes with gaps between. A
190
199
  theme with a floor makes it frosted (--sheet, --glass). */
@@ -215,6 +224,14 @@ table.facts th { text-align: left; padding-right: 1.5rem; color: var(--muted); f
215
224
  details.panel > summary { cursor: pointer; }
216
225
  details.panel[open] > summary { margin-bottom: .75rem; }
217
226
  .log-line { font-size: .8125rem; padding: .1rem 0; overflow-wrap: anywhere; }
227
+ /* Clients (serve.clients_page): each snippet under its label and Copy button */
228
+ .snip { margin-top: .9rem; }
229
+ .snip-head { display: flex; align-items: center; justify-content: space-between; gap: .5rem;
230
+ font-size: .875rem; color: var(--muted); overflow-wrap: anywhere; }
231
+ .snip-head .btn { flex: none; }
232
+ .snip pre.log { max-height: 30rem; margin-top: .3rem; overflow-wrap: anywhere; }
233
+ #tab-clients :is(.where, .watch) { font-size: .9375rem; max-width: 75ch; overflow-wrap: anywhere; }
234
+ #tab-clients .watch { background: var(--step-1); border-radius: var(--radius); padding: .5rem .75rem; margin: 1rem 0 0; }
218
235
 
219
236
  .panel { background: var(--panel); border-radius: var(--radius); padding: 1.1rem 1.25rem 1.25rem; }
220
237
  .panel > h2 { margin: 0; font-size: 1.375rem; font-weight: 700; }
@@ -248,6 +265,20 @@ details.key dl { margin: .5rem 0 0; display: grid; grid-template-columns: max-co
248
265
  details.key dt { font-weight: 700; }
249
266
  details.key dd { margin: 0; color: var(--ink); }
250
267
 
268
+ /* Share (serve.SHARE, bench.js): top right of a report, its menu under it. */
269
+ .panel:has(> .share) { position: relative; }
270
+ .panel:has(> .share) > h2 { padding-right: 4.5rem; }
271
+ details.share { position: absolute; top: 1.2rem; right: 1.4rem; font-size: .9375rem; }
272
+ details.share > summary { cursor: pointer; color: var(--link); list-style: none; text-decoration: underline;
273
+ text-underline-offset: .18em; }
274
+ details.share > summary::-webkit-details-marker { display: none; }
275
+ .share-menu { position: absolute; right: 0; top: 1.7rem; z-index: 5; display: flex; flex-direction: column;
276
+ align-items: flex-start; gap: .5rem; min-width: 14rem; padding: .8rem 1rem; border: 1px solid var(--ink);
277
+ border-radius: var(--radius); background: var(--paper-2); box-shadow: 4px 4px 0 var(--ink); }
278
+ .share-to { color: var(--muted); }
279
+ .no-clip [data-share=copy], .no-clip [data-share=md], .no-share [data-share=native],
280
+ .panel:not(:has(table)) [data-share=md], .panel:not(:has(table)) [data-share=csv] { display: none; }
281
+
251
282
  /* ---- tables --------------------------------------------------------- */
252
283
  .scroll { overflow-x: auto; }
253
284
  table { border-collapse: collapse; width: 100%; font-size: .9375rem; }
@@ -297,6 +328,7 @@ button.chip:hover { outline: 1px solid currentColor; }
297
328
  .meter { width: 4.5rem; height: .5rem; background: var(--band); border-radius: 999px; overflow: hidden; }
298
329
  .meter span { display: block; height: 100%; background: var(--brand); }
299
330
  .meter span.low { background: var(--err); }
331
+ td.why { white-space: nowrap; } /* Uptime's down / left out, with their reasons */
300
332
 
301
333
  /* ---- dialogs -------------------------------------------------------- */
302
334
  dialog { border: 1px solid var(--ink); border-radius: var(--radius); padding: 1.25rem 1.4rem;
@@ -364,6 +396,8 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
364
396
  padding: .5rem .75rem; margin: 0 -.75rem; border-radius: var(--radius); }
365
397
  .endpoint:hover { background: var(--step-1); }
366
398
  .endpoint .base { color: var(--muted); font-size: .875rem; overflow-wrap: anywhere; }
399
+ .found { margin: 0 0 1rem; }
400
+ .found h3 { margin: 0 0 .25rem; font-size: 1rem; font-weight: 700; }
367
401
  :is(dialog, .form-panel) h3.form-head { margin: 1.25rem 0 0; font-size: 1rem; font-weight: 700; }
368
402
  :is(dialog, .form-panel) input[readonly] { background: var(--step-1); }
369
403
  /* Add an endpoint's choices: API key, then each provider; the chosen one pressed */
@@ -554,6 +588,16 @@ dialog .pick { display: flex; gap: 1.5rem; }
554
588
  .a-table .said { white-space: normal; } /* the height to read it whole */
555
589
  .a-table .said-line > a { max-width: none; }
556
590
  }
591
+ /* A reload under way (bench.js marks the page as it goes), or a page being
592
+ fetched (.loading): a bar sweeping the top, over an open popup too, so the
593
+ seconds the server takes to render never look like a frozen page. */
594
+ body:is(.leaving, .loading) { cursor: progress; }
595
+ body:is(.leaving, .loading)::before, body:is(.leaving, .loading) dialog[open]::before {
596
+ content: ""; position: fixed; inset: 0 0 auto; height: 3px; z-index: 1000; pointer-events: none;
597
+ background: linear-gradient(90deg, transparent, var(--brand), transparent) 0 0 / 40% 100% no-repeat;
598
+ animation: leaving 1.1s ease-in-out infinite; }
599
+ @keyframes leaving { from { background-position: -70% 0; } to { background-position: 170% 0; } }
557
600
  @media (prefers-reduced-motion: reduce) {
558
601
  * { transition: none !important; }
602
+ body:is(.leaving, .loading)::before, body:is(.leaving, .loading) dialog[open]::before { animation: none; background: var(--brand); }
559
603
  }