@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/logo-dark.svg +49 -0
- package/.github/logo.svg +49 -0
- package/BENCHMARKS.md +160 -0
- package/GATEWAY.md +33 -11
- package/PRIVACY.md +20 -6
- package/README.md +46 -11
- package/SECURITY.md +20 -10
- package/USAGE.md +71 -9
- package/VERSION +1 -1
- package/gateway.py +93 -82
- package/metrics/bench_collections.py +27 -3
- package/metrics/catalog.py +52 -4
- package/metrics/client.py +51 -1
- package/metrics/connect.py +225 -0
- package/metrics/decide.py +690 -27
- package/metrics/endpoints.py +4 -6
- package/metrics/guard.py +6 -4
- package/metrics/ledger.py +3 -1
- package/metrics/locked.py +102 -1
- package/metrics/offload.py +36 -19
- package/metrics/route.py +9 -1
- package/metrics/scheduler.py +11 -1
- package/metrics/test_decide.py +275 -7
- package/metrics/wrong.py +21 -9
- package/package.json +3 -1
- package/scripts/auto_retire.py +96 -21
- package/scripts/onboard.py +44 -3
- package/scripts/test_connect.py +152 -3
- package/scripts/test_first_run.py +4 -2
- package/scripts/test_formats.py +39 -0
- package/scripts/test_serve_edit.py +308 -4
- package/serve.py +1672 -141
- package/suite/README.md +5 -2
- package/suite/run.py +226 -6
- package/suite/test_pick.py +15 -0
- package/suite/test_score.py +82 -1
- package/ui/bench.css +44 -0
- package/ui/bench.js +910 -203
package/suite/README.md
CHANGED
|
@@ -34,8 +34,11 @@ python suite/run.py --model gemini-flash-3.8
|
|
|
34
34
|
python suite/run.py --dry-run # print the prompts, call nothing
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
Results append to `runs/suite-<date>.md` and print a table.
|
|
38
|
-
|
|
37
|
+
Results append to `runs/suite-<date>.md` and print a table. Each answer is
|
|
38
|
+
also kept at `runs/answers/<task>/<model>/<run start>.md` (since 2026-11-08).
|
|
39
|
+
Nothing there is overwritten — a run is evidence, and evidence accumulates.
|
|
40
|
+
The Benchmarks page's Trend compares each model's scores with its own earlier
|
|
41
|
+
ones ([BENCHMARKS.md](../BENCHMARKS.md)).
|
|
39
42
|
|
|
40
43
|
## Claim verification, and tasks mined from real work
|
|
41
44
|
|
package/suite/run.py
CHANGED
|
@@ -29,6 +29,9 @@ sys.path.insert(0, str(ROOT / "metrics"))
|
|
|
29
29
|
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
30
30
|
TASKS = DATA / "suite" / "tasks"
|
|
31
31
|
RUNS = DATA / "runs"
|
|
32
|
+
# Every answer, kept: output/<model>.md is replaced by the next run, so a
|
|
33
|
+
# score in an old run note had no answer left behind it (before 2026-11-08).
|
|
34
|
+
ANSWERS = RUNS / "answers"
|
|
32
35
|
|
|
33
36
|
# THE CODEBASES THIS BENCHMARK IS DRAWN FROM.
|
|
34
37
|
#
|
|
@@ -647,8 +650,9 @@ CAPACITY = (
|
|
|
647
650
|
# than a wrong class: `check_deprecations` skips blank failures and keeps
|
|
648
651
|
# whatever it saw last, so two retired models silently kept their old
|
|
649
652
|
# classification. Imported rather than repeated, so the next class added
|
|
650
|
-
# cannot reintroduce this.
|
|
651
|
-
|
|
653
|
+
# cannot reintroduce this. `signin` was missing until 2026-11-08 (the same
|
|
654
|
+
# blank failure); it leads, as it does in `failure_class`.
|
|
655
|
+
FAILURE_CLASSES = ("signin", "structural", "paywalled", "entitlement", "capacity",
|
|
652
656
|
"timeout", "empty", "other")
|
|
653
657
|
|
|
654
658
|
|
|
@@ -710,12 +714,202 @@ PROVIDER_NOTICES = (
|
|
|
710
714
|
)
|
|
711
715
|
|
|
712
716
|
|
|
713
|
-
|
|
714
|
-
|
|
717
|
+
# A short reply the phrases miss is put to a decision model only when it LOOKS
|
|
718
|
+
# like a notice: under NOTICE_LIKE_MAX characters (a notice is a sentence or
|
|
719
|
+
# two; the probe's one-word "ready" has none of these words) and speaking of
|
|
720
|
+
# the account: credit, quota, balance, billing, a plan or subscription, an
|
|
721
|
+
# upgrade, a top-up, a limit or allowance used up. A short answer that happens
|
|
722
|
+
# to use those words costs one question, once per shape (decide.notice keeps
|
|
723
|
+
# every answer); any other answer costs nothing, not even a file read.
|
|
724
|
+
NOTICE_LIKE = re.compile(r"\b(credits?|quota|balance|billing|subscription|plan|upgrade|top[ -]?up|limit|"
|
|
725
|
+
r"allowance|exhausted|exceeded|insufficient|payment)\b")
|
|
726
|
+
NOTICE_LIKE_MAX = 300
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def provider_notice(output: str, ask: bool = False) -> str | None:
|
|
730
|
+
"""The notice phrase, if a short reply is a provider's quota/credit notice.
|
|
731
|
+
When no phrase matches but the reply looks like one (NOTICE_LIKE): what a
|
|
732
|
+
decision model said of its shape ("decided by <model> at <p>"), read from
|
|
733
|
+
metrics/decided.json; asked now only with `ask` (the gateway's seats, as
|
|
734
|
+
the reply comes in), never while a page or a score is worked out."""
|
|
715
735
|
text = output.strip().lower()
|
|
716
736
|
if len(text) > 400:
|
|
717
737
|
return None
|
|
718
|
-
|
|
738
|
+
if hit := next((m for m in PROVIDER_NOTICES if m in text), None):
|
|
739
|
+
return hit
|
|
740
|
+
if len(text) > NOTICE_LIKE_MAX or not NOTICE_LIKE.search(text):
|
|
741
|
+
return None
|
|
742
|
+
try:
|
|
743
|
+
import decide
|
|
744
|
+
return decide.notice(output.strip(), ask_now=ask)
|
|
745
|
+
except Exception: # noqa: BLE001 -- the fallback never fails a reply
|
|
746
|
+
return None
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
# WAS THE PROVIDER THERE? One call at a time, for the Uptime report (serve.py).
|
|
750
|
+
#
|
|
751
|
+
# `failure_class` asks what to do about a MODEL; this asks something narrower
|
|
752
|
+
# about one CALL: the provider failed us (down), it answered (up), or the call
|
|
753
|
+
# failed for a reason that is ours and says nothing about the provider (left
|
|
754
|
+
# out): our own quota or rate allowance, our pacer, our client, our LAN hub,
|
|
755
|
+
# or a setup problem (sign-in, no such model, no access, our own request).
|
|
756
|
+
# The owner's rules, 2026-11-08. Read from the stored error text every time,
|
|
757
|
+
# so a better rule reclassifies the past too. Generic first (status codes,
|
|
758
|
+
# standard codes and phrases), so a provider the bench does not use yet is
|
|
759
|
+
# classified; first match wins. Shapes and cases: suite/test_score.py.
|
|
760
|
+
UPTIME_UP = ("ok", "empty") # an empty reply is still an answer
|
|
761
|
+
UPTIME_DOWN = ("busy", "timeout", "unreachable", "unrecognised")
|
|
762
|
+
UPTIME_OUT = ("quota", "rate", "setup", "notsent", "client", "relay")
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def _rx(*parts: str) -> re.Pattern:
|
|
766
|
+
return re.compile("|".join(parts))
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
_NOTSENT = _rx(r"\(not sent;", r"paced by the bench", r"free-tier guard")
|
|
770
|
+
_CLIENT = _rx(r"no model is registered", r"no key named", r"endpoint base must be", r"codec can't (en|de)code",
|
|
771
|
+
r"traceback \(most recent", r"database is locked", r"sqlite3?\.")
|
|
772
|
+
_TIMEOUT = _rx(r"<<timed out", r"no answer in \d+s", r"\btimed? ?out\b", r"read timeout", r"deadline[ _]exceeded",
|
|
773
|
+
r"gateway time-?out", r"request_timeout", r"timeout_error", r"stream_timeout",
|
|
774
|
+
r"error code: (408|504|524)\b")
|
|
775
|
+
_SIGNIN = _rx(r"error code: 401\b", r"['\"](code|status)['\"]: ?['\"]?401\b", r"invalid[ _]api[ _]key",
|
|
776
|
+
r"incorrect api key", r"unauthenticated", r"authentication[ _](error|failed)", r"sign in again",
|
|
777
|
+
r"token (has )?expired", r"invalid credentials", r"sharing_usage_unavailable", r"api key not valid",
|
|
778
|
+
r"api_key_invalid", r"invalid username or password", r"(didn'?t|did not) provide an api key",
|
|
779
|
+
r"missing api key")
|
|
780
|
+
_UPSTREAM_WRAP = _rx(r"provider returned error", r"upstream error", r"upstream provider")
|
|
781
|
+
_PAYWALLED = _rx(*map(re.escape, PAYWALLED))
|
|
782
|
+
_NOTFOUND = _rx(r"error code: 404\b", r"model_not_found", r"model not found", r"no endpoints found",
|
|
783
|
+
r"does not exist", r"unknown model", r"invalid model", r"no model found", r"no such model",
|
|
784
|
+
r"not_found_error", r"unavailable_model", r"decommissioned", r"has been deprecated",
|
|
785
|
+
r"model '[^']*' not found", r"try pulling it first", r"model identifier is invalid")
|
|
786
|
+
_ENTITLEMENT = _rx(r"error code: 40[23]\b", r"payment required", r"insufficient[ _]balance", r"insufficient[ _]credit",
|
|
787
|
+
r"not available in your subscription", r"permission[ _]denied", r"not authori[sz]ed",
|
|
788
|
+
r"unauthori[sz]ed", r"no access to model", r"only available on", r"consent_required",
|
|
789
|
+
r"needs your consent", r"paid model", r"credits required", r"zero balance",
|
|
790
|
+
r"failed_precondition", r"not available in your (country|region)",
|
|
791
|
+
r"user location is not supported", r"accessdeniedexception",
|
|
792
|
+
r"(don'?t|do not|does not) have (access|permission)", r"not allowed to access")
|
|
793
|
+
# A request the provider refused as OURS: wins over busy words ("capacity",
|
|
794
|
+
# "upstream") in the same text, so a wrapped "prompt is too long" is setup.
|
|
795
|
+
_REQUEST = _rx(r"request[ _]too[ _]large", r"context[ _]length", r"maximum context", r"too many tokens(?! (per|in the last))",
|
|
796
|
+
r"prompt is too long", r"payload too large", r"invalid_argument", r"invalid_request_error",
|
|
797
|
+
r"content_filter", r"content management policy", r"input validation error",
|
|
798
|
+
r"must have less than \d+ tokens", r"extra inputs are not permitted", r"validationexception",
|
|
799
|
+
r"malformed input", r"error code: 413\b")
|
|
800
|
+
_REQUEST_STATUS = _rx(r"error code: (400|422)\b", r"['\"]status['\"]: ?['\"]?(400|422)\b")
|
|
801
|
+
# Our own LONG-window allowance used up: per day, month or hour, credits.
|
|
802
|
+
_QUOTA = _rx(r"insufficient_quota", r"quota[ _]?exceeded", r"exceeded your (current|service) quota",
|
|
803
|
+
r"quota has been exhausted", r"free usage quota", r"usage_limit_(exceeded|reached)",
|
|
804
|
+
r"usage limit has been reached", r"free-models-per-day", r"per[ -](day|month|hour)\b",
|
|
805
|
+
r"(\d|calls?|requests?|tokens?) ?/ ?(day|month|hour)\b", r"\brpd\b", r"\btpd\b", r"(requests?|tokens?|calls?)[ _]?per[ _]?day",
|
|
806
|
+
r"allowance renews", r"(?<!nothing is )(?<!not )(?<!n't )\bused up\b", r"out of credits",
|
|
807
|
+
r"no credits", r"add (\d+ )?credits", r"credit balance", r"spend(ing)? limit", r"budget exceeded",
|
|
808
|
+
r"limit_source[\"': ]+\w*(daily|monthly)", r"for the hour")
|
|
809
|
+
# Words that mean quota only beside a limit word ("daily maintenance window" is not one).
|
|
810
|
+
_QUOTA_WEAK = _rx(r"\bdaily\b", r"\bmonthly\b", r"\bweekly\b", r"\bhourly\b", r"today'?s")
|
|
811
|
+
_LIMIT_WORD = _rx(r"quota", r"limit", r"allowance", r"(?<!connection )\breset", r"exceeded", r"reached", r"\bused\b")
|
|
812
|
+
# Our own SHORT-window allowance: per account, key or org, per minute, concurrency.
|
|
813
|
+
_RATE = _rx(r"per account", r"per key", r"per[ -]user\b", r"limit for (this|the) user",
|
|
814
|
+
r"(your|this) (account|organi[sz]ation).{0,80}(limit|\brate\b|concurrent|per[ -])",
|
|
815
|
+
r"in organi[sz]ation", r"account per-minute", r"per[ -](second|minute|min)\b", r"free-models-per-min",
|
|
816
|
+
r"\b(rpm|tpm|itpm|qpm|qps|rps)\b", r"(requests?|tokens?|queries|calls)[ _]?per[ _]?(minute|second)",
|
|
817
|
+
r"(calls|requests) ?/ ?(min|minute|sec|second)\b", r"limit:? *\d+, *used:? *\d+",
|
|
818
|
+
r"allows \d+ requests?(\(s\))? every", r"concurrent requests", r"rate limit for free users",
|
|
819
|
+
r"to unlock higher limits")
|
|
820
|
+
# A pool shared with other people: its 429 is the provider busy, not our allowance.
|
|
821
|
+
_SHARED = _rx(r"shared_capacity", r"rate-limited upstream", r"shared pool", r"all providers", r"no upstreams")
|
|
822
|
+
_BUSY = _rx(r"overloaded", r"high demand", r"\bbusy\b", r"capacity", r"temporarily", r"upstream", r"saturated",
|
|
823
|
+
r"try again (later|shortly|in a)", r"retry shortly", r"service[ _]unavailable", r"\bunavailable\b",
|
|
824
|
+
r"resource_?exhausted", r"request limit reached", r"queue is full", r"max connections",
|
|
825
|
+
r"rate[ _-]?limit", r"rate exceeded", r"too many requests", r"throttl", r"provider returned error",
|
|
826
|
+
r"internal server error", r"server[ _]error", r"internal_error", r"bad[ _]gateway",
|
|
827
|
+
r"failed to generate", r"provider_unavailable", r"get_channel_failed", r"model_cooldown",
|
|
828
|
+
r"modelnotreadyexception", r"not ready for inference", r"currently loading",
|
|
829
|
+
r"error code: (429|50\d|529)\b", r"['\"](code|status)['\"]: ?['\"]?(429|50\d|529)\b",
|
|
830
|
+
r"status code '?(429|50\d|529)\b")
|
|
831
|
+
_UNREACHABLE = _rx(r"connection error", r"connection refused", r"actively refused", r"no connection could be made",
|
|
832
|
+
r"connection reset",
|
|
833
|
+
r"forcibly closed", r"connection aborted", r"remote end closed", r"getaddrinfo",
|
|
834
|
+
r"name resolution", r"\bssl\b", r"certificate", r"eof occurred", r"broke off", r"had no choices",
|
|
835
|
+
r"was not json", r"stream failed", r"cloudflare", r"cf-footer", r"<html", r"<!doctype",
|
|
836
|
+
r"protocol violation", r"network (connection lost|is unreachable)", r"no route to host",
|
|
837
|
+
r"error code: 52[0-8]\b")
|
|
838
|
+
_RELAY_DOWN = _rx(r"actively refused", r"connection refused", r"no connection could be made", r"getaddrinfo")
|
|
839
|
+
_RETRY_AFTER = re.compile(r"retry-after:? *(\d+)|retry after (\d+) ?s")
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
def uptime_class(text: str, status: str = "error", private_endpoint: bool = False,
|
|
843
|
+
decided: bool = True) -> tuple[str, str]:
|
|
844
|
+
"""(class, reason) for one call: a class from UPTIME_UP, UPTIME_DOWN or
|
|
845
|
+
UPTIME_OUT. `status` is the call's own (ok, empty, error, timeout, or
|
|
846
|
+
refused: not sent by the bench); `private_endpoint`: the call went to a
|
|
847
|
+
hub on our own network, so a refused connection is that hub, not the
|
|
848
|
+
provider. When no rule matches, the class a decision model gave the
|
|
849
|
+
error's shape, read from metrics/decided.json (decide.classify_errors
|
|
850
|
+
asks, a few new shapes a day; nothing is asked here), its reason
|
|
851
|
+
"decided by <model> at <confidence>"; `decided=False` is the rules alone,
|
|
852
|
+
which the decision benchmark labels with."""
|
|
853
|
+
if status in ("ok", "empty"):
|
|
854
|
+
return status, "answered"
|
|
855
|
+
if status == "refused":
|
|
856
|
+
return "notsent", "not sent by the bench"
|
|
857
|
+
if status == "timeout":
|
|
858
|
+
return "timeout", "no answer in time"
|
|
859
|
+
low = " ".join((text or "").lower().split())
|
|
860
|
+
if not low:
|
|
861
|
+
return "unrecognised", "no error text"
|
|
862
|
+
if _NOTSENT.search(low):
|
|
863
|
+
return "notsent", "the bench's own pacing"
|
|
864
|
+
# Not a provider's own crash sent back with its status (a 500 carrying a traceback).
|
|
865
|
+
if _CLIENT.search(low) and not re.search(r"error code: \d{3}\b", low):
|
|
866
|
+
return "client", "our client failed"
|
|
867
|
+
if _TIMEOUT.search(low):
|
|
868
|
+
return "timeout", "timed out"
|
|
869
|
+
if _SIGNIN.search(low):
|
|
870
|
+
# A router relaying ITS upstream's refused key: not our key, the provider's problem.
|
|
871
|
+
return ("busy", "upstream's own key refused") if _UPSTREAM_WRAP.search(low) else ("setup", "sign-in")
|
|
872
|
+
if _PAYWALLED.search(low) and not re.search(r"high demand|overloaded|temporarily", low):
|
|
873
|
+
return "setup", "paywalled"
|
|
874
|
+
if _NOTFOUND.search(low) and not re.search(r"temporarily|overloaded|high demand|capacity|saturated", low):
|
|
875
|
+
return "setup", "not found"
|
|
876
|
+
if _REQUEST.search(low):
|
|
877
|
+
return "setup", "our request"
|
|
878
|
+
if not _SHARED.search(low) or "free usage quota" in low:
|
|
879
|
+
# A weak word ("daily limit") beside the provider's own trouble is its advice, not our quota.
|
|
880
|
+
weak = (_QUOTA_WEAK.search(low) and _LIMIT_WORD.search(low)
|
|
881
|
+
and not re.search(r"high demand|overloaded|temporarily", low))
|
|
882
|
+
if _QUOTA.search(low) or weak:
|
|
883
|
+
if "credits required" in low:
|
|
884
|
+
return "setup", "no access"
|
|
885
|
+
return "quota", "our quota"
|
|
886
|
+
if _RATE.search(low):
|
|
887
|
+
return "rate", "our rate limit"
|
|
888
|
+
if _ENTITLEMENT.search(low):
|
|
889
|
+
# As with sign-in: a router relaying its upstream's refused key is the provider's problem.
|
|
890
|
+
if _UPSTREAM_WRAP.search(low) and re.search(r"unauthori[sz]ed|permission[ _]denied", low):
|
|
891
|
+
return "busy", "upstream's own key refused"
|
|
892
|
+
return "setup", "no access"
|
|
893
|
+
if _REQUEST_STATUS.search(low) and not _BUSY.search(low):
|
|
894
|
+
return "setup", "our request"
|
|
895
|
+
# An otherwise silent 429 that asks for an hour or more is a long window: ours.
|
|
896
|
+
if ((m := _RETRY_AFTER.search(low)) and int(m.group(1) or m.group(2)) >= 3600
|
|
897
|
+
and re.search(r"\b429\b", low) and not _SHARED.search(low)):
|
|
898
|
+
return "quota", "our quota (a wait of an hour or more)"
|
|
899
|
+
if _BUSY.search(low):
|
|
900
|
+
return "busy", "busy"
|
|
901
|
+
if _UNREACHABLE.search(low):
|
|
902
|
+
if private_endpoint and _RELAY_DOWN.search(low):
|
|
903
|
+
return "relay", "our LAN hub refused the connection"
|
|
904
|
+
return "unreachable", "unreachable"
|
|
905
|
+
if decided:
|
|
906
|
+
try:
|
|
907
|
+
import decide
|
|
908
|
+
if hit := decide.cached_error_class(text):
|
|
909
|
+
return hit
|
|
910
|
+
except Exception: # noqa: BLE001 -- a broken cache is no decision
|
|
911
|
+
pass
|
|
912
|
+
return "unrecognised", "nothing matched"
|
|
719
913
|
|
|
720
914
|
|
|
721
915
|
def outcome(output: str, min_length: int = 40) -> tuple[str, str] | None:
|
|
@@ -1063,7 +1257,8 @@ def main() -> int:
|
|
|
1063
1257
|
return 0
|
|
1064
1258
|
|
|
1065
1259
|
RUNS.mkdir(exist_ok=True)
|
|
1066
|
-
|
|
1260
|
+
started = dt.datetime.now()
|
|
1261
|
+
stamp = started.strftime("%Y-%m-%d")
|
|
1067
1262
|
report = RUNS / f"suite-{stamp}.md"
|
|
1068
1263
|
rows = []
|
|
1069
1264
|
|
|
@@ -1123,12 +1318,37 @@ def main() -> int:
|
|
|
1123
1318
|
f" {seconds:6.1f}s{tries}{flag}")
|
|
1124
1319
|
(spec["dir"] / "output").mkdir(exist_ok=True)
|
|
1125
1320
|
(spec["dir"] / "output" / f"{model}.md").write_text(body, encoding="utf-8")
|
|
1321
|
+
keep_answer(spec["id"], model, started, body)
|
|
1126
1322
|
|
|
1127
1323
|
_write_report(report, stamp, rows)
|
|
1128
1324
|
print(f"\nwritten: {report}")
|
|
1129
1325
|
return 0
|
|
1130
1326
|
|
|
1131
1327
|
|
|
1328
|
+
def keep_answer(task: str, model: str, started: dt.datetime, body: str) -> pathlib.Path | None:
|
|
1329
|
+
"""A copy of the answer that the next run cannot replace:
|
|
1330
|
+
runs/answers/<task>/<model>/<run start>.md. The run's start ties it to its
|
|
1331
|
+
score, a row of that run's section in runs/suite-<its date>.md (the
|
|
1332
|
+
sections are in start order); the file's own time is when it was saved.
|
|
1333
|
+
No reader of the saved answers looks there (they read suite/tasks/*/output).
|
|
1334
|
+
A copy that cannot be written costs the copy, not the run's note."""
|
|
1335
|
+
folder = ANSWERS / task / model
|
|
1336
|
+
name = started.strftime("%Y-%m-%dT%H%M%S")
|
|
1337
|
+
try:
|
|
1338
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
1339
|
+
for n in range(1, 100): # two runs started in the same second ("_" sorts after ".")
|
|
1340
|
+
path = folder / (f"{name}.md" if n == 1 else f"{name}_{n}.md")
|
|
1341
|
+
try:
|
|
1342
|
+
with path.open("x", encoding="utf-8") as fh:
|
|
1343
|
+
fh.write(body)
|
|
1344
|
+
return path
|
|
1345
|
+
except FileExistsError:
|
|
1346
|
+
continue
|
|
1347
|
+
except OSError as exc:
|
|
1348
|
+
print(f" (answer not kept: {exc})")
|
|
1349
|
+
return None
|
|
1350
|
+
|
|
1351
|
+
|
|
1132
1352
|
def _learn(model: str, body: str, where: str) -> None:
|
|
1133
1353
|
"""Pass a failed call's text to metrics/learn.py: a context window or rate
|
|
1134
1354
|
limit the error states, and a cooldown if it was rate-limited. Never lets
|
package/suite/test_pick.py
CHANGED
|
@@ -44,6 +44,21 @@ def main() -> None:
|
|
|
44
44
|
(tasks[3]["dir"] / "output" / "m.md").write_text("<<TIMED OUT after 300s>>", encoding="utf-8")
|
|
45
45
|
assert [t["id"] for t in pick_tasks(tasks, "m")] == ["t2"], pick_tasks(tasks, "m")
|
|
46
46
|
assert [t["id"] for t in pick_tasks(tasks, "other", n=2)] == ["t0", "t1"] # per model
|
|
47
|
+
|
|
48
|
+
# Every answer is also kept, by its run's start, and never replaced: two runs
|
|
49
|
+
# that started in the same second keep both.
|
|
50
|
+
import datetime as dt
|
|
51
|
+
import run
|
|
52
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
53
|
+
kept, run.ANSWERS = run.ANSWERS, pathlib.Path(tmp)
|
|
54
|
+
try:
|
|
55
|
+
at = dt.datetime(2026, 11, 8, 9, 30, 5)
|
|
56
|
+
a, b = run.keep_answer("t", "m", at, "first"), run.keep_answer("t", "m", at, "second")
|
|
57
|
+
assert (a.relative_to(tmp).as_posix(), b.name) == ("t/m/2026-11-08T093005.md", "2026-11-08T093005_2.md"), (a, b)
|
|
58
|
+
assert sorted(a.parent.iterdir()) == [a, b] # by name is in order
|
|
59
|
+
assert (a.read_text(encoding="utf-8"), b.read_text(encoding="utf-8")) == ("first", "second")
|
|
60
|
+
finally:
|
|
61
|
+
run.ANSWERS = kept
|
|
47
62
|
print("ok")
|
|
48
63
|
|
|
49
64
|
|
package/suite/test_score.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
"""python suite/test_score.py — the scorer's negation guard and first-line scoring,
|
|
1
|
+
"""python suite/test_score.py — the scorer's negation guard and first-line scoring, and the
|
|
2
|
+
uptime classifier's cases; no calls."""
|
|
2
3
|
import pathlib
|
|
3
4
|
import sys
|
|
4
5
|
|
|
@@ -27,4 +28,84 @@ seeded = {"must_find": [{"regex": r"^fault:\s*line\s*1\b", "label": "fault: line
|
|
|
27
28
|
assert run.score(seeded, "FAULT: line 1\nthe comparison ...")["found"] == 1
|
|
28
29
|
assert run.score(seeded, "FAULT: line 12\n...")["found"] == 0 # not a prefix match
|
|
29
30
|
assert run.score(seeded, "NO FAULT")["invented"] == ["no fault"]
|
|
31
|
+
|
|
32
|
+
# Every class failure_class returns has a place in the worst-first order (perf/run.py
|
|
33
|
+
# wrote a blank failure for signin while it had none).
|
|
34
|
+
assert run.failure_class("Error: Error code: 401 - {'error': 'invalid key'}") in run.FAILURE_CLASSES
|
|
35
|
+
|
|
36
|
+
# run.uptime_class: shapes from the bench's own data first, then public providers'
|
|
37
|
+
# documented formats and the heavy-hitters panel's cases (2026-11-08).
|
|
38
|
+
UPTIME = {
|
|
39
|
+
"Error: Error code: 429 - {'error': {'message': 'Provider returned error', 'code': 429, 'metadata': {'raw': 'x/y:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits'}}}": "busy",
|
|
40
|
+
"Error: Error code: 429 - {'error': {'message': 'Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day', 'code': 429}}": "quota",
|
|
41
|
+
"s. Credits don't affect this cap. Add your own keys to get a boost.', 'code': 429, 'metadata': {'limit_source': 'openrouter_shared_capacity'}}": "busy",
|
|
42
|
+
"Error: Error code: 429 - {'error': {'message': 'Too many requests. The free tier allows 1 request(s) every 1 min per account on m:free - nothing is used up, retry in 60s.'}}": "rate",
|
|
43
|
+
"Error: Error code: 503 - {'error': {'message': 'The free usage quota for this model is used up and it is currently unavailable.'}}": "quota",
|
|
44
|
+
"Error: Error code: 503 - {'error': {'code': 'get_channel_failed', 'message': 'All providers for model m are busy right now (they hit their rate limit).'}}": "busy",
|
|
45
|
+
"Error: Error code: 500 - ResourceExhausted: Worker local total request limit reached (33/32)": "busy",
|
|
46
|
+
"Error: Error code: 404 - {'error': 'The requested model is currently unavailable.', 'error_type': 'unavailable_model'}": "setup",
|
|
47
|
+
"Error: Error code: 400 - {'error': {'code': 400, 'message': 'Request contains an invalid argument.', 'status': 'INVALID_ARGUMENT'}}": "setup",
|
|
48
|
+
"Error: Error code: 413 - Request too large for model in organization on input tokens per minute (ITPM): Limit 7000, Requested 9000": "setup",
|
|
49
|
+
"Error: Error code: 429 - Your organization reached its limit of concurrent requests. top up at <url>": "rate",
|
|
50
|
+
"Error: Error code: 402 - {'error': {'title': 'Paid Model - Credits Required', 'message': 'Add credits to continue'}}": "setup",
|
|
51
|
+
"Error: Error code: 429 - {'error': {'message': \"You've reached today's free-model token quota. Your plan's paid allowance is separate, or wait for the daily reset.\"}} Retry-After: 14652": "quota",
|
|
52
|
+
"Error: client failed: Connection error. ([WinError 10061] No connection could be made because the target machine actively refused it)": "relay",
|
|
53
|
+
"Error: client failed: the reply broke off part way ([WinError 10054] An existing connection was forcibly closed by the remote host)": "unreachable",
|
|
54
|
+
"Error: client failed: the reply had no choices: the stream sent none": "unreachable",
|
|
55
|
+
"<<TIMED OUT after 90s>>": "timeout",
|
|
56
|
+
"no answer in 420s": "timeout",
|
|
57
|
+
"Error: client failed: rate limit (not sent; paced by the bench): x allows 50 requests per day": "notsent",
|
|
58
|
+
"sqlite3.OperationalError: database is locked": "client",
|
|
59
|
+
"Error: Error code: 429 - {'error': {'code': 'model_cooldown', 'last_upstream_error': 'usage_limit_reached: The usage limit has been reached', 'message': 'All credentials for model m are cooling down via provider codex'}} Retry-After: 2528616": "quota",
|
|
60
|
+
"Error: Error code: 503 - {'error': {'message': 'The model is temporarily unavailable due to high demand. Please retry in a moment.', 'type': 'server_error'}}": "busy",
|
|
61
|
+
"Error: Error code: 418 - {'error': 'teapot'}": "unrecognised",
|
|
62
|
+
"Error: Error code: 529 - {'type': 'error', 'error': {'type': 'overloaded_error', 'message': 'Overloaded'}}": "busy",
|
|
63
|
+
"Error: Error code: 429 - {'error': {'code': 429, 'message': 'You exceeded your current quota', 'status': 'RESOURCE_EXHAUSTED'}}": "quota",
|
|
64
|
+
"Error: Error code: 429 - {'error': {'code': 429, 'message': 'Resource has been exhausted (e.g. check quota).', 'status': 'RESOURCE_EXHAUSTED'}}": "busy",
|
|
65
|
+
"Error: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of request tokens has exceeded your per-minute rate limit'}}": "rate",
|
|
66
|
+
"Error: Error code: 429 - {'error': {'message': 'Rate limit reached for gpt-x in organization org-x on requests per min (RPM): Limit 3, Used 3, Requested 1.', 'type': 'requests', 'code': 'rate_limit_exceeded'}}": "rate",
|
|
67
|
+
"Error: Error code: 429 - {'message': \"You are using a Trial key, which is limited to 1000 API calls / month.\"}": "quota",
|
|
68
|
+
"Error: Error code: 429 - {'error': {'message': 'Rate limit exceeded: free-models-per-min.', 'code': 429}}": "rate",
|
|
69
|
+
"Error: Error code: 400 - {'error': {'message': 'Provider returned error: prompt is too long: 210000 tokens > 200000 maximum', 'code': 400}}": "setup",
|
|
70
|
+
"Error: Error code: 400 - {'message': 'Prompt exceeds the capacity of this model: too many tokens'}": "setup",
|
|
71
|
+
"Error: Error code: 503 - {'error': 'Service temporarily unavailable during daily maintenance window'}": "busy",
|
|
72
|
+
"Error: Error code: 503 - Your account is temporarily overloaded.": "busy",
|
|
73
|
+
"Error: Error code: 500 - Your account failed to generate a reply.": "busy",
|
|
74
|
+
"403: Your account does not have access to this model.": "setup",
|
|
75
|
+
"An error occurred (ServiceQuotaExceededException) when calling the InvokeModel operation: Rate exceeded.": "quota",
|
|
76
|
+
"An error occurred (AccessDeniedException) when calling the InvokeModel operation: You don't have access to the model.": "setup",
|
|
77
|
+
"An error occurred (ValidationException) when calling the InvokeModel operation: The provided model identifier is invalid.": "setup",
|
|
78
|
+
"An error occurred (ModelNotReadyException) when calling the Converse operation: Model is not ready for inference.": "busy",
|
|
79
|
+
"An error occurred (ThrottlingException) when calling the InvokeModel operation: Too many requests, please wait.": "busy",
|
|
80
|
+
"{\"error\": \"model 'llama3:70b' not found, try pulling it first\"}": "setup",
|
|
81
|
+
"{\"success\": false, \"errors\": [{\"code\": 10000, \"message\": \"Authentication error\"}]}": "setup",
|
|
82
|
+
"... 'code': 'content_filter', 'status': 400}}": "setup",
|
|
83
|
+
"Error: Error code: 401 - {'error': {'message': 'Provider returned error', 'metadata': {'raw': 'invalid api key'}}}": "busy",
|
|
84
|
+
"Error: Error code: 429 - {'error': {'message': 'Too many requests'}} Retry-After: 86400": "quota",
|
|
85
|
+
"Error: Error code: 429 - {'error': {'message': 'Too many requests'}} Retry-After: 30": "busy",
|
|
86
|
+
"Error: Error code: 503 - model unavailable for free users due to high demand": "busy",
|
|
87
|
+
"provider notice in a success reply (out of credits): You are out of credits.": "quota",
|
|
88
|
+
# the heavy-hitters review of the build (p20261108-104415-db82)
|
|
89
|
+
"Error: Error code: 403 - {'error': {'message': 'Provider returned error', 'metadata': {'raw': 'Unauthorized'}}}": "busy",
|
|
90
|
+
"Error: Error code: 403 - {'error': {'message': 'Unauthorized'}}": "setup",
|
|
91
|
+
"Error: Error code: 429 - {'error': {'message': 'Too many tokens per minute'}}": "rate",
|
|
92
|
+
"Error: Error code: 502 - Bad Gateway at https://example.com/api/day/status": "busy",
|
|
93
|
+
"Error: client failed: the reply broke off part way (connection reset while fetching today's list)": "unreachable",
|
|
94
|
+
"Error: Error code: 429 - Model is experiencing high demand. Please retry shortly, or add your own key for a higher daily limit.": "busy",
|
|
95
|
+
"[WinError 10065] No connection could be made to the host": "relay",
|
|
96
|
+
"ne 12, in handler ... Traceback (most recent call last): ... 'status': 500, 'error code: 500'": "busy",
|
|
97
|
+
}
|
|
98
|
+
# The rules alone (decided=False): what this install's decision models said of a shape is not a rule.
|
|
99
|
+
bad = [(t[:80], run.uptime_class(t, private_endpoint=True, decided=False)[0], want) for t, want in UPTIME.items()
|
|
100
|
+
if run.uptime_class(t, private_endpoint=True, decided=False)[0] != want]
|
|
101
|
+
assert not bad, bad
|
|
102
|
+
# The call's own status decides first; an error with no text is never an answer.
|
|
103
|
+
assert run.uptime_class("", "ok")[0] == "ok" and run.uptime_class("", "empty")[0] == "empty"
|
|
104
|
+
assert run.uptime_class("", "refused")[0] == "notsent" and run.uptime_class("", "timeout")[0] == "timeout"
|
|
105
|
+
assert run.uptime_class("", "error")[0] == "unrecognised"
|
|
106
|
+
# A refused connection is our hub only when the call went to one.
|
|
107
|
+
assert run.uptime_class("Error: client failed: Connection error. (connection refused)")[0] == "unreachable"
|
|
108
|
+
assert set(run.UPTIME_UP + run.UPTIME_DOWN + run.UPTIME_OUT) >= {run.uptime_class(t, decided=False)[0] for t in UPTIME}
|
|
109
|
+
# A normal answer never reaches the notice fallback (metrics/test_decide.py tests the fallback itself).
|
|
110
|
+
assert not run.NOTICE_LIKE.search("ready") and run.NOTICE_LIKE.search("your credit balance is low")
|
|
30
111
|
print("ok")
|
package/ui/bench.css
CHANGED
|
@@ -185,6 +185,15 @@ table.a-table col.c-acts { width: 7.5rem; }
|
|
|
185
185
|
padding: .05rem .4rem; border-radius: 999px; font: 700 .75rem/1.25 var(--font-mono);
|
|
186
186
|
background: var(--err-ground); color: var(--err); }
|
|
187
187
|
.tab { display: none; } .tab.active { display: block; }
|
|
188
|
+
/* A page not shown keeps its layout (content-visibility), so showing it again
|
|
189
|
+
is cheap: with display none, Discover's 1,200 rows were laid out again on
|
|
190
|
+
every switch to it (0.2–0.3 s), now 15 ms. Its contents stay out of the
|
|
191
|
+
accessibility tree and focus order, as with display none. */
|
|
192
|
+
@supports (content-visibility: hidden) {
|
|
193
|
+
.tab { display: block; content-visibility: hidden; } .tab.active { content-visibility: visible; }
|
|
194
|
+
}
|
|
195
|
+
/* A page not fetched yet (bench.js load), while the bar sweeps the top. */
|
|
196
|
+
.tab[data-lazy]:empty::before { content: "Loading…"; display: block; padding: 1.4rem 1.6rem; color: var(--muted); }
|
|
188
197
|
/* One surface for the page's content: its panels are sections of it,
|
|
189
198
|
divided by a hairline, rather than separate boxes with gaps between. A
|
|
190
199
|
theme with a floor makes it frosted (--sheet, --glass). */
|
|
@@ -215,6 +224,14 @@ table.facts th { text-align: left; padding-right: 1.5rem; color: var(--muted); f
|
|
|
215
224
|
details.panel > summary { cursor: pointer; }
|
|
216
225
|
details.panel[open] > summary { margin-bottom: .75rem; }
|
|
217
226
|
.log-line { font-size: .8125rem; padding: .1rem 0; overflow-wrap: anywhere; }
|
|
227
|
+
/* Clients (serve.clients_page): each snippet under its label and Copy button */
|
|
228
|
+
.snip { margin-top: .9rem; }
|
|
229
|
+
.snip-head { display: flex; align-items: center; justify-content: space-between; gap: .5rem;
|
|
230
|
+
font-size: .875rem; color: var(--muted); overflow-wrap: anywhere; }
|
|
231
|
+
.snip-head .btn { flex: none; }
|
|
232
|
+
.snip pre.log { max-height: 30rem; margin-top: .3rem; overflow-wrap: anywhere; }
|
|
233
|
+
#tab-clients :is(.where, .watch) { font-size: .9375rem; max-width: 75ch; overflow-wrap: anywhere; }
|
|
234
|
+
#tab-clients .watch { background: var(--step-1); border-radius: var(--radius); padding: .5rem .75rem; margin: 1rem 0 0; }
|
|
218
235
|
|
|
219
236
|
.panel { background: var(--panel); border-radius: var(--radius); padding: 1.1rem 1.25rem 1.25rem; }
|
|
220
237
|
.panel > h2 { margin: 0; font-size: 1.375rem; font-weight: 700; }
|
|
@@ -248,6 +265,20 @@ details.key dl { margin: .5rem 0 0; display: grid; grid-template-columns: max-co
|
|
|
248
265
|
details.key dt { font-weight: 700; }
|
|
249
266
|
details.key dd { margin: 0; color: var(--ink); }
|
|
250
267
|
|
|
268
|
+
/* Share (serve.SHARE, bench.js): top right of a report, its menu under it. */
|
|
269
|
+
.panel:has(> .share) { position: relative; }
|
|
270
|
+
.panel:has(> .share) > h2 { padding-right: 4.5rem; }
|
|
271
|
+
details.share { position: absolute; top: 1.2rem; right: 1.4rem; font-size: .9375rem; }
|
|
272
|
+
details.share > summary { cursor: pointer; color: var(--link); list-style: none; text-decoration: underline;
|
|
273
|
+
text-underline-offset: .18em; }
|
|
274
|
+
details.share > summary::-webkit-details-marker { display: none; }
|
|
275
|
+
.share-menu { position: absolute; right: 0; top: 1.7rem; z-index: 5; display: flex; flex-direction: column;
|
|
276
|
+
align-items: flex-start; gap: .5rem; min-width: 14rem; padding: .8rem 1rem; border: 1px solid var(--ink);
|
|
277
|
+
border-radius: var(--radius); background: var(--paper-2); box-shadow: 4px 4px 0 var(--ink); }
|
|
278
|
+
.share-to { color: var(--muted); }
|
|
279
|
+
.no-clip [data-share=copy], .no-clip [data-share=md], .no-share [data-share=native],
|
|
280
|
+
.panel:not(:has(table)) [data-share=md], .panel:not(:has(table)) [data-share=csv] { display: none; }
|
|
281
|
+
|
|
251
282
|
/* ---- tables --------------------------------------------------------- */
|
|
252
283
|
.scroll { overflow-x: auto; }
|
|
253
284
|
table { border-collapse: collapse; width: 100%; font-size: .9375rem; }
|
|
@@ -297,6 +328,7 @@ button.chip:hover { outline: 1px solid currentColor; }
|
|
|
297
328
|
.meter { width: 4.5rem; height: .5rem; background: var(--band); border-radius: 999px; overflow: hidden; }
|
|
298
329
|
.meter span { display: block; height: 100%; background: var(--brand); }
|
|
299
330
|
.meter span.low { background: var(--err); }
|
|
331
|
+
td.why { white-space: nowrap; } /* Uptime's down / left out, with their reasons */
|
|
300
332
|
|
|
301
333
|
/* ---- dialogs -------------------------------------------------------- */
|
|
302
334
|
dialog { border: 1px solid var(--ink); border-radius: var(--radius); padding: 1.25rem 1.4rem;
|
|
@@ -364,6 +396,8 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
|
|
|
364
396
|
padding: .5rem .75rem; margin: 0 -.75rem; border-radius: var(--radius); }
|
|
365
397
|
.endpoint:hover { background: var(--step-1); }
|
|
366
398
|
.endpoint .base { color: var(--muted); font-size: .875rem; overflow-wrap: anywhere; }
|
|
399
|
+
.found { margin: 0 0 1rem; }
|
|
400
|
+
.found h3 { margin: 0 0 .25rem; font-size: 1rem; font-weight: 700; }
|
|
367
401
|
:is(dialog, .form-panel) h3.form-head { margin: 1.25rem 0 0; font-size: 1rem; font-weight: 700; }
|
|
368
402
|
:is(dialog, .form-panel) input[readonly] { background: var(--step-1); }
|
|
369
403
|
/* Add an endpoint's choices: API key, then each provider; the chosen one pressed */
|
|
@@ -554,6 +588,16 @@ dialog .pick { display: flex; gap: 1.5rem; }
|
|
|
554
588
|
.a-table .said { white-space: normal; } /* the height to read it whole */
|
|
555
589
|
.a-table .said-line > a { max-width: none; }
|
|
556
590
|
}
|
|
591
|
+
/* A reload under way (bench.js marks the page as it goes), or a page being
|
|
592
|
+
fetched (.loading): a bar sweeping the top, over an open popup too, so the
|
|
593
|
+
seconds the server takes to render never look like a frozen page. */
|
|
594
|
+
body:is(.leaving, .loading) { cursor: progress; }
|
|
595
|
+
body:is(.leaving, .loading)::before, body:is(.leaving, .loading) dialog[open]::before {
|
|
596
|
+
content: ""; position: fixed; inset: 0 0 auto; height: 3px; z-index: 1000; pointer-events: none;
|
|
597
|
+
background: linear-gradient(90deg, transparent, var(--brand), transparent) 0 0 / 40% 100% no-repeat;
|
|
598
|
+
animation: leaving 1.1s ease-in-out infinite; }
|
|
599
|
+
@keyframes leaving { from { background-position: -70% 0; } to { background-position: 170% 0; } }
|
|
557
600
|
@media (prefers-reduced-motion: reduce) {
|
|
558
601
|
* { transition: none !important; }
|
|
602
|
+
body:is(.leaving, .loading)::before, body:is(.leaving, .loading) dialog[open]::before { animation: none; background: var(--brand); }
|
|
559
603
|
}
|