@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +152 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +415 -272
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +408 -314
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +186 -163
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -74
- package/cli.py +674 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +939 -755
- package/launcher.py +120 -0
- package/mcp_server.py +266 -266
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +304 -295
- package/metrics/client.py +804 -394
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +171 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/formats.py +1021 -0
- package/metrics/guard.py +647 -592
- package/metrics/learn.py +234 -221
- package/metrics/ledger.py +148 -0
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +94 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/pacing.py +248 -0
- package/metrics/paths.py +60 -60
- package/metrics/quota.py +383 -0
- package/metrics/route.py +581 -579
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +69 -65
- package/perf/README.md +89 -89
- package/perf/run.py +284 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +113 -106
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +410 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +277 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_formats.py +549 -0
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -81
- package/scripts/test_mcp.py +178 -178
- package/scripts/test_merged.py +215 -214
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_quota.py +334 -0
- package/scripts/test_serve_edit.py +453 -438
- package/scripts/test_server_json.py +79 -0
- package/scripts/test_verdict.py +102 -98
- package/scripts/test_vision.py +231 -228
- package/serve.py +3972 -3764
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1042
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -425
- package/ui/bench.js +1757 -1576
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/scripts/notice.py
CHANGED
|
@@ -1,78 +1,78 @@
|
|
|
1
|
-
"""Write NOTICE from what the repository actually ships.
|
|
2
|
-
|
|
3
|
-
python scripts/notice.py # rewrite NOTICE
|
|
4
|
-
python scripts/notice.py --check # fail if NOTICE is out of date (scripts/check.py runs this)
|
|
5
|
-
|
|
6
|
-
RELEASE-PLAN.md Phase 4: the credits are generated, not kept by hand. Each
|
|
7
|
-
public real-bug task records the project it came from, its licence and the
|
|
8
|
-
pull request (`upstream`, `license`, `source_url` in task.json), so adding a
|
|
9
|
-
task adds its credit and dropping one removes it. The fonts and the dataset
|
|
10
|
-
are listed from what is in the tree.
|
|
11
|
-
|
|
12
|
-
Still to do before release (Phase 7): the full licence texts the projects'
|
|
13
|
-
licences require (Apache-2.0, BSD-3-Clause with its copyright lines) in a
|
|
14
|
-
LICENSES/ folder.
|
|
15
|
-
"""
|
|
16
|
-
|
|
17
|
-
from __future__ import annotations
|
|
18
|
-
|
|
19
|
-
import collections
|
|
20
|
-
import json
|
|
21
|
-
import pathlib
|
|
22
|
-
import sys
|
|
23
|
-
|
|
24
|
-
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
25
|
-
NOTICE = ROOT / "NOTICE"
|
|
26
|
-
DATASET = "nebius/SWE-rebench-V2" # suite/rebench.py DATASET
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def render() -> str:
|
|
30
|
-
projects: dict[tuple[str, str], list[str]] = collections.defaultdict(list)
|
|
31
|
-
for path in sorted((ROOT / "suite" / "tasks").glob("rebench-*/task.json")):
|
|
32
|
-
spec = json.loads(path.read_text(encoding="utf-8"))
|
|
33
|
-
key = (spec.get("upstream") or "unknown", spec.get("license") or "unknown licence")
|
|
34
|
-
if spec.get("source_url") and spec["source_url"] not in projects[key]:
|
|
35
|
-
projects[key].append(spec["source_url"])
|
|
36
|
-
lines = [
|
|
37
|
-
"Benchwarmers, by Wendell Research",
|
|
38
|
-
"Copyright (c) 2026 Wendell Research. Licensed under the MIT License (LICENSE).",
|
|
39
|
-
"",
|
|
40
|
-
"This repository also contains material from others, under their own licences:",
|
|
41
|
-
"",
|
|
42
|
-
]
|
|
43
|
-
if projects:
|
|
44
|
-
lines += [f"Real-bug tasks (suite/tasks/rebench-*) are drawn from the {DATASET} dataset",
|
|
45
|
-
"on Hugging Face (CC-BY-4.0). Each task holds code from the project it names,",
|
|
46
|
-
"under that project's licence:", ""]
|
|
47
|
-
for (upstream, licence), urls in sorted(projects.items(), key=lambda kv: kv[0][0].lower()):
|
|
48
|
-
lines.append(f" {upstream} ({licence})")
|
|
49
|
-
lines += [f" {u}" for u in urls]
|
|
50
|
-
lines.append("")
|
|
51
|
-
if sorted((ROOT / "ui" / "fonts").glob("chivo*.woff2")):
|
|
52
|
-
lines += ["Fonts (ui/fonts): Chivo and Chivo Mono, Copyright The Chivo Project Authors",
|
|
53
|
-
"(https://github.com/Omnibus-Type/Chivo), under the SIL Open Font License 1.1",
|
|
54
|
-
"(ui/fonts/OFL-*.txt).", ""]
|
|
55
|
-
if (ROOT / "ui" / "mascot.png").exists():
|
|
56
|
-
lines += ["The Wendell Research mascot (ui/mascot.png, drawn at centre court in the",
|
|
57
|
-
"Hardwood theme) is a registered trademark of Wendell Research. It is not",
|
|
58
|
-
"covered by the MIT License: a fork may keep it in place unchanged, but should",
|
|
59
|
-
"not use it to name or brand another product.", ""]
|
|
60
|
-
return "\n".join(lines)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def main() -> int:
|
|
64
|
-
text = render()
|
|
65
|
-
if "--check" in sys.argv:
|
|
66
|
-
current = NOTICE.read_text(encoding="utf-8") if NOTICE.exists() else ""
|
|
67
|
-
if current.replace("\r\n", "\n") != text:
|
|
68
|
-
print("NOTICE is out of date: run python scripts/notice.py")
|
|
69
|
-
return 1
|
|
70
|
-
print("NOTICE is current")
|
|
71
|
-
return 0
|
|
72
|
-
NOTICE.write_text(text, encoding="utf-8", newline="\n")
|
|
73
|
-
print(f"wrote {NOTICE.name}: {text.count(chr(10))} lines")
|
|
74
|
-
return 0
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if __name__ == "__main__":
|
|
78
|
-
sys.exit(main())
|
|
1
|
+
"""Write NOTICE from what the repository actually ships.
|
|
2
|
+
|
|
3
|
+
python scripts/notice.py # rewrite NOTICE
|
|
4
|
+
python scripts/notice.py --check # fail if NOTICE is out of date (scripts/check.py runs this)
|
|
5
|
+
|
|
6
|
+
RELEASE-PLAN.md Phase 4: the credits are generated, not kept by hand. Each
|
|
7
|
+
public real-bug task records the project it came from, its licence and the
|
|
8
|
+
pull request (`upstream`, `license`, `source_url` in task.json), so adding a
|
|
9
|
+
task adds its credit and dropping one removes it. The fonts and the dataset
|
|
10
|
+
are listed from what is in the tree.
|
|
11
|
+
|
|
12
|
+
Still to do before release (Phase 7): the full licence texts the projects'
|
|
13
|
+
licences require (Apache-2.0, BSD-3-Clause with its copyright lines) in a
|
|
14
|
+
LICENSES/ folder.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import collections
|
|
20
|
+
import json
|
|
21
|
+
import pathlib
|
|
22
|
+
import sys
|
|
23
|
+
|
|
24
|
+
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
25
|
+
NOTICE = ROOT / "NOTICE"
|
|
26
|
+
DATASET = "nebius/SWE-rebench-V2" # suite/rebench.py DATASET
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def render() -> str:
|
|
30
|
+
projects: dict[tuple[str, str], list[str]] = collections.defaultdict(list)
|
|
31
|
+
for path in sorted((ROOT / "suite" / "tasks").glob("rebench-*/task.json")):
|
|
32
|
+
spec = json.loads(path.read_text(encoding="utf-8"))
|
|
33
|
+
key = (spec.get("upstream") or "unknown", spec.get("license") or "unknown licence")
|
|
34
|
+
if spec.get("source_url") and spec["source_url"] not in projects[key]:
|
|
35
|
+
projects[key].append(spec["source_url"])
|
|
36
|
+
lines = [
|
|
37
|
+
"Benchwarmers, by Wendell Research",
|
|
38
|
+
"Copyright (c) 2026 Wendell Research. Licensed under the MIT License (LICENSE).",
|
|
39
|
+
"",
|
|
40
|
+
"This repository also contains material from others, under their own licences:",
|
|
41
|
+
"",
|
|
42
|
+
]
|
|
43
|
+
if projects:
|
|
44
|
+
lines += [f"Real-bug tasks (suite/tasks/rebench-*) are drawn from the {DATASET} dataset",
|
|
45
|
+
"on Hugging Face (CC-BY-4.0). Each task holds code from the project it names,",
|
|
46
|
+
"under that project's licence:", ""]
|
|
47
|
+
for (upstream, licence), urls in sorted(projects.items(), key=lambda kv: kv[0][0].lower()):
|
|
48
|
+
lines.append(f" {upstream} ({licence})")
|
|
49
|
+
lines += [f" {u}" for u in urls]
|
|
50
|
+
lines.append("")
|
|
51
|
+
if sorted((ROOT / "ui" / "fonts").glob("chivo*.woff2")):
|
|
52
|
+
lines += ["Fonts (ui/fonts): Chivo and Chivo Mono, Copyright The Chivo Project Authors",
|
|
53
|
+
"(https://github.com/Omnibus-Type/Chivo), under the SIL Open Font License 1.1",
|
|
54
|
+
"(ui/fonts/OFL-*.txt).", ""]
|
|
55
|
+
if (ROOT / "ui" / "mascot.png").exists():
|
|
56
|
+
lines += ["The Wendell Research mascot (ui/mascot.png, drawn at centre court in the",
|
|
57
|
+
"Hardwood theme) is a registered trademark of Wendell Research. It is not",
|
|
58
|
+
"covered by the MIT License: a fork may keep it in place unchanged, but should",
|
|
59
|
+
"not use it to name or brand another product.", ""]
|
|
60
|
+
return "\n".join(lines)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main() -> int:
|
|
64
|
+
text = render()
|
|
65
|
+
if "--check" in sys.argv:
|
|
66
|
+
current = NOTICE.read_text(encoding="utf-8") if NOTICE.exists() else ""
|
|
67
|
+
if current.replace("\r\n", "\n") != text:
|
|
68
|
+
print("NOTICE is out of date: run python scripts/notice.py")
|
|
69
|
+
return 1
|
|
70
|
+
print("NOTICE is current")
|
|
71
|
+
return 0
|
|
72
|
+
NOTICE.write_text(text, encoding="utf-8", newline="\n")
|
|
73
|
+
print(f"wrote {NOTICE.name}: {text.count(chr(10))} lines")
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
sys.exit(main())
|