@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +152 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +415 -272
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +408 -314
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +186 -163
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -74
- package/cli.py +674 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +939 -755
- package/launcher.py +120 -0
- package/mcp_server.py +266 -266
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +304 -295
- package/metrics/client.py +804 -394
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +171 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/formats.py +1021 -0
- package/metrics/guard.py +647 -592
- package/metrics/learn.py +234 -221
- package/metrics/ledger.py +148 -0
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +94 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/pacing.py +248 -0
- package/metrics/paths.py +60 -60
- package/metrics/quota.py +383 -0
- package/metrics/route.py +581 -579
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +69 -65
- package/perf/README.md +89 -89
- package/perf/run.py +284 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +113 -106
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +410 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +277 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_formats.py +549 -0
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -81
- package/scripts/test_mcp.py +178 -178
- package/scripts/test_merged.py +215 -214
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_quota.py +334 -0
- package/scripts/test_serve_edit.py +453 -438
- package/scripts/test_server_json.py +79 -0
- package/scripts/test_verdict.py +102 -98
- package/scripts/test_vision.py +231 -228
- package/serve.py +3972 -3764
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1042
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -425
- package/ui/bench.js +1757 -1576
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/NOTICE
CHANGED
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
Benchwarmers, by Wendell Research
|
|
2
|
-
Copyright (c) 2026 Wendell Research. Licensed under the MIT License (LICENSE).
|
|
3
|
-
|
|
4
|
-
This repository also contains material from others, under their own licences:
|
|
5
|
-
|
|
6
|
-
Real-bug tasks (suite/tasks/rebench-*) are drawn from the nebius/SWE-rebench-V2 dataset
|
|
7
|
-
on Hugging Face (CC-BY-4.0). Each task holds code from the project it names,
|
|
8
|
-
under that project's licence:
|
|
9
|
-
|
|
10
|
-
matchms/matchms (Apache-2.0)
|
|
11
|
-
https://github.com/matchms/matchms/pull/809
|
|
12
|
-
more-itertools/more-itertools (MIT)
|
|
13
|
-
https://github.com/more-itertools/more-itertools/pull/1028
|
|
14
|
-
narwhals-dev/narwhals (MIT)
|
|
15
|
-
https://github.com/narwhals-dev/narwhals/pull/2864
|
|
16
|
-
QuantEcon/QuantEcon.py (MIT)
|
|
17
|
-
https://github.com/QuantEcon/QuantEcon.py/pull/769
|
|
18
|
-
scrapy/scrapy (BSD-3-Clause)
|
|
19
|
-
https://github.com/scrapy/scrapy/pull/6867
|
|
20
|
-
|
|
21
|
-
Fonts (ui/fonts): Chivo and Chivo Mono, Copyright The Chivo Project Authors
|
|
22
|
-
(https://github.com/Omnibus-Type/Chivo), under the SIL Open Font License 1.1
|
|
23
|
-
(ui/fonts/OFL-*.txt).
|
|
24
|
-
|
|
25
|
-
The Wendell Research mascot (ui/mascot.png, drawn at centre court in the
|
|
26
|
-
Hardwood theme) is a registered trademark of Wendell Research. It is not
|
|
27
|
-
covered by the MIT License: a fork may keep it in place unchanged, but should
|
|
28
|
-
not use it to name or brand another product.
|
|
1
|
+
Benchwarmers, by Wendell Research
|
|
2
|
+
Copyright (c) 2026 Wendell Research. Licensed under the MIT License (LICENSE).
|
|
3
|
+
|
|
4
|
+
This repository also contains material from others, under their own licences:
|
|
5
|
+
|
|
6
|
+
Real-bug tasks (suite/tasks/rebench-*) are drawn from the nebius/SWE-rebench-V2 dataset
|
|
7
|
+
on Hugging Face (CC-BY-4.0). Each task holds code from the project it names,
|
|
8
|
+
under that project's licence:
|
|
9
|
+
|
|
10
|
+
matchms/matchms (Apache-2.0)
|
|
11
|
+
https://github.com/matchms/matchms/pull/809
|
|
12
|
+
more-itertools/more-itertools (MIT)
|
|
13
|
+
https://github.com/more-itertools/more-itertools/pull/1028
|
|
14
|
+
narwhals-dev/narwhals (MIT)
|
|
15
|
+
https://github.com/narwhals-dev/narwhals/pull/2864
|
|
16
|
+
QuantEcon/QuantEcon.py (MIT)
|
|
17
|
+
https://github.com/QuantEcon/QuantEcon.py/pull/769
|
|
18
|
+
scrapy/scrapy (BSD-3-Clause)
|
|
19
|
+
https://github.com/scrapy/scrapy/pull/6867
|
|
20
|
+
|
|
21
|
+
Fonts (ui/fonts): Chivo and Chivo Mono, Copyright The Chivo Project Authors
|
|
22
|
+
(https://github.com/Omnibus-Type/Chivo), under the SIL Open Font License 1.1
|
|
23
|
+
(ui/fonts/OFL-*.txt).
|
|
24
|
+
|
|
25
|
+
The Wendell Research mascot (ui/mascot.png, drawn at centre court in the
|
|
26
|
+
Hardwood theme) is a registered trademark of Wendell Research. It is not
|
|
27
|
+
covered by the MIT License: a fork may keep it in place unchanged, but should
|
|
28
|
+
not use it to name or brand another product.
|
package/PRIVACY.md
CHANGED
|
@@ -1,46 +1,46 @@
|
|
|
1
|
-
# Privacy policy
|
|
2
|
-
|
|
3
|
-
*Effective 22 October 2026.*
|
|
4
|
-
|
|
5
|
-
Benchwarmers runs on your own machine or server. It has no account, no
|
|
6
|
-
telemetry and no server of its own, and nothing you do with it reaches
|
|
7
|
-
Wendell Research.
|
|
8
|
-
|
|
9
|
-
## What leaves your machine
|
|
10
|
-
|
|
11
|
-
- **Your prompts**, only to the endpoints you add
|
|
12
|
-
(a provider, an aggregator, a hub such as Open WebUI, or a local runtime).
|
|
13
|
-
That includes what the gateway relays for your apps, and the calls the
|
|
14
|
-
probes, the suite and onboarding make. Each endpoint's own privacy policy
|
|
15
|
-
applies to what it receives; some free tiers keep prompts or train on them.
|
|
16
|
-
- **Catalog reads**, carrying nothing of yours: models.dev and FreeLLMAPI
|
|
17
|
-
(at most once a day) and OpenRouter's public model list.
|
|
18
|
-
- **Sign-ins**, only when you choose one: the provider's own sign-in page, in
|
|
19
|
-
your browser, and the provider's token endpoint.
|
|
20
|
-
- **Charts**: the dashboard's page loads Observable Plot from jsDelivr, at a
|
|
21
|
-
pinned version, in your browser.
|
|
22
|
-
- **Real-bug task seeding**, only when you run `suite/rebench.py`: dataset rows
|
|
23
|
-
from Hugging Face and source files from GitHub.
|
|
24
|
-
|
|
25
|
-
Keys and sign-in tokens stay in your environment or the `llm` CLI's key
|
|
26
|
-
store. [SECURITY.md](SECURITY.md) has the full detail.
|
|
27
|
-
|
|
28
|
-
## What stays on your machine
|
|
29
|
-
|
|
30
|
-
Everything else: your endpoints, collections, verdicts, call logs, probe
|
|
31
|
-
history and task answers live in your data folder, which is yours to back up,
|
|
32
|
-
move or delete.
|
|
33
|
-
|
|
34
|
-
## Third parties
|
|
35
|
-
|
|
36
|
-
The endpoints you add, the catalogs above, jsDelivr, GitHub, Hugging Face and
|
|
37
|
-
any app you connect to the gateway each have their own privacy policies.
|
|
38
|
-
|
|
39
|
-
## Changes
|
|
40
|
-
|
|
41
|
-
Changes to this policy are made in this file, and its history is in the
|
|
42
|
-
repository's commits.
|
|
43
|
-
|
|
44
|
-
## Contact
|
|
45
|
-
|
|
46
|
-
Questions: [open an issue](https://github.com/WendellResearch-AI/benchwarmers/issues).
|
|
1
|
+
# Privacy policy
|
|
2
|
+
|
|
3
|
+
*Effective 22 October 2026.*
|
|
4
|
+
|
|
5
|
+
Benchwarmers runs on your own machine or server. It has no account, no
|
|
6
|
+
telemetry and no server of its own, and nothing you do with it reaches
|
|
7
|
+
Wendell Research.
|
|
8
|
+
|
|
9
|
+
## What leaves your machine
|
|
10
|
+
|
|
11
|
+
- **Your prompts**, only to the endpoints you add
|
|
12
|
+
(a provider, an aggregator, a hub such as Open WebUI, or a local runtime).
|
|
13
|
+
That includes what the gateway relays for your apps, and the calls the
|
|
14
|
+
probes, the suite and onboarding make. Each endpoint's own privacy policy
|
|
15
|
+
applies to what it receives; some free tiers keep prompts or train on them.
|
|
16
|
+
- **Catalog reads**, carrying nothing of yours: models.dev and FreeLLMAPI
|
|
17
|
+
(at most once a day) and OpenRouter's public model list.
|
|
18
|
+
- **Sign-ins**, only when you choose one: the provider's own sign-in page, in
|
|
19
|
+
your browser, and the provider's token endpoint.
|
|
20
|
+
- **Charts**: the dashboard's page loads Observable Plot from jsDelivr, at a
|
|
21
|
+
pinned version, in your browser.
|
|
22
|
+
- **Real-bug task seeding**, only when you run `suite/rebench.py`: dataset rows
|
|
23
|
+
from Hugging Face and source files from GitHub.
|
|
24
|
+
|
|
25
|
+
Keys and sign-in tokens stay in your environment or the `llm` CLI's key
|
|
26
|
+
store. [SECURITY.md](SECURITY.md) has the full detail.
|
|
27
|
+
|
|
28
|
+
## What stays on your machine
|
|
29
|
+
|
|
30
|
+
Everything else: your endpoints, collections, verdicts, call logs, probe
|
|
31
|
+
history and task answers live in your data folder, which is yours to back up,
|
|
32
|
+
move or delete.
|
|
33
|
+
|
|
34
|
+
## Third parties
|
|
35
|
+
|
|
36
|
+
The endpoints you add, the catalogs above, jsDelivr, GitHub, Hugging Face and
|
|
37
|
+
any app you connect to the gateway each have their own privacy policies.
|
|
38
|
+
|
|
39
|
+
## Changes
|
|
40
|
+
|
|
41
|
+
Changes to this policy are made in this file, and its history is in the
|
|
42
|
+
repository's commits.
|
|
43
|
+
|
|
44
|
+
## Contact
|
|
45
|
+
|
|
46
|
+
Questions: [open an issue](https://github.com/WendellResearch-AI/benchwarmers/issues).
|