@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/AGENTS.md +147 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +272 -271
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +368 -275
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +170 -155
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -72
  12. package/cli.py +666 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +755 -712
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -0
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +295 -295
  22. package/metrics/client.py +394 -373
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +152 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/guard.py +617 -592
  30. package/metrics/learn.py +221 -221
  31. package/metrics/litellm_hub.py +206 -206
  32. package/metrics/model_roles.py +84 -84
  33. package/metrics/offload.py +166 -166
  34. package/metrics/owui.py +205 -205
  35. package/metrics/paths.py +60 -60
  36. package/metrics/route.py +579 -574
  37. package/metrics/scheduler.py +275 -275
  38. package/metrics/sources.py +167 -167
  39. package/metrics/wrong.py +144 -144
  40. package/package.json +68 -64
  41. package/perf/README.md +89 -89
  42. package/perf/run.py +261 -261
  43. package/scripts/auto_retire.py +367 -367
  44. package/scripts/check.py +108 -104
  45. package/scripts/check_deprecations.py +378 -378
  46. package/scripts/check_themes.py +103 -103
  47. package/scripts/init.py +124 -124
  48. package/scripts/list_served.py +78 -78
  49. package/scripts/notice.py +78 -78
  50. package/scripts/onboard.py +401 -331
  51. package/scripts/provider_tags.py +129 -129
  52. package/scripts/reprefix.py +136 -136
  53. package/scripts/sync_config.py +105 -105
  54. package/scripts/test_auth.py +255 -255
  55. package/scripts/test_cli.py +189 -0
  56. package/scripts/test_client.py +233 -233
  57. package/scripts/test_connect.py +222 -222
  58. package/scripts/test_first_run.py +180 -180
  59. package/scripts/test_hubs.py +328 -328
  60. package/scripts/test_launcher.py +67 -0
  61. package/scripts/test_llm_log.py +85 -0
  62. package/scripts/test_mcp.py +178 -0
  63. package/scripts/test_merged.py +214 -193
  64. package/scripts/test_no_owui.py +291 -291
  65. package/scripts/test_owui.py +90 -90
  66. package/scripts/test_serve_edit.py +448 -386
  67. package/scripts/test_verdict.py +98 -98
  68. package/scripts/test_vision.py +228 -228
  69. package/serve.py +3904 -3662
  70. package/suite/mine.py +154 -154
  71. package/suite/pace.py +145 -145
  72. package/suite/run.py +1046 -1040
  73. package/suite/seed.py +421 -421
  74. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  75. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  76. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  77. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  78. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  79. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  80. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  81. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  82. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  83. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  84. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  85. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  86. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  87. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  88. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  89. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  90. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  91. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  92. package/ui/bench.css +430 -424
  93. package/ui/bench.js +1756 -1555
  94. package/ui/themes/dark.css +35 -35
  95. package/ui/themes/hardwood.css +52 -52
  96. package/ui/themes/rainforest.css +32 -32
package/PRIVACY.md CHANGED
@@ -1,46 +1,46 @@
1
- # Privacy policy
2
-
3
- *Effective 22 October 2026.*
4
-
5
- Benchwarmers runs on your own machine or server. It has no account, no
6
- telemetry and no server of its own, and nothing you do with it reaches
7
- Wendell Research.
8
-
9
- ## What leaves your machine
10
-
11
- - **Your prompts**, only to the endpoints you add
12
- (a provider, an aggregator, a hub such as Open WebUI, or a local runtime).
13
- That includes what the gateway relays for your apps, and the calls the
14
- probes, the suite and onboarding make. Each endpoint's own privacy policy
15
- applies to what it receives; some free tiers keep prompts or train on them.
16
- - **Catalog reads**, carrying nothing of yours: models.dev and FreeLLMAPI
17
- (at most once a day) and OpenRouter's public model list.
18
- - **Sign-ins**, only when you choose one: the provider's own sign-in page, in
19
- your browser, and the provider's token endpoint.
20
- - **Charts**: the dashboard's page loads Observable Plot from jsDelivr, at a
21
- pinned version, in your browser.
22
- - **Real-bug task seeding**, only when you run `suite/rebench.py`: dataset rows
23
- from Hugging Face and source files from GitHub.
24
-
25
- Keys and sign-in tokens stay in your environment or the `llm` CLI's key
26
- store. [SECURITY.md](SECURITY.md) has the full detail.
27
-
28
- ## What stays on your machine
29
-
30
- Everything else: your endpoints, collections, verdicts, call logs, probe
31
- history and task answers live in your data folder, which is yours to back up,
32
- move or delete.
33
-
34
- ## Third parties
35
-
36
- The endpoints you add, the catalogs above, jsDelivr, GitHub, Hugging Face and
37
- any app you connect to the gateway each have their own privacy policies.
38
-
39
- ## Changes
40
-
41
- Changes to this policy are made in this file, and its history is in the
42
- repository's commits.
43
-
44
- ## Contact
45
-
46
- Questions: [open an issue](https://github.com/WendellResearch-AI/benchwarmers/issues).
1
+ # Privacy policy
2
+
3
+ *Effective 22 October 2026.*
4
+
5
+ Benchwarmers runs on your own machine or server. It has no account, no
6
+ telemetry and no server of its own, and nothing you do with it reaches
7
+ Wendell Research.
8
+
9
+ ## What leaves your machine
10
+
11
+ - **Your prompts**, only to the endpoints you add
12
+ (a provider, an aggregator, a hub such as Open WebUI, or a local runtime).
13
+ That includes what the gateway relays for your apps, and the calls the
14
+ probes, the suite and onboarding make. Each endpoint's own privacy policy
15
+ applies to what it receives; some free tiers keep prompts or train on them.
16
+ - **Catalog reads**, carrying nothing of yours: models.dev and FreeLLMAPI
17
+ (at most once a day) and OpenRouter's public model list.
18
+ - **Sign-ins**, only when you choose one: the provider's own sign-in page, in
19
+ your browser, and the provider's token endpoint.
20
+ - **Charts**: the dashboard's page loads Observable Plot from jsDelivr, at a
21
+ pinned version, in your browser.
22
+ - **Real-bug task seeding**, only when you run `suite/rebench.py`: dataset rows
23
+ from Hugging Face and source files from GitHub.
24
+
25
+ Keys and sign-in tokens stay in your environment or the `llm` CLI's key
26
+ store. [SECURITY.md](SECURITY.md) has the full detail.
27
+
28
+ ## What stays on your machine
29
+
30
+ Everything else: your endpoints, collections, verdicts, call logs, probe
31
+ history and task answers live in your data folder, which is yours to back up,
32
+ move or delete.
33
+
34
+ ## Third parties
35
+
36
+ The endpoints you add, the catalogs above, jsDelivr, GitHub, Hugging Face and
37
+ any app you connect to the gateway each have their own privacy policies.
38
+
39
+ ## Changes
40
+
41
+ Changes to this policy are made in this file, and its history is in the
42
+ repository's commits.
43
+
44
+ ## Contact
45
+
46
+ Questions: [open an issue](https://github.com/WendellResearch-AI/benchwarmers/issues).