@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/AGENTS.md +147 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +272 -271
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +368 -275
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +170 -155
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -72
  12. package/cli.py +666 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +755 -712
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -0
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +295 -295
  22. package/metrics/client.py +394 -373
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +152 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/guard.py +617 -592
  30. package/metrics/learn.py +221 -221
  31. package/metrics/litellm_hub.py +206 -206
  32. package/metrics/model_roles.py +84 -84
  33. package/metrics/offload.py +166 -166
  34. package/metrics/owui.py +205 -205
  35. package/metrics/paths.py +60 -60
  36. package/metrics/route.py +579 -574
  37. package/metrics/scheduler.py +275 -275
  38. package/metrics/sources.py +167 -167
  39. package/metrics/wrong.py +144 -144
  40. package/package.json +68 -64
  41. package/perf/README.md +89 -89
  42. package/perf/run.py +261 -261
  43. package/scripts/auto_retire.py +367 -367
  44. package/scripts/check.py +108 -104
  45. package/scripts/check_deprecations.py +378 -378
  46. package/scripts/check_themes.py +103 -103
  47. package/scripts/init.py +124 -124
  48. package/scripts/list_served.py +78 -78
  49. package/scripts/notice.py +78 -78
  50. package/scripts/onboard.py +401 -331
  51. package/scripts/provider_tags.py +129 -129
  52. package/scripts/reprefix.py +136 -136
  53. package/scripts/sync_config.py +105 -105
  54. package/scripts/test_auth.py +255 -255
  55. package/scripts/test_cli.py +189 -0
  56. package/scripts/test_client.py +233 -233
  57. package/scripts/test_connect.py +222 -222
  58. package/scripts/test_first_run.py +180 -180
  59. package/scripts/test_hubs.py +328 -328
  60. package/scripts/test_launcher.py +67 -0
  61. package/scripts/test_llm_log.py +85 -0
  62. package/scripts/test_mcp.py +178 -0
  63. package/scripts/test_merged.py +214 -193
  64. package/scripts/test_no_owui.py +291 -291
  65. package/scripts/test_owui.py +90 -90
  66. package/scripts/test_serve_edit.py +448 -386
  67. package/scripts/test_verdict.py +98 -98
  68. package/scripts/test_vision.py +228 -228
  69. package/serve.py +3904 -3662
  70. package/suite/mine.py +154 -154
  71. package/suite/pace.py +145 -145
  72. package/suite/run.py +1046 -1040
  73. package/suite/seed.py +421 -421
  74. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  75. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  76. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  77. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  78. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  79. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  80. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  81. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  82. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  83. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  84. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  85. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  86. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  87. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  88. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  89. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  90. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  91. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  92. package/ui/bench.css +430 -424
  93. package/ui/bench.js +1756 -1555
  94. package/ui/themes/dark.css +35 -35
  95. package/ui/themes/hardwood.css +52 -52
  96. package/ui/themes/rainforest.css +32 -32
package/examples/config/gateway.json CHANGED
@@ -1,5 +1,5 @@
1
- {
2
- "_why": "Bench gateway settings, also set on the Collections page. A request runs a collection's members plus explorer seats picked by staleness: explorers = min(max_explorers, max_calls - members), so a 2- or 3-model collection gets 2 explorers, a 4-model one 1, a 5-model one none. There is no default collection: a request that names none is told what each one is for and asked to choose.",
3
- "max_calls": 5,
4
- "max_explorers": 2
5
- }
1
+ {
2
+ "_why": "Bench gateway settings, also set on the Collections page. A request runs a collection's members plus explorer seats picked by staleness: explorers = min(max_explorers, max_calls - members), so a 2- or 3-model collection gets 2 explorers, a 4-model one 1, a 5-model one none. There is no default collection: a request that names none is told what each one is for and asked to choose.",
3
+ "max_calls": 5,
4
+ "max_explorers": 2
5
+ }
package/examples/config/model-roles.json CHANGED
@@ -1,11 +1,11 @@
1
- {
2
- "_comment": [
3
- "Your models' roles, and the ones never chosen automatically (ad_hoc).",
4
- "An alias not listed under models is general. The roles and task shapes",
5
- "themselves ship with Benchwarmers (metrics/model_roles.py says what each",
6
- "means); add \"roles\" or \"shapes\" here to define your own, which win over",
7
- "a shipped one of the same name."
8
- ],
9
- "models": {},
10
- "ad_hoc": []
11
- }
1
+ {
2
+ "_comment": [
3
+ "Your models' roles, and the ones never chosen automatically (ad_hoc).",
4
+ "An alias not listed under models is general. The roles and task shapes",
5
+ "themselves ship with Benchwarmers (metrics/model_roles.py says what each",
6
+ "means); add \"roles\" or \"shapes\" here to define your own, which win over",
7
+ "a shipped one of the same name."
8
+ ],
9
+ "models": {},
10
+ "ad_hoc": []
11
+ }