@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/AGENTS.md +152 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +415 -272
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +408 -314
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +186 -163
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -74
  12. package/cli.py +674 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +939 -755
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -266
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +304 -295
  22. package/metrics/client.py +804 -394
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +171 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/formats.py +1021 -0
  30. package/metrics/guard.py +647 -592
  31. package/metrics/learn.py +234 -221
  32. package/metrics/ledger.py +148 -0
  33. package/metrics/litellm_hub.py +206 -206
  34. package/metrics/model_roles.py +94 -84
  35. package/metrics/offload.py +166 -166
  36. package/metrics/owui.py +205 -205
  37. package/metrics/pacing.py +248 -0
  38. package/metrics/paths.py +60 -60
  39. package/metrics/quota.py +383 -0
  40. package/metrics/route.py +581 -579
  41. package/metrics/scheduler.py +275 -275
  42. package/metrics/sources.py +167 -167
  43. package/metrics/wrong.py +144 -144
  44. package/package.json +69 -65
  45. package/perf/README.md +89 -89
  46. package/perf/run.py +284 -261
  47. package/scripts/auto_retire.py +367 -367
  48. package/scripts/check.py +113 -106
  49. package/scripts/check_deprecations.py +378 -378
  50. package/scripts/check_themes.py +103 -103
  51. package/scripts/init.py +124 -124
  52. package/scripts/list_served.py +78 -78
  53. package/scripts/notice.py +78 -78
  54. package/scripts/onboard.py +410 -331
  55. package/scripts/provider_tags.py +129 -129
  56. package/scripts/reprefix.py +136 -136
  57. package/scripts/sync_config.py +105 -105
  58. package/scripts/test_auth.py +255 -255
  59. package/scripts/test_cli.py +189 -0
  60. package/scripts/test_client.py +277 -233
  61. package/scripts/test_connect.py +222 -222
  62. package/scripts/test_first_run.py +180 -180
  63. package/scripts/test_formats.py +549 -0
  64. package/scripts/test_hubs.py +328 -328
  65. package/scripts/test_launcher.py +67 -0
  66. package/scripts/test_llm_log.py +85 -81
  67. package/scripts/test_mcp.py +178 -178
  68. package/scripts/test_merged.py +215 -214
  69. package/scripts/test_no_owui.py +291 -291
  70. package/scripts/test_owui.py +90 -90
  71. package/scripts/test_quota.py +334 -0
  72. package/scripts/test_serve_edit.py +453 -438
  73. package/scripts/test_server_json.py +79 -0
  74. package/scripts/test_verdict.py +102 -98
  75. package/scripts/test_vision.py +231 -228
  76. package/serve.py +3972 -3764
  77. package/suite/mine.py +154 -154
  78. package/suite/pace.py +145 -145
  79. package/suite/run.py +1046 -1042
  80. package/suite/seed.py +421 -421
  81. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  82. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  83. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  84. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  85. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  86. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  87. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  88. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  89. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  90. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  91. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  92. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  93. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  94. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  95. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  96. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  97. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  98. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  99. package/ui/bench.css +430 -425
  100. package/ui/bench.js +1757 -1576
  101. package/ui/themes/dark.css +35 -35
  102. package/ui/themes/hardwood.css +52 -52
  103. package/ui/themes/rainforest.css +32 -32
@@ -1,35 +1,35 @@
1
- /* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
2
- established dark themes are (Material, GitHub, Notion, VS Code):
3
- - no pure black: the page is a soft charcoal;
4
- - depth by lighter surfaces, not lines: panels sit a clear step above the
5
- page, chips and bands a step above those;
6
- - text about 91% white, not pure white; secondary text about 74% (64%,
7
- the usual figure, read as too grey here);
8
- - greys nearly neutral (a strong tint reads as muddy), the colour kept for
9
- the accents, which stay vivid but not neon. */
10
- :root {
11
- --scheme: dark;
12
- --ink: #e8ebe6;
13
- --ink-2: #d0d5ce;
14
- --ink-deep: #0b0d0c;
15
- --paper: #191c1a;
16
- --paper-2: #232724;
17
- --band: #2e332f;
18
- --signal: #e0c050;
19
- --link: #7cc0f7;
20
- --muted: #bcc3b9;
21
- --muted-dark: #727a70;
22
- --line: #363c38;
23
- --line-dark: #2b302c;
24
- --brand: #66b84e;
25
- --primary: #74c75e;
26
- --primary-bright: #93dc80;
27
- --primary-deep: #b0e8a3;
28
- --sky: #69bdeb;
29
- --warn: #e7b25a;
30
- --err: #ff8f85;
31
- /* Light text on these, unlike on paper. */
32
- --on-err: #1b1e1c;
33
- --log-ground: #0f1110;
34
- --log-ink: #e8ebe6;
35
- }
1
+ /* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
2
+ established dark themes are (Material, GitHub, Notion, VS Code):
3
+ - no pure black: the page is a soft charcoal;
4
+ - depth by lighter surfaces, not lines: panels sit a clear step above the
5
+ page, chips and bands a step above those;
6
+ - text about 91% white, not pure white; secondary text about 74% (64%,
7
+ the usual figure, read as too grey here);
8
+ - greys nearly neutral (a strong tint reads as muddy), the colour kept for
9
+ the accents, which stay vivid but not neon. */
10
+ :root {
11
+ --scheme: dark;
12
+ --ink: #e8ebe6;
13
+ --ink-2: #d0d5ce;
14
+ --ink-deep: #0b0d0c;
15
+ --paper: #191c1a;
16
+ --paper-2: #232724;
17
+ --band: #2e332f;
18
+ --signal: #e0c050;
19
+ --link: #7cc0f7;
20
+ --muted: #bcc3b9;
21
+ --muted-dark: #727a70;
22
+ --line: #363c38;
23
+ --line-dark: #2b302c;
24
+ --brand: #66b84e;
25
+ --primary: #74c75e;
26
+ --primary-bright: #93dc80;
27
+ --primary-deep: #b0e8a3;
28
+ --sky: #69bdeb;
29
+ --warn: #e7b25a;
30
+ --err: #ff8f85;
31
+ /* Light text on these, unlike on paper. */
32
+ --on-err: #1b1e1c;
33
+ --log-ground: #0f1110;
34
+ --log-ink: #e8ebe6;
35
+ }
@@ -1,52 +1,52 @@
1
- /* Hardwood: a basketball gym. The page is a maple court seen from above:
2
- boards with staggered joints and grain, and the court painted on them
3
- (boundary, half-court line, centre circle, crimson lanes, free-throw
4
- circles, three-point arcs, backboards), held still behind the page as it
5
- scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
6
- across the top; the name sits on a crimson team bench. The sidebar and the
7
- page's content sit on frosted cream so the court shows without fighting
8
- the text. Named for the gym, not a film.
9
- Crimson marks the team (button shadows, the bench, the court line, links);
10
- OK stays green and errors a burnt orange, so no state reads as the brand.
11
- --floor is the page's ground; --floor-ground its darkest board, which the
12
- theme check holds the page's text to. */
13
- :root {
14
- --ink: #1a1410;
15
- --ink-2: #2b211a;
16
- --ink-deep: #120d09;
17
- --paper: #f6efe3;
18
- --paper-2: #fffaf3;
19
- --band: #efe0c4;
20
- --signal: #ffb000;
21
- --link: #8f0a1a;
22
- --muted: #4c4136;
23
- --muted-dark: #c4b59c;
24
- --line: #e2cba3;
25
- --line-dark: #4a3e31;
26
- --brand: #990000;
27
- --primary: #2f6b34;
28
- --primary-bright: #4f9a55;
29
- --primary-deep: #1d4a22;
30
- --sky: #7fb2de;
31
- --warn: #7a4d00;
32
- --err: #b0400c;
33
- --bench-seat: #990000;
34
- --bench-leg: #2b211a;
35
- --court-line: #990000;
36
- --floor-ground: #e1ba80;
37
- --rail: rgb(255 250 243 / .9);
38
- --rail-pad: .9rem 1rem;
39
- --sheet: rgb(255 250 243 / .9);
40
- --glass: blur(5px);
41
- /* Every layer is fixed, so the whole floor holds still as the page scrolls.
42
- The court starts just under the header (its top sideline 5rem down, its
43
- height the window's less 6rem), so the name's bench sits on the
44
- sideline; Wendell, the Wendell Research mascot, is at centre court. */
45
- --floor:
46
- url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
47
- url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
48
- url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
49
- repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
50
- repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
51
- repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
52
- }
1
+ /* Hardwood: a basketball gym. The page is a maple court seen from above:
2
+ boards with staggered joints and grain, and the court painted on them
3
+ (boundary, half-court line, centre circle, crimson lanes, free-throw
4
+ circles, three-point arcs, backboards), held still behind the page as it
5
+ scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
6
+ across the top; the name sits on a crimson team bench. The sidebar and the
7
+ page's content sit on frosted cream so the court shows without fighting
8
+ the text. Named for the gym, not a film.
9
+ Crimson marks the team (button shadows, the bench, the court line, links);
10
+ OK stays green and errors a burnt orange, so no state reads as the brand.
11
+ --floor is the page's ground; --floor-ground its darkest board, which the
12
+ theme check holds the page's text to. */
13
+ :root {
14
+ --ink: #1a1410;
15
+ --ink-2: #2b211a;
16
+ --ink-deep: #120d09;
17
+ --paper: #f6efe3;
18
+ --paper-2: #fffaf3;
19
+ --band: #efe0c4;
20
+ --signal: #ffb000;
21
+ --link: #8f0a1a;
22
+ --muted: #4c4136;
23
+ --muted-dark: #c4b59c;
24
+ --line: #e2cba3;
25
+ --line-dark: #4a3e31;
26
+ --brand: #990000;
27
+ --primary: #2f6b34;
28
+ --primary-bright: #4f9a55;
29
+ --primary-deep: #1d4a22;
30
+ --sky: #7fb2de;
31
+ --warn: #7a4d00;
32
+ --err: #b0400c;
33
+ --bench-seat: #990000;
34
+ --bench-leg: #2b211a;
35
+ --court-line: #990000;
36
+ --floor-ground: #e1ba80;
37
+ --rail: rgb(255 250 243 / .9);
38
+ --rail-pad: .9rem 1rem;
39
+ --sheet: rgb(255 250 243 / .9);
40
+ --glass: blur(5px);
41
+ /* Every layer is fixed, so the whole floor holds still as the page scrolls.
42
+ The court starts just under the header (its top sideline 5rem down, its
43
+ height the window's less 6rem), so the name's bench sits on the
44
+ sideline; Wendell, the Wendell Research mascot, is at centre court. */
45
+ --floor:
46
+ url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
47
+ url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
48
+ url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
49
+ repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
50
+ repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
51
+ repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
52
+ }
@@ -1,32 +1,32 @@
1
- /* Rainforest Light: the Wendell Research palette, and the base theme: it sets
2
- every token (faces, measure, radius), and the others lay theirs over it.
3
- Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
4
- grid tokens), which the dashboard used to read live from the sibling repo.
5
- Colours only: layout, fonts and components are the same in every theme. */
6
- :root {
7
- --ink: #24331f;
8
- --ink-2: #2d3d28;
9
- --ink-deep: #121a10;
10
- --paper: #faf8f2;
11
- --paper-2: #ffffff;
12
- --band: #f3ecdc;
13
- --signal: #fae66c;
14
- --link: #01709c;
15
- --muted: #6b685a;
16
- --muted-dark: #b5b19f;
17
- --line: #e4dfd2;
18
- --line-dark: #42523d;
19
- --brand: #3e8625;
20
- --primary: #327a16;
21
- --primary-bright: #63ac4d;
22
- --primary-deep: #1b5501;
23
- --sky: #69bdeb;
24
- /* States the brand has no word for (were in ui/bench.css). */
25
- --warn: #7a4d00;
26
- --err: #a12b2b;
27
- --font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
28
- --font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
29
- --measure: 76rem;
30
- --gutter: clamp(1rem, 4vw, 2.5rem);
31
- --radius: 4px;
32
- }
1
+ /* Rainforest Light: the Wendell Research palette, and the base theme: it sets
2
+ every token (faces, measure, radius), and the others lay theirs over it.
3
+ Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
4
+ grid tokens), which the dashboard used to read live from the sibling repo.
5
+ Colours only: layout, fonts and components are the same in every theme. */
6
+ :root {
7
+ --ink: #24331f;
8
+ --ink-2: #2d3d28;
9
+ --ink-deep: #121a10;
10
+ --paper: #faf8f2;
11
+ --paper-2: #ffffff;
12
+ --band: #f3ecdc;
13
+ --signal: #fae66c;
14
+ --link: #01709c;
15
+ --muted: #6b685a;
16
+ --muted-dark: #b5b19f;
17
+ --line: #e4dfd2;
18
+ --line-dark: #42523d;
19
+ --brand: #3e8625;
20
+ --primary: #327a16;
21
+ --primary-bright: #63ac4d;
22
+ --primary-deep: #1b5501;
23
+ --sky: #69bdeb;
24
+ /* States the brand has no word for (were in ui/bench.css). */
25
+ --warn: #7a4d00;
26
+ --err: #a12b2b;
27
+ --font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
28
+ --font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
29
+ --measure: 76rem;
30
+ --gutter: clamp(1rem, 4vw, 2.5rem);
31
+ --radius: 4px;
32
+ }