@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/AGENTS.md +147 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +272 -271
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +368 -275
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +170 -155
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -72
  12. package/cli.py +666 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +755 -712
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -0
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +295 -295
  22. package/metrics/client.py +394 -373
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +152 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/guard.py +617 -592
  30. package/metrics/learn.py +221 -221
  31. package/metrics/litellm_hub.py +206 -206
  32. package/metrics/model_roles.py +84 -84
  33. package/metrics/offload.py +166 -166
  34. package/metrics/owui.py +205 -205
  35. package/metrics/paths.py +60 -60
  36. package/metrics/route.py +579 -574
  37. package/metrics/scheduler.py +275 -275
  38. package/metrics/sources.py +167 -167
  39. package/metrics/wrong.py +144 -144
  40. package/package.json +68 -64
  41. package/perf/README.md +89 -89
  42. package/perf/run.py +261 -261
  43. package/scripts/auto_retire.py +367 -367
  44. package/scripts/check.py +108 -104
  45. package/scripts/check_deprecations.py +378 -378
  46. package/scripts/check_themes.py +103 -103
  47. package/scripts/init.py +124 -124
  48. package/scripts/list_served.py +78 -78
  49. package/scripts/notice.py +78 -78
  50. package/scripts/onboard.py +401 -331
  51. package/scripts/provider_tags.py +129 -129
  52. package/scripts/reprefix.py +136 -136
  53. package/scripts/sync_config.py +105 -105
  54. package/scripts/test_auth.py +255 -255
  55. package/scripts/test_cli.py +189 -0
  56. package/scripts/test_client.py +233 -233
  57. package/scripts/test_connect.py +222 -222
  58. package/scripts/test_first_run.py +180 -180
  59. package/scripts/test_hubs.py +328 -328
  60. package/scripts/test_launcher.py +67 -0
  61. package/scripts/test_llm_log.py +85 -0
  62. package/scripts/test_mcp.py +178 -0
  63. package/scripts/test_merged.py +214 -193
  64. package/scripts/test_no_owui.py +291 -291
  65. package/scripts/test_owui.py +90 -90
  66. package/scripts/test_serve_edit.py +448 -386
  67. package/scripts/test_verdict.py +98 -98
  68. package/scripts/test_vision.py +228 -228
  69. package/serve.py +3904 -3662
  70. package/suite/mine.py +154 -154
  71. package/suite/pace.py +145 -145
  72. package/suite/run.py +1046 -1040
  73. package/suite/seed.py +421 -421
  74. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  75. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  76. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  77. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  78. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  79. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  80. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  81. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  82. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  83. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  84. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  85. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  86. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  87. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  88. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  89. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  90. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  91. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  92. package/ui/bench.css +430 -424
  93. package/ui/bench.js +1756 -1555
  94. package/ui/themes/dark.css +35 -35
  95. package/ui/themes/hardwood.css +52 -52
  96. package/ui/themes/rainforest.css +32 -32
@@ -1,35 +1,35 @@
1
- /* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
2
- established dark themes are (Material, GitHub, Notion, VS Code):
3
- - no pure black: the page is a soft charcoal;
4
- - depth by lighter surfaces, not lines: panels sit a clear step above the
5
- page, chips and bands a step above those;
6
- - text about 91% white, not pure white; secondary text about 74% (64%,
7
- the usual figure, read as too grey here);
8
- - greys nearly neutral (a strong tint reads as muddy), the colour kept for
9
- the accents, which stay vivid but not neon. */
10
- :root {
11
- --scheme: dark;
12
- --ink: #e8ebe6;
13
- --ink-2: #d0d5ce;
14
- --ink-deep: #0b0d0c;
15
- --paper: #191c1a;
16
- --paper-2: #232724;
17
- --band: #2e332f;
18
- --signal: #e0c050;
19
- --link: #7cc0f7;
20
- --muted: #bcc3b9;
21
- --muted-dark: #727a70;
22
- --line: #363c38;
23
- --line-dark: #2b302c;
24
- --brand: #66b84e;
25
- --primary: #74c75e;
26
- --primary-bright: #93dc80;
27
- --primary-deep: #b0e8a3;
28
- --sky: #69bdeb;
29
- --warn: #e7b25a;
30
- --err: #ff8f85;
31
- /* Light text on these, unlike on paper. */
32
- --on-err: #1b1e1c;
33
- --log-ground: #0f1110;
34
- --log-ink: #e8ebe6;
35
- }
1
+ /* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
2
+ established dark themes are (Material, GitHub, Notion, VS Code):
3
+ - no pure black: the page is a soft charcoal;
4
+ - depth by lighter surfaces, not lines: panels sit a clear step above the
5
+ page, chips and bands a step above those;
6
+ - text about 91% white, not pure white; secondary text about 74% (64%,
7
+ the usual figure, read as too grey here);
8
+ - greys nearly neutral (a strong tint reads as muddy), the colour kept for
9
+ the accents, which stay vivid but not neon. */
10
+ :root {
11
+ --scheme: dark;
12
+ --ink: #e8ebe6;
13
+ --ink-2: #d0d5ce;
14
+ --ink-deep: #0b0d0c;
15
+ --paper: #191c1a;
16
+ --paper-2: #232724;
17
+ --band: #2e332f;
18
+ --signal: #e0c050;
19
+ --link: #7cc0f7;
20
+ --muted: #bcc3b9;
21
+ --muted-dark: #727a70;
22
+ --line: #363c38;
23
+ --line-dark: #2b302c;
24
+ --brand: #66b84e;
25
+ --primary: #74c75e;
26
+ --primary-bright: #93dc80;
27
+ --primary-deep: #b0e8a3;
28
+ --sky: #69bdeb;
29
+ --warn: #e7b25a;
30
+ --err: #ff8f85;
31
+ /* Light text on these, unlike on paper. */
32
+ --on-err: #1b1e1c;
33
+ --log-ground: #0f1110;
34
+ --log-ink: #e8ebe6;
35
+ }
@@ -1,52 +1,52 @@
1
- /* Hardwood: a basketball gym. The page is a maple court seen from above:
2
- boards with staggered joints and grain, and the court painted on them
3
- (boundary, half-court line, centre circle, crimson lanes, free-throw
4
- circles, three-point arcs, backboards), held still behind the page as it
5
- scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
6
- across the top; the name sits on a crimson team bench. The sidebar and the
7
- page's content sit on frosted cream so the court shows without fighting
8
- the text. Named for the gym, not a film.
9
- Crimson marks the team (button shadows, the bench, the court line, links);
10
- OK stays green and errors a burnt orange, so no state reads as the brand.
11
- --floor is the page's ground; --floor-ground its darkest board, which the
12
- theme check holds the page's text to. */
13
- :root {
14
- --ink: #1a1410;
15
- --ink-2: #2b211a;
16
- --ink-deep: #120d09;
17
- --paper: #f6efe3;
18
- --paper-2: #fffaf3;
19
- --band: #efe0c4;
20
- --signal: #ffb000;
21
- --link: #8f0a1a;
22
- --muted: #4c4136;
23
- --muted-dark: #c4b59c;
24
- --line: #e2cba3;
25
- --line-dark: #4a3e31;
26
- --brand: #990000;
27
- --primary: #2f6b34;
28
- --primary-bright: #4f9a55;
29
- --primary-deep: #1d4a22;
30
- --sky: #7fb2de;
31
- --warn: #7a4d00;
32
- --err: #b0400c;
33
- --bench-seat: #990000;
34
- --bench-leg: #2b211a;
35
- --court-line: #990000;
36
- --floor-ground: #e1ba80;
37
- --rail: rgb(255 250 243 / .9);
38
- --rail-pad: .9rem 1rem;
39
- --sheet: rgb(255 250 243 / .9);
40
- --glass: blur(5px);
41
- /* Every layer is fixed, so the whole floor holds still as the page scrolls.
42
- The court starts just under the header (its top sideline 5rem down, its
43
- height the window's less 6rem), so the name's bench sits on the
44
- sideline; Wendell, the Wendell Research mascot, is at centre court. */
45
- --floor:
46
- url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
47
- url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
48
- url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
49
- repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
50
- repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
51
- repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
52
- }
1
+ /* Hardwood: a basketball gym. The page is a maple court seen from above:
2
+ boards with staggered joints and grain, and the court painted on them
3
+ (boundary, half-court line, centre circle, crimson lanes, free-throw
4
+ circles, three-point arcs, backboards), held still behind the page as it
5
+ scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
6
+ across the top; the name sits on a crimson team bench. The sidebar and the
7
+ page's content sit on frosted cream so the court shows without fighting
8
+ the text. Named for the gym, not a film.
9
+ Crimson marks the team (button shadows, the bench, the court line, links);
10
+ OK stays green and errors a burnt orange, so no state reads as the brand.
11
+ --floor is the page's ground; --floor-ground its darkest board, which the
12
+ theme check holds the page's text to. */
13
+ :root {
14
+ --ink: #1a1410;
15
+ --ink-2: #2b211a;
16
+ --ink-deep: #120d09;
17
+ --paper: #f6efe3;
18
+ --paper-2: #fffaf3;
19
+ --band: #efe0c4;
20
+ --signal: #ffb000;
21
+ --link: #8f0a1a;
22
+ --muted: #4c4136;
23
+ --muted-dark: #c4b59c;
24
+ --line: #e2cba3;
25
+ --line-dark: #4a3e31;
26
+ --brand: #990000;
27
+ --primary: #2f6b34;
28
+ --primary-bright: #4f9a55;
29
+ --primary-deep: #1d4a22;
30
+ --sky: #7fb2de;
31
+ --warn: #7a4d00;
32
+ --err: #b0400c;
33
+ --bench-seat: #990000;
34
+ --bench-leg: #2b211a;
35
+ --court-line: #990000;
36
+ --floor-ground: #e1ba80;
37
+ --rail: rgb(255 250 243 / .9);
38
+ --rail-pad: .9rem 1rem;
39
+ --sheet: rgb(255 250 243 / .9);
40
+ --glass: blur(5px);
41
+ /* Every layer is fixed, so the whole floor holds still as the page scrolls.
42
+ The court starts just under the header (its top sideline 5rem down, its
43
+ height the window's less 6rem), so the name's bench sits on the
44
+ sideline; Wendell, the Wendell Research mascot, is at centre court. */
45
+ --floor:
46
+ url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
47
+ url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
48
+ url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
49
+ repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
50
+ repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
51
+ repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
52
+ }
@@ -1,32 +1,32 @@
1
- /* Rainforest Light: the Wendell Research palette, and the base theme: it sets
2
- every token (faces, measure, radius), and the others lay theirs over it.
3
- Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
4
- grid tokens), which the dashboard used to read live from the sibling repo.
5
- Colours only: layout, fonts and components are the same in every theme. */
6
- :root {
7
- --ink: #24331f;
8
- --ink-2: #2d3d28;
9
- --ink-deep: #121a10;
10
- --paper: #faf8f2;
11
- --paper-2: #ffffff;
12
- --band: #f3ecdc;
13
- --signal: #fae66c;
14
- --link: #01709c;
15
- --muted: #6b685a;
16
- --muted-dark: #b5b19f;
17
- --line: #e4dfd2;
18
- --line-dark: #42523d;
19
- --brand: #3e8625;
20
- --primary: #327a16;
21
- --primary-bright: #63ac4d;
22
- --primary-deep: #1b5501;
23
- --sky: #69bdeb;
24
- /* States the brand has no word for (were in ui/bench.css). */
25
- --warn: #7a4d00;
26
- --err: #a12b2b;
27
- --font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
28
- --font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
29
- --measure: 76rem;
30
- --gutter: clamp(1rem, 4vw, 2.5rem);
31
- --radius: 4px;
32
- }
1
+ /* Rainforest Light: the Wendell Research palette, and the base theme: it sets
2
+ every token (faces, measure, radius), and the others lay theirs over it.
3
+ Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
4
+ grid tokens), which the dashboard used to read live from the sibling repo.
5
+ Colours only: layout, fonts and components are the same in every theme. */
6
+ :root {
7
+ --ink: #24331f;
8
+ --ink-2: #2d3d28;
9
+ --ink-deep: #121a10;
10
+ --paper: #faf8f2;
11
+ --paper-2: #ffffff;
12
+ --band: #f3ecdc;
13
+ --signal: #fae66c;
14
+ --link: #01709c;
15
+ --muted: #6b685a;
16
+ --muted-dark: #b5b19f;
17
+ --line: #e4dfd2;
18
+ --line-dark: #42523d;
19
+ --brand: #3e8625;
20
+ --primary: #327a16;
21
+ --primary-bright: #63ac4d;
22
+ --primary-deep: #1b5501;
23
+ --sky: #69bdeb;
24
+ /* States the brand has no word for (were in ui/bench.css). */
25
+ --warn: #7a4d00;
26
+ --err: #a12b2b;
27
+ --font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
28
+ --font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
29
+ --measure: 76rem;
30
+ --gutter: clamp(1rem, 4vw, 2.5rem);
31
+ --radius: 4px;
32
+ }