@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +152 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +415 -272
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +408 -314
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +186 -163
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -74
- package/cli.py +674 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +939 -755
- package/launcher.py +120 -0
- package/mcp_server.py +266 -266
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +304 -295
- package/metrics/client.py +804 -394
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +171 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/formats.py +1021 -0
- package/metrics/guard.py +647 -592
- package/metrics/learn.py +234 -221
- package/metrics/ledger.py +148 -0
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +94 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/pacing.py +248 -0
- package/metrics/paths.py +60 -60
- package/metrics/quota.py +383 -0
- package/metrics/route.py +581 -579
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +69 -65
- package/perf/README.md +89 -89
- package/perf/run.py +284 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +113 -106
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +410 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +277 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_formats.py +549 -0
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -81
- package/scripts/test_mcp.py +178 -178
- package/scripts/test_merged.py +215 -214
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_quota.py +334 -0
- package/scripts/test_serve_edit.py +453 -438
- package/scripts/test_server_json.py +79 -0
- package/scripts/test_verdict.py +102 -98
- package/scripts/test_vision.py +231 -228
- package/serve.py +3972 -3764
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1042
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -425
- package/ui/bench.js +1757 -1576
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/ui/themes/dark.css
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
/* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
|
|
2
|
-
established dark themes are (Material, GitHub, Notion, VS Code):
|
|
3
|
-
- no pure black: the page is a soft charcoal;
|
|
4
|
-
- depth by lighter surfaces, not lines: panels sit a clear step above the
|
|
5
|
-
page, chips and bands a step above those;
|
|
6
|
-
- text about 91% white, not pure white; secondary text about 74% (64%,
|
|
7
|
-
the usual figure, read as too grey here);
|
|
8
|
-
- greys nearly neutral (a strong tint reads as muddy), the colour kept for
|
|
9
|
-
the accents, which stay vivid but not neon. */
|
|
10
|
-
:root {
|
|
11
|
-
--scheme: dark;
|
|
12
|
-
--ink: #e8ebe6;
|
|
13
|
-
--ink-2: #d0d5ce;
|
|
14
|
-
--ink-deep: #0b0d0c;
|
|
15
|
-
--paper: #191c1a;
|
|
16
|
-
--paper-2: #232724;
|
|
17
|
-
--band: #2e332f;
|
|
18
|
-
--signal: #e0c050;
|
|
19
|
-
--link: #7cc0f7;
|
|
20
|
-
--muted: #bcc3b9;
|
|
21
|
-
--muted-dark: #727a70;
|
|
22
|
-
--line: #363c38;
|
|
23
|
-
--line-dark: #2b302c;
|
|
24
|
-
--brand: #66b84e;
|
|
25
|
-
--primary: #74c75e;
|
|
26
|
-
--primary-bright: #93dc80;
|
|
27
|
-
--primary-deep: #b0e8a3;
|
|
28
|
-
--sky: #69bdeb;
|
|
29
|
-
--warn: #e7b25a;
|
|
30
|
-
--err: #ff8f85;
|
|
31
|
-
/* Light text on these, unlike on paper. */
|
|
32
|
-
--on-err: #1b1e1c;
|
|
33
|
-
--log-ground: #0f1110;
|
|
34
|
-
--log-ink: #e8ebe6;
|
|
35
|
-
}
|
|
1
|
+
/* Rainforest Dark: the rainforest greens on a near-neutral dark ground, tuned the way
|
|
2
|
+
established dark themes are (Material, GitHub, Notion, VS Code):
|
|
3
|
+
- no pure black: the page is a soft charcoal;
|
|
4
|
+
- depth by lighter surfaces, not lines: panels sit a clear step above the
|
|
5
|
+
page, chips and bands a step above those;
|
|
6
|
+
- text about 91% white, not pure white; secondary text about 74% (64%,
|
|
7
|
+
the usual figure, read as too grey here);
|
|
8
|
+
- greys nearly neutral (a strong tint reads as muddy), the colour kept for
|
|
9
|
+
the accents, which stay vivid but not neon. */
|
|
10
|
+
:root {
|
|
11
|
+
--scheme: dark;
|
|
12
|
+
--ink: #e8ebe6;
|
|
13
|
+
--ink-2: #d0d5ce;
|
|
14
|
+
--ink-deep: #0b0d0c;
|
|
15
|
+
--paper: #191c1a;
|
|
16
|
+
--paper-2: #232724;
|
|
17
|
+
--band: #2e332f;
|
|
18
|
+
--signal: #e0c050;
|
|
19
|
+
--link: #7cc0f7;
|
|
20
|
+
--muted: #bcc3b9;
|
|
21
|
+
--muted-dark: #727a70;
|
|
22
|
+
--line: #363c38;
|
|
23
|
+
--line-dark: #2b302c;
|
|
24
|
+
--brand: #66b84e;
|
|
25
|
+
--primary: #74c75e;
|
|
26
|
+
--primary-bright: #93dc80;
|
|
27
|
+
--primary-deep: #b0e8a3;
|
|
28
|
+
--sky: #69bdeb;
|
|
29
|
+
--warn: #e7b25a;
|
|
30
|
+
--err: #ff8f85;
|
|
31
|
+
/* Light text on these, unlike on paper. */
|
|
32
|
+
--on-err: #1b1e1c;
|
|
33
|
+
--log-ground: #0f1110;
|
|
34
|
+
--log-ink: #e8ebe6;
|
|
35
|
+
}
|
package/ui/themes/hardwood.css
CHANGED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
/* Hardwood: a basketball gym. The page is a maple court seen from above:
|
|
2
|
-
boards with staggered joints and grain, and the court painted on them
|
|
3
|
-
(boundary, half-court line, centre circle, crimson lanes, free-throw
|
|
4
|
-
circles, three-point arcs, backboards), held still behind the page as it
|
|
5
|
-
scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
|
|
6
|
-
across the top; the name sits on a crimson team bench. The sidebar and the
|
|
7
|
-
page's content sit on frosted cream so the court shows without fighting
|
|
8
|
-
the text. Named for the gym, not a film.
|
|
9
|
-
Crimson marks the team (button shadows, the bench, the court line, links);
|
|
10
|
-
OK stays green and errors a burnt orange, so no state reads as the brand.
|
|
11
|
-
--floor is the page's ground; --floor-ground its darkest board, which the
|
|
12
|
-
theme check holds the page's text to. */
|
|
13
|
-
:root {
|
|
14
|
-
--ink: #1a1410;
|
|
15
|
-
--ink-2: #2b211a;
|
|
16
|
-
--ink-deep: #120d09;
|
|
17
|
-
--paper: #f6efe3;
|
|
18
|
-
--paper-2: #fffaf3;
|
|
19
|
-
--band: #efe0c4;
|
|
20
|
-
--signal: #ffb000;
|
|
21
|
-
--link: #8f0a1a;
|
|
22
|
-
--muted: #4c4136;
|
|
23
|
-
--muted-dark: #c4b59c;
|
|
24
|
-
--line: #e2cba3;
|
|
25
|
-
--line-dark: #4a3e31;
|
|
26
|
-
--brand: #990000;
|
|
27
|
-
--primary: #2f6b34;
|
|
28
|
-
--primary-bright: #4f9a55;
|
|
29
|
-
--primary-deep: #1d4a22;
|
|
30
|
-
--sky: #7fb2de;
|
|
31
|
-
--warn: #7a4d00;
|
|
32
|
-
--err: #b0400c;
|
|
33
|
-
--bench-seat: #990000;
|
|
34
|
-
--bench-leg: #2b211a;
|
|
35
|
-
--court-line: #990000;
|
|
36
|
-
--floor-ground: #e1ba80;
|
|
37
|
-
--rail: rgb(255 250 243 / .9);
|
|
38
|
-
--rail-pad: .9rem 1rem;
|
|
39
|
-
--sheet: rgb(255 250 243 / .9);
|
|
40
|
-
--glass: blur(5px);
|
|
41
|
-
/* Every layer is fixed, so the whole floor holds still as the page scrolls.
|
|
42
|
-
The court starts just under the header (its top sideline 5rem down, its
|
|
43
|
-
height the window's less 6rem), so the name's bench sits on the
|
|
44
|
-
sideline; Wendell, the Wendell Research mascot, is at centre court. */
|
|
45
|
-
--floor:
|
|
46
|
-
url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
|
|
47
|
-
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
|
|
48
|
-
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
|
|
49
|
-
repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
|
|
50
|
-
repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
|
|
51
|
-
repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
|
|
52
|
-
}
|
|
1
|
+
/* Hardwood: a basketball gym. The page is a maple court seen from above:
|
|
2
|
+
boards with staggered joints and grain, and the court painted on them
|
|
3
|
+
(boundary, half-court line, centre circle, crimson lanes, free-throw
|
|
4
|
+
circles, three-point arcs, backboards), held still behind the page as it
|
|
5
|
+
scrolls. Hoosier crimson and cream; scoreboard amber; a crimson court line
|
|
6
|
+
across the top; the name sits on a crimson team bench. The sidebar and the
|
|
7
|
+
page's content sit on frosted cream so the court shows without fighting
|
|
8
|
+
the text. Named for the gym, not a film.
|
|
9
|
+
Crimson marks the team (button shadows, the bench, the court line, links);
|
|
10
|
+
OK stays green and errors a burnt orange, so no state reads as the brand.
|
|
11
|
+
--floor is the page's ground; --floor-ground its darkest board, which the
|
|
12
|
+
theme check holds the page's text to. */
|
|
13
|
+
:root {
|
|
14
|
+
--ink: #1a1410;
|
|
15
|
+
--ink-2: #2b211a;
|
|
16
|
+
--ink-deep: #120d09;
|
|
17
|
+
--paper: #f6efe3;
|
|
18
|
+
--paper-2: #fffaf3;
|
|
19
|
+
--band: #efe0c4;
|
|
20
|
+
--signal: #ffb000;
|
|
21
|
+
--link: #8f0a1a;
|
|
22
|
+
--muted: #4c4136;
|
|
23
|
+
--muted-dark: #c4b59c;
|
|
24
|
+
--line: #e2cba3;
|
|
25
|
+
--line-dark: #4a3e31;
|
|
26
|
+
--brand: #990000;
|
|
27
|
+
--primary: #2f6b34;
|
|
28
|
+
--primary-bright: #4f9a55;
|
|
29
|
+
--primary-deep: #1d4a22;
|
|
30
|
+
--sky: #7fb2de;
|
|
31
|
+
--warn: #7a4d00;
|
|
32
|
+
--err: #b0400c;
|
|
33
|
+
--bench-seat: #990000;
|
|
34
|
+
--bench-leg: #2b211a;
|
|
35
|
+
--court-line: #990000;
|
|
36
|
+
--floor-ground: #e1ba80;
|
|
37
|
+
--rail: rgb(255 250 243 / .9);
|
|
38
|
+
--rail-pad: .9rem 1rem;
|
|
39
|
+
--sheet: rgb(255 250 243 / .9);
|
|
40
|
+
--glass: blur(5px);
|
|
41
|
+
/* Every layer is fixed, so the whole floor holds still as the page scrolls.
|
|
42
|
+
The court starts just under the header (its top sideline 5rem down, its
|
|
43
|
+
height the window's less 6rem), so the name's bench sits on the
|
|
44
|
+
sideline; Wendell, the Wendell Research mascot, is at centre court. */
|
|
45
|
+
--floor:
|
|
46
|
+
url("/ui/mascot.png") center calc(5rem + (100vh - 6rem) * .4235) / auto calc((100vh - 6rem) * .153) no-repeat fixed,
|
|
47
|
+
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 940 500'%3E%3Cg fill='%23990000' fill-opacity='.4'%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3C/g%3E%3Cg fill='none' stroke='%231a1410' stroke-opacity='.55' stroke-width='4'%3E%3Crect x='10' y='10' width='920' height='480'/%3E%3Cpath d='M470 10V190M470 310V490M10 39H160A232 232 0 0 1 160 461H10M930 39H780A232 232 0 0 0 780 461H930M50 220V280M890 220V280'/%3E%3Crect x='10' y='170' width='190' height='160'/%3E%3Crect x='740' y='170' width='190' height='160'/%3E%3Ccircle cx='470' cy='250' r='60'/%3E%3Ccircle cx='200' cy='250' r='60'/%3E%3Ccircle cx='740' cy='250' r='60'/%3E%3Ccircle cx='62' cy='250' r='9'/%3E%3Ccircle cx='878' cy='250' r='9'/%3E%3C/g%3E%3C/svg%3E") center top 5rem / auto calc(100vh - 6rem) no-repeat fixed,
|
|
48
|
+
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='340' height='88'%3E%3Cpath d='M0 0v22M200 22v22M90 44v22M280 66v22' stroke='%23704214' stroke-opacity='.32'/%3E%3C/svg%3E") fixed,
|
|
49
|
+
repeating-linear-gradient(0deg, rgba(112, 66, 20, .26) 0 1px, transparent 1px 22px) fixed,
|
|
50
|
+
repeating-linear-gradient(.5deg, transparent 0 5px, rgba(140, 86, 30, .07) 5px 6px, transparent 6px 13px) fixed,
|
|
51
|
+
repeating-linear-gradient(0deg, #ecce9b 0 22px, #e6c48c 22px 44px, #efd4a5 44px 66px, #e1ba80 66px 88px) fixed;
|
|
52
|
+
}
|
package/ui/themes/rainforest.css
CHANGED
|
@@ -1,32 +1,32 @@
|
|
|
1
|
-
/* Rainforest Light: the Wendell Research palette, and the base theme: it sets
|
|
2
|
-
every token (faces, measure, radius), and the others lay theirs over it.
|
|
3
|
-
Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
|
|
4
|
-
grid tokens), which the dashboard used to read live from the sibling repo.
|
|
5
|
-
Colours only: layout, fonts and components are the same in every theme. */
|
|
6
|
-
:root {
|
|
7
|
-
--ink: #24331f;
|
|
8
|
-
--ink-2: #2d3d28;
|
|
9
|
-
--ink-deep: #121a10;
|
|
10
|
-
--paper: #faf8f2;
|
|
11
|
-
--paper-2: #ffffff;
|
|
12
|
-
--band: #f3ecdc;
|
|
13
|
-
--signal: #fae66c;
|
|
14
|
-
--link: #01709c;
|
|
15
|
-
--muted: #6b685a;
|
|
16
|
-
--muted-dark: #b5b19f;
|
|
17
|
-
--line: #e4dfd2;
|
|
18
|
-
--line-dark: #42523d;
|
|
19
|
-
--brand: #3e8625;
|
|
20
|
-
--primary: #327a16;
|
|
21
|
-
--primary-bright: #63ac4d;
|
|
22
|
-
--primary-deep: #1b5501;
|
|
23
|
-
--sky: #69bdeb;
|
|
24
|
-
/* States the brand has no word for (were in ui/bench.css). */
|
|
25
|
-
--warn: #7a4d00;
|
|
26
|
-
--err: #a12b2b;
|
|
27
|
-
--font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
|
|
28
|
-
--font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
|
|
29
|
-
--measure: 76rem;
|
|
30
|
-
--gutter: clamp(1rem, 4vw, 2.5rem);
|
|
31
|
-
--radius: 4px;
|
|
32
|
-
}
|
|
1
|
+
/* Rainforest Light: the Wendell Research palette, and the base theme: it sets
|
|
2
|
+
every token (faces, measure, radius), and the others lay theirs over it.
|
|
3
|
+
Copied 2026-10-19 from the AI site's src/styles/global.css (:root, less the
|
|
4
|
+
grid tokens), which the dashboard used to read live from the sibling repo.
|
|
5
|
+
Colours only: layout, fonts and components are the same in every theme. */
|
|
6
|
+
:root {
|
|
7
|
+
--ink: #24331f;
|
|
8
|
+
--ink-2: #2d3d28;
|
|
9
|
+
--ink-deep: #121a10;
|
|
10
|
+
--paper: #faf8f2;
|
|
11
|
+
--paper-2: #ffffff;
|
|
12
|
+
--band: #f3ecdc;
|
|
13
|
+
--signal: #fae66c;
|
|
14
|
+
--link: #01709c;
|
|
15
|
+
--muted: #6b685a;
|
|
16
|
+
--muted-dark: #b5b19f;
|
|
17
|
+
--line: #e4dfd2;
|
|
18
|
+
--line-dark: #42523d;
|
|
19
|
+
--brand: #3e8625;
|
|
20
|
+
--primary: #327a16;
|
|
21
|
+
--primary-bright: #63ac4d;
|
|
22
|
+
--primary-deep: #1b5501;
|
|
23
|
+
--sky: #69bdeb;
|
|
24
|
+
/* States the brand has no word for (were in ui/bench.css). */
|
|
25
|
+
--warn: #7a4d00;
|
|
26
|
+
--err: #a12b2b;
|
|
27
|
+
--font-display: 'Chivo', system-ui, -apple-system, 'Segoe UI', sans-serif;
|
|
28
|
+
--font-mono: 'Chivo Mono', ui-monospace, 'Cascadia Mono', Consolas, monospace;
|
|
29
|
+
--measure: 76rem;
|
|
30
|
+
--gutter: clamp(1rem, 4vw, 2.5rem);
|
|
31
|
+
--radius: 4px;
|
|
32
|
+
}
|