@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +152 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +415 -272
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +408 -314
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +186 -163
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -74
- package/cli.py +674 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +939 -755
- package/launcher.py +120 -0
- package/mcp_server.py +266 -266
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +304 -295
- package/metrics/client.py +804 -394
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +171 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/formats.py +1021 -0
- package/metrics/guard.py +647 -592
- package/metrics/learn.py +234 -221
- package/metrics/ledger.py +148 -0
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +94 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/pacing.py +248 -0
- package/metrics/paths.py +60 -60
- package/metrics/quota.py +383 -0
- package/metrics/route.py +581 -579
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +69 -65
- package/perf/README.md +89 -89
- package/perf/run.py +284 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +113 -106
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +410 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +277 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_formats.py +549 -0
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -81
- package/scripts/test_mcp.py +178 -178
- package/scripts/test_merged.py +215 -214
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_quota.py +334 -0
- package/scripts/test_serve_edit.py +453 -438
- package/scripts/test_server_json.py +79 -0
- package/scripts/test_verdict.py +102 -98
- package/scripts/test_vision.py +231 -228
- package/serve.py +3972 -3764
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1042
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -425
- package/ui/bench.js +1757 -1576
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/bin/benchwarmers.js
CHANGED
|
@@ -1,41 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// Benchwarmers from npm:
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// standard library only; nothing is installed beside it.
|
|
2
|
+
// Benchwarmers from npm: finds Python 3.11 or later and hands over to
|
|
3
|
+
// launcher.py, the same launcher a pip install runs, which knows the commands,
|
|
4
|
+
// the help and the first run. The app itself is the Python standard library
|
|
5
|
+
// only; nothing is installed beside it.
|
|
7
6
|
'use strict';
|
|
8
7
|
|
|
9
8
|
const { spawn, spawnSync } = require('child_process');
|
|
10
|
-
const fs = require('fs');
|
|
11
|
-
const os = require('os');
|
|
12
9
|
const path = require('path');
|
|
13
10
|
|
|
14
11
|
const ROOT = path.resolve(__dirname, '..');
|
|
15
12
|
|
|
16
|
-
// The same folder as scripts/init.py's default_folder(), so a checkout and an
|
|
17
|
-
// npm install on one machine find the same data.
|
|
18
|
-
function defaultHome() {
|
|
19
|
-
if (process.platform === 'win32') return path.join(process.env.LOCALAPPDATA || os.homedir(), 'benchwarmers');
|
|
20
|
-
if (process.platform === 'darwin') return path.join(os.homedir(), 'Library', 'Application Support', 'benchwarmers');
|
|
21
|
-
return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'benchwarmers');
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function usage() {
|
|
25
|
-
return [
|
|
26
|
-
'Usage: benchwarmers [command]',
|
|
27
|
-
'',
|
|
28
|
-
' serve the dashboard, with the gateway at /v1: http://localhost:8122 (the default)',
|
|
29
|
-
' gateway the gateway alone: http://127.0.0.1:8123/v1',
|
|
30
|
-
' init [...] set up a data folder, optionally with a first endpoint (init --help)',
|
|
31
|
-
' mcp an MCP server on stdio for agents, a client of a running Benchwarmers (BENCHWARMERS_URL)',
|
|
32
|
-
' --version the version',
|
|
33
|
-
'',
|
|
34
|
-
'Your data lives in BENCHWARMERS_HOME, else ' + defaultHome() + '.',
|
|
35
|
-
'BENCH_PORT changes the port. Docs: https://github.com/WendellResearch-AI/benchwarmers',
|
|
36
|
-
].join('\n');
|
|
37
|
-
}
|
|
38
|
-
|
|
39
13
|
// The first Python that is 3.11 or later: `py -3` on Windows, then python3, python.
|
|
40
14
|
function findPython() {
|
|
41
15
|
const tries = process.platform === 'win32'
|
|
@@ -50,48 +24,16 @@ function findPython() {
|
|
|
50
24
|
return null;
|
|
51
25
|
}
|
|
52
26
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return 0;
|
|
58
|
-
}
|
|
59
|
-
if (command !== 'init' && (argv.includes('--help') || argv.includes('-h'))) {
|
|
60
|
-
console.log(usage());
|
|
61
|
-
return 0;
|
|
62
|
-
}
|
|
63
|
-
const scripts = { serve: 'serve.py', gateway: 'gateway.py', mcp: 'mcp_server.py', init: path.join('scripts', 'init.py') };
|
|
64
|
-
if (!scripts[command]) {
|
|
65
|
-
console.error('benchwarmers: no command "' + command + '"\n\n' + usage());
|
|
66
|
-
return 2;
|
|
67
|
-
}
|
|
68
|
-
const python = findPython();
|
|
69
|
-
if (!python) {
|
|
70
|
-
console.error('benchwarmers needs Python 3.11 or later on PATH (https://www.python.org/downloads/).');
|
|
71
|
-
return 1;
|
|
72
|
-
}
|
|
73
|
-
const home = (process.env.BENCHWARMERS_HOME || '').trim() || defaultHome();
|
|
74
|
-
// No bytecode beside the code: an npm folder may not be writable, and an
|
|
75
|
-
// upgrade replaces it anyway.
|
|
76
|
-
const env = Object.assign({}, process.env, {
|
|
77
|
-
BENCHWARMERS_HOME: home, BENCHWARMERS_LAUNCHER: 'npm', PYTHONDONTWRITEBYTECODE: '1', PYTHONUTF8: '1' });
|
|
78
|
-
const rest = command === argv[0] ? argv.slice(1) : argv;
|
|
79
|
-
const py = (script, args) => python[1].concat([path.join(ROOT, script)], args);
|
|
80
|
-
|
|
81
|
-
// The first run makes the data folder from examples/ (init.py never writes
|
|
82
|
-
// over a file already there, and with BENCHWARMERS_HOME set it leaves the
|
|
83
|
-
// package folder alone). Not for mcp: it keeps no data, and its stdout
|
|
84
|
-
// carries protocol messages only.
|
|
85
|
-
if (command !== 'init' && command !== 'mcp' && !fs.existsSync(home)) {
|
|
86
|
-
console.log('First run: making your data folder at ' + home);
|
|
87
|
-
const r = spawnSync(python[0], py(scripts.init, [home]), { stdio: 'inherit', env, cwd: ROOT });
|
|
88
|
-
if (r.status !== 0) return r.status || 1;
|
|
89
|
-
}
|
|
90
|
-
const child = spawn(python[0], py(scripts[command], rest), { stdio: 'inherit', env, cwd: ROOT });
|
|
91
|
-
for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig));
|
|
92
|
-
child.on('exit', (code, signal) => process.exit(code === null ? (signal ? 1 : 0) : code));
|
|
93
|
-
return null;
|
|
27
|
+
const python = findPython();
|
|
28
|
+
if (!python) {
|
|
29
|
+
console.error('benchwarmers needs Python 3.11 or later on PATH (https://www.python.org/downloads/).');
|
|
30
|
+
process.exit(1);
|
|
94
31
|
}
|
|
95
|
-
|
|
96
|
-
const
|
|
97
|
-
|
|
32
|
+
// `bench` and `benchwarmers` are the same program; usage names the one typed.
|
|
33
|
+
const env = Object.assign({}, process.env, {
|
|
34
|
+
BENCHWARMERS_LAUNCHER: 'npm', BENCHWARMERS_PROG: path.basename(process.argv[1] || 'benchwarmers').replace(/\.js$/, ''),
|
|
35
|
+
PYTHONDONTWRITEBYTECODE: '1', PYTHONUTF8: '1' });
|
|
36
|
+
const child = spawn(python[0], python[1].concat([path.join(ROOT, 'launcher.py')], process.argv.slice(2)),
|
|
37
|
+
{ stdio: 'inherit', env });
|
|
38
|
+
for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig));
|
|
39
|
+
child.on('exit', (code, signal) => process.exit(code === null ? (signal ? 1 : 0) : code));
|