@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/AGENTS.md +147 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +272 -272
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +368 -314
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +170 -163
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -74
  12. package/cli.py +666 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +755 -755
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -266
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +295 -295
  22. package/metrics/client.py +394 -394
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +152 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/guard.py +617 -592
  30. package/metrics/learn.py +221 -221
  31. package/metrics/litellm_hub.py +206 -206
  32. package/metrics/model_roles.py +84 -84
  33. package/metrics/offload.py +166 -166
  34. package/metrics/owui.py +205 -205
  35. package/metrics/paths.py +60 -60
  36. package/metrics/route.py +579 -579
  37. package/metrics/scheduler.py +275 -275
  38. package/metrics/sources.py +167 -167
  39. package/metrics/wrong.py +144 -144
  40. package/package.json +68 -65
  41. package/perf/README.md +89 -89
  42. package/perf/run.py +261 -261
  43. package/scripts/auto_retire.py +367 -367
  44. package/scripts/check.py +108 -106
  45. package/scripts/check_deprecations.py +378 -378
  46. package/scripts/check_themes.py +103 -103
  47. package/scripts/init.py +124 -124
  48. package/scripts/list_served.py +78 -78
  49. package/scripts/notice.py +78 -78
  50. package/scripts/onboard.py +401 -331
  51. package/scripts/provider_tags.py +129 -129
  52. package/scripts/reprefix.py +136 -136
  53. package/scripts/sync_config.py +105 -105
  54. package/scripts/test_auth.py +255 -255
  55. package/scripts/test_cli.py +189 -0
  56. package/scripts/test_client.py +233 -233
  57. package/scripts/test_connect.py +222 -222
  58. package/scripts/test_first_run.py +180 -180
  59. package/scripts/test_hubs.py +328 -328
  60. package/scripts/test_launcher.py +67 -0
  61. package/scripts/test_llm_log.py +85 -81
  62. package/scripts/test_mcp.py +178 -178
  63. package/scripts/test_merged.py +214 -214
  64. package/scripts/test_no_owui.py +291 -291
  65. package/scripts/test_owui.py +90 -90
  66. package/scripts/test_serve_edit.py +448 -438
  67. package/scripts/test_verdict.py +98 -98
  68. package/scripts/test_vision.py +228 -228
  69. package/serve.py +3904 -3764
  70. package/suite/mine.py +154 -154
  71. package/suite/pace.py +145 -145
  72. package/suite/run.py +1046 -1042
  73. package/suite/seed.py +421 -421
  74. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  75. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  76. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  77. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  78. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  79. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  80. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  81. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  82. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  83. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  84. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  85. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  86. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  87. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  88. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  89. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  90. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  91. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  92. package/ui/bench.css +430 -425
  93. package/ui/bench.js +1756 -1576
  94. package/ui/themes/dark.css +35 -35
  95. package/ui/themes/hardwood.css +52 -52
  96. package/ui/themes/rainforest.css +32 -32
@@ -1,41 +1,15 @@
1
1
  #!/usr/bin/env node
2
- // Benchwarmers from npm: a launcher for the Python app this package carries.
3
- // It finds Python 3.11 or later, keeps your data in your user profile (the
4
- // folder scripts/init.py picks, or BENCHWARMERS_HOME), sets that folder up on
5
- // the first run, then starts the dashboard. The app itself is the Python
6
- // standard library only; nothing is installed beside it.
2
+ // Benchwarmers from npm: finds Python 3.11 or later and hands over to
3
+ // launcher.py, the same launcher a pip install runs, which knows the commands,
4
+ // the help and the first run. The app itself is the Python standard library
5
+ // only; nothing is installed beside it.
7
6
  'use strict';
8
7
 
9
8
  const { spawn, spawnSync } = require('child_process');
10
- const fs = require('fs');
11
- const os = require('os');
12
9
  const path = require('path');
13
10
 
14
11
  const ROOT = path.resolve(__dirname, '..');
15
12
 
16
- // The same folder as scripts/init.py's default_folder(), so a checkout and an
17
- // npm install on one machine find the same data.
18
- function defaultHome() {
19
- if (process.platform === 'win32') return path.join(process.env.LOCALAPPDATA || os.homedir(), 'benchwarmers');
20
- if (process.platform === 'darwin') return path.join(os.homedir(), 'Library', 'Application Support', 'benchwarmers');
21
- return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'benchwarmers');
22
- }
23
-
24
- function usage() {
25
- return [
26
- 'Usage: benchwarmers [command]',
27
- '',
28
- ' serve the dashboard, with the gateway at /v1: http://localhost:8122 (the default)',
29
- ' gateway the gateway alone: http://127.0.0.1:8123/v1',
30
- ' init [...] set up a data folder, optionally with a first endpoint (init --help)',
31
- ' mcp an MCP server on stdio for agents, a client of a running Benchwarmers (BENCHWARMERS_URL)',
32
- ' --version the version',
33
- '',
34
- 'Your data lives in BENCHWARMERS_HOME, else ' + defaultHome() + '.',
35
- 'BENCH_PORT changes the port. Docs: https://github.com/WendellResearch-AI/benchwarmers',
36
- ].join('\n');
37
- }
38
-
39
13
  // The first Python that is 3.11 or later: `py -3` on Windows, then python3, python.
40
14
  function findPython() {
41
15
  const tries = process.platform === 'win32'
@@ -50,48 +24,16 @@ function findPython() {
50
24
  return null;
51
25
  }
52
26
 
53
- function main(argv) {
54
- const command = argv[0] && !argv[0].startsWith('-') ? argv[0] : 'serve';
55
- if (argv.includes('--version') || argv.includes('-v')) {
56
- console.log(fs.readFileSync(path.join(ROOT, 'VERSION'), 'utf8').trim());
57
- return 0;
58
- }
59
- if (command !== 'init' && (argv.includes('--help') || argv.includes('-h'))) {
60
- console.log(usage());
61
- return 0;
62
- }
63
- const scripts = { serve: 'serve.py', gateway: 'gateway.py', mcp: 'mcp_server.py', init: path.join('scripts', 'init.py') };
64
- if (!scripts[command]) {
65
- console.error('benchwarmers: no command "' + command + '"\n\n' + usage());
66
- return 2;
67
- }
68
- const python = findPython();
69
- if (!python) {
70
- console.error('benchwarmers needs Python 3.11 or later on PATH (https://www.python.org/downloads/).');
71
- return 1;
72
- }
73
- const home = (process.env.BENCHWARMERS_HOME || '').trim() || defaultHome();
74
- // No bytecode beside the code: an npm folder may not be writable, and an
75
- // upgrade replaces it anyway.
76
- const env = Object.assign({}, process.env, {
77
- BENCHWARMERS_HOME: home, BENCHWARMERS_LAUNCHER: 'npm', PYTHONDONTWRITEBYTECODE: '1', PYTHONUTF8: '1' });
78
- const rest = command === argv[0] ? argv.slice(1) : argv;
79
- const py = (script, args) => python[1].concat([path.join(ROOT, script)], args);
80
-
81
- // The first run makes the data folder from examples/ (init.py never writes
82
- // over a file already there, and with BENCHWARMERS_HOME set it leaves the
83
- // package folder alone). Not for mcp: it keeps no data, and its stdout
84
- // carries protocol messages only.
85
- if (command !== 'init' && command !== 'mcp' && !fs.existsSync(home)) {
86
- console.log('First run: making your data folder at ' + home);
87
- const r = spawnSync(python[0], py(scripts.init, [home]), { stdio: 'inherit', env, cwd: ROOT });
88
- if (r.status !== 0) return r.status || 1;
89
- }
90
- const child = spawn(python[0], py(scripts[command], rest), { stdio: 'inherit', env, cwd: ROOT });
91
- for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig));
92
- child.on('exit', (code, signal) => process.exit(code === null ? (signal ? 1 : 0) : code));
93
- return null;
27
+ const python = findPython();
28
+ if (!python) {
29
+ console.error('benchwarmers needs Python 3.11 or later on PATH (https://www.python.org/downloads/).');
30
+ process.exit(1);
94
31
  }
95
-
96
- const code = main(process.argv.slice(2));
97
- if (code !== null) process.exit(code);
32
+ // `bench` and `benchwarmers` are the same program; usage names the one typed.
33
+ const env = Object.assign({}, process.env, {
34
+ BENCHWARMERS_LAUNCHER: 'npm', BENCHWARMERS_PROG: path.basename(process.argv[1] || 'benchwarmers').replace(/\.js$/, ''),
35
+ PYTHONDONTWRITEBYTECODE: '1', PYTHONUTF8: '1' });
36
+ const child = spawn(python[0], python[1].concat([path.join(ROOT, 'launcher.py')], process.argv.slice(2)),
37
+ { stdio: 'inherit', env });
38
+ for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, () => child.kill(sig));
39
+ child.on('exit', (code, signal) => process.exit(code === null ? (signal ? 1 : 0) : code));