@wendellresearch-ai/benchwarmers 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GATEWAY.md +9 -5
- package/PRIVACY.md +16 -3
- package/README.md +20 -8
- package/USAGE.md +105 -14
- package/VERSION +1 -1
- package/cli.py +13 -4
- package/gateway.py +34 -2
- package/launcher.py +2 -1
- package/metrics/catalog.py +5 -0
- package/metrics/discover.py +68 -18
- package/metrics/learn.py +25 -1
- package/metrics/pacing.py +23 -3
- package/metrics/quota.py +41 -4
- package/metrics/quotaview.py +792 -0
- package/metrics/scheduler.py +2 -1
- package/metrics/sources.py +29 -6
- package/package.json +1 -1
- package/scripts/check.py +1 -0
- package/scripts/test_hubs.py +10 -5
- package/scripts/test_no_owui.py +9 -0
- package/scripts/test_quota.py +12 -2
- package/scripts/test_serve_edit.py +64 -2
- package/serve.py +311 -102
- package/ui/bench.css +92 -6
- package/ui/bench.js +544 -40
package/ui/bench.css
CHANGED
|
@@ -283,10 +283,28 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
|
|
|
283
283
|
:is(dialog, .form-panel) .rules { font-size: .875rem; color: var(--muted); margin: .35rem 0 0; }
|
|
284
284
|
:is(dialog, .form-panel) label.check { display: flex; align-items: center; gap: .5rem; margin-top: 1rem; cursor: pointer; }
|
|
285
285
|
:is(dialog, .form-panel) fieldset { border: 0; padding: 0; margin: 1rem 0 0; }
|
|
286
|
-
:is(dialog, .form-panel) .members {
|
|
287
|
-
|
|
288
|
-
|
|
286
|
+
:is(dialog, .form-panel) .members { max-height: 18rem; overflow: auto; border-top: 1px solid var(--line); }
|
|
287
|
+
/* One row per model: its name and what it holds, then a chip per provider it
|
|
288
|
+
is on, each ticking that alias (bench.js fillForm). */
|
|
289
|
+
:is(dialog, .form-panel) .members .m-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .2rem 1rem;
|
|
290
|
+
align-items: center; padding: .4rem 0; border-bottom: 1px solid var(--line); }
|
|
291
|
+
:is(dialog, .form-panel) .members .m-name { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0 .5rem;
|
|
292
|
+
font-size: .875rem; overflow-wrap: anywhere; }
|
|
293
|
+
:is(dialog, .form-panel) .members .m-facts { color: var(--muted); font-size: .8125rem; }
|
|
294
|
+
:is(dialog, .form-panel) .members .m-routes { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: .3rem; }
|
|
295
|
+
:is(dialog, .form-panel) .members label.check { margin-top: 0; font-size: .8125rem; gap: .3rem; padding: .15rem .5rem;
|
|
296
|
+
border: 1px solid var(--line); border-radius: var(--radius); background: var(--paper-2); }
|
|
297
|
+
:is(dialog, .form-panel) .members label.check:has(input:checked) { border-color: var(--bench-seat, var(--brand));
|
|
298
|
+
background: var(--band); }
|
|
299
|
+
#c-form .seats { min-height: 2rem; }
|
|
300
|
+
#c-form .seat .k.after { margin: 0 0 0 .4rem; }
|
|
289
301
|
:is(dialog, .form-panel) input.member-filter { font-size: .9375rem; padding: .4rem .6rem; margin-bottom: .2rem; }
|
|
302
|
+
#c-form .member-find { display: flex; gap: .5rem; }
|
|
303
|
+
#c-form .member-find input { flex: 1; min-width: 0; }
|
|
304
|
+
#c-form .member-find select { flex: none; width: auto; max-width: 45%; margin-bottom: .2rem; }
|
|
305
|
+
#c-form .seat .seat-x { border: 0; background: none; padding: 0 0 0 .45rem; font: inherit; font-size: 1rem; line-height: 1;
|
|
306
|
+
color: var(--muted); cursor: pointer; }
|
|
307
|
+
#c-form .seat .seat-x:hover, #c-form .seat .seat-x:focus-visible { color: var(--err); }
|
|
290
308
|
:is(dialog, .form-panel) .members input:disabled + span { color: var(--muted); }
|
|
291
309
|
/* Ruled out by the collection's tags (bench.js applyTags): greyed; a ticked
|
|
292
310
|
one stays tickable and is marked, so it can be taken off. */
|
|
@@ -300,7 +318,8 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
|
|
|
300
318
|
:is(dialog, .form-panel) .profile .chip { margin-right: .25rem; }
|
|
301
319
|
:is(dialog, .form-panel) .profile .src { font-size: .75rem; color: var(--muted); }
|
|
302
320
|
:is(dialog, .form-panel) .settings { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 1rem; }
|
|
303
|
-
@media (max-width: 40rem) { :is(dialog, .form-panel) .settings { grid-template-columns: minmax(0, 1fr); }
|
|
321
|
+
@media (max-width: 40rem) { :is(dialog, .form-panel) :is(.settings, .members .m-row) { grid-template-columns: minmax(0, 1fr); }
|
|
322
|
+
:is(dialog, .form-panel) .members .m-routes { justify-content: flex-start; } }
|
|
304
323
|
:is(dialog, .form-panel) fieldset label.check { margin-top: .4rem; flex-wrap: wrap; }
|
|
305
324
|
:is(dialog, .form-panel) fieldset label.check .muted { font-size: .875rem; }
|
|
306
325
|
:is(dialog, .form-panel) .errors { color: var(--err); font-size: .9375rem; margin: .8rem 0 0; }
|
|
@@ -384,6 +403,55 @@ pre.log { background: var(--log-ground, var(--ink-deep)); color: var(--log-ink,
|
|
|
384
403
|
white-space: pre-wrap; margin: .5rem 0 0; }
|
|
385
404
|
pre.log.answer { max-height: 40vh; }
|
|
386
405
|
|
|
406
|
+
/* Quota: a tile per provider, as many columns as fit; a group of windows (a
|
|
407
|
+
5-hour and a weekly limit of one model group) boxed together; a meter of
|
|
408
|
+
ten segments filled like a bar. */
|
|
409
|
+
/* Tiles in a row share a height, so every footer sits on one line; nothing in
|
|
410
|
+
a tile is wider than it: names shorten, values never wrap. */
|
|
411
|
+
.q-tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(17rem, 1fr)); gap: .75rem; align-items: stretch; margin-top: .75rem; }
|
|
412
|
+
.q-tile { display: flex; flex-direction: column; gap: .5rem; padding: .75rem .85rem; background: var(--panel); min-width: 0;
|
|
413
|
+
border: 1px solid var(--line); border-radius: var(--radius); cursor: grab; }
|
|
414
|
+
.q-tile:focus-visible { outline: 2px solid var(--ink); outline-offset: 2px; }
|
|
415
|
+
.q-tile.out { border-color: var(--err); }
|
|
416
|
+
.q-tile.dragging { opacity: .45; }
|
|
417
|
+
.q-tile .q-head { display: flex; justify-content: space-between; align-items: center; gap: .5rem; min-width: 0; }
|
|
418
|
+
.q-tile .nm { font-size: 1.0625rem; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
419
|
+
.q-tile .q-head .chip { flex: none; white-space: nowrap; font-size: .75rem; }
|
|
420
|
+
.q-about { display: flex; flex-direction: column; gap: .1rem; margin-top: -.3rem; font-size: .8125rem; color: var(--muted); }
|
|
421
|
+
.q-about .q-from { font-style: italic; }
|
|
422
|
+
.q-small { font-size: .8125rem; color: var(--muted); }
|
|
423
|
+
.q-body { display: flex; flex-direction: column; gap: .5rem; min-width: 0; }
|
|
424
|
+
.q-group { border: 1px solid var(--line); border-radius: var(--radius); padding: .45rem .55rem .55rem; display: flex; flex-direction: column; gap: .45rem; }
|
|
425
|
+
.q-group.bare { border: 0; padding: 0; }
|
|
426
|
+
.q-group h4 { margin: 0; font-size: .8125rem; font-weight: 500; color: var(--muted); }
|
|
427
|
+
.q-meter { display: flex; flex-direction: column; gap: .2rem; min-width: 0; }
|
|
428
|
+
.q-line { display: flex; justify-content: space-between; align-items: baseline; gap: .5rem; font-size: .875rem; min-width: 0; }
|
|
429
|
+
.q-line b { font-weight: 600; }
|
|
430
|
+
.q-line > :first-child { min-width: 0; }
|
|
431
|
+
.q-line > a.model { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
432
|
+
.q-line .q-val { flex: none; white-space: nowrap; }.segs.none span { background: var(--step-1); opacity: .6; }
|
|
433
|
+
/* Calls: green for what answered, red for what failed, out of every call. */
|
|
434
|
+
.segs.calls span { background: var(--err); }
|
|
435
|
+
.segs.calls i { background: var(--ok); }
|
|
436
|
+
.q-calls { white-space: nowrap; }
|
|
437
|
+
.q-calls .segs { display: inline-flex; width: 5.5rem; margin-right: .5rem; vertical-align: middle; }
|
|
438
|
+
.segs { display: flex; gap: 2px; }
|
|
439
|
+
.segs span { flex: 1; height: .5rem; border-radius: 2px; background: var(--step-1); overflow: hidden; }
|
|
440
|
+
.segs i { display: block; height: 100%; background: var(--ok); }
|
|
441
|
+
.segs.warn i { background: var(--warn); }
|
|
442
|
+
.segs.err i { background: var(--err); }
|
|
443
|
+
.q-rest { font-size: .8125rem; background: var(--warn-ground); border-radius: var(--radius); padding: .2rem .45rem; }
|
|
444
|
+
.warn-t { color: var(--warn); }
|
|
445
|
+
.q-msg { font-size: .8125rem; }
|
|
446
|
+
.q-msg:empty { display: none; }
|
|
447
|
+
.q-tile .q-foot { display: flex; gap: .4rem; justify-content: space-between; margin-top: auto;
|
|
448
|
+
padding-top: .5rem; border-top: 1px solid var(--line); }
|
|
449
|
+
.q-tile .q-foot .chip { margin-left: .2rem; font-size: .75rem; }
|
|
450
|
+
.q-legend { display: flex; flex-wrap: wrap; gap: 1rem; font-size: .875rem; margin: 0 0 .4rem; }
|
|
451
|
+
.q-legend i { display: inline-block; width: .8rem; height: .7rem; border-radius: 2px; margin-right: .35rem; vertical-align: -1px; }
|
|
452
|
+
#q-models { width: min(48rem, 94vw); max-width: none; }
|
|
453
|
+
#q-models td:first-child { white-space: nowrap; }
|
|
454
|
+
|
|
387
455
|
/* Benchmarks charts (Observable Plot) */
|
|
388
456
|
.chart { margin: .5rem 0 .75rem; overflow-x: auto; }
|
|
389
457
|
.chart.muted { font-size: .875rem; }
|
|
@@ -394,10 +462,26 @@ h3.chart-head { font-size: 1rem; margin: 1rem 0 .2rem; }
|
|
|
394
462
|
.list-actions { display: flex; align-items: center; gap: 1rem; margin: .75rem 0; flex-wrap: wrap; }
|
|
395
463
|
.list-actions .done { margin: 0; color: var(--ok); font-size: .9375rem; }
|
|
396
464
|
.list-actions .errors { color: var(--err); font-size: .9375rem; }
|
|
397
|
-
button.c-row { width: 100
|
|
398
|
-
border-
|
|
465
|
+
button.c-row { width: calc(100% + 1.5rem); padding: .9rem .75rem; text-align: left; font: inherit; color: inherit;
|
|
466
|
+
background: none; border: 0; border-top: 1px solid var(--line); border-radius: 0; cursor: pointer; }
|
|
467
|
+
button.c-row:first-child { border-top: 0; }
|
|
399
468
|
button.c-row:hover, button.c-row:focus-visible { background: var(--step-1); }
|
|
400
469
|
.c-row .open-hint { color: var(--muted); font-size: .875rem; white-space: nowrap; }
|
|
470
|
+
.c-row .top { display: flex; flex-wrap: wrap; align-items: baseline; gap: .3rem; }
|
|
471
|
+
.c-row .top .alias { font-size: 1.125rem; font-weight: 500; margin-right: .5rem; }
|
|
472
|
+
.c-row .desc { margin: .1rem 0 0; max-width: 75ch; }
|
|
473
|
+
.c-row .together { margin: 0; font-size: .9375rem; max-width: 80ch; }
|
|
474
|
+
/* The panel as it sits: each member a seat topped in the bench's colour, as
|
|
475
|
+
the header's bench is; an explorer seat empty and dashed; the merger outlined
|
|
476
|
+
after an arrow. The bordered seats trade padding for border, so all are one size. */
|
|
477
|
+
:is(.c-row, #c-form) .seats { display: flex; flex-wrap: wrap; align-items: center; gap: .4rem; margin: .6rem 0 .5rem; }
|
|
478
|
+
:is(.c-row, #c-form) .seat { font-family: var(--font-mono); font-size: .8125rem; padding: .3rem .6rem; background: var(--band);
|
|
479
|
+
border-top: 3px solid var(--bench-seat, var(--brand)); border-radius: var(--radius); }
|
|
480
|
+
:is(.c-row, #c-form) .seat:is(.explorer, .merger) { background: transparent; border: 1px dashed var(--muted); color: var(--muted);
|
|
481
|
+
padding: calc(.3rem + 2px) calc(.6rem - 1px) calc(.3rem - 1px); }
|
|
482
|
+
.c-row .seat.merger { border-style: solid; border-color: var(--ink); color: var(--ink); }
|
|
483
|
+
:is(.c-row, #c-form) .seat .k { font-family: var(--font-display); color: var(--muted); margin-right: .35rem; }
|
|
484
|
+
.c-row .merge { display: inline-flex; align-items: center; gap: .4rem; color: var(--muted); }
|
|
401
485
|
/* suggested collection tags under the field */
|
|
402
486
|
.tag-picks { display: flex; flex-wrap: wrap; gap: .35rem; margin-top: .45rem; }
|
|
403
487
|
.tag-picks .chip { cursor: pointer; border: 0; font-family: inherit; }
|
|
@@ -424,6 +508,8 @@ dialog .pick { display: flex; gap: 1.5rem; }
|
|
|
424
508
|
.ep .acts { justify-content: flex-start; }
|
|
425
509
|
details.key dl { grid-template-columns: minmax(0, 1fr); }
|
|
426
510
|
.toolbar input[type=search] { min-width: 0; width: 100%; }
|
|
511
|
+
button.c-row { grid-template-columns: minmax(0, 1fr); } /* the seats get the width; the whole row opens */
|
|
512
|
+
.c-row .open-hint { display: none; }
|
|
427
513
|
}
|
|
428
514
|
@media (prefers-reduced-motion: reduce) {
|
|
429
515
|
* { transition: none !important; }
|