@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +147 -147
- package/BENCHMARKS.md +262 -262
- package/GATEWAY.md +272 -271
- package/NOTICE +28 -28
- package/PRIVACY.md +46 -46
- package/README.md +368 -275
- package/SECURITY.md +83 -83
- package/TERMS.md +22 -22
- package/USAGE.md +170 -155
- package/VERSION +1 -1
- package/bin/benchwarmers.js +16 -72
- package/cli.py +666 -0
- package/examples/config/gateway.json +5 -5
- package/examples/config/model-roles.json +11 -11
- package/gateway.py +755 -712
- package/launcher.py +120 -0
- package/mcp_server.py +266 -0
- package/metrics/alerts.py +116 -116
- package/metrics/aliases.py +41 -41
- package/metrics/bench_collections.py +141 -141
- package/metrics/catalog.py +295 -295
- package/metrics/client.py +394 -373
- package/metrics/cliproxy.py +191 -191
- package/metrics/connect.py +691 -691
- package/metrics/cost.py +152 -152
- package/metrics/decide.py +227 -227
- package/metrics/discover.py +318 -318
- package/metrics/endpoints.py +436 -436
- package/metrics/guard.py +617 -592
- package/metrics/learn.py +221 -221
- package/metrics/litellm_hub.py +206 -206
- package/metrics/model_roles.py +84 -84
- package/metrics/offload.py +166 -166
- package/metrics/owui.py +205 -205
- package/metrics/paths.py +60 -60
- package/metrics/route.py +579 -574
- package/metrics/scheduler.py +275 -275
- package/metrics/sources.py +167 -167
- package/metrics/wrong.py +144 -144
- package/package.json +68 -64
- package/perf/README.md +89 -89
- package/perf/run.py +261 -261
- package/scripts/auto_retire.py +367 -367
- package/scripts/check.py +108 -104
- package/scripts/check_deprecations.py +378 -378
- package/scripts/check_themes.py +103 -103
- package/scripts/init.py +124 -124
- package/scripts/list_served.py +78 -78
- package/scripts/notice.py +78 -78
- package/scripts/onboard.py +401 -331
- package/scripts/provider_tags.py +129 -129
- package/scripts/reprefix.py +136 -136
- package/scripts/sync_config.py +105 -105
- package/scripts/test_auth.py +255 -255
- package/scripts/test_cli.py +189 -0
- package/scripts/test_client.py +233 -233
- package/scripts/test_connect.py +222 -222
- package/scripts/test_first_run.py +180 -180
- package/scripts/test_hubs.py +328 -328
- package/scripts/test_launcher.py +67 -0
- package/scripts/test_llm_log.py +85 -0
- package/scripts/test_mcp.py +178 -0
- package/scripts/test_merged.py +214 -193
- package/scripts/test_no_owui.py +291 -291
- package/scripts/test_owui.py +90 -90
- package/scripts/test_serve_edit.py +448 -386
- package/scripts/test_verdict.py +98 -98
- package/scripts/test_vision.py +228 -228
- package/serve.py +3904 -3662
- package/suite/mine.py +154 -154
- package/suite/pace.py +145 -145
- package/suite/run.py +1046 -1040
- package/suite/seed.py +421 -421
- package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
- package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
- package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
- package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
- package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
- package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
- package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
- package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
- package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
- package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
- package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
- package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
- package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
- package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
- package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
- package/ui/bench.css +430 -424
- package/ui/bench.js +1756 -1555
- package/ui/themes/dark.css +35 -35
- package/ui/themes/hardwood.css +52 -52
- package/ui/themes/rainforest.css +32 -32
package/ui/bench.js
CHANGED
|
@@ -1,1555 +1,1756 @@
|
|
|
1
|
-
// Benchwarmers. Plain DOM, no framework: serve.py renders the page, this only
|
|
2
|
-
// wires the controls. Every write goes through a POST that re-checks its own
|
|
3
|
-
// rules server-side, so nothing here is trusted to enforce anything.
|
|
4
|
-
(function () {
|
|
5
|
-
'use strict';
|
|
6
|
-
var D = window.BENCH || {models: {}, roles: {}, retired: {}, stamp: 0};
|
|
7
|
-
function $(id) { return document.getElementById(id); }
|
|
8
|
-
function el(tag, attrs, text) {
|
|
9
|
-
var e = document.createElement(tag);
|
|
10
|
-
Object.keys(attrs || {}).forEach(function (k) { e.setAttribute(k, attrs[k]); });
|
|
11
|
-
if (text != null) e.textContent = text;
|
|
12
|
-
return e;
|
|
13
|
-
}
|
|
14
|
-
function store(k, v) { try { if (v === undefined) return localStorage.getItem(k); localStorage.setItem(k, v); } catch (e) { return null; } }
|
|
15
|
-
function post(url, body) {
|
|
16
|
-
return fetch(url, {method: 'POST', headers: {'Content-Type': 'application/json'},
|
|
17
|
-
body: JSON.stringify(body)}).then(function (r) { return r.json(); });
|
|
18
|
-
}
|
|
19
|
-
function listErrors(node, errors) { node.textContent = errors.map(function (x) { return '• ' + x; }).join('\n'); node.style.whiteSpace = 'pre-line'; }
|
|
20
|
-
|
|
21
|
-
//
|
|
22
|
-
var
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
var
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
th.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
m.
|
|
174
|
-
m.
|
|
175
|
-
m.
|
|
176
|
-
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
m.
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
var
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
var
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
//
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
//
|
|
379
|
-
function
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
});
|
|
441
|
-
if (
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
o.
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
});
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
});
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (!od.open)
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
if (
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
$('oq-
|
|
623
|
-
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
+ '
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
.
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
$('
|
|
723
|
-
function
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
if
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
});
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
$('
|
|
944
|
-
|
|
945
|
-
if (
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
function
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
}
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
if (
|
|
1129
|
-
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
}
|
|
1156
|
-
//
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
}
|
|
1212
|
-
function
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
}
|
|
1239
|
-
box.
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
$('
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
var
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
}
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
});
|
|
1407
|
-
});
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
var
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
if (!
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1
|
+
// Benchwarmers. Plain DOM, no framework: serve.py renders the page, this only
|
|
2
|
+
// wires the controls. Every write goes through a POST that re-checks its own
|
|
3
|
+
// rules server-side, so nothing here is trusted to enforce anything.
|
|
4
|
+
(function () {
|
|
5
|
+
'use strict';
|
|
6
|
+
var D = window.BENCH || {models: {}, roles: {}, retired: {}, stamp: 0};
|
|
7
|
+
function $(id) { return document.getElementById(id); }
|
|
8
|
+
function el(tag, attrs, text) {
|
|
9
|
+
var e = document.createElement(tag);
|
|
10
|
+
Object.keys(attrs || {}).forEach(function (k) { e.setAttribute(k, attrs[k]); });
|
|
11
|
+
if (text != null) e.textContent = text;
|
|
12
|
+
return e;
|
|
13
|
+
}
|
|
14
|
+
function store(k, v) { try { if (v === undefined) return localStorage.getItem(k); localStorage.setItem(k, v); } catch (e) { return null; } }
|
|
15
|
+
function post(url, body) {
|
|
16
|
+
return fetch(url, {method: 'POST', headers: {'Content-Type': 'application/json'},
|
|
17
|
+
body: JSON.stringify(body)}).then(function (r) { return r.json(); });
|
|
18
|
+
}
|
|
19
|
+
function listErrors(node, errors) { node.textContent = errors.map(function (x) { return '• ' + x; }).join('\n'); node.style.whiteSpace = 'pre-line'; }
|
|
20
|
+
// The page's own confirm, a Promise of true/false: window.confirm is
|
|
21
|
+
// silently false where a browser blocks dialogs. Opens over any open popup.
|
|
22
|
+
var askD = $('ask-dialog'), askDone = null;
|
|
23
|
+
function ask(text, yes, danger) {
|
|
24
|
+
if (askDone) askDone(false);
|
|
25
|
+
$('ask-text').textContent = text;
|
|
26
|
+
$('ask-yes').textContent = yes || 'OK';
|
|
27
|
+
$('ask-yes').className = 'btn ' + (danger ? 'danger' : 'primary');
|
|
28
|
+
askD.showModal(); $('ask-no').focus();
|
|
29
|
+
return new Promise(function (resolve) { askDone = resolve; });
|
|
30
|
+
}
|
|
31
|
+
function answer(yes) { var r = askDone; askDone = null; if (askD.open) askD.close(); if (r) r(yes); }
|
|
32
|
+
$('ask-yes').addEventListener('click', function () { answer(true); });
|
|
33
|
+
$('ask-no').addEventListener('click', function () { answer(false); });
|
|
34
|
+
askD.addEventListener('close', function () { answer(false); }); // Escape
|
|
35
|
+
|
|
36
|
+
// ---- tabs, remembered per browser ----------------------------------
|
|
37
|
+
var tabs = document.querySelectorAll('.tabs button[data-tab]'); // not the sidebar's own toggle
|
|
38
|
+
var onShow = {}; // a page's loader, run each time it is shown
|
|
39
|
+
function show(id) {
|
|
40
|
+
tabs.forEach(function (b) { b.setAttribute('aria-selected', String(b.dataset.tab === id)); });
|
|
41
|
+
document.querySelectorAll('.tab').forEach(function (s) { s.classList.toggle('active', s.id === 'tab-' + id); });
|
|
42
|
+
store('bench.tab', id);
|
|
43
|
+
if (onShow[id]) onShow[id]();
|
|
44
|
+
}
|
|
45
|
+
tabs.forEach(function (b) { b.addEventListener('click', function () { show(b.dataset.tab); }); });
|
|
46
|
+
var wanted = store('bench.tab');
|
|
47
|
+
show(wanted && $('tab-' + wanted) ? wanted : 'glance');
|
|
48
|
+
|
|
49
|
+
// ---- sortable tables -------------------------------------------------
|
|
50
|
+
function key(cell) {
|
|
51
|
+
var raw = cell.dataset.sort != null ? cell.dataset.sort : cell.textContent.trim();
|
|
52
|
+
var n = parseFloat(raw);
|
|
53
|
+
return isNaN(n) || !/^-?[\d.]+/.test(raw) ? raw.toLowerCase() : n;
|
|
54
|
+
}
|
|
55
|
+
document.querySelectorAll('th[data-sortable]').forEach(function (th) {
|
|
56
|
+
th.tabIndex = 0;
|
|
57
|
+
function sort() {
|
|
58
|
+
var table = th.closest('table'), body = table.tBodies[0];
|
|
59
|
+
var idx = Array.prototype.indexOf.call(th.parentNode.children, th);
|
|
60
|
+
var dir = th.getAttribute('aria-sort') === 'descending' ? 'ascending' : 'descending';
|
|
61
|
+
table.querySelectorAll('th[aria-sort]').forEach(function (o) { o.removeAttribute('aria-sort'); });
|
|
62
|
+
th.setAttribute('aria-sort', dir);
|
|
63
|
+
var rows = Array.prototype.slice.call(body.rows);
|
|
64
|
+
rows.sort(function (a, b) {
|
|
65
|
+
var x = key(a.cells[idx]), y = key(b.cells[idx]);
|
|
66
|
+
var c = (typeof x === typeof y) ? (x < y ? -1 : x > y ? 1 : 0) : (typeof x === 'number' ? -1 : 1);
|
|
67
|
+
return dir === 'ascending' ? c : -c;
|
|
68
|
+
});
|
|
69
|
+
rows.forEach(function (r) { body.appendChild(r); });
|
|
70
|
+
}
|
|
71
|
+
th.addEventListener('click', sort);
|
|
72
|
+
th.addEventListener('keydown', function (e) { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); sort(); } });
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
// ---- filter boxes: one per panel (or the whole page: data-filter="page"),
|
|
76
|
+
// matching any text in a table row or a log line, and the panel's facet
|
|
77
|
+
// selects (serve.facet), each matching a row's data-<facet> exactly ------
|
|
78
|
+
function applyFilter(scope) {
|
|
79
|
+
var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
|
|
80
|
+
var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
|
|
81
|
+
scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
|
|
82
|
+
r.hidden = (q && r.textContent.toLowerCase().indexOf(q) === -1)
|
|
83
|
+
|| facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
|
|
84
|
+
});
|
|
85
|
+
// A match inside a folded list (an endpoint's models) opens it.
|
|
86
|
+
if (q) scope.querySelectorAll('details.models').forEach(function (d) {
|
|
87
|
+
if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
document.querySelectorAll('input[data-filter], select[data-facet]').forEach(function (c) {
|
|
91
|
+
var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
|
|
92
|
+
var scope = box && box.dataset.filter === 'page' ? c.closest('.tab') : c.closest('.panel');
|
|
93
|
+
c.addEventListener(c.tagName === 'SELECT' ? 'change' : 'input', function () { applyFilter(scope); });
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
// ---- Action needed: Hide folds it to its heading, per browser, until an
|
|
97
|
+
// action it has not seen appears or 12 hours pass, whichever is first ----
|
|
98
|
+
var acts = $('actions');
|
|
99
|
+
if (acts) {
|
|
100
|
+
var keys = Array.prototype.map.call(acts.querySelectorAll('.problem[data-key]'), function (p) { return p.dataset.key; });
|
|
101
|
+
var hid = null;
|
|
102
|
+
try { hid = JSON.parse(store('bench.actions.hidden') || 'null'); } catch (err) { hid = null; }
|
|
103
|
+
var fold = function (on) {
|
|
104
|
+
acts.classList.toggle('collapsed', on);
|
|
105
|
+
$('actions-hide').textContent = on ? 'show' : 'Hide';
|
|
106
|
+
$('actions-hide').setAttribute('aria-expanded', String(!on));
|
|
107
|
+
};
|
|
108
|
+
fold(!!(hid && hid.keys && Date.now() - hid.at < 12 * 3600 * 1000
|
|
109
|
+
&& keys.every(function (k) { return hid.keys.indexOf(k) !== -1; })));
|
|
110
|
+
$('actions-hide').addEventListener('click', function () {
|
|
111
|
+
var on = !acts.classList.contains('collapsed');
|
|
112
|
+
store('bench.actions.hidden', on ? JSON.stringify({at: Date.now(), keys: keys}) : '');
|
|
113
|
+
fold(on);
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ---- retired rows: hidden unless asked for ---------------------------
|
|
118
|
+
var showRetired = store('bench.retired') === '1';
|
|
119
|
+
function applyRetired() {
|
|
120
|
+
document.body.classList.toggle('show-retired', showRetired);
|
|
121
|
+
document.querySelectorAll('input[data-retired]').forEach(function (c) { c.checked = showRetired; });
|
|
122
|
+
}
|
|
123
|
+
document.querySelectorAll('input[data-retired]').forEach(function (c) {
|
|
124
|
+
c.addEventListener('change', function () { showRetired = c.checked; store('bench.retired', showRetired ? '1' : '0'); applyRetired(); });
|
|
125
|
+
});
|
|
126
|
+
applyRetired();
|
|
127
|
+
|
|
128
|
+
// ---- Discover: paid models hidden unless asked for -------------------
|
|
129
|
+
var showPaid = store('bench.paid') === '1';
|
|
130
|
+
function applyPaid() {
|
|
131
|
+
document.body.classList.toggle('show-paid', showPaid);
|
|
132
|
+
document.querySelectorAll('input[data-paid]').forEach(function (c) { c.checked = showPaid; });
|
|
133
|
+
}
|
|
134
|
+
document.querySelectorAll('input[data-paid]').forEach(function (c) {
|
|
135
|
+
c.addEventListener('change', function () { showPaid = c.checked; store('bench.paid', showPaid ? '1' : '0'); applyPaid(); });
|
|
136
|
+
});
|
|
137
|
+
applyPaid();
|
|
138
|
+
|
|
139
|
+
// ---- fresh results: offered, never forced ----------------------------
|
|
140
|
+
// Not while an onboarding batch runs: each model's probe writes results, so
|
|
141
|
+
// it would be offered after every model. The first poll after the batch
|
|
142
|
+
// ends offers it once.
|
|
143
|
+
var fresh = $('fresh'), batchRunning = false;
|
|
144
|
+
setInterval(function () {
|
|
145
|
+
if (document.hidden || !fresh.hidden || batchRunning) return;
|
|
146
|
+
fetch('/api/stamp').then(function (r) { return r.json(); }).then(function (s) {
|
|
147
|
+
if (s.stamp > D.stamp + 0.5) fresh.hidden = false;
|
|
148
|
+
}).catch(function () {});
|
|
149
|
+
}, 15000);
|
|
150
|
+
fresh.addEventListener('click', function () { location.reload(); });
|
|
151
|
+
|
|
152
|
+
// ---- the model dialog ------------------------------------------------
|
|
153
|
+
var md = $('model-dialog');
|
|
154
|
+
var m = {title: $('m-title'), served: $('m-served'), editor: $('m-editor'), gone: $('m-gone'),
|
|
155
|
+
alias: $('m-alias'), role: $('m-role'), roleWhy: $('m-role-why'), adhoc: $('m-adhoc'),
|
|
156
|
+
danger: $('m-danger'), errors: $('m-errors'), done: $('m-done'), save: $('m-save'),
|
|
157
|
+
restore: $('m-restore'), close: $('m-close'), removeBox: $('m-remove-box'),
|
|
158
|
+
reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
|
|
159
|
+
qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why')};
|
|
160
|
+
m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
|
|
161
|
+
var editing = null;
|
|
162
|
+
|
|
163
|
+
function openModel(alias) {
|
|
164
|
+
var model = D.models[alias], gone = (D.retired || {})[alias];
|
|
165
|
+
if (!model && !gone) return;
|
|
166
|
+
editing = model ? alias : null;
|
|
167
|
+
m.title.textContent = alias;
|
|
168
|
+
m.errors.textContent = ''; m.done.textContent = '';
|
|
169
|
+
m.editor.hidden = !model; m.save.hidden = !model; m.removeBox.hidden = !model;
|
|
170
|
+
m.gone.hidden = !!model; m.restore.hidden = !!model;
|
|
171
|
+
showOnboardIssue(model ? (D.onboard || {})[alias] : null);
|
|
172
|
+
if (gone) {
|
|
173
|
+
m.served.textContent = 'served as ' + gone.served + (gone.tag ? ' · ' + gone.tag : '');
|
|
174
|
+
m.gone.textContent = '';
|
|
175
|
+
m.gone.appendChild(el('strong', {}, 'Retired ' + gone.at.slice(0, 10) + (gone.by === 'auto' ? ' automatically' : ' by hand')));
|
|
176
|
+
m.gone.appendChild(el('p', {style: 'margin:.3rem 0'}, gone.why || ''));
|
|
177
|
+
m.gone.appendChild(el('p', {class: 'muted', style: 'margin:0'},
|
|
178
|
+
'Its evidence is untouched. Restore puts back the same alias, role and endpoint; failures recorded before the restore are ignored.'));
|
|
179
|
+
m.restore.dataset.alias = alias; m.restore.disabled = false;
|
|
180
|
+
} else {
|
|
181
|
+
m.served.textContent = 'serves ' + model.served + (model.tag ? ' · ' + model.tag : '');
|
|
182
|
+
m.alias.value = alias;
|
|
183
|
+
m.role.textContent = '';
|
|
184
|
+
Object.keys(D.roles).forEach(function (r) {
|
|
185
|
+
var o = el('option', {value: r}, r); if (r === model.role) o.selected = true; m.role.appendChild(o);
|
|
186
|
+
});
|
|
187
|
+
// A decision model answers typed questions, never chat: its role is fixed both ways.
|
|
188
|
+
var decision = model.kind === 'decision';
|
|
189
|
+
m.role.disabled = decision;
|
|
190
|
+
m.qtypes.hidden = !decision;
|
|
191
|
+
m.qtypes.querySelectorAll('input').forEach(function (c) {
|
|
192
|
+
c.checked = decision && model.question_types.indexOf(c.value) !== -1;
|
|
193
|
+
});
|
|
194
|
+
fillFacts(alias, decision); fillOptions(alias, decision);
|
|
195
|
+
m.aliasWhy.textContent = decision
|
|
196
|
+
? 'The name decide.py and the gateway call it by, and every record is keyed by. Letters, digits, dot, dash, underscore; unused, and never a retired name.'
|
|
197
|
+
: m.aliasWhy.dataset.chat;
|
|
198
|
+
Array.prototype.forEach.call(m.role.options, function (o) {
|
|
199
|
+
o.disabled = model.kind !== 'decision' && o.value === 'decision';
|
|
200
|
+
});
|
|
201
|
+
m.adhoc.checked = !!model.ad_hoc;
|
|
202
|
+
m.save.disabled = false; m.save.textContent = 'Save';
|
|
203
|
+
m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
|
|
204
|
+
}
|
|
205
|
+
md.showModal();
|
|
206
|
+
}
|
|
207
|
+
// How its onboarding stopped (serve.onboard_issues): only while it has not
|
|
208
|
+
// answered a probe since. The error as the provider gave it, and the log.
|
|
209
|
+
var ONBOARD_NEXT = {
|
|
210
|
+
capacity: 'A limit on the provider side, not the model. It stays registered and the daily probe tries again; once it answers, run its suite.',
|
|
211
|
+
timeout: 'It did not answer within the budget. The daily probe tries again.',
|
|
212
|
+
empty: 'It answered nothing, usually an output budget spent thinking: raise Max tokens under Call options.',
|
|
213
|
+
paywalled: 'Its free tier has ended. Retire it below.',
|
|
214
|
+
structural: 'The provider has no such model now. Retire it below.',
|
|
215
|
+
entitlement: 'The account may not call it (plan, payment or permission).',
|
|
216
|
+
signin: 'The key was refused: sign in again on the Endpoints page.'
|
|
217
|
+
};
|
|
218
|
+
function showOnboardIssue(x) {
|
|
219
|
+
m.onboard.hidden = !x; m.onboard.textContent = '';
|
|
220
|
+
if (!x) return;
|
|
221
|
+
m.onboard.appendChild(el('strong', {}, 'Onboarding stopped at the ' + (x.step || 'probe') + ' on '
|
|
222
|
+
+ (x.at || '').slice(0, 10) + ': ' + x.label));
|
|
223
|
+
m.onboard.appendChild(el('p', {style: 'margin:.3rem 0'}, ONBOARD_NEXT[x.failure] || x.reading || ''));
|
|
224
|
+
if (x.detail) m.onboard.appendChild(el('pre', {class: 'log'}, x.detail));
|
|
225
|
+
if (x.log) {
|
|
226
|
+
var d = el('details'), pre = el('pre', {class: 'log'});
|
|
227
|
+
d.appendChild(el('summary', {class: 'muted'}, 'Onboarding log'));
|
|
228
|
+
d.appendChild(pre);
|
|
229
|
+
d.addEventListener('toggle', function () {
|
|
230
|
+
if (!d.open || pre.textContent) return;
|
|
231
|
+
fetch('/api/onboard?log=' + encodeURIComponent(x.log)).then(function (r) { return r.json(); })
|
|
232
|
+
.then(function (s) { pre.textContent = s.ok ? (s.tail || '(empty)') : s.errors.join(' '); });
|
|
233
|
+
});
|
|
234
|
+
m.onboard.appendChild(d);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
// ---- model facts: yours in the boxes, the sources' as placeholders -----
|
|
238
|
+
var FACT_NUM = ['context_window', 'max_output'], FACT_CAP = ['tools', 'vision', 'structured_output', 'reasoning'];
|
|
239
|
+
function fillFacts(alias, decision) {
|
|
240
|
+
var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
|
|
241
|
+
box.hidden = profile.hidden = decision || !f;
|
|
242
|
+
profile.textContent = '';
|
|
243
|
+
$('f-errors').textContent = ''; $('f-done').textContent = '';
|
|
244
|
+
if (box.hidden) return;
|
|
245
|
+
var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
|
|
246
|
+
// At a glance, as models.dev shows a model: what goes in, what comes out.
|
|
247
|
+
[['Input', 'input_modalities'], ['Output', 'output_modalities']].forEach(function (p) {
|
|
248
|
+
var row = el('div', {class: 'prof'}, null);
|
|
249
|
+
row.appendChild(el('span', {class: 'muted'}, p[0] + ' '));
|
|
250
|
+
(f[p[1]] && f[p[1]].length ? f[p[1]] : ['unknown']).forEach(function (x) { row.appendChild(el('span', {class: 'chip'}, x)); });
|
|
251
|
+
if (src[p[1]]) row.appendChild(el('span', {class: 'src'}, ' ' + src[p[1]]));
|
|
252
|
+
profile.appendChild(row);
|
|
253
|
+
});
|
|
254
|
+
var bits = [], c = f.cost || {};
|
|
255
|
+
if (f.release_date) bits.push('released ' + f.release_date);
|
|
256
|
+
if (f.knowledge_cutoff) bits.push('knowledge to ' + f.knowledge_cutoff);
|
|
257
|
+
if (f.open_weights != null) bits.push(f.open_weights ? 'open weights' : 'closed weights');
|
|
258
|
+
if (c.input != null && c.output != null) {
|
|
259
|
+
bits.push(c.input === 0 && c.output === 0 ? 'list price free'
|
|
260
|
+
: 'list price $' + c.input + ' in / $' + c.output + ' out per 1M tokens (a free tier may still apply)');
|
|
261
|
+
}
|
|
262
|
+
if (bits.length) profile.appendChild(el('div', {class: 'prof muted'}, bits.join(' · ')));
|
|
263
|
+
var show = function (k, v) { $('s-' + k).textContent = v == null ? 'unknown' : v + (src[k] ? ' · ' + src[k] : ''); };
|
|
264
|
+
FACT_NUM.forEach(function (k) {
|
|
265
|
+
$('f-' + k).value = mine(k) ? f[k] : '';
|
|
266
|
+
$('f-' + k).placeholder = f[k] != null && !mine(k) ? f[k] : '';
|
|
267
|
+
show(k, f[k] != null ? Number(f[k]).toLocaleString() : null);
|
|
268
|
+
});
|
|
269
|
+
FACT_CAP.forEach(function (k) {
|
|
270
|
+
$('f-' + k).value = mine(k) ? (f[k] ? 'yes' : 'no') : '';
|
|
271
|
+
// The blank choice is "whatever the sources say": say what that is.
|
|
272
|
+
$('f-' + k).options[0].textContent = mine(k) ? 'auto (hand back to the sources)'
|
|
273
|
+
: 'auto: ' + (f[k] == null ? 'unknown' : (f[k] ? 'yes' : 'no'));
|
|
274
|
+
show(k, f[k] == null ? null : (f[k] ? 'yes' : 'no'));
|
|
275
|
+
});
|
|
276
|
+
$('f-knowledge_cutoff').value = mine('knowledge_cutoff') ? f.knowledge_cutoff : '';
|
|
277
|
+
$('f-knowledge_cutoff').placeholder = !mine('knowledge_cutoff') && f.knowledge_cutoff ? f.knowledge_cutoff : '';
|
|
278
|
+
show('knowledge_cutoff', f.knowledge_cutoff || null);
|
|
279
|
+
var own = src.limits === 'you (this model)';
|
|
280
|
+
document.querySelectorAll('#f-limits input').forEach(function (b) {
|
|
281
|
+
var v = (f.limits || {})[b.dataset.k];
|
|
282
|
+
b.value = own && v ? v : ''; b.placeholder = !own && v ? v : '—';
|
|
283
|
+
});
|
|
284
|
+
var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
|
|
285
|
+
$('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
|
|
286
|
+
}
|
|
287
|
+
$('f-save').addEventListener('click', function () {
|
|
288
|
+
if (!editing) return;
|
|
289
|
+
var facts = {limits: {}};
|
|
290
|
+
FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
|
|
291
|
+
FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
|
|
292
|
+
if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
|
|
293
|
+
document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
|
|
294
|
+
$('f-errors').textContent = ''; $('f-done').textContent = '';
|
|
295
|
+
post('/api/model-facts', {alias: editing, facts: facts}).then(function (r) {
|
|
296
|
+
if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
|
|
297
|
+
$('f-done').textContent = 'Saved. Reloading…'; setTimeout(function () { location.reload(); }, 700);
|
|
298
|
+
});
|
|
299
|
+
});
|
|
300
|
+
// ---- call options: how the bench asks this model (serve.save_model_options)
|
|
301
|
+
function fillOptions(alias, decision) {
|
|
302
|
+
var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
|
|
303
|
+
$('m-opts').hidden = decision;
|
|
304
|
+
$('co-errors').textContent = ''; $('co-done').textContent = '';
|
|
305
|
+
$('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
|
|
306
|
+
$('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
|
|
307
|
+
}
|
|
308
|
+
$('co-save').addEventListener('click', function () {
|
|
309
|
+
if (!editing) return;
|
|
310
|
+
var v = $('co-max').value.trim();
|
|
311
|
+
$('co-errors').textContent = ''; $('co-done').textContent = '';
|
|
312
|
+
post('/api/model-options', {alias: editing, max_tokens: v ? Number(v) : null}).then(function (r) {
|
|
313
|
+
if (!r.ok) { listErrors($('co-errors'), r.errors); return; }
|
|
314
|
+
$('co-done').textContent = 'Saved' + (r.note ? '; ' + r.note : '') + '. Reloading…';
|
|
315
|
+
setTimeout(function () { location.reload(); }, 900);
|
|
316
|
+
});
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
function refresh() {
|
|
320
|
+
m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
|
|
321
|
+
m.roleWhy.textContent = D.roles[m.role.value] || '';
|
|
322
|
+
}
|
|
323
|
+
document.addEventListener('click', function (e) {
|
|
324
|
+
var a = e.target.closest('a.model');
|
|
325
|
+
if (a) { e.preventDefault(); openModel(a.dataset.alias); }
|
|
326
|
+
});
|
|
327
|
+
m.close.addEventListener('click', function () { md.close(); });
|
|
328
|
+
m.alias.addEventListener('input', refresh);
|
|
329
|
+
m.role.addEventListener('change', refresh);
|
|
330
|
+
|
|
331
|
+
// A success is said, left up for two seconds, then the popup closes and
|
|
332
|
+
// the page reloads with the change in it.
|
|
333
|
+
function closeAndReload(dialog) {
|
|
334
|
+
setTimeout(function () { dialog.close(); location.reload(); }, 2000);
|
|
335
|
+
}
|
|
336
|
+
function after(node, button, label) {
|
|
337
|
+
return function (res) {
|
|
338
|
+
if (!res.ok) { listErrors(node, res.errors); if (button) { button.disabled = false; button.textContent = label; } return; }
|
|
339
|
+
m.done.textContent = 'Done: ' + (res.changed || []).join(', ') + (res.sync ? ' · ' + res.sync : '') + '. Closing…';
|
|
340
|
+
closeAndReload(md);
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
m.save.addEventListener('click', function () {
|
|
344
|
+
if (!editing) return;
|
|
345
|
+
var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
|
|
346
|
+
if (!m.qtypes.hidden) {
|
|
347
|
+
body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
|
|
348
|
+
function (c) { return c.checked; }).map(function (c) { return c.value; });
|
|
349
|
+
}
|
|
350
|
+
(body.new_alias === editing ? Promise.resolve(true)
|
|
351
|
+
: ask('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
|
|
352
|
+
).then(function (yes) {
|
|
353
|
+
if (!yes) return;
|
|
354
|
+
m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
|
|
355
|
+
post('/api/model', body).then(after(m.errors, m.save, 'Save')).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
|
|
356
|
+
});
|
|
357
|
+
});
|
|
358
|
+
// The confirm is the last word: the popup closes at once and the page
|
|
359
|
+
// reloads when the retirement is done. A refusal reopens it with the reason.
|
|
360
|
+
m.remove.addEventListener('click', function () {
|
|
361
|
+
if (!editing) return;
|
|
362
|
+
var alias = editing;
|
|
363
|
+
ask('Retire ' + alias + '? The registry entry goes; the evidence stays, and Restore brings it back.', 'Retire', true)
|
|
364
|
+
.then(function (yes) {
|
|
365
|
+
if (!yes) return;
|
|
366
|
+
m.remove.disabled = true; m.errors.textContent = ''; md.close();
|
|
367
|
+
function refused(errors) { md.showModal(); listErrors(m.errors, errors); m.remove.disabled = false; }
|
|
368
|
+
post('/api/remove', {alias: alias, reason: m.reason.value}).then(function (res) {
|
|
369
|
+
if (res.ok) location.reload(); else refused(res.errors);
|
|
370
|
+
}).catch(function (err) { refused([String(err)]); });
|
|
371
|
+
});
|
|
372
|
+
});
|
|
373
|
+
m.restore.addEventListener('click', function () {
|
|
374
|
+
m.restore.disabled = true; m.restore.textContent = 'Restoring…';
|
|
375
|
+
post('/api/restore', {alias: m.restore.dataset.alias}).then(after(m.errors, m.restore, 'Restore')).catch(function (err) { m.errors.textContent = String(err); });
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
// ---- problem actions ---------------------------------------------------
|
|
379
|
+
document.querySelectorAll('[data-act]').forEach(function (b) {
|
|
380
|
+
b.addEventListener('click', function () {
|
|
381
|
+
var act = b.dataset.act, aliases = (b.dataset.aliases || b.dataset.alias || '').split(' ').filter(Boolean);
|
|
382
|
+
var status = b.closest('.problem').querySelector('.why');
|
|
383
|
+
if (act === 'open') { openModel(aliases[0]); return; }
|
|
384
|
+
if (act === 'retry') { // no confirm: one call, and only an answer changes anything
|
|
385
|
+
b.disabled = true; b.textContent = 'Asking…';
|
|
386
|
+
post('/api/retry', {alias: aliases[0]}).then(function (res) {
|
|
387
|
+
if (!res.ok) { status.textContent = res.errors.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = 'Retry'; return; }
|
|
388
|
+
status.textContent = res.changed.join(' · ') + '. Reloading…'; setTimeout(function () { location.reload(); }, 900);
|
|
389
|
+
}).catch(function (err) { status.textContent = String(err); b.disabled = false; b.textContent = 'Retry'; });
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
var verb = act === 'restore' ? 'Restore' : 'Retire';
|
|
393
|
+
ask(verb + ' ' + aliases.join(', ') + '?' + (act === 'restore' ? '' : ' The evidence stays and Restore brings it back.'),
|
|
394
|
+
verb, act !== 'restore').then(function (yes) {
|
|
395
|
+
if (!yes) return;
|
|
396
|
+
b.disabled = true; b.textContent = verb === 'Restore' ? 'Restoring…' : 'Retiring…';
|
|
397
|
+
var chain = Promise.resolve([]);
|
|
398
|
+
aliases.forEach(function (alias) {
|
|
399
|
+
chain = chain.then(function (errs) {
|
|
400
|
+
var call = act === 'restore' ? post('/api/restore', {alias: alias})
|
|
401
|
+
: post('/api/remove', {alias: alias, reason: 'from the dashboard: ' + (b.dataset.why || 'retired by hand')});
|
|
402
|
+
return call.then(function (res) { return errs.concat(res.ok ? [] : res.errors); });
|
|
403
|
+
});
|
|
404
|
+
});
|
|
405
|
+
chain.then(function (errs) {
|
|
406
|
+
if (errs.length) { status.textContent = errs.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = verb; return; }
|
|
407
|
+
status.textContent = 'Done. Reloading…'; setTimeout(function () { location.reload(); }, 700);
|
|
408
|
+
});
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
// ---- onboarding --------------------------------------------------------
|
|
414
|
+
var od = $('onboard-dialog');
|
|
415
|
+
var o = {find: $('o-find'), picker: $('o-picker'), errors: $('o-errors'), start: $('o-start'),
|
|
416
|
+
close: $('o-close'), choose: $('o-choose'), run: $('o-run'), steps: $('o-steps'),
|
|
417
|
+
log: $('o-log'), title: $('o-title'), result: $('o-result'), dismiss: $('o-dismiss')};
|
|
418
|
+
var STEPS = ['Served by the endpoint','Registered', 'Synced to the llm CLI', 'Availability probe, 4 calls', 'Internal suite'];
|
|
419
|
+
var timer = null, data = null;
|
|
420
|
+
// What is ticked, by the candidate's key (served@endpoint): kept while the
|
|
421
|
+
// search redraws the list, so you can search, tick, search again, tick.
|
|
422
|
+
var chosen = {};
|
|
423
|
+
// Folded groups, per browser: openrouter (direct) alone is hundreds of rows.
|
|
424
|
+
var NEW = 'New since the daily check', folded = {};
|
|
425
|
+
try { (JSON.parse(store('bench.picker.folded') || '[]') || []).forEach(function (g) { folded[g] = 1; }); } catch (e) { folded = {}; }
|
|
426
|
+
|
|
427
|
+
function armStart() {
|
|
428
|
+
var n = Object.keys(chosen).length;
|
|
429
|
+
o.start.disabled = o.dismiss.disabled = !n;
|
|
430
|
+
o.start.textContent = n > 1 ? 'Onboard ' + n + ' models' : 'Onboard model';
|
|
431
|
+
o.dismiss.textContent = n > 1 ? 'Dismiss ' + n + ' models' : 'Dismiss model';
|
|
432
|
+
}
|
|
433
|
+
// Rows move between the lists here rather than by asking every endpoint again.
|
|
434
|
+
function moveRows(keys, from, to) {
|
|
435
|
+
data[from] = data[from].filter(function (c) { if (keys.indexOf(c.key) === -1) return true; data[to].push(c); return false; });
|
|
436
|
+
data[to].sort(function (a, b) { return a.served < b.served ? -1 : a.served > b.served ? 1 : 0; });
|
|
437
|
+
}
|
|
438
|
+
function pickRow(c, dismissed) {
|
|
439
|
+
var row = el('label', {class: 'pick'});
|
|
440
|
+
var box = el('input', {type: 'checkbox', name: 'o-pick', value: c.served, 'data-endpoint': c.endpoint});
|
|
441
|
+
if (dismissed) box.disabled = true;
|
|
442
|
+
box.checked = !dismissed && !!chosen[c.key];
|
|
443
|
+
row.appendChild(box);
|
|
444
|
+
var name = el('span', {class: 'alias'}, c.served);
|
|
445
|
+
// What it takes besides text (every model takes text, so that says
|
|
446
|
+
// nothing), and whether it makes images: an image generator is no seat.
|
|
447
|
+
// One order on every row, so the rows scan alike.
|
|
448
|
+
var ORDER = ['image', 'pdf', 'file', 'audio', 'video'];
|
|
449
|
+
(c.inputs || []).filter(function (m) { return m !== 'text'; }).sort(function (a, b) {
|
|
450
|
+
return (ORDER.indexOf(a) + 1 || 99) - (ORDER.indexOf(b) + 1 || 99);
|
|
451
|
+
}).forEach(function (m) { name.appendChild(el('span', {class: 'chip mod', title: 'takes ' + m}, m)); });
|
|
452
|
+
if ((c.outputs || []).indexOf('image') !== -1) {
|
|
453
|
+
name.appendChild(el('span', {class: 'chip mod warn', title: 'an image generator: never a panel seat'}, 'makes images'));
|
|
454
|
+
}
|
|
455
|
+
row.appendChild(name);
|
|
456
|
+
var side = el('span');
|
|
457
|
+
if (c.new && !dismissed) side.appendChild(el('span', {class: 'muted'}, c.group)); // under New: where it comes from
|
|
458
|
+
if (dismissed) {
|
|
459
|
+
var b = el('button', {type: 'button', class: 'linkish', style: 'margin-left:.6rem'}, 'Bring back');
|
|
460
|
+
b.addEventListener('click', function (e) {
|
|
461
|
+
e.preventDefault();
|
|
462
|
+
post('/api/dismiss', {served: c.key, undo: true}).then(function (res) {
|
|
463
|
+
if (!res.ok) { listErrors(o.errors, res.errors); return; }
|
|
464
|
+
moveRows([c.key], 'dismissed', 'candidates'); renderPicker();
|
|
465
|
+
});
|
|
466
|
+
});
|
|
467
|
+
side.appendChild(b);
|
|
468
|
+
}
|
|
469
|
+
row.appendChild(side);
|
|
470
|
+
box.addEventListener('change', function () {
|
|
471
|
+
if (box.checked) chosen[c.key] = {served: c.served, endpoint: c.endpoint};
|
|
472
|
+
else delete chosen[c.key];
|
|
473
|
+
armStart();
|
|
474
|
+
});
|
|
475
|
+
return row;
|
|
476
|
+
}
|
|
477
|
+
function renderPicker() {
|
|
478
|
+
o.picker.textContent = '';
|
|
479
|
+
if (!data) return;
|
|
480
|
+
var q = o.find.value.trim().toLowerCase();
|
|
481
|
+
var groups = {};
|
|
482
|
+
data.candidates.forEach(function (c) {
|
|
483
|
+
// The endpoint too: "Onboard a model from it" filters on it, and the default
|
|
484
|
+
// endpoint's models are grouped by connection, not by its name.
|
|
485
|
+
if (q && [c.served, c.group, c.endpoint].every(function (s) { return s.toLowerCase().indexOf(q) === -1; })) return;
|
|
486
|
+
// New ones in a group of their own, first: just added to the hub is what
|
|
487
|
+
// you came to onboard, and it is there without scrolling or unfolding.
|
|
488
|
+
var g = c.new ? NEW : c.group;
|
|
489
|
+
(groups[g] = groups[g] || []).push(c);
|
|
490
|
+
});
|
|
491
|
+
var names = Object.keys(groups).sort(function (a, b) { return (b === NEW) - (a === NEW) || (a < b ? -1 : a > b ? 1 : 0); });
|
|
492
|
+
if (!names.length) o.picker.appendChild(el('p', {class: 'empty'}, data.candidates.length ? 'Nothing matches.' : 'Everything the endpoints serve is already on the bench, retired or dismissed.'));
|
|
493
|
+
names.forEach(function (g) {
|
|
494
|
+
// Folds, remembered per browser; a search opens every group it matches in.
|
|
495
|
+
var d = el('details', {class: 'grp'});
|
|
496
|
+
d.open = !!q || !folded[g];
|
|
497
|
+
d.appendChild(el('summary', {}, g + ' · ' + groups[g].length));
|
|
498
|
+
groups[g].forEach(function (c) { d.appendChild(pickRow(c, false)); });
|
|
499
|
+
d.addEventListener('toggle', function () {
|
|
500
|
+
if (q) return; // opened by the search, not by you
|
|
501
|
+
if (d.open) delete folded[g]; else folded[g] = 1;
|
|
502
|
+
store('bench.picker.folded', JSON.stringify(Object.keys(folded)));
|
|
503
|
+
});
|
|
504
|
+
o.picker.appendChild(d);
|
|
505
|
+
});
|
|
506
|
+
if (data.dismissed.length) {
|
|
507
|
+
var d = el('details'); d.appendChild(el('summary', {}, data.dismissed.length + ' dismissed'));
|
|
508
|
+
data.dismissed.forEach(function (c) { d.appendChild(pickRow(c, true)); });
|
|
509
|
+
o.picker.appendChild(d);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
function loadCandidates() {
|
|
513
|
+
o.picker.textContent = ''; o.picker.appendChild(el('p', {class: 'empty'}, 'Asking the endpoints what they serve…'));
|
|
514
|
+
o.start.disabled = o.dismiss.disabled = true;
|
|
515
|
+
o.find.readOnly = true; // typing before the list arrives is lost, or lands on a row
|
|
516
|
+
fetch('/api/candidates').then(function (r) { return r.json(); }).then(function (res) {
|
|
517
|
+
o.find.readOnly = false; o.find.focus();
|
|
518
|
+
if (!res.ok) { o.picker.textContent = ''; listErrors(o.errors, res.errors); return; }
|
|
519
|
+
if (res.errors.length) listErrors(o.errors, res.errors); // one endpoint down; the rest listed
|
|
520
|
+
data = res;
|
|
521
|
+
// Only what is still offered stays ticked.
|
|
522
|
+
var offered = {};
|
|
523
|
+
res.candidates.forEach(function (c) { offered[c.key] = 1; });
|
|
524
|
+
Object.keys(chosen).forEach(function (k) { if (!offered[k]) delete chosen[k]; });
|
|
525
|
+
renderPicker(); armStart();
|
|
526
|
+
}).catch(function (err) { o.find.readOnly = false; o.errors.textContent = String(err); });
|
|
527
|
+
}
|
|
528
|
+
$('onboard-open').addEventListener('click', function () {
|
|
529
|
+
preStep = null;
|
|
530
|
+
o.errors.textContent = ''; o.find.value = ''; o.choose.hidden = false; o.run.hidden = true;
|
|
531
|
+
$('o-queue').hidden = true; chosen = {}; waitToken++;
|
|
532
|
+
o.start.hidden = o.dismiss.hidden = false; armStart(); o.title.textContent = 'Onboard a model';
|
|
533
|
+
od.showModal(); o.find.focus(); loadCandidates();
|
|
534
|
+
});
|
|
535
|
+
o.find.addEventListener('input', renderPicker);
|
|
536
|
+
o.close.addEventListener('click', function () { od.close(); });
|
|
537
|
+
od.addEventListener('close', function () { if (timer) { clearInterval(timer); timer = null; } });
|
|
538
|
+
|
|
539
|
+
// Discover's Add through a hub: the hub's own steps come before onboard.py's
|
|
540
|
+
// five, as one more step at the top, {label, state: now|done|fail, note}.
|
|
541
|
+
var preStep = null;
|
|
542
|
+
function renderSteps(text, finished, exit) {
|
|
543
|
+
var reached = 0;
|
|
544
|
+
for (var i = 1; i <= 5; i++) if (text.indexOf('[' + i + '/5]') !== -1) reached = i;
|
|
545
|
+
var stopped = /STOPPING|is not served|cannot reach/.test(text) || (finished && exit !== 0);
|
|
546
|
+
var lines = text.split('\n');
|
|
547
|
+
o.steps.textContent = '';
|
|
548
|
+
if (preStep) {
|
|
549
|
+
var li0 = el('li', {class: preStep.state}, preStep.label);
|
|
550
|
+
if (preStep.note) li0.appendChild(el('small', {}, preStep.note));
|
|
551
|
+
o.steps.appendChild(li0);
|
|
552
|
+
if (preStep.state !== 'done') { // onboard.py has not started: its steps wait
|
|
553
|
+
STEPS.forEach(function (label) { o.steps.appendChild(el('li', {}, label)); });
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
STEPS.forEach(function (label, i) {
|
|
558
|
+
var n = i + 1, li = el('li', {}, label);
|
|
559
|
+
var line = lines.filter(function (l) { return l.indexOf('[' + n + '/5]') !== -1; })[0];
|
|
560
|
+
if (line) li.appendChild(el('small', {}, line.replace(/^\s*\[\d\/5\]\s*/, '')));
|
|
561
|
+
if (n < reached || (n === reached && finished && !stopped)) li.className = 'done';
|
|
562
|
+
else if (n === reached && stopped) {
|
|
563
|
+
li.className = 'fail';
|
|
564
|
+
// Why it stopped, under the step that failed, not only in the full log.
|
|
565
|
+
var at = text.search(/STOPPING|is not served|cannot reach/);
|
|
566
|
+
if (at !== -1) li.appendChild(el('small', {}, text.slice(at).split('\n').slice(0, 3).join(' ').trim()));
|
|
567
|
+
}
|
|
568
|
+
else if (n === reached || (n === 1 && !reached && !finished)) li.className = 'now';
|
|
569
|
+
o.steps.appendChild(li);
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
// Everything ticked goes to the queue (serve.queue_onboard), which runs one
|
|
573
|
+
// model at a time. The popup closes: the header's chip shows the batch, and
|
|
574
|
+
// its Details the queue. Refused, it stays open with the reasons.
|
|
575
|
+
o.start.addEventListener('click', function () {
|
|
576
|
+
var items = Object.keys(chosen).map(function (k) { return chosen[k]; });
|
|
577
|
+
if (!items.length) return;
|
|
578
|
+
o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true; o.start.textContent = 'Starting…';
|
|
579
|
+
post('/api/onboard-queue', {items: items}).then(function (res) {
|
|
580
|
+
if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
|
|
581
|
+
od.close(); oqPoll();
|
|
582
|
+
}).catch(function (err) { o.errors.textContent = String(err); armStart(); });
|
|
583
|
+
});
|
|
584
|
+
// Every ticked model at once (serve.dismiss); the dismissed list brings one back.
|
|
585
|
+
o.dismiss.addEventListener('click', function () {
|
|
586
|
+
var keys = Object.keys(chosen);
|
|
587
|
+
if (!keys.length) return;
|
|
588
|
+
o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true;
|
|
589
|
+
post('/api/dismiss', {served: keys}).then(function (res) {
|
|
590
|
+
if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
|
|
591
|
+
moveRows(keys, 'candidates', 'dismissed'); chosen = {}; renderPicker(); armStart();
|
|
592
|
+
}).catch(function (err) { o.errors.textContent = String(err); armStart(); });
|
|
593
|
+
});
|
|
594
|
+
// After Discover's Add queues a model: with nothing ahead of it, it is
|
|
595
|
+
// followed step by step, otherwise the queue shows.
|
|
596
|
+
var waitToken = 0;
|
|
597
|
+
function afterQueued(res, one, title, before) {
|
|
598
|
+
oqPoll();
|
|
599
|
+
var at = -1; // its LAST entry: an earlier, finished one may share its name
|
|
600
|
+
res.items.forEach(function (i, n) { if (one && i.served === one.served && i.endpoint === one.endpoint) at = n; });
|
|
601
|
+
var ahead = at === -1 ? [] : res.items.slice(0, at).filter(function (i) { return i.state === 'queued' || i.state === 'running'; });
|
|
602
|
+
if (one && at !== -1 && !ahead.length) waitAndFollow(one, at, title || 'Onboarding ' + one.served + ' from ' + one.endpoint, before || '', ++waitToken, 20);
|
|
603
|
+
else showQueue(res);
|
|
604
|
+
}
|
|
605
|
+
// The queue starts it within a moment; follow its log once it has one,
|
|
606
|
+
// unless the popup was closed or reopened meanwhile (the header has it then).
|
|
607
|
+
function waitAndFollow(one, at, title, before, token, tries) {
|
|
608
|
+
if (token !== waitToken || !od.open) return;
|
|
609
|
+
fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
|
|
610
|
+
if (!q.ok || token !== waitToken || !od.open) return;
|
|
611
|
+
var it = q.items[at];
|
|
612
|
+
if (!it || it.served !== one.served || it.endpoint !== one.endpoint) { showQueue(q); return; }
|
|
613
|
+
if (it.log) { follow(it.log, title, before); return; }
|
|
614
|
+
if (it.state !== 'queued' && it.state !== 'running' || tries <= 0) { showQueue(q); return; }
|
|
615
|
+
setTimeout(function () { waitAndFollow(one, at, title, before, token, tries - 1); }, 500);
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
// ---- the onboarding queue: progress in the header (serve.onboard_queue) --
|
|
620
|
+
// Polled while it runs, and once on every page load, so a batch started
|
|
621
|
+
// before a reload still shows. Its results stay until Clear.
|
|
622
|
+
var oq = {box: $('oq'), text: $('oq-text'), chip: $('oq-chip'), stop: $('oq-stop'), clear: $('oq-clear'),
|
|
623
|
+
view: $('o-queue'), sum: $('o-queue-sum'), list: $('o-queue-list')};
|
|
624
|
+
var OQ_CHIP = {queued: '', running: 'new', done: 'ok', 'no answer': 'warn', failed: 'err', skipped: ''};
|
|
625
|
+
var oqTimer = null, oqLast = '';
|
|
626
|
+
function oqCount(q, state) { return q.items.filter(function (i) { return i.state === state; }).length; }
|
|
627
|
+
function oqRender(q) {
|
|
628
|
+
var n = q.items.length;
|
|
629
|
+
oq.box.hidden = !n; batchRunning = !!q.running;
|
|
630
|
+
var now = q.items.filter(function (i) { return i.state === 'running'; })[0];
|
|
631
|
+
var finished = n - oqCount(q, 'queued') - (now ? 1 : 0);
|
|
632
|
+
// The chip fills as models finish, and by fifths of the running one as its
|
|
633
|
+
// log reaches each of onboard.py's five steps (serve.onboard_queue `step`):
|
|
634
|
+
// a step is half a fifth while it runs, so step 5/5 is 90%, and full
|
|
635
|
+
// means finished. At the end it is green when every one is on the bench,
|
|
636
|
+
// red (with one shake) when any failed or did not answer.
|
|
637
|
+
var step = now && now.step ? now.step : 0;
|
|
638
|
+
oq.chip.style.setProperty('--fill', (n ? Math.round(100 * (finished + (step ? step - 0.5 : 0) / 5) / n) : 0) + '%');
|
|
639
|
+
var bad = oqCount(q, 'failed') + oqCount(q, 'no answer');
|
|
640
|
+
oq.chip.classList.toggle('done', !q.running && n > 0 && oqCount(q, 'done') === n);
|
|
641
|
+
oq.chip.classList.toggle('fail', !q.running && bad > 0);
|
|
642
|
+
if (q.running) {
|
|
643
|
+
oq.text.textContent = (q.stopping ? 'Stopping after ' : 'Onboarding ' + (finished + 1) + ' of ' + n
|
|
644
|
+
+ (step ? ', step ' + step + '/5' : '') + ': ') + (now ? now.served : '…');
|
|
645
|
+
} else {
|
|
646
|
+
oq.text.textContent = 'Onboarded ' + oqCount(q, 'done') + ' of ' + n
|
|
647
|
+
+ [['failed', 'failed'], ['no answer', 'did not answer'], ['skipped', 'skipped']].map(function (s) {
|
|
648
|
+
var c = oqCount(q, s[0]); return c ? ' · ' + c + ' ' + s[1] : '';
|
|
649
|
+
}).join('');
|
|
650
|
+
}
|
|
651
|
+
oq.text.title = oq.text.textContent; // cut short beside the button; whole on hover
|
|
652
|
+
oq.stop.hidden = !q.running || q.stopping; oq.clear.hidden = q.running;
|
|
653
|
+
var sig = JSON.stringify(q);
|
|
654
|
+
if (sig !== oqLast) { oqLast = sig; oqLive = null; if (!oq.view.hidden) oqList(q); }
|
|
655
|
+
// The log grows between steps, which the queue's state does not show.
|
|
656
|
+
if (oqLive && !oq.view.hidden && od.open && oqLive.d.open) oqTail(oqLive.log, oqLive.pre);
|
|
657
|
+
}
|
|
658
|
+
function oqList(q) {
|
|
659
|
+
oq.sum.textContent = q.running
|
|
660
|
+
? 'One at a time, in this order. Closing this does not stop it; Stop in the header lets the model running now finish and skips the rest.'
|
|
661
|
+
: q.items.length ? 'Finished. Each log is in runs/.' : 'Nothing queued.';
|
|
662
|
+
oq.list.textContent = '';
|
|
663
|
+
q.items.forEach(function (i) {
|
|
664
|
+
var li = el('li'), head = el('div', {class: 'what'});
|
|
665
|
+
head.appendChild(el('span', {class: 'alias'}, i.served));
|
|
666
|
+
head.appendChild(el('span', {class: 'chip ' + (OQ_CHIP[i.state] || '')}, i.state));
|
|
667
|
+
if (i.alias && i.state !== 'failed') head.appendChild(el('span', {class: 'muted'}, 'as ' + i.alias));
|
|
668
|
+
li.appendChild(head);
|
|
669
|
+
if (i.note) li.appendChild(el('small', {}, i.note));
|
|
670
|
+
if (i.log) {
|
|
671
|
+
var d = el('details'), pre = el('pre', {class: 'log'});
|
|
672
|
+
d.appendChild(el('summary', {class: 'muted'}, 'Log'));
|
|
673
|
+
d.appendChild(pre);
|
|
674
|
+
d.addEventListener('toggle', function () {
|
|
675
|
+
if (i.state === 'running') { if (d.open) delete oqFolded[i.log]; else oqFolded[i.log] = 1; }
|
|
676
|
+
if (d.open) oqTail(i.log, pre);
|
|
677
|
+
});
|
|
678
|
+
// The running one's log is open and follows along (oqRender), unless
|
|
679
|
+
// you folded it; queued ones have none yet, finished ones stay folded.
|
|
680
|
+
if (i.state === 'running') { oqLive = {log: i.log, pre: pre, d: d}; d.open = !oqFolded[i.log]; }
|
|
681
|
+
li.appendChild(d);
|
|
682
|
+
}
|
|
683
|
+
oq.list.appendChild(li);
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
var oqLive = null, oqFolded = {};
|
|
687
|
+
function oqTail(log, pre) {
|
|
688
|
+
fetch('/api/onboard?log=' + encodeURIComponent(log)).then(function (r) { return r.json(); }).then(function (s) {
|
|
689
|
+
var end = pre.scrollHeight - pre.scrollTop - pre.clientHeight < 24; // at the bottom: stay there
|
|
690
|
+
pre.textContent = s.ok ? (s.tail || '(no output yet)') : s.errors.join(' ');
|
|
691
|
+
if (end) pre.scrollTop = pre.scrollHeight;
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
function showQueue(q) {
|
|
695
|
+
if (timer) { clearInterval(timer); timer = null; } // a model followed step by step: the queue view replaces it
|
|
696
|
+
waitToken++;
|
|
697
|
+
o.errors.textContent = ''; o.choose.hidden = true; o.run.hidden = true; o.start.hidden = o.dismiss.hidden = true;
|
|
698
|
+
oq.view.hidden = false; o.title.textContent = 'Onboarding queue';
|
|
699
|
+
if (!od.open) od.showModal();
|
|
700
|
+
oqLast = ''; oqRender(q);
|
|
701
|
+
}
|
|
702
|
+
function oqPoll() {
|
|
703
|
+
fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
|
|
704
|
+
if (!q.ok) return;
|
|
705
|
+
oqRender(q);
|
|
706
|
+
if (q.running && !oqTimer) oqTimer = setInterval(oqPoll, 3000);
|
|
707
|
+
if (!q.running && oqTimer) { clearInterval(oqTimer); oqTimer = null; }
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
function oqAct(path) {
|
|
711
|
+
post(path, {}).then(function (r) { if (r.ok) oqRender(r); else oq.text.textContent = r.errors.join(' '); });
|
|
712
|
+
}
|
|
713
|
+
$('oq-open').addEventListener('click', function () {
|
|
714
|
+
fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) { showQueue(q); oqPoll(); });
|
|
715
|
+
});
|
|
716
|
+
oq.stop.addEventListener('click', function () { oqAct('/api/onboard-queue/stop'); });
|
|
717
|
+
oq.clear.addEventListener('click', function () { oqAct('/api/onboard-queue/clear'); if (!oq.view.hidden) od.close(); });
|
|
718
|
+
oqPoll();
|
|
719
|
+
// ---- Discover: a row opens its model, with every route to add it by -----
|
|
720
|
+
// The rest of what is known comes from /api/discover-model, so the page does
|
|
721
|
+
// not carry 1,200 models' details it will mostly never show.
|
|
722
|
+
var dd = $('discover-dialog');
|
|
723
|
+
function price(x) {
|
|
724
|
+
var c = x.cost || {};
|
|
725
|
+
return x.free ? 'free' : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
|
|
726
|
+
}
|
|
727
|
+
function limits(l) {
|
|
728
|
+
return Object.keys(l || {}).map(function (k) { return Number(l[k]).toLocaleString() + ' ' + k; }).join(' · ');
|
|
729
|
+
}
|
|
730
|
+
function addVia(x, b) {
|
|
731
|
+
var direct = x.via === 'direct';
|
|
732
|
+
ask('Add ' + x.model + ' via ' + x.label + (x.free ? '' : ' (PAID)') + '?\n\n'
|
|
733
|
+
+ (direct ? 'The bench only: register it on the ' + x.endpoint + ' endpoint, a 4-call probe, then the suite.'
|
|
734
|
+
// A LiteLLM or CLI Proxy API route says its own steps: only what that hub takes.
|
|
735
|
+
: x.steps ? x.steps.map(function (s, i) { return (i + 1) + '. ' + s; }).join('\n')
|
|
736
|
+
: '1. Open WebUI: add it to the ' + x.connection + ' connection\'s Model IDs (if that connection lists them)\n'
|
|
737
|
+
+ '2. Open WebUI: a public model record, tagged and with the logo of the connection\'s other models\n'
|
|
738
|
+
+ '3. The bench: register, a 4-call probe, then the suite'), 'Add').then(function (yes) { if (yes) addNow(x, b, direct); });
|
|
739
|
+
}
|
|
740
|
+
function addNow(x, b, direct) {
|
|
741
|
+
var title = 'Adding ' + x.model + ' via ' + x.label;
|
|
742
|
+
$('d-errors').textContent = '';
|
|
743
|
+
if (direct) {
|
|
744
|
+
b.disabled = true; b.textContent = 'Adding…';
|
|
745
|
+
post('/api/onboard-queue', {items: [{served: x.model, endpoint: x.endpoint}]}).then(function (res) {
|
|
746
|
+
if (!res.ok) { listErrors($('d-errors'), res.errors); b.disabled = false; b.textContent = 'Add via this'; return; }
|
|
747
|
+
dd.close(); preStep = null; o.errors.textContent = ''; od.showModal();
|
|
748
|
+
afterQueued(res, {served: x.model, endpoint: x.endpoint}, title, '');
|
|
749
|
+
}).catch(function (err) { $('d-errors').textContent = String(err); b.disabled = false; b.textContent = 'Add via this'; });
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
// A hub's own steps take a while before the onboarding is even queued, so
|
|
753
|
+
// the progress starts now: the onboarding popup with the hub as its first
|
|
754
|
+
// step, and the header chip. The queue's own poll takes the chip over.
|
|
755
|
+
dd.close(); waitToken++;
|
|
756
|
+
preStep = {label: 'Added to ' + x.label, state: 'now', note: ''};
|
|
757
|
+
runView(title, 'Adding it to ' + x.label + '…');
|
|
758
|
+
oq.box.hidden = false; oq.chip.classList.remove('done', 'fail'); oq.chip.style.setProperty('--fill', '3%');
|
|
759
|
+
oq.text.textContent = oq.text.title = title; oq.stop.hidden = oq.clear.hidden = true;
|
|
760
|
+
post('/api/discover/add', {connection: x.connection, model: x.model, endpoint: x.endpoint}).then(function (res) {
|
|
761
|
+
var hubDone = !!(res.served && res.steps && res.steps.length); // in the hub, even if queueing it was refused
|
|
762
|
+
preStep.state = hubDone ? 'done' : 'fail';
|
|
763
|
+
preStep.note = (res.steps || []).join('. ') + (hubDone ? '' : (res.steps && res.steps.length ? '. ' : '') + res.errors.join(' '));
|
|
764
|
+
renderSteps('', false, null);
|
|
765
|
+
if (!res.ok) {
|
|
766
|
+
o.result.textContent = hubDone ? res.errors.join(' ') : 'Not added. Nothing was queued.';
|
|
767
|
+
o.result.className = 'errors';
|
|
768
|
+
oqPoll(); // the chip back to what the queue holds
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
afterQueued(res, {served: res.served || x.model, endpoint: res.endpoint || x.endpoint}, title, '');
|
|
772
|
+
}).catch(function (err) {
|
|
773
|
+
preStep.state = 'fail'; preStep.note = String(err); renderSteps('', false, null); oqPoll();
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
// The onboarding popup in its step-by-step view, before there is a log.
|
|
777
|
+
function runView(title, message) {
|
|
778
|
+
o.choose.hidden = true; o.run.hidden = false; o.start.hidden = o.dismiss.hidden = true; $('o-queue').hidden = true;
|
|
779
|
+
o.title.textContent = title; o.result.className = 'served'; o.result.textContent = message; o.errors.textContent = '';
|
|
780
|
+
o.log.textContent = ''; delete o.log.dataset.opened; o.log.parentNode.open = false;
|
|
781
|
+
renderSteps('', false, null);
|
|
782
|
+
if (!od.open) od.showModal();
|
|
783
|
+
}
|
|
784
|
+
function openDiscover(provider, model) {
|
|
785
|
+
$('d-errors').textContent = '';
|
|
786
|
+
fetch('/api/discover-model?provider=' + encodeURIComponent(provider) + '&model=' + encodeURIComponent(model))
|
|
787
|
+
.then(function (r) { return r.json(); }).then(function (res) {
|
|
788
|
+
if (!res.ok) { listErrors($('d-errors'), res.errors); dd.showModal(); return; }
|
|
789
|
+
if (res.errors.length) listErrors($('d-errors'), res.errors); // a hub that could not be asked
|
|
790
|
+
var r = res.model, p = $('d-profile'), list = $('d-routes');
|
|
791
|
+
$('d-title').textContent = r.name;
|
|
792
|
+
$('d-served').textContent = r.provider_name + ' · ' + r.model + ' · listed by ' + r.sources.join(' and ')
|
|
793
|
+
+ (r.on_bench_as ? ' · on the bench as ' + r.on_bench_as : '');
|
|
794
|
+
p.textContent = '';
|
|
795
|
+
[['Input', r.in_mods], ['Output', r.out_mods]].forEach(function (x) {
|
|
796
|
+
var row = el('div', {class: 'prof'});
|
|
797
|
+
row.appendChild(el('span', {class: 'muted'}, x[0] + ' '));
|
|
798
|
+
(x[1] && x[1].length ? x[1] : ['unknown']).forEach(function (v) { row.appendChild(el('span', {class: 'chip'}, v)); });
|
|
799
|
+
p.appendChild(row);
|
|
800
|
+
});
|
|
801
|
+
var can = [['tools', r.tools], ['vision', r.vision], ['reasons', r.reasoning]]
|
|
802
|
+
.filter(function (x) { return x[1]; }).map(function (x) { return x[0]; });
|
|
803
|
+
var facts = [
|
|
804
|
+
r.context ? 'context ' + Number(r.context).toLocaleString() + ' tokens' : 'context unknown',
|
|
805
|
+
r.output ? 'output up to ' + Number(r.output).toLocaleString() : '',
|
|
806
|
+
can.length ? 'can: ' + can.join(', ') : '',
|
|
807
|
+
price(r),
|
|
808
|
+
r.limits ? 'limits ' + limits(r.limits) : '',
|
|
809
|
+
r.released ? 'released ' + r.released : '',
|
|
810
|
+
r.knowledge ? 'knowledge to ' + r.knowledge : '',
|
|
811
|
+
r.open_weights != null ? (r.open_weights ? 'open weights' : 'closed weights') : '',
|
|
812
|
+
'first seen ' + (r.first_seen === 'baseline' ? 'before the list began' : r.first_seen)];
|
|
813
|
+
p.appendChild(el('div', {class: 'prof'}, facts.filter(Boolean).join(' · ')));
|
|
814
|
+
if (r.also_on && r.also_on.length) p.appendChild(el('div', {class: 'prof muted'}, 'Also on the bench as ' + r.also_on.join(', ')));
|
|
815
|
+
list.textContent = '';
|
|
816
|
+
res.routes.forEach(function (x) {
|
|
817
|
+
var row = el('div', {class: 'endpoint'}), left = el('div');
|
|
818
|
+
left.appendChild(el('div', {class: 'what'}, x.label));
|
|
819
|
+
left.appendChild(el('div', {class: 'base'}, x.model + ' · ' + price(x) + (x.limits ? ' · ' + limits(x.limits) : '')));
|
|
820
|
+
left.appendChild(el('div', {class: 'base'}, x.note));
|
|
821
|
+
row.appendChild(left);
|
|
822
|
+
var acts = el('div', {class: 'acts'});
|
|
823
|
+
if (x.on_bench_as) acts.appendChild(el('span', {class: 'chip ok'}, 'on the bench as ' + x.on_bench_as));
|
|
824
|
+
else {
|
|
825
|
+
var b = el('button', {class: 'btn small', type: 'button'}, 'Add via this');
|
|
826
|
+
b.addEventListener('click', function () { addVia(x, b); });
|
|
827
|
+
acts.appendChild(b);
|
|
828
|
+
}
|
|
829
|
+
row.appendChild(acts);
|
|
830
|
+
list.appendChild(row);
|
|
831
|
+
});
|
|
832
|
+
if (!res.routes.length) list.appendChild(el('p', {class: 'rules'}, 'None: ' + r.provider_name
|
|
833
|
+
+ ' needs a connection in a hub or an endpoint of its own first'
|
|
834
|
+
+ (r.api ? ' (its OpenAI-compatible base URL is ' + r.api + ')' : '') + '.'));
|
|
835
|
+
dd.showModal();
|
|
836
|
+
}).catch(function (err) { $('d-errors').textContent = String(err); dd.showModal(); });
|
|
837
|
+
}
|
|
838
|
+
document.addEventListener('click', function (e) {
|
|
839
|
+
var row = e.target.closest('tr.pick-row'); // the whole row: its name, its Add…, anywhere
|
|
840
|
+
if (!row) return;
|
|
841
|
+
e.preventDefault(); openDiscover(row.dataset.provider, row.dataset.model);
|
|
842
|
+
});
|
|
843
|
+
$('d-close').addEventListener('click', function () { dd.close(); });
|
|
844
|
+
function follow(log, title, before) {
|
|
845
|
+
runView(title, before + 'Running in the background; closing this does not stop it. Log: runs/' + log);
|
|
846
|
+
timer = setInterval(function () {
|
|
847
|
+
fetch('/api/onboard?log=' + encodeURIComponent(log)).then(function (r) { return r.json(); }).then(function (s) {
|
|
848
|
+
if (!s.ok) return;
|
|
849
|
+
o.log.textContent = s.tail || '(no output yet)';
|
|
850
|
+
// Open the log the first time it has something in it; after that the user decides.
|
|
851
|
+
if (s.tail && !o.log.dataset.opened) { o.log.parentNode.open = true; o.log.dataset.opened = '1'; }
|
|
852
|
+
renderSteps(s.tail || '', s.done, s.exit);
|
|
853
|
+
if (s.done) {
|
|
854
|
+
clearInterval(timer); timer = null;
|
|
855
|
+
var alias = (/->\s+alias\s+(\S+)/.exec(s.tail) || [])[1];
|
|
856
|
+
if (s.exit === 0) {
|
|
857
|
+
o.result.textContent = 'Finished. ' + (alias ? alias + ' is on the bench. ' : '') + 'The suite scores are in the dated suite note; whether to route work there is your call. Closing…';
|
|
858
|
+
o.result.className = 'done';
|
|
859
|
+
closeAndReload(od);
|
|
860
|
+
} else {
|
|
861
|
+
o.result.textContent = 'Stopped with exit ' + s.exit + '. The last lines of the log say why.';
|
|
862
|
+
fresh.hidden = false;
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
});
|
|
866
|
+
}, 3000);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
// ---- theme: applied the moment it is picked, and saved ------------------
|
|
870
|
+
// Every theme's colours are on the page (serve.brand_css), keyed by the
|
|
871
|
+
// data-theme on <html>, so switching is one attribute: no reload. The
|
|
872
|
+
// charts read their colours when drawn, so they are drawn again.
|
|
873
|
+
document.querySelectorAll('#themes input[name=theme]').forEach(function (input) {
|
|
874
|
+
input.addEventListener('change', function () {
|
|
875
|
+
document.documentElement.setAttribute('data-theme', input.value);
|
|
876
|
+
$('t-errors').textContent = ''; $('t-done').textContent = '';
|
|
877
|
+
if (typeof drawCharts === 'function') drawCharts();
|
|
878
|
+
post('/api/theme', {theme: input.value}).then(function (r) {
|
|
879
|
+
if (!r.ok) { listErrors($('t-errors'), r.errors); return; }
|
|
880
|
+
$('t-done').textContent = 'Saved: ' + input.value + ' is the theme from now on.';
|
|
881
|
+
if (location.search.indexOf('theme=') !== -1) history.replaceState(null, '', '/'); // drop a ?theme preview
|
|
882
|
+
});
|
|
883
|
+
});
|
|
884
|
+
});
|
|
885
|
+
|
|
886
|
+
// ---- gateway keys: shown once, then only their names ---------------------
|
|
887
|
+
if ($('k-make')) $('k-make').addEventListener('click', function () {
|
|
888
|
+
$('k-errors').textContent = '';
|
|
889
|
+
var scope = Array.prototype.map.call(document.querySelectorAll('input[name=k-scope]:checked'),
|
|
890
|
+
function (c) { return c.value; });
|
|
891
|
+
post('/api/gateway-key', {name: $('k-name').value, collections: scope, rpm: $('k-rpm').value}).then(function (r) {
|
|
892
|
+
if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
|
|
893
|
+
$('k-value').value = r.key;
|
|
894
|
+
$('k-done').hidden = false;
|
|
895
|
+
$('k-value').select();
|
|
896
|
+
$('k-make').disabled = true; // reload to see it listed; the value is gone then
|
|
897
|
+
});
|
|
898
|
+
});
|
|
899
|
+
document.querySelectorAll('[data-revoke-key]').forEach(function (b) {
|
|
900
|
+
b.addEventListener('click', function () {
|
|
901
|
+
var name = b.dataset.revokeKey;
|
|
902
|
+
ask('Revoke the gateway key ' + name + '? Anything using it is refused from now on.', 'Revoke', true).then(function (yes) {
|
|
903
|
+
if (!yes) return;
|
|
904
|
+
post('/api/gateway-key/remove', {name: name}).then(function (r) {
|
|
905
|
+
if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
|
|
906
|
+
location.reload();
|
|
907
|
+
});
|
|
908
|
+
});
|
|
909
|
+
});
|
|
910
|
+
});
|
|
911
|
+
|
|
912
|
+
// ---- scheduler: its mode and jobs (serve.save_scheduler) -----------------
|
|
913
|
+
var sMode = $('s-mode');
|
|
914
|
+
function modeWhy() { $('s-mode-why').textContent = sMode.selectedOptions[0].dataset.why; }
|
|
915
|
+
sMode.addEventListener('change', modeWhy); modeWhy();
|
|
916
|
+
$('s-save').addEventListener('click', function () {
|
|
917
|
+
var jobs = {};
|
|
918
|
+
document.querySelectorAll('input[name=s-job]').forEach(function (c) { jobs[c.value] = c.checked; });
|
|
919
|
+
(sMode.value === 'run' && sMode.dataset.was !== 'run'
|
|
920
|
+
? ask('Run the jobs for real? They make model calls on their own schedule. If something else'
|
|
921
|
+
+ ' also runs them (cron, Windows Task Scheduler), turn it off now, or every job runs twice.', 'Run them')
|
|
922
|
+
: Promise.resolve(true)).then(function (yes) {
|
|
923
|
+
if (!yes) return;
|
|
924
|
+
$('s-errors').textContent = ''; $('s-done').textContent = '';
|
|
925
|
+
post('/api/scheduler', {mode: sMode.value, jobs: jobs}).then(function (r) {
|
|
926
|
+
if (!r.ok) { listErrors($('s-errors'), r.errors); return; }
|
|
927
|
+
sMode.dataset.was = sMode.value; $('s-done').textContent = 'Saved. It applies within a minute.';
|
|
928
|
+
});
|
|
929
|
+
});
|
|
930
|
+
});
|
|
931
|
+
|
|
932
|
+
// ---- model client (serve.save_client): the next call uses it -------------
|
|
933
|
+
$('mc-save').addEventListener('click', function () {
|
|
934
|
+
$('mc-errors').textContent = ''; $('mc-done').textContent = '';
|
|
935
|
+
post('/api/client', {client: $('mc-client').value}).then(function (r) {
|
|
936
|
+
if (!r.ok) { listErrors($('mc-errors'), r.errors); return; }
|
|
937
|
+
$('mc-done').textContent = 'Saved. The next model call uses it.';
|
|
938
|
+
});
|
|
939
|
+
});
|
|
940
|
+
|
|
941
|
+
// ---- free only in their own app (serve.save_own_client_free) -------------
|
|
942
|
+
if ($('oc-save')) $('oc-save').addEventListener('click', function () {
|
|
943
|
+
var providers = {}, add = $('oc-add').value.trim().toLowerCase();
|
|
944
|
+
document.querySelectorAll('input[name=oc-prov]').forEach(function (c) { providers[c.value] = c.checked; });
|
|
945
|
+
if (add) providers[add] = true;
|
|
946
|
+
$('oc-errors').textContent = ''; $('oc-done').textContent = '';
|
|
947
|
+
post('/api/own-client-free', {providers: providers}).then(function (r) {
|
|
948
|
+
if (!r.ok) { listErrors($('oc-errors'), r.errors); return; }
|
|
949
|
+
if (add) { location.reload(); return; } // the new provider gets its row
|
|
950
|
+
$('oc-done').textContent = 'Saved. Discover uses it on its next load.';
|
|
951
|
+
});
|
|
952
|
+
});
|
|
953
|
+
|
|
954
|
+
// ---- endpoints: keys go in, never come back out -------------------------
|
|
955
|
+
// The Endpoints page lists them (serve.endpoints_page, each row's data in
|
|
956
|
+
// data-ep); Add an endpoint and Change key are one dialog. The page is
|
|
957
|
+
// drawn by the server, so a change reloads it, as the model popup does.
|
|
958
|
+
var ed = $('endpoint-dialog');
|
|
959
|
+
var e = {name: $('e-name'), base: $('e-base'), key: $('e-key'),
|
|
960
|
+
kind: $('e-kind'), hub: $('e-hub'), admin: $('e-admin'),
|
|
961
|
+
errors: $('e-errors'), done: $('e-done'), test: $('e-test'), save: $('e-save'),
|
|
962
|
+
listErrors: $('el-errors'), listDone: $('el-done'), changed: false};
|
|
963
|
+
e.kind.addEventListener('change', function () {
|
|
964
|
+
$('e-hub-row').hidden = $('e-admin-row').hidden = e.kind.value !== 'hub';
|
|
965
|
+
});
|
|
966
|
+
ed.querySelectorAll('[data-close]').forEach(function (b) { b.addEventListener('click', function () { ed.close(); }); });
|
|
967
|
+
ed.addEventListener('close', function () { if (e.changed) location.reload(); });
|
|
968
|
+
function reloadSoon(message) {
|
|
969
|
+
e.listErrors.textContent = ''; e.listDone.textContent = message + ' Reloading…';
|
|
970
|
+
setTimeout(function () { location.reload(); }, 900);
|
|
971
|
+
}
|
|
972
|
+
var rowActs = {
|
|
973
|
+
change: function (x) { openEndpoint('change', x); },
|
|
974
|
+
again: function (x) { // this endpoint, not a new one
|
|
975
|
+
openEndpoint('again', x);
|
|
976
|
+
connectStart({id: x.connect.provider, flow: x.connect.flow, label: x.connect.label}, x.name);
|
|
977
|
+
},
|
|
978
|
+
'default': function (x) {
|
|
979
|
+
ask('Make ' + x.name + ' the default endpoint? The default is where a model is taken'
|
|
980
|
+
+ ' to be when nothing says otherwise, and the one endpoint that cannot be removed.', 'Make default').then(function (yes) {
|
|
981
|
+
if (!yes) return;
|
|
982
|
+
post('/api/endpoint/default', {name: x.name}).then(function (r) {
|
|
983
|
+
if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
|
|
984
|
+
reloadSoon(r.note + '.');
|
|
985
|
+
});
|
|
986
|
+
});
|
|
987
|
+
},
|
|
988
|
+
// Its models go with it, after saying how many: each is retired as by
|
|
989
|
+
// hand (evidence kept, restorable once the endpoint is back).
|
|
990
|
+
remove: function (x) {
|
|
991
|
+
var n = x.models, them = n + ' model' + (n === 1 ? '' : 's');
|
|
992
|
+
ask(n ? 'Remove the ' + x.name + ' endpoint and the ' + them + ' on the bench through it?\n\n'
|
|
993
|
+
+ 'The ' + them + ' will be retired too. Their results and history are kept, and'
|
|
994
|
+
+ ' Restore brings them back if an endpoint named ' + x.name + ' is added again.'
|
|
995
|
+
: 'Remove the ' + x.name + ' endpoint?', 'Remove', true).then(function (yes) {
|
|
996
|
+
if (!yes) return;
|
|
997
|
+
post('/api/endpoint/remove', {name: x.name, with_models: n > 0}).then(function (r) {
|
|
998
|
+
if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
|
|
999
|
+
reloadSoon('Removed ' + x.name + '; ' + r.note + '.');
|
|
1000
|
+
});
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
};
|
|
1004
|
+
$('tab-endpoints').addEventListener('click', function (ev) {
|
|
1005
|
+
var b = ev.target.closest('.ep button[data-ep-act]');
|
|
1006
|
+
if (b) rowActs[b.dataset.epAct](JSON.parse(b.closest('.ep').dataset.ep));
|
|
1007
|
+
});
|
|
1008
|
+
function loadLimits() {
|
|
1009
|
+
var list = $('l-list');
|
|
1010
|
+
fetch('/api/connections').then(function (r) { return r.json(); }).then(function (res) {
|
|
1011
|
+
list.textContent = '';
|
|
1012
|
+
res.connections.forEach(function (c) {
|
|
1013
|
+
var row = el('div', {class: 'endpoint'});
|
|
1014
|
+
var left = el('div');
|
|
1015
|
+
left.appendChild(el('div', {class: 'what'}, c.name));
|
|
1016
|
+
left.appendChild(el('div', {class: 'base'}, c.models.join(', ')));
|
|
1017
|
+
var inputs = el('div', {class: 'limits'});
|
|
1018
|
+
res.keys.forEach(function (k) {
|
|
1019
|
+
var lab = el('label', {}, k + ' ');
|
|
1020
|
+
var box = el('input', {type: 'number', min: '1', 'data-k': k, placeholder: '—'});
|
|
1021
|
+
if (c.limits[k]) box.value = c.limits[k];
|
|
1022
|
+
lab.appendChild(box); inputs.appendChild(lab);
|
|
1023
|
+
});
|
|
1024
|
+
left.appendChild(inputs);
|
|
1025
|
+
row.appendChild(left);
|
|
1026
|
+
var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
|
|
1027
|
+
save.addEventListener('click', function () {
|
|
1028
|
+
var limits = {};
|
|
1029
|
+
inputs.querySelectorAll('input').forEach(function (b) { if (b.value) limits[b.dataset.k] = parseInt(b.value, 10); });
|
|
1030
|
+
$('l-errors').textContent = ''; $('l-done').textContent = '';
|
|
1031
|
+
post('/api/connection-limits', {connection: c.name, limits: limits}).then(function (r) {
|
|
1032
|
+
if (!r.ok) { listErrors($('l-errors'), r.errors); return; }
|
|
1033
|
+
$('l-done').textContent = 'Saved limits for ' + c.name + '.';
|
|
1034
|
+
});
|
|
1035
|
+
});
|
|
1036
|
+
var acts = el('div', {class: 'acts'}); acts.appendChild(save); row.appendChild(acts);
|
|
1037
|
+
list.appendChild(row);
|
|
1038
|
+
});
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
onShow.endpoints = function () { // its own-app and rate-limit sections
|
|
1042
|
+
$('cn-app-errors').textContent = $('cn-app-done').textContent = '';
|
|
1043
|
+
loadProviders(); loadLimits();
|
|
1044
|
+
};
|
|
1045
|
+
// The dialog, in one of three modes. add: the choices, API key first, then
|
|
1046
|
+
// each provider's sign-in; change: one endpoint's key form; again: a
|
|
1047
|
+
// sign-in under way for an endpoint that has one.
|
|
1048
|
+
function openEndpoint(mode, x) {
|
|
1049
|
+
cn.run++;
|
|
1050
|
+
e.adding = mode === 'add'; // Add refuses a name in use; Change key edits that one
|
|
1051
|
+
$('e-title').textContent = mode === 'change' ? 'Change the key for ' + x.name
|
|
1052
|
+
: mode === 'again' ? 'Sign in to ' + x.connect.label + ' again' : 'Add an endpoint';
|
|
1053
|
+
$('e-blurb').hidden = cn.list.hidden = $('e-kinds').hidden = !e.adding;
|
|
1054
|
+
e.key.placeholder = e.adding ? 'blank if it needs none, as a local runtime' : 'leave blank to keep the current one';
|
|
1055
|
+
cn.wait.hidden = true; $('e-after').hidden = true;
|
|
1056
|
+
resetKeyForm();
|
|
1057
|
+
choose(mode === 'again' ? null : 'key');
|
|
1058
|
+
if (mode === 'change') {
|
|
1059
|
+
e.name.value = x.name; e.base.value = x.base; e.name.readOnly = true;
|
|
1060
|
+
$('e-admin-row').hidden = !x.hub; // a hub's admin key changes here too
|
|
1061
|
+
}
|
|
1062
|
+
if (e.adding) loadProviders();
|
|
1063
|
+
if (!ed.open) ed.showModal();
|
|
1064
|
+
(mode === 'change' ? e.key : e.name).focus();
|
|
1065
|
+
}
|
|
1066
|
+
// Which form shows: 'key' (paste an API key), a provider (sign in), or none.
|
|
1067
|
+
function choose(p) {
|
|
1068
|
+
cn.pick = p && p !== 'key' ? p : null;
|
|
1069
|
+
$('e-form').hidden = p !== 'key';
|
|
1070
|
+
$('cn-form').hidden = !cn.pick;
|
|
1071
|
+
if (cn.pick) {
|
|
1072
|
+
$('cn-name').value = cn.pick.suggest;
|
|
1073
|
+
$('cn-go').textContent = (cn.pick.flow === 'local' ? 'Add ' : 'Sign in to ') + cn.pick.label;
|
|
1074
|
+
}
|
|
1075
|
+
e.errors.textContent = ''; e.done.textContent = '';
|
|
1076
|
+
press();
|
|
1077
|
+
}
|
|
1078
|
+
function press() {
|
|
1079
|
+
var on = cn.pick ? cn.pick.id : $('e-form').hidden ? '' : 'key';
|
|
1080
|
+
cn.list.querySelectorAll('button').forEach(function (b) { b.setAttribute('aria-pressed', String(b.dataset.choice === on)); });
|
|
1081
|
+
}
|
|
1082
|
+
function resetKeyForm() {
|
|
1083
|
+
e.name.value = e.base.value = e.key.value = e.admin.value = ''; e.name.readOnly = false;
|
|
1084
|
+
e.kind.value = 'provider'; e.hub.selectedIndex = 0; $('e-hub-row').hidden = $('e-admin-row').hidden = true;
|
|
1085
|
+
untested();
|
|
1086
|
+
}
|
|
1087
|
+
// Save opens only after Test passes, and closes again when anything in the
|
|
1088
|
+
// form changes. The server tests again on Save: this is the form's manners,
|
|
1089
|
+
// not the guarantee.
|
|
1090
|
+
function untested() { e.save.disabled = true; e.save.title = 'Test it first'; }
|
|
1091
|
+
$('e-form').addEventListener('input', function () { untested(); e.done.textContent = ''; });
|
|
1092
|
+
function keyBody() {
|
|
1093
|
+
var body = {name: e.name.value, base: e.base.value, key: e.key.value, new: e.adding,
|
|
1094
|
+
admin: $('e-admin-row').hidden ? '' : e.admin.value};
|
|
1095
|
+
if (e.adding) { body.kind = e.kind.value; body.hub = e.kind.value === 'hub' ? e.hub.value : ''; } // Change key keeps its kind
|
|
1096
|
+
return body;
|
|
1097
|
+
}
|
|
1098
|
+
$('e-add').addEventListener('click', function () { openEndpoint('add'); });
|
|
1099
|
+
e.test.addEventListener('click', function () {
|
|
1100
|
+
e.errors.textContent = ''; e.done.textContent = ''; untested();
|
|
1101
|
+
e.test.disabled = true; e.test.textContent = 'Testing…';
|
|
1102
|
+
var body = keyBody(), sent = JSON.stringify(body), run = cn.run; body.test = true;
|
|
1103
|
+
function end() { e.test.disabled = false; e.test.textContent = 'Test'; }
|
|
1104
|
+
post('/api/endpoint', body).then(function (r) {
|
|
1105
|
+
end();
|
|
1106
|
+
if (run !== cn.run || JSON.stringify(keyBody()) !== sent) return; // opened afresh, or edited, meanwhile
|
|
1107
|
+
if (!r.ok) { listErrors(e.errors, r.errors); return; }
|
|
1108
|
+
e.done.textContent = 'It answers: ' + r.served + ' models listed. ' + (r.note || '');
|
|
1109
|
+
e.save.disabled = false; e.save.title = '';
|
|
1110
|
+
}).catch(function (err) { end(); e.errors.textContent = String(err); });
|
|
1111
|
+
});
|
|
1112
|
+
e.save.addEventListener('click', function () {
|
|
1113
|
+
e.errors.textContent = ''; e.save.disabled = e.test.disabled = true; e.save.textContent = 'Saving…';
|
|
1114
|
+
function end() { e.save.textContent = 'Save'; e.test.disabled = false; }
|
|
1115
|
+
post('/api/endpoint', keyBody()).then(function (r) {
|
|
1116
|
+
end();
|
|
1117
|
+
if (!r.ok) { listErrors(e.errors, r.errors); return; } // Save stays shut: test again
|
|
1118
|
+
var name = e.name.value.trim().toLowerCase();
|
|
1119
|
+
endpointDone('Saved. ' + name + ' lists ' + r.served + ' models; they appear in the onboarding picker. '
|
|
1120
|
+
+ (r.note || ''), e.adding ? name : '');
|
|
1121
|
+
}).catch(function (err) { end(); e.errors.textContent = String(err); });
|
|
1122
|
+
});
|
|
1123
|
+
// Saved or signed in: the forms fold, the message stays, with the next step.
|
|
1124
|
+
function endpointDone(message, endpoint) {
|
|
1125
|
+
resetKeyForm(); choose(null);
|
|
1126
|
+
cn.list.hidden = $('e-blurb').hidden = cn.wait.hidden = true; $('e-after').hidden = false;
|
|
1127
|
+
e.done.textContent = message;
|
|
1128
|
+
if (endpoint) e.done.appendChild(onboardFrom(endpoint));
|
|
1129
|
+
e.changed = true; // the page reloads when the dialog closes
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// ---- connect: sign in on the provider's page (metrics/connect.py) --------
|
|
1133
|
+
// `run` counts the dialog's openings: a sign-in still finishing after the
|
|
1134
|
+
// dialog was closed or opened again reports on the Endpoints page instead,
|
|
1135
|
+
// and reloads it once nothing is open over it.
|
|
1136
|
+
var cn = {list: $('cn-list'), wait: $('cn-wait'), msg: $('cn-msg'), timer: null, run: 0};
|
|
1137
|
+
// Reached from another machine (a hosted install): a callback to this
|
|
1138
|
+
// machine's 127.0.0.1 could not arrive, so sign-ins go by code instead.
|
|
1139
|
+
var remote = !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(location.hostname);
|
|
1140
|
+
function connectDone(r, run) {
|
|
1141
|
+
clearInterval(cn.timer); cn.timer = null;
|
|
1142
|
+
var said = r.ok && r.label + ' is connected as ' + r.endpoint + ': it lists ' + r.served + ' models. ' + (r.note || '');
|
|
1143
|
+
if (!ed.open || run !== cn.run) {
|
|
1144
|
+
if (!r.ok) listErrors(e.listErrors, r.errors);
|
|
1145
|
+
else if (ed.open) { e.listDone.textContent = said; e.changed = true; }
|
|
1146
|
+
else reloadSoon(said);
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
cn.wait.hidden = true;
|
|
1150
|
+
if (!r.ok) { // back to the provider's form to try again, or just Close
|
|
1151
|
+
$('cn-form').hidden = !cn.pick; $('e-after').hidden = !!cn.pick;
|
|
1152
|
+
listErrors(e.errors, r.errors); return;
|
|
1153
|
+
}
|
|
1154
|
+
endpointDone(said, r.endpoint);
|
|
1155
|
+
}
|
|
1156
|
+
// The next step for a new endpoint, signed in or pasted: onboard what it serves.
|
|
1157
|
+
function onboardFrom(endpoint) {
|
|
1158
|
+
var go = el('button', {class: 'btn small', type: 'button', style: 'margin-left:.6rem'}, 'Onboard a model from it');
|
|
1159
|
+
go.addEventListener('click', function () {
|
|
1160
|
+
// No reload on the way: onboarding opens over the page, and the page's
|
|
1161
|
+
// fresh-results button offers the reload once the files have changed.
|
|
1162
|
+
e.changed = false; ed.close(); $('onboard-open').click(); o.find.value = endpoint; // the picker filters on it once loaded
|
|
1163
|
+
});
|
|
1164
|
+
return go;
|
|
1165
|
+
}
|
|
1166
|
+
function connectStart(p, endpoint) {
|
|
1167
|
+
e.errors.textContent = ''; e.done.textContent = '';
|
|
1168
|
+
var run = cn.run;
|
|
1169
|
+
// Opened now, while this is still the click: a window opened after the
|
|
1170
|
+
// request returns is a popup the browser blocks.
|
|
1171
|
+
var byCode = remote && p.flow !== 'local';
|
|
1172
|
+
var tab = !byCode && (p.flow === 'key' || p.flow === 'oauth') ? window.open('', '_blank') : null;
|
|
1173
|
+
if (tab) tab.opener = null;
|
|
1174
|
+
post('/api/connect/start', {provider: p.id, endpoint: endpoint, code: byCode}).then(function (r) {
|
|
1175
|
+
if (!r.ok || r.state) { if (tab) tab.close(); connectDone(r, run); return; } // refused, or local: saved at once
|
|
1176
|
+
cn.wait.hidden = false; cn.msg.textContent = ''; $('e-after').hidden = false;
|
|
1177
|
+
if (r.url) {
|
|
1178
|
+
if (tab) tab.location = r.url; else window.open(r.url, '_blank', 'noopener');
|
|
1179
|
+
cn.msg.textContent = 'Sign in to ' + p.label + ' in the tab that opened. This updates when it is done.';
|
|
1180
|
+
} else {
|
|
1181
|
+
cn.msg.appendChild(document.createTextNode('Open '));
|
|
1182
|
+
cn.msg.appendChild(el('a', {href: r.verify, target: '_blank', rel: 'noopener'}, r.verify));
|
|
1183
|
+
cn.msg.appendChild(document.createTextNode(' and enter the code '));
|
|
1184
|
+
cn.msg.appendChild(el('strong', {class: 'mono'}, r.code));
|
|
1185
|
+
}
|
|
1186
|
+
var tries = 0;
|
|
1187
|
+
clearInterval(cn.timer);
|
|
1188
|
+
cn.timer = setInterval(function () {
|
|
1189
|
+
if (++tries > 450) { connectDone({ok: false, errors: ['No sign-in after fifteen minutes; start again.']}, run); return; }
|
|
1190
|
+
post('/api/connect/poll', {id: r.id}).then(function (s) { if (s.state !== 'pending') connectDone(s, run); });
|
|
1191
|
+
}, 2000);
|
|
1192
|
+
}).catch(function (err) { if (tab) tab.close(); e.errors.textContent = String(err); });
|
|
1193
|
+
}
|
|
1194
|
+
// The dialog's choices (API key, then each provider) and the Manage page's
|
|
1195
|
+
// own-app rows, from one listing.
|
|
1196
|
+
function loadProviders() {
|
|
1197
|
+
fetch('/api/connect').then(function (r) { return r.json(); }).then(function (res) {
|
|
1198
|
+
cn.list.textContent = ''; $('cn-apps').textContent = '';
|
|
1199
|
+
choice('key', 'API key', '').addEventListener('click', function () { choose('key'); e.name.focus(); });
|
|
1200
|
+
res.providers.forEach(function (p) {
|
|
1201
|
+
// A new endpoint, named first (a provider can be several: a pasted key
|
|
1202
|
+
// and a sign-in, or two accounts). Signing an existing one in again is
|
|
1203
|
+
// its row's button.
|
|
1204
|
+
var why = p.missing || (remote && !p.code ? p.label + ' signs in only through a page on the machine'
|
|
1205
|
+
+ ' Benchwarmers runs on; from here, choose API key' : '');
|
|
1206
|
+
choice(p.id, p.label, why).addEventListener('click', function () { choose(p); $('cn-name').select(); });
|
|
1207
|
+
if (p.register) $('cn-apps').appendChild(appRow(p));
|
|
1208
|
+
});
|
|
1209
|
+
press();
|
|
1210
|
+
});
|
|
1211
|
+
}
|
|
1212
|
+
function choice(id, label, why) {
|
|
1213
|
+
var b = el('button', {class: 'btn', type: 'button', 'data-choice': id, 'aria-pressed': 'false'}, label);
|
|
1214
|
+
if (why) { b.disabled = true; b.title = why; }
|
|
1215
|
+
cn.list.appendChild(b);
|
|
1216
|
+
return b;
|
|
1217
|
+
}
|
|
1218
|
+
// The Sign in click itself opens the provider's tab (connectStart), so the
|
|
1219
|
+
// browser does not block it as a popup.
|
|
1220
|
+
$('cn-go').addEventListener('click', function () { $('cn-form').hidden = true; connectStart(cn.pick, $('cn-name').value); });
|
|
1221
|
+
// Use your own app: a provider's own client id, instead of the shipped one.
|
|
1222
|
+
function appRow(p) {
|
|
1223
|
+
var row = el('div', {class: 'endpoint'});
|
|
1224
|
+
var left = el('div');
|
|
1225
|
+
left.appendChild(el('div', {class: 'what'}, p.label));
|
|
1226
|
+
left.appendChild(el('div', {class: 'base'}, 'Register ' + p.register));
|
|
1227
|
+
var box = el('input', {spellcheck: 'false', autocomplete: 'off', 'aria-label': p.label + ' client id',
|
|
1228
|
+
placeholder: p.shipped ? "Wendell Research's app (leave blank)" : 'client id, needed to sign in'});
|
|
1229
|
+
box.value = p.own_id || '';
|
|
1230
|
+
var line = el('div', {class: 'row-input'}); // the box and its Save on one line
|
|
1231
|
+
line.appendChild(box);
|
|
1232
|
+
left.appendChild(line);
|
|
1233
|
+
row.appendChild(left);
|
|
1234
|
+
var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
|
|
1235
|
+
save.addEventListener('click', function () {
|
|
1236
|
+
$('cn-app-errors').textContent = ''; $('cn-app-done').textContent = '';
|
|
1237
|
+
post('/api/connect/app', {provider: p.id, client_id: box.value}).then(function (r) {
|
|
1238
|
+
if (!r.ok) { listErrors($('cn-app-errors'), r.errors); return; }
|
|
1239
|
+
$('cn-app-done').textContent = box.value.trim() ? p.label + ' signs in with your app now.'
|
|
1240
|
+
: p.label + (p.shipped ? " is back to Wendell Research's app." : ' has no app until one is added.');
|
|
1241
|
+
loadProviders();
|
|
1242
|
+
});
|
|
1243
|
+
});
|
|
1244
|
+
line.appendChild(save);
|
|
1245
|
+
return row;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
if ($('tab-endpoints').classList.contains('active')) onShow.endpoints(); // the page opened on it
|
|
1249
|
+
|
|
1250
|
+
// ---- first run: no endpoint yet, no model, no collection (serve.first_run) -
|
|
1251
|
+
var first = $('first-run');
|
|
1252
|
+
function firstStep() {
|
|
1253
|
+
if (first.dataset.step === 'onboard') { $('onboard-open').click(); return; }
|
|
1254
|
+
if (first.dataset.step === 'collection') { show('collections'); $('c-new').click(); return; }
|
|
1255
|
+
show('endpoints'); openEndpoint('add');
|
|
1256
|
+
}
|
|
1257
|
+
if (first) {
|
|
1258
|
+
$('first-go').addEventListener('click', firstStep);
|
|
1259
|
+
if (first.dataset.step === 'endpoint') firstStep(); // straight into Add: an empty bench offers nothing else
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
// ---- collections: what the bench gateway runs together (its own tab) -----
|
|
1263
|
+
var c = {list: $('c-list'), name: $('c-name'), members: $('c-members'), count: $('c-count'),
|
|
1264
|
+
shape: $('c-shape'), desc: $('c-desc'), tags: $('c-tags'), errors: $('c-errors'), done: $('c-done'),
|
|
1265
|
+
head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
|
|
1266
|
+
gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
|
|
1267
|
+
del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter')};
|
|
1268
|
+
var C = null, open = null; // the collection whose details are showing
|
|
1269
|
+
function explorersFor(n) {
|
|
1270
|
+
return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
|
|
1271
|
+
}
|
|
1272
|
+
function picked() {
|
|
1273
|
+
return Array.prototype.filter.call(c.members.querySelectorAll('input'), function (x) { return x.checked; })
|
|
1274
|
+
.map(function (x) { return x.value; });
|
|
1275
|
+
}
|
|
1276
|
+
function recount() {
|
|
1277
|
+
var n = picked().length, ex = explorersFor(n);
|
|
1278
|
+
var rows = c.members.querySelectorAll('label.check'), shown = c.members.querySelectorAll('label.check:not([hidden])');
|
|
1279
|
+
var greyed = c.members.querySelectorAll('label.check.blocked:not(.clash):not([hidden])').length;
|
|
1280
|
+
var clash = c.members.querySelectorAll('label.check.clash').length;
|
|
1281
|
+
var paid = c.members.querySelectorAll('label.check[data-paid="1"][hidden]').length;
|
|
1282
|
+
c.count.textContent = n + ' member' + (n === 1 ? '' : 's') + ' + ' + ex + ' explorer' + (ex === 1 ? '' : 's') +
|
|
1283
|
+
' = ' + (n + ex) + ' calls per request' + (n && (n < 2 || n > 4) ? ' · two to four members is recommended' : '') +
|
|
1284
|
+
(shown.length < rows.length ? ' · showing ' + shown.length + ' of ' + rows.length + ' models' : '') +
|
|
1285
|
+
(paid ? ' (' + paid + ' paid hidden by the free tag)' : '') +
|
|
1286
|
+
(greyed ? ' · ' + greyed + ' greyed out by the tags (hover one for why)' : '') +
|
|
1287
|
+
(clash ? ' · ' + clash + ' ticked member' + (clash === 1 ? ' does' : 's do') + ' not fit the tags' : '');
|
|
1288
|
+
c.members.querySelectorAll('input').forEach(function (x) {
|
|
1289
|
+
x.disabled = !x.checked && (n >= C.settings.max_calls || !!x.dataset.blocked);
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
// Tags that say what every member must do: why a model does not fit them
|
|
1293
|
+
// (none: it fits). From collections_view's facts; an unknown context is
|
|
1294
|
+
// let through, an unknown capability is not, as the gateway's vision seats.
|
|
1295
|
+
var CAP_TAGS = {vision: ['vision', 'take images'], tools: ['tools', 'call tools'], json: ['structured_output', 'give structured output']};
|
|
1296
|
+
function tagBlocks(alias) {
|
|
1297
|
+
var f = (C.facts || {})[alias], why = [];
|
|
1298
|
+
if (!f) return why;
|
|
1299
|
+
if (f.role === 'image') why.push('an image generator: it never answers a panel');
|
|
1300
|
+
tagList().forEach(function (t) {
|
|
1301
|
+
var cap = CAP_TAGS[t], roles = (C.shape_roles || {})[t];
|
|
1302
|
+
if (cap && !f.caps[cap[0]]) {
|
|
1303
|
+
why.push(f.caps[cap[0]] === false ? 'does not ' + cap[1]
|
|
1304
|
+
: 'the catalog does not say it can ' + cap[1] + '; tick it in its model popup if it does');
|
|
1305
|
+
}
|
|
1306
|
+
else if (t === 'long-context' && f.context && f.context < C.long_context) {
|
|
1307
|
+
why.push('holds ' + Math.round(f.context / 1000) + 'k tokens, under ' + Math.round(C.long_context / 1000) + 'k');
|
|
1308
|
+
}
|
|
1309
|
+
// general admits only the general role, but as a tag it means "for anything".
|
|
1310
|
+
else if (roles && t !== 'general' && roles.indexOf(f.role) === -1) {
|
|
1311
|
+
why.push('role ' + f.role + ': not for ' + t + ' tasks');
|
|
1312
|
+
}
|
|
1313
|
+
});
|
|
1314
|
+
return why;
|
|
1315
|
+
}
|
|
1316
|
+
// Greyed out, not hidden, so you see what the tags rule out; a ticked one
|
|
1317
|
+
// stays tickable (marked) so it can be taken off. `free` hides the paid ones.
|
|
1318
|
+
function applyTags() {
|
|
1319
|
+
var free = tagList().indexOf('free') !== -1;
|
|
1320
|
+
c.members.querySelectorAll('label.check').forEach(function (lab) {
|
|
1321
|
+
var box = lab.querySelector('input'), why = tagBlocks(box.value);
|
|
1322
|
+
var paid = free && ((C.facts || {})[box.value] || {}).price === 'paid';
|
|
1323
|
+
if (paid) why.push('paid, and the collection is tagged free');
|
|
1324
|
+
lab.classList.toggle('blocked', why.length > 0);
|
|
1325
|
+
lab.classList.toggle('clash', why.length > 0 && box.checked);
|
|
1326
|
+
lab.dataset.paid = paid ? '1' : '';
|
|
1327
|
+
if (why.length) box.dataset.blocked = '1'; else delete box.dataset.blocked;
|
|
1328
|
+
var served = lab.dataset.served;
|
|
1329
|
+
lab.title = (served && served !== box.value ? served : '') + (why.length ? (served && served !== box.value ? '\n' : '') + why.join('\n') : '');
|
|
1330
|
+
});
|
|
1331
|
+
filterMembers();
|
|
1332
|
+
}
|
|
1333
|
+
// The member search: the filter boxes' match (any text, case-insensitive),
|
|
1334
|
+
// over alias and served name. It only hides rows, and never a ticked one, so
|
|
1335
|
+
// what is picked is always in view and every count still includes it.
|
|
1336
|
+
function filterMembers() {
|
|
1337
|
+
var q = c.find.value.trim().toLowerCase();
|
|
1338
|
+
c.members.querySelectorAll('label.check').forEach(function (lab) {
|
|
1339
|
+
var box = lab.querySelector('input');
|
|
1340
|
+
lab.hidden = !box.checked && (!!lab.dataset.paid ||
|
|
1341
|
+
!!q && (box.value + ' ' + (lab.dataset.served || '')).toLowerCase().indexOf(q) === -1);
|
|
1342
|
+
});
|
|
1343
|
+
recount();
|
|
1344
|
+
}
|
|
1345
|
+
c.find.addEventListener('input', filterMembers);
|
|
1346
|
+
// Escape in a box with text clears it; only an empty box lets Escape close the popup.
|
|
1347
|
+
c.find.addEventListener('keydown', function (e) {
|
|
1348
|
+
if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
|
|
1349
|
+
});
|
|
1350
|
+
// Suggested tags under the field: a click adds or removes one, and the
|
|
1351
|
+
// field stays free for any other.
|
|
1352
|
+
function tagList() {
|
|
1353
|
+
return c.tags.value.split(',').map(function (t) { return t.trim().toLowerCase(); }).filter(Boolean);
|
|
1354
|
+
}
|
|
1355
|
+
function drawTagPicks() {
|
|
1356
|
+
var box = $('c-tag-picks'), have = tagList();
|
|
1357
|
+
box.textContent = '';
|
|
1358
|
+
(C.tag_suggestions || []).forEach(function (t) {
|
|
1359
|
+
var on = have.indexOf(t) !== -1;
|
|
1360
|
+
var b = el('button', {type: 'button', class: 'chip' + (on ? ' ok' : ''), 'aria-pressed': String(on)}, (on ? '✓ ' : '+ ') + t);
|
|
1361
|
+
b.addEventListener('click', function () {
|
|
1362
|
+
var now = tagList(), i = now.indexOf(t);
|
|
1363
|
+
if (i === -1) now.push(t); else now.splice(i, 1);
|
|
1364
|
+
c.tags.value = now.join(', '); drawTagPicks(); applyTags();
|
|
1365
|
+
});
|
|
1366
|
+
box.appendChild(b);
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
c.tags.addEventListener('input', function () { drawTagPicks(); applyTags(); });
|
|
1370
|
+
function showMerger() { $('c-merger-row').hidden = c.reply.value !== 'merged'; }
|
|
1371
|
+
c.reply.addEventListener('change', showMerger);
|
|
1372
|
+
// A collection opens in a popup, as every other edit on the dashboard does:
|
|
1373
|
+
// the one clicked, or a new one from New collection.
|
|
1374
|
+
function closeForm() {
|
|
1375
|
+
if (c.form.open) c.form.close();
|
|
1376
|
+
open = null;
|
|
1377
|
+
}
|
|
1378
|
+
c.form.addEventListener('close', function () { open = null; });
|
|
1379
|
+
function fillForm(row) {
|
|
1380
|
+
c.errors.textContent = ''; c.done.textContent = ''; c.status.textContent = '';
|
|
1381
|
+
open = row ? row.name : null; c.del.hidden = !row;
|
|
1382
|
+
c.head.textContent = row ? 'bench/' + row.name : 'New collection';
|
|
1383
|
+
if (!c.form.open) c.form.showModal();
|
|
1384
|
+
c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
|
|
1385
|
+
c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
|
|
1386
|
+
drawTagPicks();
|
|
1387
|
+
c.members.textContent = ''; c.find.value = '';
|
|
1388
|
+
C.models.forEach(function (alias) {
|
|
1389
|
+
var box = el('input', {type: 'checkbox', value: alias});
|
|
1390
|
+
box.checked = !!row && row.members.indexOf(alias) !== -1;
|
|
1391
|
+
box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
|
|
1392
|
+
var served = (C.served || {})[alias] || '';
|
|
1393
|
+
var lab = el('label', {class: 'check', 'data-served': served});
|
|
1394
|
+
lab.appendChild(box); lab.appendChild(el('span', {class: 'mono'}, alias));
|
|
1395
|
+
c.members.appendChild(lab);
|
|
1396
|
+
});
|
|
1397
|
+
c.shape.textContent = '';
|
|
1398
|
+
c.shape.appendChild(el('option', {value: ''}, 'automatic'));
|
|
1399
|
+
C.shapes.forEach(function (s) {
|
|
1400
|
+
var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
|
|
1401
|
+
});
|
|
1402
|
+
c.reply.value = row ? row.reply : 'all';
|
|
1403
|
+
c.merger.textContent = '';
|
|
1404
|
+
c.merger.appendChild(el('option', {value: ''}, 'automatic'));
|
|
1405
|
+
C.models.forEach(function (alias) {
|
|
1406
|
+
var o = el('option', {value: alias}, alias); if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
|
|
1407
|
+
});
|
|
1408
|
+
showMerger(); applyTags();
|
|
1409
|
+
}
|
|
1410
|
+
function loadCollections(keep) {
|
|
1411
|
+
fetch('/api/collections').then(function (r) { return r.json(); }).then(function (res) {
|
|
1412
|
+
C = res; c.list.textContent = '';
|
|
1413
|
+
res.collections.forEach(function (row) {
|
|
1414
|
+
// The whole row opens the collection: no tight button pair on it.
|
|
1415
|
+
var box = el('button', {class: 'endpoint c-row', type: 'button', 'data-name': row.name,
|
|
1416
|
+
'aria-label': 'Open bench/' + row.name});
|
|
1417
|
+
var left = el('div');
|
|
1418
|
+
var title = el('div', {class: 'what'});
|
|
1419
|
+
title.appendChild(el('span', {class: 'alias'}, 'bench/' + row.name + ' '));
|
|
1420
|
+
row.tags.forEach(function (t) { title.appendChild(el('span', {class: 'chip', style: 'margin-left:.3rem'}, t)); });
|
|
1421
|
+
left.appendChild(title);
|
|
1422
|
+
if (row.description) left.appendChild(el('div', {}, row.description));
|
|
1423
|
+
left.appendChild(el('div', {class: 'base'}, row.members.join(', ') + ' + ' + row.explorers + ' explorer' +
|
|
1424
|
+
(row.explorers === 1 ? '' : 's') + (row.shape ? ' (' + row.shape + ')' : '') +
|
|
1425
|
+
(row.reply === 'merged' ? ' · merged by ' + (row.merger || 'verdicts') : '')));
|
|
1426
|
+
var t = row.together;
|
|
1427
|
+
if (t) {
|
|
1428
|
+
var lim = Object.keys(t.limits || {}).filter(function (k) { return t.limits[k]; })
|
|
1429
|
+
.map(function (k) { return t.limits[k].toLocaleString() + ' ' + k; });
|
|
1430
|
+
left.appendChild(el('div', {class: 'base'}, 'Together: ' +
|
|
1431
|
+
(t.context_window ? 'prompts up to ' + Math.floor(t.context_window / 1000).toLocaleString() + 'k tokens' : 'context unknown') +
|
|
1432
|
+
(t.context_unknown.length ? ' (unknown for ' + t.context_unknown.join(', ') + ')' : '') +
|
|
1433
|
+
' · all can: ' + (t.capabilities.join(', ') || 'nothing known in common') +
|
|
1434
|
+
' · ' + t.connections.length + ' connection' + (t.connections.length === 1 ? '' : 's') +
|
|
1435
|
+
(lim.length ? ' · tightest ' + lim.join(', ') : '')));
|
|
1436
|
+
}
|
|
1437
|
+
box.appendChild(left);
|
|
1438
|
+
box.appendChild(el('span', {class: 'open-hint', 'aria-hidden': 'true'}, 'Open ›'));
|
|
1439
|
+
box.addEventListener('click', function () { fillForm(row); });
|
|
1440
|
+
c.list.appendChild(box);
|
|
1441
|
+
});
|
|
1442
|
+
if (!res.collections.length) c.list.appendChild(el('p', {class: 'empty'}, 'No collections yet.'));
|
|
1443
|
+
c.maxCalls.value = res.settings.max_calls; c.maxExp.value = res.settings.max_explorers;
|
|
1444
|
+
if (!keep) closeForm();
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
loadCollections();
|
|
1448
|
+
$('c-new').addEventListener('click', function () { fillForm(null); c.name.focus(); });
|
|
1449
|
+
$('c-close').addEventListener('click', closeForm);
|
|
1450
|
+
// Done: say so in the popup, close it a moment later, and keep the message
|
|
1451
|
+
// above the list, as onboarding and retiring do.
|
|
1452
|
+
function finished(message) {
|
|
1453
|
+
c.done.textContent = message; c.status.textContent = message; loadCollections(true);
|
|
1454
|
+
setTimeout(closeForm, 1200);
|
|
1455
|
+
}
|
|
1456
|
+
c.del.addEventListener('click', function () {
|
|
1457
|
+
var name = open;
|
|
1458
|
+
if (!name) return;
|
|
1459
|
+
ask('Delete bench/' + name + '? Callers asking for it will get an error.', 'Delete', true).then(function (yes) {
|
|
1460
|
+
if (!yes) return;
|
|
1461
|
+
post('/api/collection/remove', {name: name}).then(function (r) {
|
|
1462
|
+
if (!r.ok) { listErrors(c.errors, r.errors); return; }
|
|
1463
|
+
finished('Deleted bench/' + name + '.');
|
|
1464
|
+
});
|
|
1465
|
+
});
|
|
1466
|
+
});
|
|
1467
|
+
$('c-save').addEventListener('click', function () {
|
|
1468
|
+
c.errors.textContent = ''; c.done.textContent = '';
|
|
1469
|
+
var name = c.name.value.trim().replace(/^bench\//, '');
|
|
1470
|
+
post('/api/collection', {name: name, members: picked(), shape: c.shape.value, description: c.desc.value,
|
|
1471
|
+
tags: c.tags.value.split(','), reply: c.reply.value,
|
|
1472
|
+
merger: c.merger.value}).then(function (r) {
|
|
1473
|
+
if (!r.ok) { listErrors(c.errors, r.errors); return; }
|
|
1474
|
+
finished('Saved bench/' + name + '.');
|
|
1475
|
+
}).catch(function (err) { c.errors.textContent = String(err); });
|
|
1476
|
+
});
|
|
1477
|
+
$('g-save').addEventListener('click', function () {
|
|
1478
|
+
c.gErrors.textContent = ''; c.gDone.textContent = '';
|
|
1479
|
+
post('/api/gateway-settings', {max_calls: parseInt(c.maxCalls.value, 10),
|
|
1480
|
+
max_explorers: parseInt(c.maxExp.value, 10)}).then(function (r) {
|
|
1481
|
+
if (!r.ok) { listErrors(c.gErrors, r.errors); return; }
|
|
1482
|
+
c.gDone.textContent = 'Settings saved.'; loadCollections(true);
|
|
1483
|
+
}).catch(function (err) { c.gErrors.textContent = String(err); });
|
|
1484
|
+
});
|
|
1485
|
+
|
|
1486
|
+
// ---- a click on the backdrop closes a popup, as its Close button does ----
|
|
1487
|
+
// Pressed AND released outside: a text selection dragged out of an input
|
|
1488
|
+
// must not throw the form away.
|
|
1489
|
+
document.querySelectorAll('dialog').forEach(function (d) {
|
|
1490
|
+
var downOutside = false;
|
|
1491
|
+
function outside(e) {
|
|
1492
|
+
var r = d.getBoundingClientRect();
|
|
1493
|
+
return e.target === d && (e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom);
|
|
1494
|
+
}
|
|
1495
|
+
d.addEventListener('pointerdown', function (e) { downOutside = outside(e); });
|
|
1496
|
+
d.addEventListener('click', function (e) { if (downOutside && outside(e)) d.close(); downOutside = false; });
|
|
1497
|
+
});
|
|
1498
|
+
|
|
1499
|
+
// ---- verdict on one gateway seat --------------------------------------
|
|
1500
|
+
var vd = $('verdict-dialog'), vcall = null;
|
|
1501
|
+
function lines(id) { return $(id).value.split('\n').map(function (s) { return s.trim(); }).filter(Boolean); }
|
|
1502
|
+
document.addEventListener('click', function (e) {
|
|
1503
|
+
var b = e.target.closest('button.seat');
|
|
1504
|
+
if (!b) return;
|
|
1505
|
+
vcall = b.dataset.call;
|
|
1506
|
+
$('v-errors').textContent = ''; $('v-done').textContent = '';
|
|
1507
|
+
fetch('/api/gateway-seat?call=' + encodeURIComponent(vcall)).then(function (r) { return r.json(); }).then(function (s) {
|
|
1508
|
+
if (!s.ok) { listErrors($('v-errors'), s.errors); vd.showModal(); return; }
|
|
1509
|
+
$('v-title').textContent = s.alias + ' · ' + s.seat;
|
|
1510
|
+
$('v-meta').textContent = s.model + ' · ' + s.status + ' in ' + s.seconds + 's · ' + s.at.slice(0, 16).replace('T', ' ') + ' · ' + s.call;
|
|
1511
|
+
$('v-prompt').textContent = s.prompt;
|
|
1512
|
+
$('v-answer').textContent = s.answer || (s.error ? 'Error: ' + s.error : 'Empty reply: usually a token budget, not incapacity.');
|
|
1513
|
+
document.querySelectorAll('input[name=v-useful]').forEach(function (r) {
|
|
1514
|
+
r.checked = s.useful !== null && r.value === (s.useful ? 'yes' : 'no');
|
|
1515
|
+
});
|
|
1516
|
+
$('v-note').value = s.note; $('v-real').value = s.real.join('\n'); $('v-false').value = s.false.join('\n');
|
|
1517
|
+
vd.showModal();
|
|
1518
|
+
});
|
|
1519
|
+
});
|
|
1520
|
+
$('v-close').addEventListener('click', function () { vd.close(); });
|
|
1521
|
+
$('v-save').addEventListener('click', function () {
|
|
1522
|
+
var pick = document.querySelector('input[name=v-useful]:checked');
|
|
1523
|
+
$('v-errors').textContent = ''; $('v-done').textContent = '';
|
|
1524
|
+
post('/api/verdict', {call: vcall, useful: pick ? pick.value === 'yes' : null, note: $('v-note').value,
|
|
1525
|
+
real: lines('v-real'), false: lines('v-false')}).then(function (r) {
|
|
1526
|
+
if (!r.ok) { listErrors($('v-errors'), r.errors); return; }
|
|
1527
|
+
$('v-done').textContent = 'Recorded: ' + r.changed.join(', ') + '. Reloading…';
|
|
1528
|
+
setTimeout(function () { location.reload(); }, 700);
|
|
1529
|
+
}).catch(function (err) { $('v-errors').textContent = String(err); });
|
|
1530
|
+
});
|
|
1531
|
+
|
|
1532
|
+
// ---- Benchmarks charts: Observable Plot, data from serve.charts_data ------
|
|
1533
|
+
// A function, not run once: a theme switched on Settings redraws them in its colours.
|
|
1534
|
+
function drawCharts() {
|
|
1535
|
+
var holder = $('charts-data');
|
|
1536
|
+
if (!holder) return;
|
|
1537
|
+
var C = JSON.parse(holder.textContent);
|
|
1538
|
+
var css = getComputedStyle(document.documentElement);
|
|
1539
|
+
function tok(name, fallback) { return (css.getPropertyValue(name) || '').trim() || fallback; }
|
|
1540
|
+
// `good` is the theme's OK colour, not its brand: in a theme whose brand is
|
|
1541
|
+
// red (Hardwood), a found bug drawn in the brand colour would read as bad.
|
|
1542
|
+
var ink = tok('--ink', '#24331f'), good = tok('--ok', '#327a16'), err = tok('--err', '#a12b2b'),
|
|
1543
|
+
warn = tok('--warn', '#7a4d00'),
|
|
1544
|
+
signal = tok('--signal', '#fae66c'), sky = tok('--sky', '#69bdeb'), muted = tok('--muted', '#6b685a'),
|
|
1545
|
+
line = tok('--line', '#e4dfd2'), halo = tok('--paper-2', '#ffffff');
|
|
1546
|
+
var style = {background: 'transparent', color: ink, fontSize: '13px', overflow: 'visible'};
|
|
1547
|
+
// Tabs other than the open one have no width yet, so size from the column
|
|
1548
|
+
// the pages share. It was the whole page, which made every chart wider than
|
|
1549
|
+
// its column beside the sidebar, scaled down, with smaller text.
|
|
1550
|
+
function width() {
|
|
1551
|
+
var col = document.querySelector('.panes') || document.querySelector('main');
|
|
1552
|
+
return Math.max(520, (col.clientWidth || 900) - 50);
|
|
1553
|
+
}
|
|
1554
|
+
function put(id, make) {
|
|
1555
|
+
var box = $(id);
|
|
1556
|
+
if (!box) return;
|
|
1557
|
+
if (!window.Plot) {
|
|
1558
|
+
box.className += ' muted';
|
|
1559
|
+
box.textContent = 'Charts need Observable Plot from cdn.jsdelivr.net, which did not load; the table below has the same numbers.';
|
|
1560
|
+
return;
|
|
1561
|
+
}
|
|
1562
|
+
box.textContent = ''; // a redraw replaces the chart
|
|
1563
|
+
try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
|
|
1564
|
+
}
|
|
1565
|
+
// Pixel offsets that keep scatter labels apart: each label, in order, moves
|
|
1566
|
+
// up or down until it clears the ones already placed.
|
|
1567
|
+
function spread(points, xpx, ypx) {
|
|
1568
|
+
var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
|
|
1569
|
+
return points.map(function (d) {
|
|
1570
|
+
var cx = xpx(d), half = d.model.length * 2.9;
|
|
1571
|
+
for (var k = 0; k < tries.length; k++) {
|
|
1572
|
+
var cy = ypx(d) + tries[k];
|
|
1573
|
+
if (!boxes.some(function (b) { return Math.abs(b.x - cx) < b.half + half + 4 && Math.abs(b.y - cy) < 12; })) {
|
|
1574
|
+
boxes.push({x: cx, y: cy, half: half});
|
|
1575
|
+
return tries[k];
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
return tries[0];
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
function rows(n) { return 20 * n + 80; }
|
|
1582
|
+
// Who to trust: finds real bugs (right) vs makes things up (up = less).
|
|
1583
|
+
put('chart-overview', function (w) {
|
|
1584
|
+
if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
|
|
1585
|
+
var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
|
|
1586
|
+
var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
|
|
1587
|
+
var pxW = w - 110, pxH = 370;
|
|
1588
|
+
var pts = C.overview.slice().sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
|
|
1589
|
+
var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
|
|
1590
|
+
function (d) { return d.wrong / worst * pxH; });
|
|
1591
|
+
var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; });
|
|
1592
|
+
return Plot.plot({
|
|
1593
|
+
width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
1594
|
+
x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
|
|
1595
|
+
y: {label: '↑ Makes fewer things up', domain: [0, worst], reverse: true, tickFormat: '.0%'},
|
|
1596
|
+
marks: [
|
|
1597
|
+
// The zone a reviewer should be in: outlined, so it reads on a dark ground too.
|
|
1598
|
+
Plot.rect([{x1: x0 + (1 - x0) / 2, x2: 1, y1: 0, y2: worst / 4}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
|
|
1599
|
+
fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
|
|
1600
|
+
Plot.dot(C.overview, {x: 'recall', y: 'wrong', r: 6, fill: good, stroke: ink, strokeWidth: 0.5}),
|
|
1601
|
+
// A halo in the panel's colour keeps each name legible over dots, grid lines and the zone.
|
|
1602
|
+
Plot.text(labels, {x: 'recall', y: 'ty', text: 'model', fontSize: 11, fill: ink,
|
|
1603
|
+
stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
|
|
1604
|
+
Plot.tip(C.overview, Plot.pointer({x: 'recall', y: 'wrong',
|
|
1605
|
+
title: function (d) { return d.model + '\nfinds ' + d.found + ' real findings\nconfidently wrong in ' + d.wrongN + ' answers'; }}))
|
|
1606
|
+
]
|
|
1607
|
+
});
|
|
1608
|
+
});
|
|
1609
|
+
|
|
1610
|
+
// Bug hunt: right of zero the bug tasks (found, wrong line, missed); left of
|
|
1611
|
+
// it the same files clean, where any bar is a false alarm.
|
|
1612
|
+
put('chart-hunt', function (w) {
|
|
1613
|
+
var segs = [], score = {}, tasks = {};
|
|
1614
|
+
C.hunt.forEach(function (h) {
|
|
1615
|
+
var bugs = h.found + h['wrong line'] + h.missed, clean = h.clean + h['false alarm'];
|
|
1616
|
+
tasks[h.model] = bugs + clean;
|
|
1617
|
+
var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
|
|
1618
|
+
var fa = clean ? h['false alarm'] / clean : 0;
|
|
1619
|
+
score[h.model] = f - fa;
|
|
1620
|
+
segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
|
|
1621
|
+
segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
|
|
1622
|
+
segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
|
|
1623
|
+
segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
|
|
1624
|
+
});
|
|
1625
|
+
var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
|
|
1626
|
+
return Plot.plot({
|
|
1627
|
+
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
1628
|
+
x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
|
|
1629
|
+
tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
|
|
1630
|
+
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
|
|
1631
|
+
color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
|
|
1632
|
+
marks: [
|
|
1633
|
+
Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
|
|
1634
|
+
title: function (d) { return d.model + ': ' + d.part + ', ' + d.n; }}),
|
|
1635
|
+
Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
|
|
1636
|
+
]
|
|
1637
|
+
});
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1640
|
+
// Real work: the share of judged offloads that helped, sample size beside the name.
|
|
1641
|
+
put('chart-work', function (w) {
|
|
1642
|
+
var R = C.work.slice().sort(function (a, b) { return b.useful / b.judged - a.useful / a.judged || b.judged - a.judged; });
|
|
1643
|
+
if (!R.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
|
|
1644
|
+
var by = {};
|
|
1645
|
+
R.forEach(function (r) { by[r.model] = r; });
|
|
1646
|
+
function share(r) { return r.useful / r.judged; }
|
|
1647
|
+
return Plot.plot({
|
|
1648
|
+
width: w, height: rows(R.length), style: style, marginLeft: 250, marginRight: 50,
|
|
1649
|
+
x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
|
|
1650
|
+
y: {domain: R.map(function (r) { return r.model; }), label: null,
|
|
1651
|
+
tickFormat: function (m) { return m + ' (' + by[m].judged + ')'; }},
|
|
1652
|
+
marks: [
|
|
1653
|
+
Plot.barX(R, {x: share, y: 'model', fill: function (r) { return share(r) >= 0.5 ? good : err; },
|
|
1654
|
+
title: function (r) { return r.model + ': helped ' + r.useful + ' of ' + r.judged + ' judged (' + r.offloads + ' offloads)'
|
|
1655
|
+
+ (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : ''); }}),
|
|
1656
|
+
Plot.text(R, {x: share, y: 'model', text: function (r) { return r.useful + '/' + r.judged; }, dx: 18, fill: ink}),
|
|
1657
|
+
Plot.ruleX([0], {stroke: ink})
|
|
1658
|
+
]
|
|
1659
|
+
});
|
|
1660
|
+
});
|
|
1661
|
+
|
|
1662
|
+
// Did it answer: each probe of the last 14 days, by what came back.
|
|
1663
|
+
put('chart-reliability', function (w) {
|
|
1664
|
+
if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
|
|
1665
|
+
var total = {}, ok = {};
|
|
1666
|
+
C.reliability.forEach(function (d) {
|
|
1667
|
+
total[d.model] = (total[d.model] || 0) + d.count;
|
|
1668
|
+
if (d.outcome === 'answered') ok[d.model] = d.count;
|
|
1669
|
+
});
|
|
1670
|
+
var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
|
|
1671
|
+
var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
|
|
1672
|
+
entitlement: '#c0563f', other: muted, error: ink};
|
|
1673
|
+
var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
|
|
1674
|
+
return Plot.plot({
|
|
1675
|
+
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
1676
|
+
x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
|
|
1677
|
+
y: {domain: models, label: null},
|
|
1678
|
+
color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
|
|
1679
|
+
marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
|
|
1680
|
+
title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
|
|
1681
|
+
});
|
|
1682
|
+
});
|
|
1683
|
+
|
|
1684
|
+
// Speed vs quality: median seconds on our own tasks against findings located.
|
|
1685
|
+
put('chart-speed', function (w) {
|
|
1686
|
+
if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
|
|
1687
|
+
var s = C.speed.map(function (d) { return d.seconds; });
|
|
1688
|
+
var lo = Math.max(1, Math.min.apply(null, s) * 0.8), hi = Math.max.apply(null, s) * 1.25;
|
|
1689
|
+
var y0 = Math.max(0, Math.floor((Math.min.apply(null, C.speed.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
|
|
1690
|
+
var pxW = w - 110, pxH = 340;
|
|
1691
|
+
var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
|
|
1692
|
+
var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
|
|
1693
|
+
function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
|
|
1694
|
+
var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; });
|
|
1695
|
+
return Plot.plot({
|
|
1696
|
+
width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
1697
|
+
x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
|
|
1698
|
+
y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
|
|
1699
|
+
marks: [
|
|
1700
|
+
Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
|
|
1701
|
+
Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink}),
|
|
1702
|
+
Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
|
|
1703
|
+
title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
|
|
1704
|
+
]
|
|
1705
|
+
});
|
|
1706
|
+
});
|
|
1707
|
+
|
|
1708
|
+
// Our code: one square per task and model, grouped by kind of task.
|
|
1709
|
+
put('chart-code', function (w) {
|
|
1710
|
+
var fam = {}, first = {};
|
|
1711
|
+
C.heat.forEach(function (d) { fam[d.task] = d.family; });
|
|
1712
|
+
var tasks = Object.keys(fam).sort(function (a, b) {
|
|
1713
|
+
return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
|
|
1714
|
+
});
|
|
1715
|
+
tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
|
|
1716
|
+
var label = {};
|
|
1717
|
+
Object.keys(first).forEach(function (f) { label[first[f]] = f; });
|
|
1718
|
+
// By total found, so a model that played three tasks does not top one that played forty.
|
|
1719
|
+
var sum = {};
|
|
1720
|
+
C.heat.forEach(function (d) { sum[d.model] = (sum[d.model] || 0) + (d.rate || 0); });
|
|
1721
|
+
var models = Object.keys(sum).sort(function (a, b) { return sum[b] - sum[a]; });
|
|
1722
|
+
function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
|
|
1723
|
+
return Plot.plot({
|
|
1724
|
+
width: w, height: 22 * models.length + 70, style: style, marginLeft: 230, marginTop: 30, padding: 0.08,
|
|
1725
|
+
x: {domain: tasks, axis: 'top', label: null, tickSize: 0, tickFormat: function (t) { return label[t] || ''; }},
|
|
1726
|
+
y: {domain: models, label: null, tickSize: 0},
|
|
1727
|
+
color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
|
|
1728
|
+
marks: [
|
|
1729
|
+
Plot.cell(C.heat.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
|
|
1730
|
+
Plot.cell(C.heat.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
|
|
1731
|
+
Plot.dot(C.heat.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
|
|
1732
|
+
Plot.ruleX(Object.keys(label), {x: function (t) { return t; }, stroke: ink, strokeOpacity: 0.25, dx: -2})
|
|
1733
|
+
]
|
|
1734
|
+
});
|
|
1735
|
+
});
|
|
1736
|
+
|
|
1737
|
+
// Confidently wrong: stacked by kind, worst at the top.
|
|
1738
|
+
put('chart-wrong', function (w) {
|
|
1739
|
+
var T = C.wrongTotals;
|
|
1740
|
+
var models = Object.keys(T).sort(function (a, b) { return T[b].rate - T[a].rate || T[b].answers - T[a].answers; });
|
|
1741
|
+
return Plot.plot({
|
|
1742
|
+
width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
|
|
1743
|
+
x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
|
|
1744
|
+
// The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
|
|
1745
|
+
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
|
|
1746
|
+
color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
|
|
1747
|
+
marks: [
|
|
1748
|
+
Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
|
|
1749
|
+
Plot.ruleX([0], {stroke: ink})
|
|
1750
|
+
]
|
|
1751
|
+
});
|
|
1752
|
+
});
|
|
1753
|
+
|
|
1754
|
+
}
|
|
1755
|
+
drawCharts();
|
|
1756
|
+
})();
|