@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/AGENTS.md +147 -147
  2. package/BENCHMARKS.md +262 -262
  3. package/GATEWAY.md +272 -271
  4. package/NOTICE +28 -28
  5. package/PRIVACY.md +46 -46
  6. package/README.md +368 -275
  7. package/SECURITY.md +83 -83
  8. package/TERMS.md +22 -22
  9. package/USAGE.md +170 -155
  10. package/VERSION +1 -1
  11. package/bin/benchwarmers.js +16 -72
  12. package/cli.py +666 -0
  13. package/examples/config/gateway.json +5 -5
  14. package/examples/config/model-roles.json +11 -11
  15. package/gateway.py +755 -712
  16. package/launcher.py +120 -0
  17. package/mcp_server.py +266 -0
  18. package/metrics/alerts.py +116 -116
  19. package/metrics/aliases.py +41 -41
  20. package/metrics/bench_collections.py +141 -141
  21. package/metrics/catalog.py +295 -295
  22. package/metrics/client.py +394 -373
  23. package/metrics/cliproxy.py +191 -191
  24. package/metrics/connect.py +691 -691
  25. package/metrics/cost.py +152 -152
  26. package/metrics/decide.py +227 -227
  27. package/metrics/discover.py +318 -318
  28. package/metrics/endpoints.py +436 -436
  29. package/metrics/guard.py +617 -592
  30. package/metrics/learn.py +221 -221
  31. package/metrics/litellm_hub.py +206 -206
  32. package/metrics/model_roles.py +84 -84
  33. package/metrics/offload.py +166 -166
  34. package/metrics/owui.py +205 -205
  35. package/metrics/paths.py +60 -60
  36. package/metrics/route.py +579 -574
  37. package/metrics/scheduler.py +275 -275
  38. package/metrics/sources.py +167 -167
  39. package/metrics/wrong.py +144 -144
  40. package/package.json +68 -64
  41. package/perf/README.md +89 -89
  42. package/perf/run.py +261 -261
  43. package/scripts/auto_retire.py +367 -367
  44. package/scripts/check.py +108 -104
  45. package/scripts/check_deprecations.py +378 -378
  46. package/scripts/check_themes.py +103 -103
  47. package/scripts/init.py +124 -124
  48. package/scripts/list_served.py +78 -78
  49. package/scripts/notice.py +78 -78
  50. package/scripts/onboard.py +401 -331
  51. package/scripts/provider_tags.py +129 -129
  52. package/scripts/reprefix.py +136 -136
  53. package/scripts/sync_config.py +105 -105
  54. package/scripts/test_auth.py +255 -255
  55. package/scripts/test_cli.py +189 -0
  56. package/scripts/test_client.py +233 -233
  57. package/scripts/test_connect.py +222 -222
  58. package/scripts/test_first_run.py +180 -180
  59. package/scripts/test_hubs.py +328 -328
  60. package/scripts/test_launcher.py +67 -0
  61. package/scripts/test_llm_log.py +85 -0
  62. package/scripts/test_mcp.py +178 -0
  63. package/scripts/test_merged.py +214 -193
  64. package/scripts/test_no_owui.py +291 -291
  65. package/scripts/test_owui.py +90 -90
  66. package/scripts/test_serve_edit.py +448 -386
  67. package/scripts/test_verdict.py +98 -98
  68. package/scripts/test_vision.py +228 -228
  69. package/serve.py +3904 -3662
  70. package/suite/mine.py +154 -154
  71. package/suite/pace.py +145 -145
  72. package/suite/run.py +1046 -1040
  73. package/suite/seed.py +421 -421
  74. package/suite/tasks/claim-alias-unbound/fixtures/onboard.py +286 -286
  75. package/suite/tasks/claim-error-text-string/fixtures/decide.py +187 -187
  76. package/suite/tasks/claim-list-body/fixtures/gateway.py +235 -235
  77. package/suite/tasks/claim-member-nominated-twice/fixtures/gateway.py +284 -284
  78. package/suite/tasks/claim-nominee-uses-slot/fixtures/gateway.py +284 -284
  79. package/suite/tasks/claim-silent-exit/fixtures/gateway.py +235 -235
  80. package/suite/tasks/claim-sse-hang/fixtures/gateway.py +235 -235
  81. package/suite/tasks/claim-unknown-nominee-crash/fixtures/gateway.py +284 -284
  82. package/suite/tasks/rebench-matchms-809-bug/task.json +23 -23
  83. package/suite/tasks/rebench-matchms-809-control/task.json +27 -27
  84. package/suite/tasks/rebench-more-itertools-1028-bug/task.json +23 -23
  85. package/suite/tasks/rebench-more-itertools-1028-control/task.json +27 -27
  86. package/suite/tasks/rebench-narwhals-2864-bug/task.json +23 -23
  87. package/suite/tasks/rebench-narwhals-2864-control/task.json +27 -27
  88. package/suite/tasks/rebench-quantecon-py-769-bug/task.json +23 -23
  89. package/suite/tasks/rebench-quantecon-py-769-control/task.json +27 -27
  90. package/suite/tasks/rebench-scrapy-6867-bug/task.json +23 -23
  91. package/suite/tasks/rebench-scrapy-6867-control/task.json +27 -27
  92. package/ui/bench.css +430 -424
  93. package/ui/bench.js +1756 -1555
  94. package/ui/themes/dark.css +35 -35
  95. package/ui/themes/hardwood.css +52 -52
  96. package/ui/themes/rainforest.css +32 -32
package/ui/bench.js CHANGED
@@ -1,1555 +1,1756 @@
1
- // Benchwarmers. Plain DOM, no framework: serve.py renders the page, this only
2
- // wires the controls. Every write goes through a POST that re-checks its own
3
- // rules server-side, so nothing here is trusted to enforce anything.
4
- (function () {
5
- 'use strict';
6
- var D = window.BENCH || {models: {}, roles: {}, retired: {}, stamp: 0};
7
- function $(id) { return document.getElementById(id); }
8
- function el(tag, attrs, text) {
9
- var e = document.createElement(tag);
10
- Object.keys(attrs || {}).forEach(function (k) { e.setAttribute(k, attrs[k]); });
11
- if (text != null) e.textContent = text;
12
- return e;
13
- }
14
- function store(k, v) { try { if (v === undefined) return localStorage.getItem(k); localStorage.setItem(k, v); } catch (e) { return null; } }
15
- function post(url, body) {
16
- return fetch(url, {method: 'POST', headers: {'Content-Type': 'application/json'},
17
- body: JSON.stringify(body)}).then(function (r) { return r.json(); });
18
- }
19
- function listErrors(node, errors) { node.textContent = errors.map(function (x) { return '• ' + x; }).join('\n'); node.style.whiteSpace = 'pre-line'; }
20
-
21
- // ---- tabs, remembered per browser ----------------------------------
22
- var tabs = document.querySelectorAll('.tabs button[data-tab]'); // not the sidebar's own toggle
23
- var onShow = {}; // a page's loader, run each time it is shown
24
- function show(id) {
25
- tabs.forEach(function (b) { b.setAttribute('aria-selected', String(b.dataset.tab === id)); });
26
- document.querySelectorAll('.tab').forEach(function (s) { s.classList.toggle('active', s.id === 'tab-' + id); });
27
- store('bench.tab', id);
28
- if (onShow[id]) onShow[id]();
29
- }
30
- tabs.forEach(function (b) { b.addEventListener('click', function () { show(b.dataset.tab); }); });
31
- var wanted = store('bench.tab');
32
- show(wanted && $('tab-' + wanted) ? wanted : 'glance');
33
-
34
- // ---- sortable tables -------------------------------------------------
35
- function key(cell) {
36
- var raw = cell.dataset.sort != null ? cell.dataset.sort : cell.textContent.trim();
37
- var n = parseFloat(raw);
38
- return isNaN(n) || !/^-?[\d.]+/.test(raw) ? raw.toLowerCase() : n;
39
- }
40
- document.querySelectorAll('th[data-sortable]').forEach(function (th) {
41
- th.tabIndex = 0;
42
- function sort() {
43
- var table = th.closest('table'), body = table.tBodies[0];
44
- var idx = Array.prototype.indexOf.call(th.parentNode.children, th);
45
- var dir = th.getAttribute('aria-sort') === 'descending' ? 'ascending' : 'descending';
46
- table.querySelectorAll('th[aria-sort]').forEach(function (o) { o.removeAttribute('aria-sort'); });
47
- th.setAttribute('aria-sort', dir);
48
- var rows = Array.prototype.slice.call(body.rows);
49
- rows.sort(function (a, b) {
50
- var x = key(a.cells[idx]), y = key(b.cells[idx]);
51
- var c = (typeof x === typeof y) ? (x < y ? -1 : x > y ? 1 : 0) : (typeof x === 'number' ? -1 : 1);
52
- return dir === 'ascending' ? c : -c;
53
- });
54
- rows.forEach(function (r) { body.appendChild(r); });
55
- }
56
- th.addEventListener('click', sort);
57
- th.addEventListener('keydown', function (e) { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); sort(); } });
58
- });
59
-
60
- // ---- filter boxes: one per panel (or the whole page: data-filter="page"),
61
- // matching any text in a table row or a log line, and the panel's facet
62
- // selects (serve.facet), each matching a row's data-<facet> exactly ------
63
- function applyFilter(scope) {
64
- var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
65
- var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
66
- scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
67
- r.hidden = (q && r.textContent.toLowerCase().indexOf(q) === -1)
68
- || facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
69
- });
70
- // A match inside a folded list (an endpoint's models) opens it.
71
- if (q) scope.querySelectorAll('details.models').forEach(function (d) {
72
- if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
73
- });
74
- }
75
- document.querySelectorAll('input[data-filter], select[data-facet]').forEach(function (c) {
76
- var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
77
- var scope = box && box.dataset.filter === 'page' ? c.closest('.tab') : c.closest('.panel');
78
- c.addEventListener(c.tagName === 'SELECT' ? 'change' : 'input', function () { applyFilter(scope); });
79
- });
80
-
81
- // ---- Action needed: Hide folds it to its heading, per browser, until an
82
- // action it has not seen appears or 12 hours pass, whichever is first ----
83
- var acts = $('actions');
84
- if (acts) {
85
- var keys = Array.prototype.map.call(acts.querySelectorAll('.problem[data-key]'), function (p) { return p.dataset.key; });
86
- var hid = null;
87
- try { hid = JSON.parse(store('bench.actions.hidden') || 'null'); } catch (err) { hid = null; }
88
- var fold = function (on) {
89
- acts.classList.toggle('collapsed', on);
90
- $('actions-hide').textContent = on ? 'show' : 'Hide';
91
- $('actions-hide').setAttribute('aria-expanded', String(!on));
92
- };
93
- fold(!!(hid && hid.keys && Date.now() - hid.at < 12 * 3600 * 1000
94
- && keys.every(function (k) { return hid.keys.indexOf(k) !== -1; })));
95
- $('actions-hide').addEventListener('click', function () {
96
- var on = !acts.classList.contains('collapsed');
97
- store('bench.actions.hidden', on ? JSON.stringify({at: Date.now(), keys: keys}) : '');
98
- fold(on);
99
- });
100
- }
101
-
102
- // ---- retired rows: hidden unless asked for ---------------------------
103
- var showRetired = store('bench.retired') === '1';
104
- function applyRetired() {
105
- document.body.classList.toggle('show-retired', showRetired);
106
- document.querySelectorAll('input[data-retired]').forEach(function (c) { c.checked = showRetired; });
107
- }
108
- document.querySelectorAll('input[data-retired]').forEach(function (c) {
109
- c.addEventListener('change', function () { showRetired = c.checked; store('bench.retired', showRetired ? '1' : '0'); applyRetired(); });
110
- });
111
- applyRetired();
112
-
113
- // ---- Discover: paid models hidden unless asked for -------------------
114
- var showPaid = store('bench.paid') === '1';
115
- function applyPaid() {
116
- document.body.classList.toggle('show-paid', showPaid);
117
- document.querySelectorAll('input[data-paid]').forEach(function (c) { c.checked = showPaid; });
118
- }
119
- document.querySelectorAll('input[data-paid]').forEach(function (c) {
120
- c.addEventListener('change', function () { showPaid = c.checked; store('bench.paid', showPaid ? '1' : '0'); applyPaid(); });
121
- });
122
- applyPaid();
123
-
124
- // ---- fresh results: offered, never forced ----------------------------
125
- // Not while an onboarding batch runs: each model's probe writes results, so
126
- // it would be offered after every model. The first poll after the batch
127
- // ends offers it once.
128
- var fresh = $('fresh'), batchRunning = false;
129
- setInterval(function () {
130
- if (document.hidden || !fresh.hidden || batchRunning) return;
131
- fetch('/api/stamp').then(function (r) { return r.json(); }).then(function (s) {
132
- if (s.stamp > D.stamp + 0.5) fresh.hidden = false;
133
- }).catch(function () {});
134
- }, 15000);
135
- fresh.addEventListener('click', function () { location.reload(); });
136
-
137
- // ---- the model dialog ------------------------------------------------
138
- var md = $('model-dialog');
139
- var m = {title: $('m-title'), served: $('m-served'), editor: $('m-editor'), gone: $('m-gone'),
140
- alias: $('m-alias'), role: $('m-role'), roleWhy: $('m-role-why'), adhoc: $('m-adhoc'),
141
- danger: $('m-danger'), errors: $('m-errors'), done: $('m-done'), save: $('m-save'),
142
- restore: $('m-restore'), close: $('m-close'), removeBox: $('m-remove-box'),
143
- reason: $('m-reason'), typed: $('m-typed'), remove: $('m-remove'),
144
- qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why')};
145
- m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
146
- var editing = null;
147
-
148
- function openModel(alias) {
149
- var model = D.models[alias], gone = (D.retired || {})[alias];
150
- if (!model && !gone) return;
151
- editing = model ? alias : null;
152
- m.title.textContent = alias;
153
- m.errors.textContent = ''; m.done.textContent = '';
154
- m.editor.hidden = !model; m.save.hidden = !model; m.removeBox.hidden = !model;
155
- m.gone.hidden = !!model; m.restore.hidden = !!model;
156
- if (gone) {
157
- m.served.textContent = 'served as ' + gone.served + (gone.tag ? ' · ' + gone.tag : '');
158
- m.gone.textContent = '';
159
- m.gone.appendChild(el('strong', {}, 'Retired ' + gone.at.slice(0, 10) + (gone.by === 'auto' ? ' automatically' : ' by hand')));
160
- m.gone.appendChild(el('p', {style: 'margin:.3rem 0'}, gone.why || ''));
161
- m.gone.appendChild(el('p', {class: 'muted', style: 'margin:0'},
162
- 'Its evidence is untouched. Restore puts back the same alias, role and endpoint; failures recorded before the restore are ignored.'));
163
- m.restore.dataset.alias = alias; m.restore.disabled = false;
164
- } else {
165
- m.served.textContent = 'serves ' + model.served + (model.tag ? ' · ' + model.tag : '');
166
- m.alias.value = alias;
167
- m.role.textContent = '';
168
- Object.keys(D.roles).forEach(function (r) {
169
- var o = el('option', {value: r}, r); if (r === model.role) o.selected = true; m.role.appendChild(o);
170
- });
171
- // A decision model answers typed questions, never chat: its role is fixed both ways.
172
- var decision = model.kind === 'decision';
173
- m.role.disabled = decision;
174
- m.qtypes.hidden = !decision;
175
- m.qtypes.querySelectorAll('input').forEach(function (c) {
176
- c.checked = decision && model.question_types.indexOf(c.value) !== -1;
177
- });
178
- fillFacts(alias, decision); fillOptions(alias, decision);
179
- m.aliasWhy.textContent = decision
180
- ? 'The name decide.py and the gateway call it by, and every record is keyed by. Letters, digits, dot, dash, underscore; unused, and never a retired name.'
181
- : m.aliasWhy.dataset.chat;
182
- Array.prototype.forEach.call(m.role.options, function (o) {
183
- o.disabled = model.kind !== 'decision' && o.value === 'decision';
184
- });
185
- m.adhoc.checked = !!model.ad_hoc;
186
- m.save.disabled = false; m.save.textContent = 'Save';
187
- m.reason.value = ''; m.typed.value = ''; m.removeBox.open = false; armRemove(); refresh();
188
- }
189
- md.showModal();
190
- }
191
- // ---- model facts: yours in the boxes, the sources' as placeholders -----
192
- var FACT_NUM = ['context_window', 'max_output'], FACT_CAP = ['tools', 'vision', 'structured_output', 'reasoning'];
193
- function fillFacts(alias, decision) {
194
- var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
195
- box.hidden = profile.hidden = decision || !f;
196
- profile.textContent = '';
197
- $('f-errors').textContent = ''; $('f-done').textContent = '';
198
- if (box.hidden) return;
199
- var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
200
- // At a glance, as models.dev shows a model: what goes in, what comes out.
201
- [['Input', 'input_modalities'], ['Output', 'output_modalities']].forEach(function (p) {
202
- var row = el('div', {class: 'prof'}, null);
203
- row.appendChild(el('span', {class: 'muted'}, p[0] + ' '));
204
- (f[p[1]] && f[p[1]].length ? f[p[1]] : ['unknown']).forEach(function (x) { row.appendChild(el('span', {class: 'chip'}, x)); });
205
- if (src[p[1]]) row.appendChild(el('span', {class: 'src'}, ' ' + src[p[1]]));
206
- profile.appendChild(row);
207
- });
208
- var bits = [], c = f.cost || {};
209
- if (f.release_date) bits.push('released ' + f.release_date);
210
- if (f.knowledge_cutoff) bits.push('knowledge to ' + f.knowledge_cutoff);
211
- if (f.open_weights != null) bits.push(f.open_weights ? 'open weights' : 'closed weights');
212
- if (c.input != null && c.output != null) {
213
- bits.push(c.input === 0 && c.output === 0 ? 'list price free'
214
- : 'list price $' + c.input + ' in / $' + c.output + ' out per 1M tokens (a free tier may still apply)');
215
- }
216
- if (bits.length) profile.appendChild(el('div', {class: 'prof muted'}, bits.join(' · ')));
217
- var show = function (k, v) { $('s-' + k).textContent = v == null ? 'unknown' : v + (src[k] ? ' · ' + src[k] : ''); };
218
- FACT_NUM.forEach(function (k) {
219
- $('f-' + k).value = mine(k) ? f[k] : '';
220
- $('f-' + k).placeholder = f[k] != null && !mine(k) ? f[k] : '';
221
- show(k, f[k] != null ? Number(f[k]).toLocaleString() : null);
222
- });
223
- FACT_CAP.forEach(function (k) {
224
- $('f-' + k).value = mine(k) ? (f[k] ? 'yes' : 'no') : '';
225
- // The blank choice is "whatever the sources say": say what that is.
226
- $('f-' + k).options[0].textContent = mine(k) ? 'auto (hand back to the sources)'
227
- : 'auto: ' + (f[k] == null ? 'unknown' : (f[k] ? 'yes' : 'no'));
228
- show(k, f[k] == null ? null : (f[k] ? 'yes' : 'no'));
229
- });
230
- $('f-knowledge_cutoff').value = mine('knowledge_cutoff') ? f.knowledge_cutoff : '';
231
- $('f-knowledge_cutoff').placeholder = !mine('knowledge_cutoff') && f.knowledge_cutoff ? f.knowledge_cutoff : '';
232
- show('knowledge_cutoff', f.knowledge_cutoff || null);
233
- var own = src.limits === 'you (this model)';
234
- document.querySelectorAll('#f-limits input').forEach(function (b) {
235
- var v = (f.limits || {})[b.dataset.k];
236
- b.value = own && v ? v : ''; b.placeholder = !own && v ? v : '—';
237
- });
238
- var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
239
- $('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
240
- }
241
- $('f-save').addEventListener('click', function () {
242
- if (!editing) return;
243
- var facts = {limits: {}};
244
- FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
245
- FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
246
- if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
247
- document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
248
- $('f-errors').textContent = ''; $('f-done').textContent = '';
249
- post('/api/model-facts', {alias: editing, facts: facts}).then(function (r) {
250
- if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
251
- $('f-done').textContent = 'Saved. Reloading…'; setTimeout(function () { location.reload(); }, 700);
252
- });
253
- });
254
- // ---- call options: how the bench asks this model (serve.save_model_options)
255
- function fillOptions(alias, decision) {
256
- var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
257
- $('m-opts').hidden = decision;
258
- $('co-errors').textContent = ''; $('co-done').textContent = '';
259
- $('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
260
- $('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
261
- }
262
- $('co-save').addEventListener('click', function () {
263
- if (!editing) return;
264
- var v = $('co-max').value.trim();
265
- $('co-errors').textContent = ''; $('co-done').textContent = '';
266
- post('/api/model-options', {alias: editing, max_tokens: v ? Number(v) : null}).then(function (r) {
267
- if (!r.ok) { listErrors($('co-errors'), r.errors); return; }
268
- $('co-done').textContent = 'Saved' + (r.note ? '; ' + r.note : '') + '. Reloading…';
269
- setTimeout(function () { location.reload(); }, 900);
270
- });
271
- });
272
-
273
- function refresh() {
274
- m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
275
- m.roleWhy.textContent = D.roles[m.role.value] || '';
276
- }
277
- function armRemove() { m.remove.disabled = !(editing && m.typed.value === editing && m.reason.value.trim().length >= 5); }
278
- document.addEventListener('click', function (e) {
279
- var a = e.target.closest('a.model');
280
- if (a) { e.preventDefault(); openModel(a.dataset.alias); }
281
- });
282
- m.close.addEventListener('click', function () { md.close(); });
283
- m.alias.addEventListener('input', refresh);
284
- m.role.addEventListener('change', refresh);
285
- m.reason.addEventListener('input', armRemove);
286
- m.typed.addEventListener('input', armRemove);
287
-
288
- // A success is said, left up for two seconds, then the popup closes and
289
- // the page reloads with the change in it.
290
- function closeAndReload(dialog) {
291
- setTimeout(function () { dialog.close(); location.reload(); }, 2000);
292
- }
293
- function after(node, button, label) {
294
- return function (res) {
295
- if (!res.ok) { listErrors(node, res.errors); if (button) { button.disabled = false; button.textContent = label; } return; }
296
- m.done.textContent = 'Done: ' + (res.changed || []).join(', ') + (res.sync ? ' · ' + res.sync : '') + '. Closing…';
297
- closeAndReload(md);
298
- };
299
- }
300
- m.save.addEventListener('click', function () {
301
- if (!editing) return;
302
- var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
303
- if (!m.qtypes.hidden) {
304
- body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
305
- function (c) { return c.checked; }).map(function (c) { return c.value; });
306
- }
307
- if (body.new_alias !== editing && !confirm('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.')) return;
308
- m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
309
- post('/api/model', body).then(after(m.errors, m.save, 'Save')).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
310
- });
311
- // The confirm is the last word: the popup closes at once and the page
312
- // reloads when the retirement is done. A refusal reopens it with the reason.
313
- m.remove.addEventListener('click', function () {
314
- if (!editing || !confirm('Retire ' + editing + '? The registry entry goes; the evidence stays, and Restore brings it back.')) return;
315
- m.remove.disabled = true; m.errors.textContent = ''; md.close();
316
- function refused(errors) { md.showModal(); listErrors(m.errors, errors); armRemove(); }
317
- post('/api/remove', {alias: editing, typed: m.typed.value, reason: m.reason.value}).then(function (res) {
318
- if (res.ok) location.reload(); else refused(res.errors);
319
- }).catch(function (err) { refused([String(err)]); });
320
- });
321
- m.restore.addEventListener('click', function () {
322
- m.restore.disabled = true; m.restore.textContent = 'Restoring…';
323
- post('/api/restore', {alias: m.restore.dataset.alias}).then(after(m.errors, m.restore, 'Restore')).catch(function (err) { m.errors.textContent = String(err); });
324
- });
325
-
326
- // ---- problem actions ---------------------------------------------------
327
- document.querySelectorAll('[data-act]').forEach(function (b) {
328
- b.addEventListener('click', function () {
329
- var act = b.dataset.act, aliases = (b.dataset.aliases || b.dataset.alias || '').split(' ').filter(Boolean);
330
- var status = b.closest('.problem').querySelector('.why');
331
- if (act === 'open') { openModel(aliases[0]); return; }
332
- if (act === 'retry') { // no confirm: one call, and only an answer changes anything
333
- b.disabled = true; b.textContent = 'Asking…';
334
- post('/api/retry', {alias: aliases[0]}).then(function (res) {
335
- if (!res.ok) { status.textContent = res.errors.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = 'Retry'; return; }
336
- status.textContent = res.changed.join(' · ') + '. Reloading…'; setTimeout(function () { location.reload(); }, 900);
337
- }).catch(function (err) { status.textContent = String(err); b.disabled = false; b.textContent = 'Retry'; });
338
- return;
339
- }
340
- var verb = act === 'restore' ? 'Restore' : 'Retire';
341
- if (!confirm(verb + ' ' + aliases.join(', ') + '?' + (act === 'restore' ? '' : ' The evidence stays and Restore brings it back.'))) return;
342
- b.disabled = true; b.textContent = verb === 'Restore' ? 'Restoring…' : 'Retiring…';
343
- var chain = Promise.resolve([]);
344
- aliases.forEach(function (alias) {
345
- chain = chain.then(function (errs) {
346
- var call = act === 'restore' ? post('/api/restore', {alias: alias})
347
- : post('/api/remove', {alias: alias, typed: alias, reason: 'from the dashboard: ' + (b.dataset.why || 'retired by hand')});
348
- return call.then(function (res) { return errs.concat(res.ok ? [] : res.errors); });
349
- });
350
- });
351
- chain.then(function (errs) {
352
- if (errs.length) { status.textContent = errs.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = verb; return; }
353
- status.textContent = 'Done. Reloading…'; setTimeout(function () { location.reload(); }, 700);
354
- });
355
- });
356
- });
357
-
358
- // ---- onboarding --------------------------------------------------------
359
- var od = $('onboard-dialog');
360
- var o = {find: $('o-find'), picker: $('o-picker'), errors: $('o-errors'), start: $('o-start'),
361
- close: $('o-close'), choose: $('o-choose'), run: $('o-run'), steps: $('o-steps'),
362
- log: $('o-log'), title: $('o-title'), result: $('o-result'), dismiss: $('o-dismiss')};
363
- var STEPS = ['Served by the endpoint','Registered', 'Synced to the llm CLI', 'Availability probe, 4 calls', 'Internal suite'];
364
- var timer = null, data = null;
365
- // What is ticked, by the candidate's key (served@endpoint): kept while the
366
- // search redraws the list, so you can search, tick, search again, tick.
367
- var chosen = {};
368
- // Folded groups, per browser: openrouter (direct) alone is hundreds of rows.
369
- var NEW = 'New since the daily check', folded = {};
370
- try { (JSON.parse(store('bench.picker.folded') || '[]') || []).forEach(function (g) { folded[g] = 1; }); } catch (e) { folded = {}; }
371
-
372
- function armStart() {
373
- var n = Object.keys(chosen).length;
374
- o.start.disabled = o.dismiss.disabled = !n;
375
- o.start.textContent = n > 1 ? 'Onboard ' + n + ' models' : 'Onboard model';
376
- o.dismiss.textContent = n > 1 ? 'Dismiss ' + n + ' models' : 'Dismiss model';
377
- }
378
- // Rows move between the lists here rather than by asking every endpoint again.
379
- function moveRows(keys, from, to) {
380
- data[from] = data[from].filter(function (c) { if (keys.indexOf(c.key) === -1) return true; data[to].push(c); return false; });
381
- data[to].sort(function (a, b) { return a.served < b.served ? -1 : a.served > b.served ? 1 : 0; });
382
- }
383
- function pickRow(c, dismissed) {
384
- var row = el('label', {class: 'pick'});
385
- var box = el('input', {type: 'checkbox', name: 'o-pick', value: c.served, 'data-endpoint': c.endpoint});
386
- if (dismissed) box.disabled = true;
387
- box.checked = !dismissed && !!chosen[c.key];
388
- row.appendChild(box);
389
- var name = el('span', {class: 'alias'}, c.served);
390
- row.appendChild(name);
391
- var side = el('span');
392
- if (c.new && !dismissed) side.appendChild(el('span', {class: 'muted'}, c.group)); // under New: where it comes from
393
- if (dismissed) {
394
- var b = el('button', {type: 'button', class: 'linkish', style: 'margin-left:.6rem'}, 'Bring back');
395
- b.addEventListener('click', function (e) {
396
- e.preventDefault();
397
- post('/api/dismiss', {served: c.key, undo: true}).then(function (res) {
398
- if (!res.ok) { listErrors(o.errors, res.errors); return; }
399
- moveRows([c.key], 'dismissed', 'candidates'); renderPicker();
400
- });
401
- });
402
- side.appendChild(b);
403
- }
404
- row.appendChild(side);
405
- box.addEventListener('change', function () {
406
- if (box.checked) chosen[c.key] = {served: c.served, endpoint: c.endpoint};
407
- else delete chosen[c.key];
408
- armStart();
409
- });
410
- return row;
411
- }
412
- function renderPicker() {
413
- o.picker.textContent = '';
414
- if (!data) return;
415
- var q = o.find.value.trim().toLowerCase();
416
- var groups = {};
417
- data.candidates.forEach(function (c) {
418
- // The endpoint too: "Onboard a model from it" filters on it, and the default
419
- // endpoint's models are grouped by connection, not by its name.
420
- if (q && [c.served, c.group, c.endpoint].every(function (s) { return s.toLowerCase().indexOf(q) === -1; })) return;
421
- // New ones in a group of their own, first: just added to the hub is what
422
- // you came to onboard, and it is there without scrolling or unfolding.
423
- var g = c.new ? NEW : c.group;
424
- (groups[g] = groups[g] || []).push(c);
425
- });
426
- var names = Object.keys(groups).sort(function (a, b) { return (b === NEW) - (a === NEW) || (a < b ? -1 : a > b ? 1 : 0); });
427
- if (!names.length) o.picker.appendChild(el('p', {class: 'empty'}, data.candidates.length ? 'Nothing matches.' : 'Everything the endpoints serve is already on the bench, retired or dismissed.'));
428
- names.forEach(function (g) {
429
- // Folds, remembered per browser; a search opens every group it matches in.
430
- var d = el('details', {class: 'grp'});
431
- d.open = !!q || !folded[g];
432
- d.appendChild(el('summary', {}, g + ' · ' + groups[g].length));
433
- groups[g].forEach(function (c) { d.appendChild(pickRow(c, false)); });
434
- d.addEventListener('toggle', function () {
435
- if (q) return; // opened by the search, not by you
436
- if (d.open) delete folded[g]; else folded[g] = 1;
437
- store('bench.picker.folded', JSON.stringify(Object.keys(folded)));
438
- });
439
- o.picker.appendChild(d);
440
- });
441
- if (data.dismissed.length) {
442
- var d = el('details'); d.appendChild(el('summary', {}, data.dismissed.length + ' dismissed'));
443
- data.dismissed.forEach(function (c) { d.appendChild(pickRow(c, true)); });
444
- o.picker.appendChild(d);
445
- }
446
- }
447
- function loadCandidates() {
448
- o.picker.textContent = ''; o.picker.appendChild(el('p', {class: 'empty'}, 'Asking the endpoints what they serve…'));
449
- o.start.disabled = o.dismiss.disabled = true;
450
- o.find.readOnly = true; // typing before the list arrives is lost, or lands on a row
451
- fetch('/api/candidates').then(function (r) { return r.json(); }).then(function (res) {
452
- o.find.readOnly = false; o.find.focus();
453
- if (!res.ok) { o.picker.textContent = ''; listErrors(o.errors, res.errors); return; }
454
- if (res.errors.length) listErrors(o.errors, res.errors); // one endpoint down; the rest listed
455
- data = res;
456
- // Only what is still offered stays ticked.
457
- var offered = {};
458
- res.candidates.forEach(function (c) { offered[c.key] = 1; });
459
- Object.keys(chosen).forEach(function (k) { if (!offered[k]) delete chosen[k]; });
460
- renderPicker(); armStart();
461
- }).catch(function (err) { o.find.readOnly = false; o.errors.textContent = String(err); });
462
- }
463
- $('onboard-open').addEventListener('click', function () {
464
- o.errors.textContent = ''; o.find.value = ''; o.choose.hidden = false; o.run.hidden = true;
465
- $('o-queue').hidden = true; chosen = {}; waitToken++;
466
- o.start.hidden = o.dismiss.hidden = false; armStart(); o.title.textContent = 'Onboard a model';
467
- od.showModal(); o.find.focus(); loadCandidates();
468
- });
469
- o.find.addEventListener('input', renderPicker);
470
- o.close.addEventListener('click', function () { od.close(); });
471
- od.addEventListener('close', function () { if (timer) { clearInterval(timer); timer = null; } });
472
-
473
- function renderSteps(text, finished, exit) {
474
- var reached = 0;
475
- for (var i = 1; i <= 5; i++) if (text.indexOf('[' + i + '/5]') !== -1) reached = i;
476
- var stopped = /STOPPING|is not served|cannot reach/.test(text) || (finished && exit !== 0);
477
- var lines = text.split('\n');
478
- o.steps.textContent = '';
479
- STEPS.forEach(function (label, i) {
480
- var n = i + 1, li = el('li', {}, label);
481
- var line = lines.filter(function (l) { return l.indexOf('[' + n + '/5]') !== -1; })[0];
482
- if (line) li.appendChild(el('small', {}, line.replace(/^\s*\[\d\/5\]\s*/, '')));
483
- if (n < reached || (n === reached && finished && !stopped)) li.className = 'done';
484
- else if (n === reached && stopped) {
485
- li.className = 'fail';
486
- // Why it stopped, under the step that failed, not only in the full log.
487
- var at = text.search(/STOPPING|is not served|cannot reach/);
488
- if (at !== -1) li.appendChild(el('small', {}, text.slice(at).split('\n').slice(0, 3).join(' ').trim()));
489
- }
490
- else if (n === reached || (n === 1 && !reached && !finished)) li.className = 'now';
491
- o.steps.appendChild(li);
492
- });
493
- }
494
- // Everything ticked goes to the queue (serve.queue_onboard), which runs one
495
- // model at a time. The popup closes: the header's chip shows the batch, and
496
- // its Details the queue. Refused, it stays open with the reasons.
497
- o.start.addEventListener('click', function () {
498
- var items = Object.keys(chosen).map(function (k) { return chosen[k]; });
499
- if (!items.length) return;
500
- o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true; o.start.textContent = 'Starting…';
501
- post('/api/onboard-queue', {items: items}).then(function (res) {
502
- if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
503
- od.close(); oqPoll();
504
- }).catch(function (err) { o.errors.textContent = String(err); armStart(); });
505
- });
506
- // Every ticked model at once (serve.dismiss); the dismissed list brings one back.
507
- o.dismiss.addEventListener('click', function () {
508
- var keys = Object.keys(chosen);
509
- if (!keys.length) return;
510
- o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true;
511
- post('/api/dismiss', {served: keys}).then(function (res) {
512
- if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
513
- moveRows(keys, 'candidates', 'dismissed'); chosen = {}; renderPicker(); armStart();
514
- }).catch(function (err) { o.errors.textContent = String(err); armStart(); });
515
- });
516
- // After Discover's Add queues a model: with nothing ahead of it, it is
517
- // followed step by step, otherwise the queue shows.
518
- var waitToken = 0;
519
- function afterQueued(res, one, title, before) {
520
- oqPoll();
521
- var at = -1; // its LAST entry: an earlier, finished one may share its name
522
- res.items.forEach(function (i, n) { if (one && i.served === one.served && i.endpoint === one.endpoint) at = n; });
523
- var ahead = at === -1 ? [] : res.items.slice(0, at).filter(function (i) { return i.state === 'queued' || i.state === 'running'; });
524
- if (one && at !== -1 && !ahead.length) waitAndFollow(one, at, title || 'Onboarding ' + one.served + ' from ' + one.endpoint, before || '', ++waitToken, 20);
525
- else showQueue(res);
526
- }
527
- // The queue starts it within a moment; follow its log once it has one,
528
- // unless the popup was closed or reopened meanwhile (the header has it then).
529
- function waitAndFollow(one, at, title, before, token, tries) {
530
- if (token !== waitToken || !od.open) return;
531
- fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
532
- if (!q.ok || token !== waitToken || !od.open) return;
533
- var it = q.items[at];
534
- if (!it || it.served !== one.served || it.endpoint !== one.endpoint) { showQueue(q); return; }
535
- if (it.log) { follow(it.log, title, before); return; }
536
- if (it.state !== 'queued' && it.state !== 'running' || tries <= 0) { showQueue(q); return; }
537
- setTimeout(function () { waitAndFollow(one, at, title, before, token, tries - 1); }, 500);
538
- });
539
- }
540
-
541
- // ---- the onboarding queue: progress in the header (serve.onboard_queue) --
542
- // Polled while it runs, and once on every page load, so a batch started
543
- // before a reload still shows. Its results stay until Clear.
544
- var oq = {box: $('oq'), text: $('oq-text'), chip: $('oq-chip'), stop: $('oq-stop'), clear: $('oq-clear'),
545
- view: $('o-queue'), sum: $('o-queue-sum'), list: $('o-queue-list')};
546
- var OQ_CHIP = {queued: '', running: 'new', done: 'ok', 'no answer': 'warn', failed: 'err', skipped: ''};
547
- var oqTimer = null, oqLast = '';
548
- function oqCount(q, state) { return q.items.filter(function (i) { return i.state === state; }).length; }
549
- function oqRender(q) {
550
- var n = q.items.length;
551
- oq.box.hidden = !n; batchRunning = !!q.running;
552
- var now = q.items.filter(function (i) { return i.state === 'running'; })[0];
553
- var finished = n - oqCount(q, 'queued') - (now ? 1 : 0);
554
- // The chip fills as models finish, and by fifths of the running one as its
555
- // log reaches each of onboard.py's five steps (serve.onboard_queue `step`).
556
- // At the end it is green when every one is on the bench, red (with one
557
- // shake) when any failed or did not answer.
558
- var step = now && now.step ? now.step : 0;
559
- oq.chip.style.setProperty('--fill', (n ? Math.round(100 * (finished + step / 5) / n) : 0) + '%');
560
- var bad = oqCount(q, 'failed') + oqCount(q, 'no answer');
561
- oq.chip.classList.toggle('done', !q.running && n > 0 && oqCount(q, 'done') === n);
562
- oq.chip.classList.toggle('fail', !q.running && bad > 0);
563
- if (q.running) {
564
- oq.text.textContent = (q.stopping ? 'Stopping after ' : 'Onboarding ' + (finished + 1) + ' of ' + n
565
- + (step ? ', step ' + step + '/5' : '') + ': ') + (now ? now.served : '…');
566
- } else {
567
- oq.text.textContent = 'Onboarded ' + oqCount(q, 'done') + ' of ' + n
568
- + [['failed', 'failed'], ['no answer', 'did not answer'], ['skipped', 'skipped']].map(function (s) {
569
- var c = oqCount(q, s[0]); return c ? ' · ' + c + ' ' + s[1] : '';
570
- }).join('');
571
- }
572
- oq.text.title = oq.text.textContent; // cut short beside the button; whole on hover
573
- oq.stop.hidden = !q.running || q.stopping; oq.clear.hidden = q.running;
574
- var sig = JSON.stringify(q);
575
- if (sig !== oqLast) { oqLast = sig; if (!oq.view.hidden) oqList(q); }
576
- }
577
- function oqList(q) {
578
- oq.sum.textContent = q.running
579
- ? 'One at a time, in this order. Closing this does not stop it; Stop in the header lets the model running now finish and skips the rest.'
580
- : q.items.length ? 'Finished. Each log is in runs/.' : 'Nothing queued.';
581
- oq.list.textContent = '';
582
- q.items.forEach(function (i) {
583
- var li = el('li'), head = el('div', {class: 'what'});
584
- head.appendChild(el('span', {class: 'alias'}, i.served));
585
- head.appendChild(el('span', {class: 'chip ' + (OQ_CHIP[i.state] || '')}, i.state));
586
- if (i.alias && i.state !== 'failed') head.appendChild(el('span', {class: 'muted'}, 'as ' + i.alias));
587
- li.appendChild(head);
588
- if (i.note) li.appendChild(el('small', {}, i.note));
589
- if (i.log) {
590
- var d = el('details'), pre = el('pre', {class: 'log'});
591
- d.appendChild(el('summary', {class: 'muted'}, 'Log'));
592
- d.appendChild(pre);
593
- d.addEventListener('toggle', function () {
594
- if (!d.open) return;
595
- fetch('/api/onboard?log=' + encodeURIComponent(i.log)).then(function (r) { return r.json(); })
596
- .then(function (s) { pre.textContent = s.ok ? (s.tail || '(no output yet)') : s.errors.join(' '); });
597
- });
598
- li.appendChild(d);
599
- }
600
- oq.list.appendChild(li);
601
- });
602
- }
603
- function showQueue(q) {
604
- if (timer) { clearInterval(timer); timer = null; } // a model followed step by step: the queue view replaces it
605
- waitToken++;
606
- o.errors.textContent = ''; o.choose.hidden = true; o.run.hidden = true; o.start.hidden = o.dismiss.hidden = true;
607
- oq.view.hidden = false; o.title.textContent = 'Onboarding queue';
608
- if (!od.open) od.showModal();
609
- oqLast = ''; oqRender(q);
610
- }
611
- function oqPoll() {
612
- fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
613
- if (!q.ok) return;
614
- oqRender(q);
615
- if (q.running && !oqTimer) oqTimer = setInterval(oqPoll, 3000);
616
- if (!q.running && oqTimer) { clearInterval(oqTimer); oqTimer = null; }
617
- });
618
- }
619
- function oqAct(path) {
620
- post(path, {}).then(function (r) { if (r.ok) oqRender(r); else oq.text.textContent = r.errors.join(' '); });
621
- }
622
- $('oq-open').addEventListener('click', function () {
623
- fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) { showQueue(q); oqPoll(); });
624
- });
625
- oq.stop.addEventListener('click', function () { oqAct('/api/onboard-queue/stop'); });
626
- oq.clear.addEventListener('click', function () { oqAct('/api/onboard-queue/clear'); if (!oq.view.hidden) od.close(); });
627
- oqPoll();
628
- // ---- Discover: a row opens its model, with every route to add it by -----
629
- // The rest of what is known comes from /api/discover-model, so the page does
630
- // not carry 1,200 models' details it will mostly never show.
631
- var dd = $('discover-dialog');
632
- function price(x) {
633
- var c = x.cost || {};
634
- return x.free ? 'free' : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
635
- }
636
- function limits(l) {
637
- return Object.keys(l || {}).map(function (k) { return Number(l[k]).toLocaleString() + ' ' + k; }).join(' · ');
638
- }
639
- function addVia(x, b) {
640
- var direct = x.via === 'direct';
641
- if (!confirm('Add ' + x.model + ' via ' + x.label + (x.free ? '' : ' (PAID)') + '?\n\n'
642
- + (direct ? 'The bench only: register it on the ' + x.endpoint + ' endpoint, a 4-call probe, then the suite.'
643
- // A LiteLLM or CLI Proxy API route says its own steps: only what that hub takes.
644
- : x.steps ? x.steps.map(function (s, i) { return (i + 1) + '. ' + s; }).join('\n')
645
- : '1. Open WebUI: add it to the ' + x.connection + ' connection\'s Model IDs (if that connection lists them)\n'
646
- + '2. Open WebUI: a public model record, tagged and with the logo of the connection\'s other models\n'
647
- + '3. The bench: register, a 4-call probe, then the suite'))) return;
648
- b.disabled = true; b.textContent = 'Adding…'; $('d-errors').textContent = '';
649
- // Either way the onboarding joins the queue (serve.queue_onboard): one at a time.
650
- var call = direct ? post('/api/onboard-queue', {items: [{served: x.model, endpoint: x.endpoint}]})
651
- : post('/api/discover/add', {connection: x.connection, model: x.model, endpoint: x.endpoint});
652
- call.then(function (res) {
653
- if (!res.ok) {
654
- listErrors($('d-errors'), (res.steps || []).concat(res.errors));
655
- b.disabled = false; b.textContent = 'Add via this'; return;
656
- }
657
- dd.close(); o.errors.textContent = ''; od.showModal();
658
- afterQueued(res, {served: res.served || x.model, endpoint: res.endpoint || x.endpoint},
659
- 'Adding ' + (res.served || x.model) + ' via ' + x.label,
660
- res.steps && res.steps.length ? res.steps.join('. ') + '. ' : '');
661
- }).catch(function (err) { $('d-errors').textContent = String(err); b.disabled = false; b.textContent = 'Add via this'; });
662
- }
663
- function openDiscover(provider, model) {
664
- $('d-errors').textContent = '';
665
- fetch('/api/discover-model?provider=' + encodeURIComponent(provider) + '&model=' + encodeURIComponent(model))
666
- .then(function (r) { return r.json(); }).then(function (res) {
667
- if (!res.ok) { listErrors($('d-errors'), res.errors); dd.showModal(); return; }
668
- if (res.errors.length) listErrors($('d-errors'), res.errors); // a hub that could not be asked
669
- var r = res.model, p = $('d-profile'), list = $('d-routes');
670
- $('d-title').textContent = r.name;
671
- $('d-served').textContent = r.provider_name + ' · ' + r.model + ' · listed by ' + r.sources.join(' and ')
672
- + (r.on_bench_as ? ' · on the bench as ' + r.on_bench_as : '');
673
- p.textContent = '';
674
- [['Input', r.in_mods], ['Output', r.out_mods]].forEach(function (x) {
675
- var row = el('div', {class: 'prof'});
676
- row.appendChild(el('span', {class: 'muted'}, x[0] + ' '));
677
- (x[1] && x[1].length ? x[1] : ['unknown']).forEach(function (v) { row.appendChild(el('span', {class: 'chip'}, v)); });
678
- p.appendChild(row);
679
- });
680
- var can = [['tools', r.tools], ['vision', r.vision], ['reasons', r.reasoning]]
681
- .filter(function (x) { return x[1]; }).map(function (x) { return x[0]; });
682
- var facts = [
683
- r.context ? 'context ' + Number(r.context).toLocaleString() + ' tokens' : 'context unknown',
684
- r.output ? 'output up to ' + Number(r.output).toLocaleString() : '',
685
- can.length ? 'can: ' + can.join(', ') : '',
686
- price(r),
687
- r.limits ? 'limits ' + limits(r.limits) : '',
688
- r.released ? 'released ' + r.released : '',
689
- r.knowledge ? 'knowledge to ' + r.knowledge : '',
690
- r.open_weights != null ? (r.open_weights ? 'open weights' : 'closed weights') : '',
691
- 'first seen ' + (r.first_seen === 'baseline' ? 'before the list began' : r.first_seen)];
692
- p.appendChild(el('div', {class: 'prof'}, facts.filter(Boolean).join(' · ')));
693
- if (r.also_on && r.also_on.length) p.appendChild(el('div', {class: 'prof muted'}, 'Also on the bench as ' + r.also_on.join(', ')));
694
- list.textContent = '';
695
- res.routes.forEach(function (x) {
696
- var row = el('div', {class: 'endpoint'}), left = el('div');
697
- left.appendChild(el('div', {class: 'what'}, x.label));
698
- left.appendChild(el('div', {class: 'base'}, x.model + ' · ' + price(x) + (x.limits ? ' · ' + limits(x.limits) : '')));
699
- left.appendChild(el('div', {class: 'base'}, x.note));
700
- row.appendChild(left);
701
- var acts = el('div', {class: 'acts'});
702
- if (x.on_bench_as) acts.appendChild(el('span', {class: 'chip ok'}, 'on the bench as ' + x.on_bench_as));
703
- else {
704
- var b = el('button', {class: 'btn small', type: 'button'}, 'Add via this');
705
- b.addEventListener('click', function () { addVia(x, b); });
706
- acts.appendChild(b);
707
- }
708
- row.appendChild(acts);
709
- list.appendChild(row);
710
- });
711
- if (!res.routes.length) list.appendChild(el('p', {class: 'rules'}, 'None: ' + r.provider_name
712
- + ' needs a connection in a hub or an endpoint of its own first'
713
- + (r.api ? ' (its OpenAI-compatible base URL is ' + r.api + ')' : '') + '.'));
714
- dd.showModal();
715
- }).catch(function (err) { $('d-errors').textContent = String(err); dd.showModal(); });
716
- }
717
- document.addEventListener('click', function (e) {
718
- var row = e.target.closest('tr.pick-row'); // the whole row: its name, its Add…, anywhere
719
- if (!row) return;
720
- e.preventDefault(); openDiscover(row.dataset.provider, row.dataset.model);
721
- });
722
- $('d-close').addEventListener('click', function () { dd.close(); });
723
- function follow(log, title, before) {
724
- o.choose.hidden = true; o.run.hidden = false; o.start.hidden = o.dismiss.hidden = true; $('o-queue').hidden = true;
725
- o.title.textContent = title; o.result.className = 'served';
726
- o.result.textContent = before + 'Running in the background; closing this does not stop it. Log: runs/' + log;
727
- renderSteps('', false, null);
728
- o.log.textContent = ''; delete o.log.dataset.opened; o.log.parentNode.open = false;
729
- timer = setInterval(function () {
730
- fetch('/api/onboard?log=' + encodeURIComponent(log)).then(function (r) { return r.json(); }).then(function (s) {
731
- if (!s.ok) return;
732
- o.log.textContent = s.tail || '(no output yet)';
733
- // Open the log the first time it has something in it; after that the user decides.
734
- if (s.tail && !o.log.dataset.opened) { o.log.parentNode.open = true; o.log.dataset.opened = '1'; }
735
- renderSteps(s.tail || '', s.done, s.exit);
736
- if (s.done) {
737
- clearInterval(timer); timer = null;
738
- var alias = (/->\s+alias\s+(\S+)/.exec(s.tail) || [])[1];
739
- if (s.exit === 0) {
740
- o.result.textContent = 'Finished. ' + (alias ? alias + ' is on the bench. ' : '') + 'The suite scores are in the dated suite note; whether to route work there is your call. Closing…';
741
- o.result.className = 'done';
742
- closeAndReload(od);
743
- } else {
744
- o.result.textContent = 'Stopped with exit ' + s.exit + '. The last lines of the log say why.';
745
- fresh.hidden = false;
746
- }
747
- }
748
- });
749
- }, 3000);
750
- }
751
-
752
- // ---- theme: applied the moment it is picked, and saved ------------------
753
- // Every theme's colours are on the page (serve.brand_css), keyed by the
754
- // data-theme on <html>, so switching is one attribute: no reload. The
755
- // charts read their colours when drawn, so they are drawn again.
756
- document.querySelectorAll('#themes input[name=theme]').forEach(function (input) {
757
- input.addEventListener('change', function () {
758
- document.documentElement.setAttribute('data-theme', input.value);
759
- $('t-errors').textContent = ''; $('t-done').textContent = '';
760
- if (typeof drawCharts === 'function') drawCharts();
761
- post('/api/theme', {theme: input.value}).then(function (r) {
762
- if (!r.ok) { listErrors($('t-errors'), r.errors); return; }
763
- $('t-done').textContent = 'Saved: ' + input.value + ' is the theme from now on.';
764
- if (location.search.indexOf('theme=') !== -1) history.replaceState(null, '', '/'); // drop a ?theme preview
765
- });
766
- });
767
- });
768
-
769
- // ---- gateway keys: shown once, then only their names ---------------------
770
- if ($('k-make')) $('k-make').addEventListener('click', function () {
771
- $('k-errors').textContent = '';
772
- var scope = Array.prototype.map.call(document.querySelectorAll('input[name=k-scope]:checked'),
773
- function (c) { return c.value; });
774
- post('/api/gateway-key', {name: $('k-name').value, collections: scope, rpm: $('k-rpm').value}).then(function (r) {
775
- if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
776
- $('k-value').value = r.key;
777
- $('k-done').hidden = false;
778
- $('k-value').select();
779
- $('k-make').disabled = true; // reload to see it listed; the value is gone then
780
- });
781
- });
782
- document.querySelectorAll('[data-revoke-key]').forEach(function (b) {
783
- b.addEventListener('click', function () {
784
- var name = b.dataset.revokeKey;
785
- if (!confirm('Revoke the gateway key ' + name + '? Anything using it is refused from now on.')) return;
786
- post('/api/gateway-key/remove', {name: name}).then(function (r) {
787
- if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
788
- location.reload();
789
- });
790
- });
791
- });
792
-
793
- // ---- scheduler: its mode and jobs (serve.save_scheduler) -----------------
794
- var sMode = $('s-mode');
795
- function modeWhy() { $('s-mode-why').textContent = sMode.selectedOptions[0].dataset.why; }
796
- sMode.addEventListener('change', modeWhy); modeWhy();
797
- $('s-save').addEventListener('click', function () {
798
- var jobs = {};
799
- document.querySelectorAll('input[name=s-job]').forEach(function (c) { jobs[c.value] = c.checked; });
800
- if (sMode.value === 'run' && sMode.dataset.was !== 'run'
801
- && !confirm('Run the jobs for real? They make model calls on their own schedule. If something else'
802
- + ' also runs them (cron, Windows Task Scheduler), turn it off now, or every job runs twice.')) return;
803
- $('s-errors').textContent = ''; $('s-done').textContent = '';
804
- post('/api/scheduler', {mode: sMode.value, jobs: jobs}).then(function (r) {
805
- if (!r.ok) { listErrors($('s-errors'), r.errors); return; }
806
- sMode.dataset.was = sMode.value; $('s-done').textContent = 'Saved. It applies within a minute.';
807
- });
808
- });
809
-
810
- // ---- model client (serve.save_client): the next call uses it -------------
811
- $('mc-save').addEventListener('click', function () {
812
- $('mc-errors').textContent = ''; $('mc-done').textContent = '';
813
- post('/api/client', {client: $('mc-client').value}).then(function (r) {
814
- if (!r.ok) { listErrors($('mc-errors'), r.errors); return; }
815
- $('mc-done').textContent = 'Saved. The next model call uses it.';
816
- });
817
- });
818
-
819
- // ---- free only in their own app (serve.save_own_client_free) -------------
820
- if ($('oc-save')) $('oc-save').addEventListener('click', function () {
821
- var providers = {}, add = $('oc-add').value.trim().toLowerCase();
822
- document.querySelectorAll('input[name=oc-prov]').forEach(function (c) { providers[c.value] = c.checked; });
823
- if (add) providers[add] = true;
824
- $('oc-errors').textContent = ''; $('oc-done').textContent = '';
825
- post('/api/own-client-free', {providers: providers}).then(function (r) {
826
- if (!r.ok) { listErrors($('oc-errors'), r.errors); return; }
827
- if (add) { location.reload(); return; } // the new provider gets its row
828
- $('oc-done').textContent = 'Saved. Discover uses it on its next load.';
829
- });
830
- });
831
-
832
- // ---- endpoints: keys go in, never come back out -------------------------
833
- // The Endpoints page lists them (serve.endpoints_page, each row's data in
834
- // data-ep); Add an endpoint and Change key are one dialog. The page is
835
- // drawn by the server, so a change reloads it, as the model popup does.
836
- var ed = $('endpoint-dialog');
837
- var e = {name: $('e-name'), base: $('e-base'), key: $('e-key'),
838
- kind: $('e-kind'), hub: $('e-hub'), admin: $('e-admin'),
839
- errors: $('e-errors'), done: $('e-done'), test: $('e-test'), save: $('e-save'),
840
- listErrors: $('el-errors'), listDone: $('el-done'), changed: false};
841
- e.kind.addEventListener('change', function () {
842
- $('e-hub-row').hidden = $('e-admin-row').hidden = e.kind.value !== 'hub';
843
- });
844
- ed.querySelectorAll('[data-close]').forEach(function (b) { b.addEventListener('click', function () { ed.close(); }); });
845
- ed.addEventListener('close', function () { if (e.changed) location.reload(); });
846
- function reloadSoon(message) {
847
- e.listErrors.textContent = ''; e.listDone.textContent = message + ' Reloading…';
848
- setTimeout(function () { location.reload(); }, 900);
849
- }
850
- var rowActs = {
851
- change: function (x) { openEndpoint('change', x); },
852
- again: function (x) { // this endpoint, not a new one
853
- openEndpoint('again', x);
854
- connectStart({id: x.connect.provider, flow: x.connect.flow, label: x.connect.label}, x.name);
855
- },
856
- 'default': function (x) {
857
- if (!confirm('Make ' + x.name + ' the default endpoint? The default is where a model is taken'
858
- + ' to be when nothing says otherwise, and the one endpoint that cannot be removed.')) return;
859
- post('/api/endpoint/default', {name: x.name}).then(function (r) {
860
- if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
861
- reloadSoon(r.note + '.');
862
- });
863
- },
864
- // Its models go with it, after saying how many: each is retired as by
865
- // hand (evidence kept, restorable once the endpoint is back).
866
- remove: function (x) {
867
- var n = x.models, them = n + ' model' + (n === 1 ? '' : 's');
868
- if (!confirm(n ? 'Remove the ' + x.name + ' endpoint and the ' + them + ' on the bench through it?\n\n'
869
- + 'The ' + them + ' will be retired too. Their results and history are kept, and'
870
- + ' Restore brings them back if an endpoint named ' + x.name + ' is added again.'
871
- : 'Remove the ' + x.name + ' endpoint?')) return;
872
- post('/api/endpoint/remove', {name: x.name, with_models: n > 0}).then(function (r) {
873
- if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
874
- reloadSoon('Removed ' + x.name + '; ' + r.note + '.');
875
- });
876
- }
877
- };
878
- $('tab-endpoints').addEventListener('click', function (ev) {
879
- var b = ev.target.closest('.ep button[data-ep-act]');
880
- if (b) rowActs[b.dataset.epAct](JSON.parse(b.closest('.ep').dataset.ep));
881
- });
882
- function loadLimits() {
883
- var list = $('l-list');
884
- fetch('/api/connections').then(function (r) { return r.json(); }).then(function (res) {
885
- list.textContent = '';
886
- res.connections.forEach(function (c) {
887
- var row = el('div', {class: 'endpoint'});
888
- var left = el('div');
889
- left.appendChild(el('div', {class: 'what'}, c.name));
890
- left.appendChild(el('div', {class: 'base'}, c.models.join(', ')));
891
- var inputs = el('div', {class: 'limits'});
892
- res.keys.forEach(function (k) {
893
- var lab = el('label', {}, k + ' ');
894
- var box = el('input', {type: 'number', min: '1', 'data-k': k, placeholder: '—'});
895
- if (c.limits[k]) box.value = c.limits[k];
896
- lab.appendChild(box); inputs.appendChild(lab);
897
- });
898
- left.appendChild(inputs);
899
- row.appendChild(left);
900
- var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
901
- save.addEventListener('click', function () {
902
- var limits = {};
903
- inputs.querySelectorAll('input').forEach(function (b) { if (b.value) limits[b.dataset.k] = parseInt(b.value, 10); });
904
- $('l-errors').textContent = ''; $('l-done').textContent = '';
905
- post('/api/connection-limits', {connection: c.name, limits: limits}).then(function (r) {
906
- if (!r.ok) { listErrors($('l-errors'), r.errors); return; }
907
- $('l-done').textContent = 'Saved limits for ' + c.name + '.';
908
- });
909
- });
910
- var acts = el('div', {class: 'acts'}); acts.appendChild(save); row.appendChild(acts);
911
- list.appendChild(row);
912
- });
913
- });
914
- }
915
- onShow.endpoints = function () { // its own-app and rate-limit sections
916
- $('cn-app-errors').textContent = $('cn-app-done').textContent = '';
917
- loadProviders(); loadLimits();
918
- };
919
- // The dialog, in one of three modes. add: the choices, API key first, then
920
- // each provider's sign-in; change: one endpoint's key form; again: a
921
- // sign-in under way for an endpoint that has one.
922
- function openEndpoint(mode, x) {
923
- cn.run++;
924
- e.adding = mode === 'add'; // Add refuses a name in use; Change key edits that one
925
- $('e-title').textContent = mode === 'change' ? 'Change the key for ' + x.name
926
- : mode === 'again' ? 'Sign in to ' + x.connect.label + ' again' : 'Add an endpoint';
927
- $('e-blurb').hidden = cn.list.hidden = $('e-kinds').hidden = !e.adding;
928
- e.key.placeholder = e.adding ? 'blank if it needs none, as a local runtime' : 'leave blank to keep the current one';
929
- cn.wait.hidden = true; $('e-after').hidden = true;
930
- resetKeyForm();
931
- choose(mode === 'again' ? null : 'key');
932
- if (mode === 'change') {
933
- e.name.value = x.name; e.base.value = x.base; e.name.readOnly = true;
934
- $('e-admin-row').hidden = !x.hub; // a hub's admin key changes here too
935
- }
936
- if (e.adding) loadProviders();
937
- if (!ed.open) ed.showModal();
938
- (mode === 'change' ? e.key : e.name).focus();
939
- }
940
- // Which form shows: 'key' (paste an API key), a provider (sign in), or none.
941
- function choose(p) {
942
- cn.pick = p && p !== 'key' ? p : null;
943
- $('e-form').hidden = p !== 'key';
944
- $('cn-form').hidden = !cn.pick;
945
- if (cn.pick) {
946
- $('cn-name').value = cn.pick.suggest;
947
- $('cn-go').textContent = (cn.pick.flow === 'local' ? 'Add ' : 'Sign in to ') + cn.pick.label;
948
- }
949
- e.errors.textContent = ''; e.done.textContent = '';
950
- press();
951
- }
952
- function press() {
953
- var on = cn.pick ? cn.pick.id : $('e-form').hidden ? '' : 'key';
954
- cn.list.querySelectorAll('button').forEach(function (b) { b.setAttribute('aria-pressed', String(b.dataset.choice === on)); });
955
- }
956
- function resetKeyForm() {
957
- e.name.value = e.base.value = e.key.value = e.admin.value = ''; e.name.readOnly = false;
958
- e.kind.value = 'provider'; e.hub.selectedIndex = 0; $('e-hub-row').hidden = $('e-admin-row').hidden = true;
959
- untested();
960
- }
961
- // Save opens only after Test passes, and closes again when anything in the
962
- // form changes. The server tests again on Save: this is the form's manners,
963
- // not the guarantee.
964
- function untested() { e.save.disabled = true; e.save.title = 'Test it first'; }
965
- $('e-form').addEventListener('input', function () { untested(); e.done.textContent = ''; });
966
- function keyBody() {
967
- var body = {name: e.name.value, base: e.base.value, key: e.key.value, new: e.adding,
968
- admin: $('e-admin-row').hidden ? '' : e.admin.value};
969
- if (e.adding) { body.kind = e.kind.value; body.hub = e.kind.value === 'hub' ? e.hub.value : ''; } // Change key keeps its kind
970
- return body;
971
- }
972
- $('e-add').addEventListener('click', function () { openEndpoint('add'); });
973
- e.test.addEventListener('click', function () {
974
- e.errors.textContent = ''; e.done.textContent = ''; untested();
975
- e.test.disabled = true; e.test.textContent = 'Testing…';
976
- var body = keyBody(), sent = JSON.stringify(body), run = cn.run; body.test = true;
977
- function end() { e.test.disabled = false; e.test.textContent = 'Test'; }
978
- post('/api/endpoint', body).then(function (r) {
979
- end();
980
- if (run !== cn.run || JSON.stringify(keyBody()) !== sent) return; // opened afresh, or edited, meanwhile
981
- if (!r.ok) { listErrors(e.errors, r.errors); return; }
982
- e.done.textContent = 'It answers: ' + r.served + ' models listed. ' + (r.note || '');
983
- e.save.disabled = false; e.save.title = '';
984
- }).catch(function (err) { end(); e.errors.textContent = String(err); });
985
- });
986
- e.save.addEventListener('click', function () {
987
- e.errors.textContent = ''; e.save.disabled = e.test.disabled = true; e.save.textContent = 'Saving…';
988
- function end() { e.save.textContent = 'Save'; e.test.disabled = false; }
989
- post('/api/endpoint', keyBody()).then(function (r) {
990
- end();
991
- if (!r.ok) { listErrors(e.errors, r.errors); return; } // Save stays shut: test again
992
- var name = e.name.value.trim().toLowerCase();
993
- endpointDone('Saved. ' + name + ' lists ' + r.served + ' models; they appear in the onboarding picker. '
994
- + (r.note || ''), e.adding ? name : '');
995
- }).catch(function (err) { end(); e.errors.textContent = String(err); });
996
- });
997
- // Saved or signed in: the forms fold, the message stays, with the next step.
998
- function endpointDone(message, endpoint) {
999
- resetKeyForm(); choose(null);
1000
- cn.list.hidden = $('e-blurb').hidden = cn.wait.hidden = true; $('e-after').hidden = false;
1001
- e.done.textContent = message;
1002
- if (endpoint) e.done.appendChild(onboardFrom(endpoint));
1003
- e.changed = true; // the page reloads when the dialog closes
1004
- }
1005
-
1006
- // ---- connect: sign in on the provider's page (metrics/connect.py) --------
1007
- // `run` counts the dialog's openings: a sign-in still finishing after the
1008
- // dialog was closed or opened again reports on the Endpoints page instead,
1009
- // and reloads it once nothing is open over it.
1010
- var cn = {list: $('cn-list'), wait: $('cn-wait'), msg: $('cn-msg'), timer: null, run: 0};
1011
- // Reached from another machine (a hosted install): a callback to this
1012
- // machine's 127.0.0.1 could not arrive, so sign-ins go by code instead.
1013
- var remote = !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(location.hostname);
1014
- function connectDone(r, run) {
1015
- clearInterval(cn.timer); cn.timer = null;
1016
- var said = r.ok && r.label + ' is connected as ' + r.endpoint + ': it lists ' + r.served + ' models. ' + (r.note || '');
1017
- if (!ed.open || run !== cn.run) {
1018
- if (!r.ok) listErrors(e.listErrors, r.errors);
1019
- else if (ed.open) { e.listDone.textContent = said; e.changed = true; }
1020
- else reloadSoon(said);
1021
- return;
1022
- }
1023
- cn.wait.hidden = true;
1024
- if (!r.ok) { // back to the provider's form to try again, or just Close
1025
- $('cn-form').hidden = !cn.pick; $('e-after').hidden = !!cn.pick;
1026
- listErrors(e.errors, r.errors); return;
1027
- }
1028
- endpointDone(said, r.endpoint);
1029
- }
1030
- // The next step for a new endpoint, signed in or pasted: onboard what it serves.
1031
- function onboardFrom(endpoint) {
1032
- var go = el('button', {class: 'btn small', type: 'button', style: 'margin-left:.6rem'}, 'Onboard a model from it');
1033
- go.addEventListener('click', function () {
1034
- // No reload on the way: onboarding opens over the page, and the page's
1035
- // fresh-results button offers the reload once the files have changed.
1036
- e.changed = false; ed.close(); $('onboard-open').click(); o.find.value = endpoint; // the picker filters on it once loaded
1037
- });
1038
- return go;
1039
- }
1040
- function connectStart(p, endpoint) {
1041
- e.errors.textContent = ''; e.done.textContent = '';
1042
- var run = cn.run;
1043
- // Opened now, while this is still the click: a window opened after the
1044
- // request returns is a popup the browser blocks.
1045
- var byCode = remote && p.flow !== 'local';
1046
- var tab = !byCode && (p.flow === 'key' || p.flow === 'oauth') ? window.open('', '_blank') : null;
1047
- if (tab) tab.opener = null;
1048
- post('/api/connect/start', {provider: p.id, endpoint: endpoint, code: byCode}).then(function (r) {
1049
- if (!r.ok || r.state) { if (tab) tab.close(); connectDone(r, run); return; } // refused, or local: saved at once
1050
- cn.wait.hidden = false; cn.msg.textContent = ''; $('e-after').hidden = false;
1051
- if (r.url) {
1052
- if (tab) tab.location = r.url; else window.open(r.url, '_blank', 'noopener');
1053
- cn.msg.textContent = 'Sign in to ' + p.label + ' in the tab that opened. This updates when it is done.';
1054
- } else {
1055
- cn.msg.appendChild(document.createTextNode('Open '));
1056
- cn.msg.appendChild(el('a', {href: r.verify, target: '_blank', rel: 'noopener'}, r.verify));
1057
- cn.msg.appendChild(document.createTextNode(' and enter the code '));
1058
- cn.msg.appendChild(el('strong', {class: 'mono'}, r.code));
1059
- }
1060
- var tries = 0;
1061
- clearInterval(cn.timer);
1062
- cn.timer = setInterval(function () {
1063
- if (++tries > 450) { connectDone({ok: false, errors: ['No sign-in after fifteen minutes; start again.']}, run); return; }
1064
- post('/api/connect/poll', {id: r.id}).then(function (s) { if (s.state !== 'pending') connectDone(s, run); });
1065
- }, 2000);
1066
- }).catch(function (err) { if (tab) tab.close(); e.errors.textContent = String(err); });
1067
- }
1068
- // The dialog's choices (API key, then each provider) and the Manage page's
1069
- // own-app rows, from one listing.
1070
- function loadProviders() {
1071
- fetch('/api/connect').then(function (r) { return r.json(); }).then(function (res) {
1072
- cn.list.textContent = ''; $('cn-apps').textContent = '';
1073
- choice('key', 'API key', '').addEventListener('click', function () { choose('key'); e.name.focus(); });
1074
- res.providers.forEach(function (p) {
1075
- // A new endpoint, named first (a provider can be several: a pasted key
1076
- // and a sign-in, or two accounts). Signing an existing one in again is
1077
- // its row's button.
1078
- var why = p.missing || (remote && !p.code ? p.label + ' signs in only through a page on the machine'
1079
- + ' Benchwarmers runs on; from here, choose API key' : '');
1080
- choice(p.id, p.label, why).addEventListener('click', function () { choose(p); $('cn-name').select(); });
1081
- if (p.register) $('cn-apps').appendChild(appRow(p));
1082
- });
1083
- press();
1084
- });
1085
- }
1086
- function choice(id, label, why) {
1087
- var b = el('button', {class: 'btn', type: 'button', 'data-choice': id, 'aria-pressed': 'false'}, label);
1088
- if (why) { b.disabled = true; b.title = why; }
1089
- cn.list.appendChild(b);
1090
- return b;
1091
- }
1092
- // The Sign in click itself opens the provider's tab (connectStart), so the
1093
- // browser does not block it as a popup.
1094
- $('cn-go').addEventListener('click', function () { $('cn-form').hidden = true; connectStart(cn.pick, $('cn-name').value); });
1095
- // Use your own app: a provider's own client id, instead of the shipped one.
1096
- function appRow(p) {
1097
- var row = el('div', {class: 'endpoint'});
1098
- var left = el('div');
1099
- left.appendChild(el('div', {class: 'what'}, p.label));
1100
- left.appendChild(el('div', {class: 'base'}, 'Register ' + p.register));
1101
- var box = el('input', {spellcheck: 'false', autocomplete: 'off', 'aria-label': p.label + ' client id',
1102
- placeholder: p.shipped ? "Wendell Research's app (leave blank)" : 'client id, needed to sign in'});
1103
- box.value = p.own_id || '';
1104
- var line = el('div', {class: 'row-input'}); // the box and its Save on one line
1105
- line.appendChild(box);
1106
- left.appendChild(line);
1107
- row.appendChild(left);
1108
- var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
1109
- save.addEventListener('click', function () {
1110
- $('cn-app-errors').textContent = ''; $('cn-app-done').textContent = '';
1111
- post('/api/connect/app', {provider: p.id, client_id: box.value}).then(function (r) {
1112
- if (!r.ok) { listErrors($('cn-app-errors'), r.errors); return; }
1113
- $('cn-app-done').textContent = box.value.trim() ? p.label + ' signs in with your app now.'
1114
- : p.label + (p.shipped ? " is back to Wendell Research's app." : ' has no app until one is added.');
1115
- loadProviders();
1116
- });
1117
- });
1118
- line.appendChild(save);
1119
- return row;
1120
- }
1121
-
1122
- if ($('tab-endpoints').classList.contains('active')) onShow.endpoints(); // the page opened on it
1123
-
1124
- // ---- first run: no endpoint yet, no model, no collection (serve.first_run) -
1125
- var first = $('first-run');
1126
- function firstStep() {
1127
- if (first.dataset.step === 'onboard') { $('onboard-open').click(); return; }
1128
- if (first.dataset.step === 'collection') { show('collections'); $('c-new').click(); return; }
1129
- show('endpoints'); openEndpoint('add');
1130
- }
1131
- if (first) {
1132
- $('first-go').addEventListener('click', firstStep);
1133
- if (first.dataset.step === 'endpoint') firstStep(); // straight into Add: an empty bench offers nothing else
1134
- }
1135
-
1136
- // ---- collections: what the bench gateway runs together (its own tab) -----
1137
- var c = {list: $('c-list'), name: $('c-name'), members: $('c-members'), count: $('c-count'),
1138
- shape: $('c-shape'), desc: $('c-desc'), tags: $('c-tags'), errors: $('c-errors'), done: $('c-done'),
1139
- head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
1140
- gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
1141
- del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger')};
1142
- var C = null, open = null; // the collection whose details are showing
1143
- function explorersFor(n) {
1144
- return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
1145
- }
1146
- function picked() {
1147
- return Array.prototype.filter.call(c.members.querySelectorAll('input'), function (x) { return x.checked; })
1148
- .map(function (x) { return x.value; });
1149
- }
1150
- function recount() {
1151
- var n = picked().length, ex = explorersFor(n);
1152
- c.count.textContent = n + ' member' + (n === 1 ? '' : 's') + ' + ' + ex + ' explorer' + (ex === 1 ? '' : 's') +
1153
- ' = ' + (n + ex) + ' calls per request' + (n && (n < 2 || n > 4) ? ' · two to four members is recommended' : '');
1154
- c.members.querySelectorAll('input').forEach(function (x) { x.disabled = !x.checked && n >= C.settings.max_calls; });
1155
- }
1156
- // Suggested tags under the field: a click adds or removes one, and the
1157
- // field stays free for any other.
1158
- function tagList() {
1159
- return c.tags.value.split(',').map(function (t) { return t.trim().toLowerCase(); }).filter(Boolean);
1160
- }
1161
- function drawTagPicks() {
1162
- var box = $('c-tag-picks'), have = tagList();
1163
- box.textContent = '';
1164
- (C.tag_suggestions || []).forEach(function (t) {
1165
- var on = have.indexOf(t) !== -1;
1166
- var b = el('button', {type: 'button', class: 'chip' + (on ? ' ok' : ''), 'aria-pressed': String(on)}, (on ? '✓ ' : '+ ') + t);
1167
- b.addEventListener('click', function () {
1168
- var now = tagList(), i = now.indexOf(t);
1169
- if (i === -1) now.push(t); else now.splice(i, 1);
1170
- c.tags.value = now.join(', '); drawTagPicks();
1171
- });
1172
- box.appendChild(b);
1173
- });
1174
- }
1175
- c.tags.addEventListener('input', drawTagPicks);
1176
- // A collection opens in a popup, as every other edit on the dashboard does:
1177
- // the one clicked, or a new one from New collection.
1178
- function closeForm() {
1179
- if (c.form.open) c.form.close();
1180
- open = null;
1181
- }
1182
- c.form.addEventListener('close', function () { open = null; });
1183
- function fillForm(row) {
1184
- c.errors.textContent = ''; c.done.textContent = ''; c.status.textContent = '';
1185
- open = row ? row.name : null; c.del.hidden = !row;
1186
- c.head.textContent = row ? 'bench/' + row.name : 'New collection';
1187
- if (!c.form.open) c.form.showModal();
1188
- c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
1189
- c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
1190
- drawTagPicks();
1191
- c.members.textContent = '';
1192
- C.models.forEach(function (alias) {
1193
- var box = el('input', {type: 'checkbox', value: alias});
1194
- box.checked = !!row && row.members.indexOf(alias) !== -1;
1195
- box.addEventListener('change', recount);
1196
- var lab = el('label', {class: 'check'}); lab.appendChild(box); lab.appendChild(el('span', {class: 'mono'}, alias));
1197
- c.members.appendChild(lab);
1198
- });
1199
- c.shape.textContent = '';
1200
- c.shape.appendChild(el('option', {value: ''}, 'automatic'));
1201
- C.shapes.forEach(function (s) {
1202
- var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
1203
- });
1204
- c.reply.value = row ? row.reply : 'all';
1205
- c.merger.textContent = '';
1206
- c.merger.appendChild(el('option', {value: ''}, 'automatic'));
1207
- C.models.forEach(function (alias) {
1208
- var o = el('option', {value: alias}, alias); if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
1209
- });
1210
- recount();
1211
- }
1212
- function loadCollections(keep) {
1213
- fetch('/api/collections').then(function (r) { return r.json(); }).then(function (res) {
1214
- C = res; c.list.textContent = '';
1215
- res.collections.forEach(function (row) {
1216
- // The whole row opens the collection: no tight button pair on it.
1217
- var box = el('button', {class: 'endpoint c-row', type: 'button', 'data-name': row.name,
1218
- 'aria-label': 'Open bench/' + row.name});
1219
- var left = el('div');
1220
- var title = el('div', {class: 'what'});
1221
- title.appendChild(el('span', {class: 'alias'}, 'bench/' + row.name + ' '));
1222
- row.tags.forEach(function (t) { title.appendChild(el('span', {class: 'chip', style: 'margin-left:.3rem'}, t)); });
1223
- left.appendChild(title);
1224
- if (row.description) left.appendChild(el('div', {}, row.description));
1225
- left.appendChild(el('div', {class: 'base'}, row.members.join(', ') + ' + ' + row.explorers + ' explorer' +
1226
- (row.explorers === 1 ? '' : 's') + (row.shape ? ' (' + row.shape + ')' : '') +
1227
- (row.reply === 'merged' ? ' · merged by ' + (row.merger || 'verdicts') : '')));
1228
- var t = row.together;
1229
- if (t) {
1230
- var lim = Object.keys(t.limits || {}).filter(function (k) { return t.limits[k]; })
1231
- .map(function (k) { return t.limits[k].toLocaleString() + ' ' + k; });
1232
- left.appendChild(el('div', {class: 'base'}, 'Together: ' +
1233
- (t.context_window ? 'prompts up to ' + Math.floor(t.context_window / 1000).toLocaleString() + 'k tokens' : 'context unknown') +
1234
- (t.context_unknown.length ? ' (unknown for ' + t.context_unknown.join(', ') + ')' : '') +
1235
- ' · all can: ' + (t.capabilities.join(', ') || 'nothing known in common') +
1236
- ' · ' + t.connections.length + ' connection' + (t.connections.length === 1 ? '' : 's') +
1237
- (lim.length ? ' · tightest ' + lim.join(', ') : '')));
1238
- }
1239
- box.appendChild(left);
1240
- box.appendChild(el('span', {class: 'open-hint', 'aria-hidden': 'true'}, 'Open ›'));
1241
- box.addEventListener('click', function () { fillForm(row); });
1242
- c.list.appendChild(box);
1243
- });
1244
- if (!res.collections.length) c.list.appendChild(el('p', {class: 'empty'}, 'No collections yet.'));
1245
- c.maxCalls.value = res.settings.max_calls; c.maxExp.value = res.settings.max_explorers;
1246
- if (!keep) closeForm();
1247
- });
1248
- }
1249
- loadCollections();
1250
- $('c-new').addEventListener('click', function () { fillForm(null); c.name.focus(); });
1251
- $('c-close').addEventListener('click', closeForm);
1252
- // Done: say so in the popup, close it a moment later, and keep the message
1253
- // above the list, as onboarding and retiring do.
1254
- function finished(message) {
1255
- c.done.textContent = message; c.status.textContent = message; loadCollections(true);
1256
- setTimeout(closeForm, 1200);
1257
- }
1258
- c.del.addEventListener('click', function () {
1259
- var name = open;
1260
- if (!name || !confirm('Delete bench/' + name + '? Callers asking for it will get an error.')) return;
1261
- post('/api/collection/remove', {name: name}).then(function (r) {
1262
- if (!r.ok) { listErrors(c.errors, r.errors); return; }
1263
- finished('Deleted bench/' + name + '.');
1264
- });
1265
- });
1266
- $('c-save').addEventListener('click', function () {
1267
- c.errors.textContent = ''; c.done.textContent = '';
1268
- var name = c.name.value.trim().replace(/^bench\//, '');
1269
- post('/api/collection', {name: name, members: picked(), shape: c.shape.value, description: c.desc.value,
1270
- tags: c.tags.value.split(','), reply: c.reply.value,
1271
- merger: c.merger.value}).then(function (r) {
1272
- if (!r.ok) { listErrors(c.errors, r.errors); return; }
1273
- finished('Saved bench/' + name + '.');
1274
- }).catch(function (err) { c.errors.textContent = String(err); });
1275
- });
1276
- $('g-save').addEventListener('click', function () {
1277
- c.gErrors.textContent = ''; c.gDone.textContent = '';
1278
- post('/api/gateway-settings', {max_calls: parseInt(c.maxCalls.value, 10),
1279
- max_explorers: parseInt(c.maxExp.value, 10)}).then(function (r) {
1280
- if (!r.ok) { listErrors(c.gErrors, r.errors); return; }
1281
- c.gDone.textContent = 'Settings saved.'; loadCollections(true);
1282
- }).catch(function (err) { c.gErrors.textContent = String(err); });
1283
- });
1284
-
1285
- // ---- a click on the backdrop closes a popup, as its Close button does ----
1286
- // Pressed AND released outside: a text selection dragged out of an input
1287
- // must not throw the form away.
1288
- document.querySelectorAll('dialog').forEach(function (d) {
1289
- var downOutside = false;
1290
- function outside(e) {
1291
- var r = d.getBoundingClientRect();
1292
- return e.target === d && (e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom);
1293
- }
1294
- d.addEventListener('pointerdown', function (e) { downOutside = outside(e); });
1295
- d.addEventListener('click', function (e) { if (downOutside && outside(e)) d.close(); downOutside = false; });
1296
- });
1297
-
1298
- // ---- verdict on one gateway seat --------------------------------------
1299
- var vd = $('verdict-dialog'), vcall = null;
1300
- function lines(id) { return $(id).value.split('\n').map(function (s) { return s.trim(); }).filter(Boolean); }
1301
- document.addEventListener('click', function (e) {
1302
- var b = e.target.closest('button.seat');
1303
- if (!b) return;
1304
- vcall = b.dataset.call;
1305
- $('v-errors').textContent = ''; $('v-done').textContent = '';
1306
- fetch('/api/gateway-seat?call=' + encodeURIComponent(vcall)).then(function (r) { return r.json(); }).then(function (s) {
1307
- if (!s.ok) { listErrors($('v-errors'), s.errors); vd.showModal(); return; }
1308
- $('v-title').textContent = s.alias + ' · ' + s.seat;
1309
- $('v-meta').textContent = s.model + ' · ' + s.status + ' in ' + s.seconds + 's · ' + s.at.slice(0, 16).replace('T', ' ') + ' · ' + s.call;
1310
- $('v-prompt').textContent = s.prompt;
1311
- $('v-answer').textContent = s.answer || (s.error ? 'Error: ' + s.error : 'Empty reply: usually a token budget, not incapacity.');
1312
- document.querySelectorAll('input[name=v-useful]').forEach(function (r) {
1313
- r.checked = s.useful !== null && r.value === (s.useful ? 'yes' : 'no');
1314
- });
1315
- $('v-note').value = s.note; $('v-real').value = s.real.join('\n'); $('v-false').value = s.false.join('\n');
1316
- vd.showModal();
1317
- });
1318
- });
1319
- $('v-close').addEventListener('click', function () { vd.close(); });
1320
- $('v-save').addEventListener('click', function () {
1321
- var pick = document.querySelector('input[name=v-useful]:checked');
1322
- $('v-errors').textContent = ''; $('v-done').textContent = '';
1323
- post('/api/verdict', {call: vcall, useful: pick ? pick.value === 'yes' : null, note: $('v-note').value,
1324
- real: lines('v-real'), false: lines('v-false')}).then(function (r) {
1325
- if (!r.ok) { listErrors($('v-errors'), r.errors); return; }
1326
- $('v-done').textContent = 'Recorded: ' + r.changed.join(', ') + '. Reloading…';
1327
- setTimeout(function () { location.reload(); }, 700);
1328
- }).catch(function (err) { $('v-errors').textContent = String(err); });
1329
- });
1330
-
1331
- // ---- Benchmarks charts: Observable Plot, data from serve.charts_data ------
1332
- // A function, not run once: a theme switched on Settings redraws them in its colours.
1333
- function drawCharts() {
1334
- var holder = $('charts-data');
1335
- if (!holder) return;
1336
- var C = JSON.parse(holder.textContent);
1337
- var css = getComputedStyle(document.documentElement);
1338
- function tok(name, fallback) { return (css.getPropertyValue(name) || '').trim() || fallback; }
1339
- // `good` is the theme's OK colour, not its brand: in a theme whose brand is
1340
- // red (Hardwood), a found bug drawn in the brand colour would read as bad.
1341
- var ink = tok('--ink', '#24331f'), good = tok('--ok', '#327a16'), err = tok('--err', '#a12b2b'),
1342
- warn = tok('--warn', '#7a4d00'),
1343
- signal = tok('--signal', '#fae66c'), sky = tok('--sky', '#69bdeb'), muted = tok('--muted', '#6b685a'),
1344
- line = tok('--line', '#e4dfd2'), halo = tok('--paper-2', '#ffffff');
1345
- var style = {background: 'transparent', color: ink, fontSize: '13px', overflow: 'visible'};
1346
- // Tabs other than the open one have no width yet, so size from the column
1347
- // the pages share. It was the whole page, which made every chart wider than
1348
- // its column beside the sidebar, scaled down, with smaller text.
1349
- function width() {
1350
- var col = document.querySelector('.panes') || document.querySelector('main');
1351
- return Math.max(520, (col.clientWidth || 900) - 50);
1352
- }
1353
- function put(id, make) {
1354
- var box = $(id);
1355
- if (!box) return;
1356
- if (!window.Plot) {
1357
- box.className += ' muted';
1358
- box.textContent = 'Charts need Observable Plot from cdn.jsdelivr.net, which did not load; the table below has the same numbers.';
1359
- return;
1360
- }
1361
- box.textContent = ''; // a redraw replaces the chart
1362
- try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
1363
- }
1364
- // Pixel offsets that keep scatter labels apart: each label, in order, moves
1365
- // up or down until it clears the ones already placed.
1366
- function spread(points, xpx, ypx) {
1367
- var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
1368
- return points.map(function (d) {
1369
- var cx = xpx(d), half = d.model.length * 2.9;
1370
- for (var k = 0; k < tries.length; k++) {
1371
- var cy = ypx(d) + tries[k];
1372
- if (!boxes.some(function (b) { return Math.abs(b.x - cx) < b.half + half + 4 && Math.abs(b.y - cy) < 12; })) {
1373
- boxes.push({x: cx, y: cy, half: half});
1374
- return tries[k];
1375
- }
1376
- }
1377
- return tries[0];
1378
- });
1379
- }
1380
- function rows(n) { return 20 * n + 80; }
1381
- // Who to trust: finds real bugs (right) vs makes things up (up = less).
1382
- put('chart-overview', function (w) {
1383
- if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
1384
- var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
1385
- var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
1386
- var pxW = w - 110, pxH = 370;
1387
- var pts = C.overview.slice().sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
1388
- var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
1389
- function (d) { return d.wrong / worst * pxH; });
1390
- var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; });
1391
- return Plot.plot({
1392
- width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
1393
- x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
1394
- y: {label: '↑ Makes fewer things up', domain: [0, worst], reverse: true, tickFormat: '.0%'},
1395
- marks: [
1396
- // The zone a reviewer should be in: outlined, so it reads on a dark ground too.
1397
- Plot.rect([{x1: x0 + (1 - x0) / 2, x2: 1, y1: 0, y2: worst / 4}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
1398
- fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
1399
- Plot.dot(C.overview, {x: 'recall', y: 'wrong', r: 6, fill: good, stroke: ink, strokeWidth: 0.5}),
1400
- // A halo in the panel's colour keeps each name legible over dots, grid lines and the zone.
1401
- Plot.text(labels, {x: 'recall', y: 'ty', text: 'model', fontSize: 11, fill: ink,
1402
- stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
1403
- Plot.tip(C.overview, Plot.pointer({x: 'recall', y: 'wrong',
1404
- title: function (d) { return d.model + '\nfinds ' + d.found + ' real findings\nconfidently wrong in ' + d.wrongN + ' answers'; }}))
1405
- ]
1406
- });
1407
- });
1408
-
1409
- // Bug hunt: right of zero the bug tasks (found, wrong line, missed); left of
1410
- // it the same files clean, where any bar is a false alarm.
1411
- put('chart-hunt', function (w) {
1412
- var segs = [], score = {}, tasks = {};
1413
- C.hunt.forEach(function (h) {
1414
- var bugs = h.found + h['wrong line'] + h.missed, clean = h.clean + h['false alarm'];
1415
- tasks[h.model] = bugs + clean;
1416
- var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
1417
- var fa = clean ? h['false alarm'] / clean : 0;
1418
- score[h.model] = f - fa;
1419
- segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
1420
- segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
1421
- segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
1422
- segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
1423
- });
1424
- var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
1425
- return Plot.plot({
1426
- width: w, height: rows(models.length), style: style, marginLeft: 250,
1427
- x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
1428
- tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
1429
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
1430
- color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
1431
- marks: [
1432
- Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
1433
- title: function (d) { return d.model + ': ' + d.part + ', ' + d.n; }}),
1434
- Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
1435
- ]
1436
- });
1437
- });
1438
-
1439
- // Real work: the share of judged offloads that helped, sample size beside the name.
1440
- put('chart-work', function (w) {
1441
- var R = C.work.slice().sort(function (a, b) { return b.useful / b.judged - a.useful / a.judged || b.judged - a.judged; });
1442
- if (!R.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
1443
- var by = {};
1444
- R.forEach(function (r) { by[r.model] = r; });
1445
- function share(r) { return r.useful / r.judged; }
1446
- return Plot.plot({
1447
- width: w, height: rows(R.length), style: style, marginLeft: 250, marginRight: 50,
1448
- x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
1449
- y: {domain: R.map(function (r) { return r.model; }), label: null,
1450
- tickFormat: function (m) { return m + ' (' + by[m].judged + ')'; }},
1451
- marks: [
1452
- Plot.barX(R, {x: share, y: 'model', fill: function (r) { return share(r) >= 0.5 ? good : err; },
1453
- title: function (r) { return r.model + ': helped ' + r.useful + ' of ' + r.judged + ' judged (' + r.offloads + ' offloads)'
1454
- + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : ''); }}),
1455
- Plot.text(R, {x: share, y: 'model', text: function (r) { return r.useful + '/' + r.judged; }, dx: 18, fill: ink}),
1456
- Plot.ruleX([0], {stroke: ink})
1457
- ]
1458
- });
1459
- });
1460
-
1461
- // Did it answer: each probe of the last 14 days, by what came back.
1462
- put('chart-reliability', function (w) {
1463
- if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
1464
- var total = {}, ok = {};
1465
- C.reliability.forEach(function (d) {
1466
- total[d.model] = (total[d.model] || 0) + d.count;
1467
- if (d.outcome === 'answered') ok[d.model] = d.count;
1468
- });
1469
- var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
1470
- var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
1471
- entitlement: '#c0563f', other: muted, error: ink};
1472
- var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
1473
- return Plot.plot({
1474
- width: w, height: rows(models.length), style: style, marginLeft: 250,
1475
- x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
1476
- y: {domain: models, label: null},
1477
- color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
1478
- marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
1479
- title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
1480
- });
1481
- });
1482
-
1483
- // Speed vs quality: median seconds on our own tasks against findings located.
1484
- put('chart-speed', function (w) {
1485
- if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
1486
- var s = C.speed.map(function (d) { return d.seconds; });
1487
- var lo = Math.max(1, Math.min.apply(null, s) * 0.8), hi = Math.max.apply(null, s) * 1.25;
1488
- var y0 = Math.max(0, Math.floor((Math.min.apply(null, C.speed.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
1489
- var pxW = w - 110, pxH = 340;
1490
- var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
1491
- var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
1492
- function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
1493
- var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; });
1494
- return Plot.plot({
1495
- width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
1496
- x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
1497
- y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
1498
- marks: [
1499
- Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
1500
- Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink}),
1501
- Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
1502
- title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
1503
- ]
1504
- });
1505
- });
1506
-
1507
- // Our code: one square per task and model, grouped by kind of task.
1508
- put('chart-code', function (w) {
1509
- var fam = {}, first = {};
1510
- C.heat.forEach(function (d) { fam[d.task] = d.family; });
1511
- var tasks = Object.keys(fam).sort(function (a, b) {
1512
- return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
1513
- });
1514
- tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
1515
- var label = {};
1516
- Object.keys(first).forEach(function (f) { label[first[f]] = f; });
1517
- // By total found, so a model that played three tasks does not top one that played forty.
1518
- var sum = {};
1519
- C.heat.forEach(function (d) { sum[d.model] = (sum[d.model] || 0) + (d.rate || 0); });
1520
- var models = Object.keys(sum).sort(function (a, b) { return sum[b] - sum[a]; });
1521
- function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
1522
- return Plot.plot({
1523
- width: w, height: 22 * models.length + 70, style: style, marginLeft: 230, marginTop: 30, padding: 0.08,
1524
- x: {domain: tasks, axis: 'top', label: null, tickSize: 0, tickFormat: function (t) { return label[t] || ''; }},
1525
- y: {domain: models, label: null, tickSize: 0},
1526
- color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
1527
- marks: [
1528
- Plot.cell(C.heat.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
1529
- Plot.cell(C.heat.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
1530
- Plot.dot(C.heat.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
1531
- Plot.ruleX(Object.keys(label), {x: function (t) { return t; }, stroke: ink, strokeOpacity: 0.25, dx: -2})
1532
- ]
1533
- });
1534
- });
1535
-
1536
- // Confidently wrong: stacked by kind, worst at the top.
1537
- put('chart-wrong', function (w) {
1538
- var T = C.wrongTotals;
1539
- var models = Object.keys(T).sort(function (a, b) { return T[b].rate - T[a].rate || T[b].answers - T[a].answers; });
1540
- return Plot.plot({
1541
- width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
1542
- x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
1543
- // The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
1544
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
1545
- color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
1546
- marks: [
1547
- Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
1548
- Plot.ruleX([0], {stroke: ink})
1549
- ]
1550
- });
1551
- });
1552
-
1553
- }
1554
- drawCharts();
1555
- })();
1
+ // Benchwarmers. Plain DOM, no framework: serve.py renders the page, this only
2
+ // wires the controls. Every write goes through a POST that re-checks its own
3
+ // rules server-side, so nothing here is trusted to enforce anything.
4
+ (function () {
5
+ 'use strict';
6
+ var D = window.BENCH || {models: {}, roles: {}, retired: {}, stamp: 0};
7
+ function $(id) { return document.getElementById(id); }
8
+ function el(tag, attrs, text) {
9
+ var e = document.createElement(tag);
10
+ Object.keys(attrs || {}).forEach(function (k) { e.setAttribute(k, attrs[k]); });
11
+ if (text != null) e.textContent = text;
12
+ return e;
13
+ }
14
+ function store(k, v) { try { if (v === undefined) return localStorage.getItem(k); localStorage.setItem(k, v); } catch (e) { return null; } }
15
+ function post(url, body) {
16
+ return fetch(url, {method: 'POST', headers: {'Content-Type': 'application/json'},
17
+ body: JSON.stringify(body)}).then(function (r) { return r.json(); });
18
+ }
19
+ function listErrors(node, errors) { node.textContent = errors.map(function (x) { return '• ' + x; }).join('\n'); node.style.whiteSpace = 'pre-line'; }
20
+ // The page's own confirm, a Promise of true/false: window.confirm is
21
+ // silently false where a browser blocks dialogs. Opens over any open popup.
22
+ var askD = $('ask-dialog'), askDone = null;
23
+ function ask(text, yes, danger) {
24
+ if (askDone) askDone(false);
25
+ $('ask-text').textContent = text;
26
+ $('ask-yes').textContent = yes || 'OK';
27
+ $('ask-yes').className = 'btn ' + (danger ? 'danger' : 'primary');
28
+ askD.showModal(); $('ask-no').focus();
29
+ return new Promise(function (resolve) { askDone = resolve; });
30
+ }
31
+ function answer(yes) { var r = askDone; askDone = null; if (askD.open) askD.close(); if (r) r(yes); }
32
+ $('ask-yes').addEventListener('click', function () { answer(true); });
33
+ $('ask-no').addEventListener('click', function () { answer(false); });
34
+ askD.addEventListener('close', function () { answer(false); }); // Escape
35
+
36
+ // ---- tabs, remembered per browser ----------------------------------
37
+ var tabs = document.querySelectorAll('.tabs button[data-tab]'); // not the sidebar's own toggle
38
+ var onShow = {}; // a page's loader, run each time it is shown
39
+ function show(id) {
40
+ tabs.forEach(function (b) { b.setAttribute('aria-selected', String(b.dataset.tab === id)); });
41
+ document.querySelectorAll('.tab').forEach(function (s) { s.classList.toggle('active', s.id === 'tab-' + id); });
42
+ store('bench.tab', id);
43
+ if (onShow[id]) onShow[id]();
44
+ }
45
+ tabs.forEach(function (b) { b.addEventListener('click', function () { show(b.dataset.tab); }); });
46
+ var wanted = store('bench.tab');
47
+ show(wanted && $('tab-' + wanted) ? wanted : 'glance');
48
+
49
+ // ---- sortable tables -------------------------------------------------
50
+ function key(cell) {
51
+ var raw = cell.dataset.sort != null ? cell.dataset.sort : cell.textContent.trim();
52
+ var n = parseFloat(raw);
53
+ return isNaN(n) || !/^-?[\d.]+/.test(raw) ? raw.toLowerCase() : n;
54
+ }
55
+ document.querySelectorAll('th[data-sortable]').forEach(function (th) {
56
+ th.tabIndex = 0;
57
+ function sort() {
58
+ var table = th.closest('table'), body = table.tBodies[0];
59
+ var idx = Array.prototype.indexOf.call(th.parentNode.children, th);
60
+ var dir = th.getAttribute('aria-sort') === 'descending' ? 'ascending' : 'descending';
61
+ table.querySelectorAll('th[aria-sort]').forEach(function (o) { o.removeAttribute('aria-sort'); });
62
+ th.setAttribute('aria-sort', dir);
63
+ var rows = Array.prototype.slice.call(body.rows);
64
+ rows.sort(function (a, b) {
65
+ var x = key(a.cells[idx]), y = key(b.cells[idx]);
66
+ var c = (typeof x === typeof y) ? (x < y ? -1 : x > y ? 1 : 0) : (typeof x === 'number' ? -1 : 1);
67
+ return dir === 'ascending' ? c : -c;
68
+ });
69
+ rows.forEach(function (r) { body.appendChild(r); });
70
+ }
71
+ th.addEventListener('click', sort);
72
+ th.addEventListener('keydown', function (e) { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); sort(); } });
73
+ });
74
+
75
+ // ---- filter boxes: one per panel (or the whole page: data-filter="page"),
76
+ // matching any text in a table row or a log line, and the panel's facet
77
+ // selects (serve.facet), each matching a row's data-<facet> exactly ------
78
+ function applyFilter(scope) {
79
+ var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
80
+ var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
81
+ scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
82
+ r.hidden = (q && r.textContent.toLowerCase().indexOf(q) === -1)
83
+ || facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
84
+ });
85
+ // A match inside a folded list (an endpoint's models) opens it.
86
+ if (q) scope.querySelectorAll('details.models').forEach(function (d) {
87
+ if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
88
+ });
89
+ }
90
+ document.querySelectorAll('input[data-filter], select[data-facet]').forEach(function (c) {
91
+ var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
92
+ var scope = box && box.dataset.filter === 'page' ? c.closest('.tab') : c.closest('.panel');
93
+ c.addEventListener(c.tagName === 'SELECT' ? 'change' : 'input', function () { applyFilter(scope); });
94
+ });
95
+
96
+ // ---- Action needed: Hide folds it to its heading, per browser, until an
97
+ // action it has not seen appears or 12 hours pass, whichever is first ----
98
+ var acts = $('actions');
99
+ if (acts) {
100
+ var keys = Array.prototype.map.call(acts.querySelectorAll('.problem[data-key]'), function (p) { return p.dataset.key; });
101
+ var hid = null;
102
+ try { hid = JSON.parse(store('bench.actions.hidden') || 'null'); } catch (err) { hid = null; }
103
+ var fold = function (on) {
104
+ acts.classList.toggle('collapsed', on);
105
+ $('actions-hide').textContent = on ? 'show' : 'Hide';
106
+ $('actions-hide').setAttribute('aria-expanded', String(!on));
107
+ };
108
+ fold(!!(hid && hid.keys && Date.now() - hid.at < 12 * 3600 * 1000
109
+ && keys.every(function (k) { return hid.keys.indexOf(k) !== -1; })));
110
+ $('actions-hide').addEventListener('click', function () {
111
+ var on = !acts.classList.contains('collapsed');
112
+ store('bench.actions.hidden', on ? JSON.stringify({at: Date.now(), keys: keys}) : '');
113
+ fold(on);
114
+ });
115
+ }
116
+
117
+ // ---- retired rows: hidden unless asked for ---------------------------
118
+ var showRetired = store('bench.retired') === '1';
119
+ function applyRetired() {
120
+ document.body.classList.toggle('show-retired', showRetired);
121
+ document.querySelectorAll('input[data-retired]').forEach(function (c) { c.checked = showRetired; });
122
+ }
123
+ document.querySelectorAll('input[data-retired]').forEach(function (c) {
124
+ c.addEventListener('change', function () { showRetired = c.checked; store('bench.retired', showRetired ? '1' : '0'); applyRetired(); });
125
+ });
126
+ applyRetired();
127
+
128
+ // ---- Discover: paid models hidden unless asked for -------------------
129
+ var showPaid = store('bench.paid') === '1';
130
+ function applyPaid() {
131
+ document.body.classList.toggle('show-paid', showPaid);
132
+ document.querySelectorAll('input[data-paid]').forEach(function (c) { c.checked = showPaid; });
133
+ }
134
+ document.querySelectorAll('input[data-paid]').forEach(function (c) {
135
+ c.addEventListener('change', function () { showPaid = c.checked; store('bench.paid', showPaid ? '1' : '0'); applyPaid(); });
136
+ });
137
+ applyPaid();
138
+
139
+ // ---- fresh results: offered, never forced ----------------------------
140
+ // Not while an onboarding batch runs: each model's probe writes results, so
141
+ // it would be offered after every model. The first poll after the batch
142
+ // ends offers it once.
143
+ var fresh = $('fresh'), batchRunning = false;
144
+ setInterval(function () {
145
+ if (document.hidden || !fresh.hidden || batchRunning) return;
146
+ fetch('/api/stamp').then(function (r) { return r.json(); }).then(function (s) {
147
+ if (s.stamp > D.stamp + 0.5) fresh.hidden = false;
148
+ }).catch(function () {});
149
+ }, 15000);
150
+ fresh.addEventListener('click', function () { location.reload(); });
151
+
152
+ // ---- the model dialog ------------------------------------------------
153
+ var md = $('model-dialog');
154
+ var m = {title: $('m-title'), served: $('m-served'), editor: $('m-editor'), gone: $('m-gone'),
155
+ alias: $('m-alias'), role: $('m-role'), roleWhy: $('m-role-why'), adhoc: $('m-adhoc'),
156
+ danger: $('m-danger'), errors: $('m-errors'), done: $('m-done'), save: $('m-save'),
157
+ restore: $('m-restore'), close: $('m-close'), removeBox: $('m-remove-box'),
158
+ reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
159
+ qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why')};
160
+ m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
161
+ var editing = null;
162
+
163
+ function openModel(alias) {
164
+ var model = D.models[alias], gone = (D.retired || {})[alias];
165
+ if (!model && !gone) return;
166
+ editing = model ? alias : null;
167
+ m.title.textContent = alias;
168
+ m.errors.textContent = ''; m.done.textContent = '';
169
+ m.editor.hidden = !model; m.save.hidden = !model; m.removeBox.hidden = !model;
170
+ m.gone.hidden = !!model; m.restore.hidden = !!model;
171
+ showOnboardIssue(model ? (D.onboard || {})[alias] : null);
172
+ if (gone) {
173
+ m.served.textContent = 'served as ' + gone.served + (gone.tag ? ' · ' + gone.tag : '');
174
+ m.gone.textContent = '';
175
+ m.gone.appendChild(el('strong', {}, 'Retired ' + gone.at.slice(0, 10) + (gone.by === 'auto' ? ' automatically' : ' by hand')));
176
+ m.gone.appendChild(el('p', {style: 'margin:.3rem 0'}, gone.why || ''));
177
+ m.gone.appendChild(el('p', {class: 'muted', style: 'margin:0'},
178
+ 'Its evidence is untouched. Restore puts back the same alias, role and endpoint; failures recorded before the restore are ignored.'));
179
+ m.restore.dataset.alias = alias; m.restore.disabled = false;
180
+ } else {
181
+ m.served.textContent = 'serves ' + model.served + (model.tag ? ' · ' + model.tag : '');
182
+ m.alias.value = alias;
183
+ m.role.textContent = '';
184
+ Object.keys(D.roles).forEach(function (r) {
185
+ var o = el('option', {value: r}, r); if (r === model.role) o.selected = true; m.role.appendChild(o);
186
+ });
187
+ // A decision model answers typed questions, never chat: its role is fixed both ways.
188
+ var decision = model.kind === 'decision';
189
+ m.role.disabled = decision;
190
+ m.qtypes.hidden = !decision;
191
+ m.qtypes.querySelectorAll('input').forEach(function (c) {
192
+ c.checked = decision && model.question_types.indexOf(c.value) !== -1;
193
+ });
194
+ fillFacts(alias, decision); fillOptions(alias, decision);
195
+ m.aliasWhy.textContent = decision
196
+ ? 'The name decide.py and the gateway call it by, and every record is keyed by. Letters, digits, dot, dash, underscore; unused, and never a retired name.'
197
+ : m.aliasWhy.dataset.chat;
198
+ Array.prototype.forEach.call(m.role.options, function (o) {
199
+ o.disabled = model.kind !== 'decision' && o.value === 'decision';
200
+ });
201
+ m.adhoc.checked = !!model.ad_hoc;
202
+ m.save.disabled = false; m.save.textContent = 'Save';
203
+ m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
204
+ }
205
+ md.showModal();
206
+ }
207
+ // How its onboarding stopped (serve.onboard_issues): only while it has not
208
+ // answered a probe since. The error as the provider gave it, and the log.
209
+ var ONBOARD_NEXT = {
210
+ capacity: 'A limit on the provider side, not the model. It stays registered and the daily probe tries again; once it answers, run its suite.',
211
+ timeout: 'It did not answer within the budget. The daily probe tries again.',
212
+ empty: 'It answered nothing, usually an output budget spent thinking: raise Max tokens under Call options.',
213
+ paywalled: 'Its free tier has ended. Retire it below.',
214
+ structural: 'The provider has no such model now. Retire it below.',
215
+ entitlement: 'The account may not call it (plan, payment or permission).',
216
+ signin: 'The key was refused: sign in again on the Endpoints page.'
217
+ };
218
+ function showOnboardIssue(x) {
219
+ m.onboard.hidden = !x; m.onboard.textContent = '';
220
+ if (!x) return;
221
+ m.onboard.appendChild(el('strong', {}, 'Onboarding stopped at the ' + (x.step || 'probe') + ' on '
222
+ + (x.at || '').slice(0, 10) + ': ' + x.label));
223
+ m.onboard.appendChild(el('p', {style: 'margin:.3rem 0'}, ONBOARD_NEXT[x.failure] || x.reading || ''));
224
+ if (x.detail) m.onboard.appendChild(el('pre', {class: 'log'}, x.detail));
225
+ if (x.log) {
226
+ var d = el('details'), pre = el('pre', {class: 'log'});
227
+ d.appendChild(el('summary', {class: 'muted'}, 'Onboarding log'));
228
+ d.appendChild(pre);
229
+ d.addEventListener('toggle', function () {
230
+ if (!d.open || pre.textContent) return;
231
+ fetch('/api/onboard?log=' + encodeURIComponent(x.log)).then(function (r) { return r.json(); })
232
+ .then(function (s) { pre.textContent = s.ok ? (s.tail || '(empty)') : s.errors.join(' '); });
233
+ });
234
+ m.onboard.appendChild(d);
235
+ }
236
+ }
237
+ // ---- model facts: yours in the boxes, the sources' as placeholders -----
238
+ var FACT_NUM = ['context_window', 'max_output'], FACT_CAP = ['tools', 'vision', 'structured_output', 'reasoning'];
239
+ function fillFacts(alias, decision) {
240
+ var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
241
+ box.hidden = profile.hidden = decision || !f;
242
+ profile.textContent = '';
243
+ $('f-errors').textContent = ''; $('f-done').textContent = '';
244
+ if (box.hidden) return;
245
+ var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
246
+ // At a glance, as models.dev shows a model: what goes in, what comes out.
247
+ [['Input', 'input_modalities'], ['Output', 'output_modalities']].forEach(function (p) {
248
+ var row = el('div', {class: 'prof'}, null);
249
+ row.appendChild(el('span', {class: 'muted'}, p[0] + ' '));
250
+ (f[p[1]] && f[p[1]].length ? f[p[1]] : ['unknown']).forEach(function (x) { row.appendChild(el('span', {class: 'chip'}, x)); });
251
+ if (src[p[1]]) row.appendChild(el('span', {class: 'src'}, ' ' + src[p[1]]));
252
+ profile.appendChild(row);
253
+ });
254
+ var bits = [], c = f.cost || {};
255
+ if (f.release_date) bits.push('released ' + f.release_date);
256
+ if (f.knowledge_cutoff) bits.push('knowledge to ' + f.knowledge_cutoff);
257
+ if (f.open_weights != null) bits.push(f.open_weights ? 'open weights' : 'closed weights');
258
+ if (c.input != null && c.output != null) {
259
+ bits.push(c.input === 0 && c.output === 0 ? 'list price free'
260
+ : 'list price $' + c.input + ' in / $' + c.output + ' out per 1M tokens (a free tier may still apply)');
261
+ }
262
+ if (bits.length) profile.appendChild(el('div', {class: 'prof muted'}, bits.join(' · ')));
263
+ var show = function (k, v) { $('s-' + k).textContent = v == null ? 'unknown' : v + (src[k] ? ' · ' + src[k] : ''); };
264
+ FACT_NUM.forEach(function (k) {
265
+ $('f-' + k).value = mine(k) ? f[k] : '';
266
+ $('f-' + k).placeholder = f[k] != null && !mine(k) ? f[k] : '';
267
+ show(k, f[k] != null ? Number(f[k]).toLocaleString() : null);
268
+ });
269
+ FACT_CAP.forEach(function (k) {
270
+ $('f-' + k).value = mine(k) ? (f[k] ? 'yes' : 'no') : '';
271
+ // The blank choice is "whatever the sources say": say what that is.
272
+ $('f-' + k).options[0].textContent = mine(k) ? 'auto (hand back to the sources)'
273
+ : 'auto: ' + (f[k] == null ? 'unknown' : (f[k] ? 'yes' : 'no'));
274
+ show(k, f[k] == null ? null : (f[k] ? 'yes' : 'no'));
275
+ });
276
+ $('f-knowledge_cutoff').value = mine('knowledge_cutoff') ? f.knowledge_cutoff : '';
277
+ $('f-knowledge_cutoff').placeholder = !mine('knowledge_cutoff') && f.knowledge_cutoff ? f.knowledge_cutoff : '';
278
+ show('knowledge_cutoff', f.knowledge_cutoff || null);
279
+ var own = src.limits === 'you (this model)';
280
+ document.querySelectorAll('#f-limits input').forEach(function (b) {
281
+ var v = (f.limits || {})[b.dataset.k];
282
+ b.value = own && v ? v : ''; b.placeholder = !own && v ? v : '—';
283
+ });
284
+ var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
285
+ $('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
286
+ }
287
+ $('f-save').addEventListener('click', function () {
288
+ if (!editing) return;
289
+ var facts = {limits: {}};
290
+ FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
291
+ FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
292
+ if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
293
+ document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
294
+ $('f-errors').textContent = ''; $('f-done').textContent = '';
295
+ post('/api/model-facts', {alias: editing, facts: facts}).then(function (r) {
296
+ if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
297
+ $('f-done').textContent = 'Saved. Reloading…'; setTimeout(function () { location.reload(); }, 700);
298
+ });
299
+ });
300
+ // ---- call options: how the bench asks this model (serve.save_model_options)
301
+ function fillOptions(alias, decision) {
302
+ var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
303
+ $('m-opts').hidden = decision;
304
+ $('co-errors').textContent = ''; $('co-done').textContent = '';
305
+ $('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
306
+ $('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
307
+ }
308
+ $('co-save').addEventListener('click', function () {
309
+ if (!editing) return;
310
+ var v = $('co-max').value.trim();
311
+ $('co-errors').textContent = ''; $('co-done').textContent = '';
312
+ post('/api/model-options', {alias: editing, max_tokens: v ? Number(v) : null}).then(function (r) {
313
+ if (!r.ok) { listErrors($('co-errors'), r.errors); return; }
314
+ $('co-done').textContent = 'Saved' + (r.note ? '; ' + r.note : '') + '. Reloading…';
315
+ setTimeout(function () { location.reload(); }, 900);
316
+ });
317
+ });
318
+
319
+ function refresh() {
320
+ m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
321
+ m.roleWhy.textContent = D.roles[m.role.value] || '';
322
+ }
323
+ document.addEventListener('click', function (e) {
324
+ var a = e.target.closest('a.model');
325
+ if (a) { e.preventDefault(); openModel(a.dataset.alias); }
326
+ });
327
+ m.close.addEventListener('click', function () { md.close(); });
328
+ m.alias.addEventListener('input', refresh);
329
+ m.role.addEventListener('change', refresh);
330
+
331
+ // A success is said, left up for two seconds, then the popup closes and
332
+ // the page reloads with the change in it.
333
+ function closeAndReload(dialog) {
334
+ setTimeout(function () { dialog.close(); location.reload(); }, 2000);
335
+ }
336
+ function after(node, button, label) {
337
+ return function (res) {
338
+ if (!res.ok) { listErrors(node, res.errors); if (button) { button.disabled = false; button.textContent = label; } return; }
339
+ m.done.textContent = 'Done: ' + (res.changed || []).join(', ') + (res.sync ? ' · ' + res.sync : '') + '. Closing…';
340
+ closeAndReload(md);
341
+ };
342
+ }
343
+ m.save.addEventListener('click', function () {
344
+ if (!editing) return;
345
+ var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
346
+ if (!m.qtypes.hidden) {
347
+ body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
348
+ function (c) { return c.checked; }).map(function (c) { return c.value; });
349
+ }
350
+ (body.new_alias === editing ? Promise.resolve(true)
351
+ : ask('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
352
+ ).then(function (yes) {
353
+ if (!yes) return;
354
+ m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
355
+ post('/api/model', body).then(after(m.errors, m.save, 'Save')).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
356
+ });
357
+ });
358
+ // The confirm is the last word: the popup closes at once and the page
359
+ // reloads when the retirement is done. A refusal reopens it with the reason.
360
+ m.remove.addEventListener('click', function () {
361
+ if (!editing) return;
362
+ var alias = editing;
363
+ ask('Retire ' + alias + '? The registry entry goes; the evidence stays, and Restore brings it back.', 'Retire', true)
364
+ .then(function (yes) {
365
+ if (!yes) return;
366
+ m.remove.disabled = true; m.errors.textContent = ''; md.close();
367
+ function refused(errors) { md.showModal(); listErrors(m.errors, errors); m.remove.disabled = false; }
368
+ post('/api/remove', {alias: alias, reason: m.reason.value}).then(function (res) {
369
+ if (res.ok) location.reload(); else refused(res.errors);
370
+ }).catch(function (err) { refused([String(err)]); });
371
+ });
372
+ });
373
+ m.restore.addEventListener('click', function () {
374
+ m.restore.disabled = true; m.restore.textContent = 'Restoring…';
375
+ post('/api/restore', {alias: m.restore.dataset.alias}).then(after(m.errors, m.restore, 'Restore')).catch(function (err) { m.errors.textContent = String(err); });
376
+ });
377
+
378
+ // ---- problem actions ---------------------------------------------------
379
+ document.querySelectorAll('[data-act]').forEach(function (b) {
380
+ b.addEventListener('click', function () {
381
+ var act = b.dataset.act, aliases = (b.dataset.aliases || b.dataset.alias || '').split(' ').filter(Boolean);
382
+ var status = b.closest('.problem').querySelector('.why');
383
+ if (act === 'open') { openModel(aliases[0]); return; }
384
+ if (act === 'retry') { // no confirm: one call, and only an answer changes anything
385
+ b.disabled = true; b.textContent = 'Asking…';
386
+ post('/api/retry', {alias: aliases[0]}).then(function (res) {
387
+ if (!res.ok) { status.textContent = res.errors.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = 'Retry'; return; }
388
+ status.textContent = res.changed.join(' · ') + '. Reloading…'; setTimeout(function () { location.reload(); }, 900);
389
+ }).catch(function (err) { status.textContent = String(err); b.disabled = false; b.textContent = 'Retry'; });
390
+ return;
391
+ }
392
+ var verb = act === 'restore' ? 'Restore' : 'Retire';
393
+ ask(verb + ' ' + aliases.join(', ') + '?' + (act === 'restore' ? '' : ' The evidence stays and Restore brings it back.'),
394
+ verb, act !== 'restore').then(function (yes) {
395
+ if (!yes) return;
396
+ b.disabled = true; b.textContent = verb === 'Restore' ? 'Restoring…' : 'Retiring…';
397
+ var chain = Promise.resolve([]);
398
+ aliases.forEach(function (alias) {
399
+ chain = chain.then(function (errs) {
400
+ var call = act === 'restore' ? post('/api/restore', {alias: alias})
401
+ : post('/api/remove', {alias: alias, reason: 'from the dashboard: ' + (b.dataset.why || 'retired by hand')});
402
+ return call.then(function (res) { return errs.concat(res.ok ? [] : res.errors); });
403
+ });
404
+ });
405
+ chain.then(function (errs) {
406
+ if (errs.length) { status.textContent = errs.join('; '); status.classList.add('err'); b.disabled = false; b.textContent = verb; return; }
407
+ status.textContent = 'Done. Reloading…'; setTimeout(function () { location.reload(); }, 700);
408
+ });
409
+ });
410
+ });
411
+ });
412
+
413
+ // ---- onboarding --------------------------------------------------------
414
+ var od = $('onboard-dialog');
415
+ var o = {find: $('o-find'), picker: $('o-picker'), errors: $('o-errors'), start: $('o-start'),
416
+ close: $('o-close'), choose: $('o-choose'), run: $('o-run'), steps: $('o-steps'),
417
+ log: $('o-log'), title: $('o-title'), result: $('o-result'), dismiss: $('o-dismiss')};
418
+ var STEPS = ['Served by the endpoint','Registered', 'Synced to the llm CLI', 'Availability probe, 4 calls', 'Internal suite'];
419
+ var timer = null, data = null;
420
+ // What is ticked, by the candidate's key (served@endpoint): kept while the
421
+ // search redraws the list, so you can search, tick, search again, tick.
422
+ var chosen = {};
423
+ // Folded groups, per browser: openrouter (direct) alone is hundreds of rows.
424
+ var NEW = 'New since the daily check', folded = {};
425
+ try { (JSON.parse(store('bench.picker.folded') || '[]') || []).forEach(function (g) { folded[g] = 1; }); } catch (e) { folded = {}; }
426
+
427
+ function armStart() {
428
+ var n = Object.keys(chosen).length;
429
+ o.start.disabled = o.dismiss.disabled = !n;
430
+ o.start.textContent = n > 1 ? 'Onboard ' + n + ' models' : 'Onboard model';
431
+ o.dismiss.textContent = n > 1 ? 'Dismiss ' + n + ' models' : 'Dismiss model';
432
+ }
433
+ // Rows move between the lists here rather than by asking every endpoint again.
434
+ function moveRows(keys, from, to) {
435
+ data[from] = data[from].filter(function (c) { if (keys.indexOf(c.key) === -1) return true; data[to].push(c); return false; });
436
+ data[to].sort(function (a, b) { return a.served < b.served ? -1 : a.served > b.served ? 1 : 0; });
437
+ }
438
+ function pickRow(c, dismissed) {
439
+ var row = el('label', {class: 'pick'});
440
+ var box = el('input', {type: 'checkbox', name: 'o-pick', value: c.served, 'data-endpoint': c.endpoint});
441
+ if (dismissed) box.disabled = true;
442
+ box.checked = !dismissed && !!chosen[c.key];
443
+ row.appendChild(box);
444
+ var name = el('span', {class: 'alias'}, c.served);
445
+ // What it takes besides text (every model takes text, so that says
446
+ // nothing), and whether it makes images: an image generator is no seat.
447
+ // One order on every row, so the rows scan alike.
448
+ var ORDER = ['image', 'pdf', 'file', 'audio', 'video'];
449
+ (c.inputs || []).filter(function (m) { return m !== 'text'; }).sort(function (a, b) {
450
+ return (ORDER.indexOf(a) + 1 || 99) - (ORDER.indexOf(b) + 1 || 99);
451
+ }).forEach(function (m) { name.appendChild(el('span', {class: 'chip mod', title: 'takes ' + m}, m)); });
452
+ if ((c.outputs || []).indexOf('image') !== -1) {
453
+ name.appendChild(el('span', {class: 'chip mod warn', title: 'an image generator: never a panel seat'}, 'makes images'));
454
+ }
455
+ row.appendChild(name);
456
+ var side = el('span');
457
+ if (c.new && !dismissed) side.appendChild(el('span', {class: 'muted'}, c.group)); // under New: where it comes from
458
+ if (dismissed) {
459
+ var b = el('button', {type: 'button', class: 'linkish', style: 'margin-left:.6rem'}, 'Bring back');
460
+ b.addEventListener('click', function (e) {
461
+ e.preventDefault();
462
+ post('/api/dismiss', {served: c.key, undo: true}).then(function (res) {
463
+ if (!res.ok) { listErrors(o.errors, res.errors); return; }
464
+ moveRows([c.key], 'dismissed', 'candidates'); renderPicker();
465
+ });
466
+ });
467
+ side.appendChild(b);
468
+ }
469
+ row.appendChild(side);
470
+ box.addEventListener('change', function () {
471
+ if (box.checked) chosen[c.key] = {served: c.served, endpoint: c.endpoint};
472
+ else delete chosen[c.key];
473
+ armStart();
474
+ });
475
+ return row;
476
+ }
477
+ function renderPicker() {
478
+ o.picker.textContent = '';
479
+ if (!data) return;
480
+ var q = o.find.value.trim().toLowerCase();
481
+ var groups = {};
482
+ data.candidates.forEach(function (c) {
483
+ // The endpoint too: "Onboard a model from it" filters on it, and the default
484
+ // endpoint's models are grouped by connection, not by its name.
485
+ if (q && [c.served, c.group, c.endpoint].every(function (s) { return s.toLowerCase().indexOf(q) === -1; })) return;
486
+ // New ones in a group of their own, first: just added to the hub is what
487
+ // you came to onboard, and it is there without scrolling or unfolding.
488
+ var g = c.new ? NEW : c.group;
489
+ (groups[g] = groups[g] || []).push(c);
490
+ });
491
+ var names = Object.keys(groups).sort(function (a, b) { return (b === NEW) - (a === NEW) || (a < b ? -1 : a > b ? 1 : 0); });
492
+ if (!names.length) o.picker.appendChild(el('p', {class: 'empty'}, data.candidates.length ? 'Nothing matches.' : 'Everything the endpoints serve is already on the bench, retired or dismissed.'));
493
+ names.forEach(function (g) {
494
+ // Folds, remembered per browser; a search opens every group it matches in.
495
+ var d = el('details', {class: 'grp'});
496
+ d.open = !!q || !folded[g];
497
+ d.appendChild(el('summary', {}, g + ' · ' + groups[g].length));
498
+ groups[g].forEach(function (c) { d.appendChild(pickRow(c, false)); });
499
+ d.addEventListener('toggle', function () {
500
+ if (q) return; // opened by the search, not by you
501
+ if (d.open) delete folded[g]; else folded[g] = 1;
502
+ store('bench.picker.folded', JSON.stringify(Object.keys(folded)));
503
+ });
504
+ o.picker.appendChild(d);
505
+ });
506
+ if (data.dismissed.length) {
507
+ var d = el('details'); d.appendChild(el('summary', {}, data.dismissed.length + ' dismissed'));
508
+ data.dismissed.forEach(function (c) { d.appendChild(pickRow(c, true)); });
509
+ o.picker.appendChild(d);
510
+ }
511
+ }
512
+ function loadCandidates() {
513
+ o.picker.textContent = ''; o.picker.appendChild(el('p', {class: 'empty'}, 'Asking the endpoints what they serve…'));
514
+ o.start.disabled = o.dismiss.disabled = true;
515
+ o.find.readOnly = true; // typing before the list arrives is lost, or lands on a row
516
+ fetch('/api/candidates').then(function (r) { return r.json(); }).then(function (res) {
517
+ o.find.readOnly = false; o.find.focus();
518
+ if (!res.ok) { o.picker.textContent = ''; listErrors(o.errors, res.errors); return; }
519
+ if (res.errors.length) listErrors(o.errors, res.errors); // one endpoint down; the rest listed
520
+ data = res;
521
+ // Only what is still offered stays ticked.
522
+ var offered = {};
523
+ res.candidates.forEach(function (c) { offered[c.key] = 1; });
524
+ Object.keys(chosen).forEach(function (k) { if (!offered[k]) delete chosen[k]; });
525
+ renderPicker(); armStart();
526
+ }).catch(function (err) { o.find.readOnly = false; o.errors.textContent = String(err); });
527
+ }
528
+ $('onboard-open').addEventListener('click', function () {
529
+ preStep = null;
530
+ o.errors.textContent = ''; o.find.value = ''; o.choose.hidden = false; o.run.hidden = true;
531
+ $('o-queue').hidden = true; chosen = {}; waitToken++;
532
+ o.start.hidden = o.dismiss.hidden = false; armStart(); o.title.textContent = 'Onboard a model';
533
+ od.showModal(); o.find.focus(); loadCandidates();
534
+ });
535
+ o.find.addEventListener('input', renderPicker);
536
+ o.close.addEventListener('click', function () { od.close(); });
537
+ od.addEventListener('close', function () { if (timer) { clearInterval(timer); timer = null; } });
538
+
539
+ // Discover's Add through a hub: the hub's own steps come before onboard.py's
540
+ // five, as one more step at the top, {label, state: now|done|fail, note}.
541
+ var preStep = null;
542
+ function renderSteps(text, finished, exit) {
543
+ var reached = 0;
544
+ for (var i = 1; i <= 5; i++) if (text.indexOf('[' + i + '/5]') !== -1) reached = i;
545
+ var stopped = /STOPPING|is not served|cannot reach/.test(text) || (finished && exit !== 0);
546
+ var lines = text.split('\n');
547
+ o.steps.textContent = '';
548
+ if (preStep) {
549
+ var li0 = el('li', {class: preStep.state}, preStep.label);
550
+ if (preStep.note) li0.appendChild(el('small', {}, preStep.note));
551
+ o.steps.appendChild(li0);
552
+ if (preStep.state !== 'done') { // onboard.py has not started: its steps wait
553
+ STEPS.forEach(function (label) { o.steps.appendChild(el('li', {}, label)); });
554
+ return;
555
+ }
556
+ }
557
+ STEPS.forEach(function (label, i) {
558
+ var n = i + 1, li = el('li', {}, label);
559
+ var line = lines.filter(function (l) { return l.indexOf('[' + n + '/5]') !== -1; })[0];
560
+ if (line) li.appendChild(el('small', {}, line.replace(/^\s*\[\d\/5\]\s*/, '')));
561
+ if (n < reached || (n === reached && finished && !stopped)) li.className = 'done';
562
+ else if (n === reached && stopped) {
563
+ li.className = 'fail';
564
+ // Why it stopped, under the step that failed, not only in the full log.
565
+ var at = text.search(/STOPPING|is not served|cannot reach/);
566
+ if (at !== -1) li.appendChild(el('small', {}, text.slice(at).split('\n').slice(0, 3).join(' ').trim()));
567
+ }
568
+ else if (n === reached || (n === 1 && !reached && !finished)) li.className = 'now';
569
+ o.steps.appendChild(li);
570
+ });
571
+ }
572
+ // Everything ticked goes to the queue (serve.queue_onboard), which runs one
573
+ // model at a time. The popup closes: the header's chip shows the batch, and
574
+ // its Details the queue. Refused, it stays open with the reasons.
575
+ o.start.addEventListener('click', function () {
576
+ var items = Object.keys(chosen).map(function (k) { return chosen[k]; });
577
+ if (!items.length) return;
578
+ o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true; o.start.textContent = 'Starting…';
579
+ post('/api/onboard-queue', {items: items}).then(function (res) {
580
+ if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
581
+ od.close(); oqPoll();
582
+ }).catch(function (err) { o.errors.textContent = String(err); armStart(); });
583
+ });
584
+ // Every ticked model at once (serve.dismiss); the dismissed list brings one back.
585
+ o.dismiss.addEventListener('click', function () {
586
+ var keys = Object.keys(chosen);
587
+ if (!keys.length) return;
588
+ o.errors.textContent = ''; o.start.disabled = o.dismiss.disabled = true;
589
+ post('/api/dismiss', {served: keys}).then(function (res) {
590
+ if (!res.ok) { listErrors(o.errors, res.errors); armStart(); return; }
591
+ moveRows(keys, 'candidates', 'dismissed'); chosen = {}; renderPicker(); armStart();
592
+ }).catch(function (err) { o.errors.textContent = String(err); armStart(); });
593
+ });
594
+ // After Discover's Add queues a model: with nothing ahead of it, it is
595
+ // followed step by step, otherwise the queue shows.
596
+ var waitToken = 0;
597
+ function afterQueued(res, one, title, before) {
598
+ oqPoll();
599
+ var at = -1; // its LAST entry: an earlier, finished one may share its name
600
+ res.items.forEach(function (i, n) { if (one && i.served === one.served && i.endpoint === one.endpoint) at = n; });
601
+ var ahead = at === -1 ? [] : res.items.slice(0, at).filter(function (i) { return i.state === 'queued' || i.state === 'running'; });
602
+ if (one && at !== -1 && !ahead.length) waitAndFollow(one, at, title || 'Onboarding ' + one.served + ' from ' + one.endpoint, before || '', ++waitToken, 20);
603
+ else showQueue(res);
604
+ }
605
+ // The queue starts it within a moment; follow its log once it has one,
606
+ // unless the popup was closed or reopened meanwhile (the header has it then).
607
+ function waitAndFollow(one, at, title, before, token, tries) {
608
+ if (token !== waitToken || !od.open) return;
609
+ fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
610
+ if (!q.ok || token !== waitToken || !od.open) return;
611
+ var it = q.items[at];
612
+ if (!it || it.served !== one.served || it.endpoint !== one.endpoint) { showQueue(q); return; }
613
+ if (it.log) { follow(it.log, title, before); return; }
614
+ if (it.state !== 'queued' && it.state !== 'running' || tries <= 0) { showQueue(q); return; }
615
+ setTimeout(function () { waitAndFollow(one, at, title, before, token, tries - 1); }, 500);
616
+ });
617
+ }
618
+
619
+ // ---- the onboarding queue: progress in the header (serve.onboard_queue) --
620
+ // Polled while it runs, and once on every page load, so a batch started
621
+ // before a reload still shows. Its results stay until Clear.
622
+ var oq = {box: $('oq'), text: $('oq-text'), chip: $('oq-chip'), stop: $('oq-stop'), clear: $('oq-clear'),
623
+ view: $('o-queue'), sum: $('o-queue-sum'), list: $('o-queue-list')};
624
+ var OQ_CHIP = {queued: '', running: 'new', done: 'ok', 'no answer': 'warn', failed: 'err', skipped: ''};
625
+ var oqTimer = null, oqLast = '';
626
+ function oqCount(q, state) { return q.items.filter(function (i) { return i.state === state; }).length; }
627
+ function oqRender(q) {
628
+ var n = q.items.length;
629
+ oq.box.hidden = !n; batchRunning = !!q.running;
630
+ var now = q.items.filter(function (i) { return i.state === 'running'; })[0];
631
+ var finished = n - oqCount(q, 'queued') - (now ? 1 : 0);
632
+ // The chip fills as models finish, and by fifths of the running one as its
633
+ // log reaches each of onboard.py's five steps (serve.onboard_queue `step`):
634
+ // a step is half a fifth while it runs, so step 5/5 is 90%, and full
635
+ // means finished. At the end it is green when every one is on the bench,
636
+ // red (with one shake) when any failed or did not answer.
637
+ var step = now && now.step ? now.step : 0;
638
+ oq.chip.style.setProperty('--fill', (n ? Math.round(100 * (finished + (step ? step - 0.5 : 0) / 5) / n) : 0) + '%');
639
+ var bad = oqCount(q, 'failed') + oqCount(q, 'no answer');
640
+ oq.chip.classList.toggle('done', !q.running && n > 0 && oqCount(q, 'done') === n);
641
+ oq.chip.classList.toggle('fail', !q.running && bad > 0);
642
+ if (q.running) {
643
+ oq.text.textContent = (q.stopping ? 'Stopping after ' : 'Onboarding ' + (finished + 1) + ' of ' + n
644
+ + (step ? ', step ' + step + '/5' : '') + ': ') + (now ? now.served : '…');
645
+ } else {
646
+ oq.text.textContent = 'Onboarded ' + oqCount(q, 'done') + ' of ' + n
647
+ + [['failed', 'failed'], ['no answer', 'did not answer'], ['skipped', 'skipped']].map(function (s) {
648
+ var c = oqCount(q, s[0]); return c ? ' · ' + c + ' ' + s[1] : '';
649
+ }).join('');
650
+ }
651
+ oq.text.title = oq.text.textContent; // cut short beside the button; whole on hover
652
+ oq.stop.hidden = !q.running || q.stopping; oq.clear.hidden = q.running;
653
+ var sig = JSON.stringify(q);
654
+ if (sig !== oqLast) { oqLast = sig; oqLive = null; if (!oq.view.hidden) oqList(q); }
655
+ // The log grows between steps, which the queue's state does not show.
656
+ if (oqLive && !oq.view.hidden && od.open && oqLive.d.open) oqTail(oqLive.log, oqLive.pre);
657
+ }
658
+ function oqList(q) {
659
+ oq.sum.textContent = q.running
660
+ ? 'One at a time, in this order. Closing this does not stop it; Stop in the header lets the model running now finish and skips the rest.'
661
+ : q.items.length ? 'Finished. Each log is in runs/.' : 'Nothing queued.';
662
+ oq.list.textContent = '';
663
+ q.items.forEach(function (i) {
664
+ var li = el('li'), head = el('div', {class: 'what'});
665
+ head.appendChild(el('span', {class: 'alias'}, i.served));
666
+ head.appendChild(el('span', {class: 'chip ' + (OQ_CHIP[i.state] || '')}, i.state));
667
+ if (i.alias && i.state !== 'failed') head.appendChild(el('span', {class: 'muted'}, 'as ' + i.alias));
668
+ li.appendChild(head);
669
+ if (i.note) li.appendChild(el('small', {}, i.note));
670
+ if (i.log) {
671
+ var d = el('details'), pre = el('pre', {class: 'log'});
672
+ d.appendChild(el('summary', {class: 'muted'}, 'Log'));
673
+ d.appendChild(pre);
674
+ d.addEventListener('toggle', function () {
675
+ if (i.state === 'running') { if (d.open) delete oqFolded[i.log]; else oqFolded[i.log] = 1; }
676
+ if (d.open) oqTail(i.log, pre);
677
+ });
678
+ // The running one's log is open and follows along (oqRender), unless
679
+ // you folded it; queued ones have none yet, finished ones stay folded.
680
+ if (i.state === 'running') { oqLive = {log: i.log, pre: pre, d: d}; d.open = !oqFolded[i.log]; }
681
+ li.appendChild(d);
682
+ }
683
+ oq.list.appendChild(li);
684
+ });
685
+ }
686
+ var oqLive = null, oqFolded = {};
687
+ function oqTail(log, pre) {
688
+ fetch('/api/onboard?log=' + encodeURIComponent(log)).then(function (r) { return r.json(); }).then(function (s) {
689
+ var end = pre.scrollHeight - pre.scrollTop - pre.clientHeight < 24; // at the bottom: stay there
690
+ pre.textContent = s.ok ? (s.tail || '(no output yet)') : s.errors.join(' ');
691
+ if (end) pre.scrollTop = pre.scrollHeight;
692
+ });
693
+ }
694
+ function showQueue(q) {
695
+ if (timer) { clearInterval(timer); timer = null; } // a model followed step by step: the queue view replaces it
696
+ waitToken++;
697
+ o.errors.textContent = ''; o.choose.hidden = true; o.run.hidden = true; o.start.hidden = o.dismiss.hidden = true;
698
+ oq.view.hidden = false; o.title.textContent = 'Onboarding queue';
699
+ if (!od.open) od.showModal();
700
+ oqLast = ''; oqRender(q);
701
+ }
702
+ function oqPoll() {
703
+ fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) {
704
+ if (!q.ok) return;
705
+ oqRender(q);
706
+ if (q.running && !oqTimer) oqTimer = setInterval(oqPoll, 3000);
707
+ if (!q.running && oqTimer) { clearInterval(oqTimer); oqTimer = null; }
708
+ });
709
+ }
710
+ function oqAct(path) {
711
+ post(path, {}).then(function (r) { if (r.ok) oqRender(r); else oq.text.textContent = r.errors.join(' '); });
712
+ }
713
+ $('oq-open').addEventListener('click', function () {
714
+ fetch('/api/onboard-queue').then(function (r) { return r.json(); }).then(function (q) { showQueue(q); oqPoll(); });
715
+ });
716
+ oq.stop.addEventListener('click', function () { oqAct('/api/onboard-queue/stop'); });
717
+ oq.clear.addEventListener('click', function () { oqAct('/api/onboard-queue/clear'); if (!oq.view.hidden) od.close(); });
718
+ oqPoll();
719
+ // ---- Discover: a row opens its model, with every route to add it by -----
720
+ // The rest of what is known comes from /api/discover-model, so the page does
721
+ // not carry 1,200 models' details it will mostly never show.
722
+ var dd = $('discover-dialog');
723
+ function price(x) {
724
+ var c = x.cost || {};
725
+ return x.free ? 'free' : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
726
+ }
727
+ function limits(l) {
728
+ return Object.keys(l || {}).map(function (k) { return Number(l[k]).toLocaleString() + ' ' + k; }).join(' · ');
729
+ }
730
+ function addVia(x, b) {
731
+ var direct = x.via === 'direct';
732
+ ask('Add ' + x.model + ' via ' + x.label + (x.free ? '' : ' (PAID)') + '?\n\n'
733
+ + (direct ? 'The bench only: register it on the ' + x.endpoint + ' endpoint, a 4-call probe, then the suite.'
734
+ // A LiteLLM or CLI Proxy API route says its own steps: only what that hub takes.
735
+ : x.steps ? x.steps.map(function (s, i) { return (i + 1) + '. ' + s; }).join('\n')
736
+ : '1. Open WebUI: add it to the ' + x.connection + ' connection\'s Model IDs (if that connection lists them)\n'
737
+ + '2. Open WebUI: a public model record, tagged and with the logo of the connection\'s other models\n'
738
+ + '3. The bench: register, a 4-call probe, then the suite'), 'Add').then(function (yes) { if (yes) addNow(x, b, direct); });
739
+ }
740
+ function addNow(x, b, direct) {
741
+ var title = 'Adding ' + x.model + ' via ' + x.label;
742
+ $('d-errors').textContent = '';
743
+ if (direct) {
744
+ b.disabled = true; b.textContent = 'Adding…';
745
+ post('/api/onboard-queue', {items: [{served: x.model, endpoint: x.endpoint}]}).then(function (res) {
746
+ if (!res.ok) { listErrors($('d-errors'), res.errors); b.disabled = false; b.textContent = 'Add via this'; return; }
747
+ dd.close(); preStep = null; o.errors.textContent = ''; od.showModal();
748
+ afterQueued(res, {served: x.model, endpoint: x.endpoint}, title, '');
749
+ }).catch(function (err) { $('d-errors').textContent = String(err); b.disabled = false; b.textContent = 'Add via this'; });
750
+ return;
751
+ }
752
+ // A hub's own steps take a while before the onboarding is even queued, so
753
+ // the progress starts now: the onboarding popup with the hub as its first
754
+ // step, and the header chip. The queue's own poll takes the chip over.
755
+ dd.close(); waitToken++;
756
+ preStep = {label: 'Added to ' + x.label, state: 'now', note: ''};
757
+ runView(title, 'Adding it to ' + x.label + '…');
758
+ oq.box.hidden = false; oq.chip.classList.remove('done', 'fail'); oq.chip.style.setProperty('--fill', '3%');
759
+ oq.text.textContent = oq.text.title = title; oq.stop.hidden = oq.clear.hidden = true;
760
+ post('/api/discover/add', {connection: x.connection, model: x.model, endpoint: x.endpoint}).then(function (res) {
761
+ var hubDone = !!(res.served && res.steps && res.steps.length); // in the hub, even if queueing it was refused
762
+ preStep.state = hubDone ? 'done' : 'fail';
763
+ preStep.note = (res.steps || []).join('. ') + (hubDone ? '' : (res.steps && res.steps.length ? '. ' : '') + res.errors.join(' '));
764
+ renderSteps('', false, null);
765
+ if (!res.ok) {
766
+ o.result.textContent = hubDone ? res.errors.join(' ') : 'Not added. Nothing was queued.';
767
+ o.result.className = 'errors';
768
+ oqPoll(); // the chip back to what the queue holds
769
+ return;
770
+ }
771
+ afterQueued(res, {served: res.served || x.model, endpoint: res.endpoint || x.endpoint}, title, '');
772
+ }).catch(function (err) {
773
+ preStep.state = 'fail'; preStep.note = String(err); renderSteps('', false, null); oqPoll();
774
+ });
775
+ }
776
+ // The onboarding popup in its step-by-step view, before there is a log.
777
+ function runView(title, message) {
778
+ o.choose.hidden = true; o.run.hidden = false; o.start.hidden = o.dismiss.hidden = true; $('o-queue').hidden = true;
779
+ o.title.textContent = title; o.result.className = 'served'; o.result.textContent = message; o.errors.textContent = '';
780
+ o.log.textContent = ''; delete o.log.dataset.opened; o.log.parentNode.open = false;
781
+ renderSteps('', false, null);
782
+ if (!od.open) od.showModal();
783
+ }
784
+ function openDiscover(provider, model) {
785
+ $('d-errors').textContent = '';
786
+ fetch('/api/discover-model?provider=' + encodeURIComponent(provider) + '&model=' + encodeURIComponent(model))
787
+ .then(function (r) { return r.json(); }).then(function (res) {
788
+ if (!res.ok) { listErrors($('d-errors'), res.errors); dd.showModal(); return; }
789
+ if (res.errors.length) listErrors($('d-errors'), res.errors); // a hub that could not be asked
790
+ var r = res.model, p = $('d-profile'), list = $('d-routes');
791
+ $('d-title').textContent = r.name;
792
+ $('d-served').textContent = r.provider_name + ' · ' + r.model + ' · listed by ' + r.sources.join(' and ')
793
+ + (r.on_bench_as ? ' · on the bench as ' + r.on_bench_as : '');
794
+ p.textContent = '';
795
+ [['Input', r.in_mods], ['Output', r.out_mods]].forEach(function (x) {
796
+ var row = el('div', {class: 'prof'});
797
+ row.appendChild(el('span', {class: 'muted'}, x[0] + ' '));
798
+ (x[1] && x[1].length ? x[1] : ['unknown']).forEach(function (v) { row.appendChild(el('span', {class: 'chip'}, v)); });
799
+ p.appendChild(row);
800
+ });
801
+ var can = [['tools', r.tools], ['vision', r.vision], ['reasons', r.reasoning]]
802
+ .filter(function (x) { return x[1]; }).map(function (x) { return x[0]; });
803
+ var facts = [
804
+ r.context ? 'context ' + Number(r.context).toLocaleString() + ' tokens' : 'context unknown',
805
+ r.output ? 'output up to ' + Number(r.output).toLocaleString() : '',
806
+ can.length ? 'can: ' + can.join(', ') : '',
807
+ price(r),
808
+ r.limits ? 'limits ' + limits(r.limits) : '',
809
+ r.released ? 'released ' + r.released : '',
810
+ r.knowledge ? 'knowledge to ' + r.knowledge : '',
811
+ r.open_weights != null ? (r.open_weights ? 'open weights' : 'closed weights') : '',
812
+ 'first seen ' + (r.first_seen === 'baseline' ? 'before the list began' : r.first_seen)];
813
+ p.appendChild(el('div', {class: 'prof'}, facts.filter(Boolean).join(' · ')));
814
+ if (r.also_on && r.also_on.length) p.appendChild(el('div', {class: 'prof muted'}, 'Also on the bench as ' + r.also_on.join(', ')));
815
+ list.textContent = '';
816
+ res.routes.forEach(function (x) {
817
+ var row = el('div', {class: 'endpoint'}), left = el('div');
818
+ left.appendChild(el('div', {class: 'what'}, x.label));
819
+ left.appendChild(el('div', {class: 'base'}, x.model + ' · ' + price(x) + (x.limits ? ' · ' + limits(x.limits) : '')));
820
+ left.appendChild(el('div', {class: 'base'}, x.note));
821
+ row.appendChild(left);
822
+ var acts = el('div', {class: 'acts'});
823
+ if (x.on_bench_as) acts.appendChild(el('span', {class: 'chip ok'}, 'on the bench as ' + x.on_bench_as));
824
+ else {
825
+ var b = el('button', {class: 'btn small', type: 'button'}, 'Add via this');
826
+ b.addEventListener('click', function () { addVia(x, b); });
827
+ acts.appendChild(b);
828
+ }
829
+ row.appendChild(acts);
830
+ list.appendChild(row);
831
+ });
832
+ if (!res.routes.length) list.appendChild(el('p', {class: 'rules'}, 'None: ' + r.provider_name
833
+ + ' needs a connection in a hub or an endpoint of its own first'
834
+ + (r.api ? ' (its OpenAI-compatible base URL is ' + r.api + ')' : '') + '.'));
835
+ dd.showModal();
836
+ }).catch(function (err) { $('d-errors').textContent = String(err); dd.showModal(); });
837
+ }
838
+ document.addEventListener('click', function (e) {
839
+ var row = e.target.closest('tr.pick-row'); // the whole row: its name, its Add…, anywhere
840
+ if (!row) return;
841
+ e.preventDefault(); openDiscover(row.dataset.provider, row.dataset.model);
842
+ });
843
+ $('d-close').addEventListener('click', function () { dd.close(); });
844
+ function follow(log, title, before) {
845
+ runView(title, before + 'Running in the background; closing this does not stop it. Log: runs/' + log);
846
+ timer = setInterval(function () {
847
+ fetch('/api/onboard?log=' + encodeURIComponent(log)).then(function (r) { return r.json(); }).then(function (s) {
848
+ if (!s.ok) return;
849
+ o.log.textContent = s.tail || '(no output yet)';
850
+ // Open the log the first time it has something in it; after that the user decides.
851
+ if (s.tail && !o.log.dataset.opened) { o.log.parentNode.open = true; o.log.dataset.opened = '1'; }
852
+ renderSteps(s.tail || '', s.done, s.exit);
853
+ if (s.done) {
854
+ clearInterval(timer); timer = null;
855
+ var alias = (/->\s+alias\s+(\S+)/.exec(s.tail) || [])[1];
856
+ if (s.exit === 0) {
857
+ o.result.textContent = 'Finished. ' + (alias ? alias + ' is on the bench. ' : '') + 'The suite scores are in the dated suite note; whether to route work there is your call. Closing…';
858
+ o.result.className = 'done';
859
+ closeAndReload(od);
860
+ } else {
861
+ o.result.textContent = 'Stopped with exit ' + s.exit + '. The last lines of the log say why.';
862
+ fresh.hidden = false;
863
+ }
864
+ }
865
+ });
866
+ }, 3000);
867
+ }
868
+
869
+ // ---- theme: applied the moment it is picked, and saved ------------------
870
+ // Every theme's colours are on the page (serve.brand_css), keyed by the
871
+ // data-theme on <html>, so switching is one attribute: no reload. The
872
+ // charts read their colours when drawn, so they are drawn again.
873
+ document.querySelectorAll('#themes input[name=theme]').forEach(function (input) {
874
+ input.addEventListener('change', function () {
875
+ document.documentElement.setAttribute('data-theme', input.value);
876
+ $('t-errors').textContent = ''; $('t-done').textContent = '';
877
+ if (typeof drawCharts === 'function') drawCharts();
878
+ post('/api/theme', {theme: input.value}).then(function (r) {
879
+ if (!r.ok) { listErrors($('t-errors'), r.errors); return; }
880
+ $('t-done').textContent = 'Saved: ' + input.value + ' is the theme from now on.';
881
+ if (location.search.indexOf('theme=') !== -1) history.replaceState(null, '', '/'); // drop a ?theme preview
882
+ });
883
+ });
884
+ });
885
+
886
+ // ---- gateway keys: shown once, then only their names ---------------------
887
+ if ($('k-make')) $('k-make').addEventListener('click', function () {
888
+ $('k-errors').textContent = '';
889
+ var scope = Array.prototype.map.call(document.querySelectorAll('input[name=k-scope]:checked'),
890
+ function (c) { return c.value; });
891
+ post('/api/gateway-key', {name: $('k-name').value, collections: scope, rpm: $('k-rpm').value}).then(function (r) {
892
+ if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
893
+ $('k-value').value = r.key;
894
+ $('k-done').hidden = false;
895
+ $('k-value').select();
896
+ $('k-make').disabled = true; // reload to see it listed; the value is gone then
897
+ });
898
+ });
899
+ document.querySelectorAll('[data-revoke-key]').forEach(function (b) {
900
+ b.addEventListener('click', function () {
901
+ var name = b.dataset.revokeKey;
902
+ ask('Revoke the gateway key ' + name + '? Anything using it is refused from now on.', 'Revoke', true).then(function (yes) {
903
+ if (!yes) return;
904
+ post('/api/gateway-key/remove', {name: name}).then(function (r) {
905
+ if (!r.ok) { listErrors($('k-errors'), r.errors); return; }
906
+ location.reload();
907
+ });
908
+ });
909
+ });
910
+ });
911
+
912
+ // ---- scheduler: its mode and jobs (serve.save_scheduler) -----------------
913
+ var sMode = $('s-mode');
914
+ function modeWhy() { $('s-mode-why').textContent = sMode.selectedOptions[0].dataset.why; }
915
+ sMode.addEventListener('change', modeWhy); modeWhy();
916
+ $('s-save').addEventListener('click', function () {
917
+ var jobs = {};
918
+ document.querySelectorAll('input[name=s-job]').forEach(function (c) { jobs[c.value] = c.checked; });
919
+ (sMode.value === 'run' && sMode.dataset.was !== 'run'
920
+ ? ask('Run the jobs for real? They make model calls on their own schedule. If something else'
921
+ + ' also runs them (cron, Windows Task Scheduler), turn it off now, or every job runs twice.', 'Run them')
922
+ : Promise.resolve(true)).then(function (yes) {
923
+ if (!yes) return;
924
+ $('s-errors').textContent = ''; $('s-done').textContent = '';
925
+ post('/api/scheduler', {mode: sMode.value, jobs: jobs}).then(function (r) {
926
+ if (!r.ok) { listErrors($('s-errors'), r.errors); return; }
927
+ sMode.dataset.was = sMode.value; $('s-done').textContent = 'Saved. It applies within a minute.';
928
+ });
929
+ });
930
+ });
931
+
932
+ // ---- model client (serve.save_client): the next call uses it -------------
933
+ $('mc-save').addEventListener('click', function () {
934
+ $('mc-errors').textContent = ''; $('mc-done').textContent = '';
935
+ post('/api/client', {client: $('mc-client').value}).then(function (r) {
936
+ if (!r.ok) { listErrors($('mc-errors'), r.errors); return; }
937
+ $('mc-done').textContent = 'Saved. The next model call uses it.';
938
+ });
939
+ });
940
+
941
+ // ---- free only in their own app (serve.save_own_client_free) -------------
942
+ if ($('oc-save')) $('oc-save').addEventListener('click', function () {
943
+ var providers = {}, add = $('oc-add').value.trim().toLowerCase();
944
+ document.querySelectorAll('input[name=oc-prov]').forEach(function (c) { providers[c.value] = c.checked; });
945
+ if (add) providers[add] = true;
946
+ $('oc-errors').textContent = ''; $('oc-done').textContent = '';
947
+ post('/api/own-client-free', {providers: providers}).then(function (r) {
948
+ if (!r.ok) { listErrors($('oc-errors'), r.errors); return; }
949
+ if (add) { location.reload(); return; } // the new provider gets its row
950
+ $('oc-done').textContent = 'Saved. Discover uses it on its next load.';
951
+ });
952
+ });
953
+
954
+ // ---- endpoints: keys go in, never come back out -------------------------
955
+ // The Endpoints page lists them (serve.endpoints_page, each row's data in
956
+ // data-ep); Add an endpoint and Change key are one dialog. The page is
957
+ // drawn by the server, so a change reloads it, as the model popup does.
958
+ var ed = $('endpoint-dialog');
959
+ var e = {name: $('e-name'), base: $('e-base'), key: $('e-key'),
960
+ kind: $('e-kind'), hub: $('e-hub'), admin: $('e-admin'),
961
+ errors: $('e-errors'), done: $('e-done'), test: $('e-test'), save: $('e-save'),
962
+ listErrors: $('el-errors'), listDone: $('el-done'), changed: false};
963
+ e.kind.addEventListener('change', function () {
964
+ $('e-hub-row').hidden = $('e-admin-row').hidden = e.kind.value !== 'hub';
965
+ });
966
+ ed.querySelectorAll('[data-close]').forEach(function (b) { b.addEventListener('click', function () { ed.close(); }); });
967
+ ed.addEventListener('close', function () { if (e.changed) location.reload(); });
968
+ function reloadSoon(message) {
969
+ e.listErrors.textContent = ''; e.listDone.textContent = message + ' Reloading…';
970
+ setTimeout(function () { location.reload(); }, 900);
971
+ }
972
+ var rowActs = {
973
+ change: function (x) { openEndpoint('change', x); },
974
+ again: function (x) { // this endpoint, not a new one
975
+ openEndpoint('again', x);
976
+ connectStart({id: x.connect.provider, flow: x.connect.flow, label: x.connect.label}, x.name);
977
+ },
978
+ 'default': function (x) {
979
+ ask('Make ' + x.name + ' the default endpoint? The default is where a model is taken'
980
+ + ' to be when nothing says otherwise, and the one endpoint that cannot be removed.', 'Make default').then(function (yes) {
981
+ if (!yes) return;
982
+ post('/api/endpoint/default', {name: x.name}).then(function (r) {
983
+ if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
984
+ reloadSoon(r.note + '.');
985
+ });
986
+ });
987
+ },
988
+ // Its models go with it, after saying how many: each is retired as by
989
+ // hand (evidence kept, restorable once the endpoint is back).
990
+ remove: function (x) {
991
+ var n = x.models, them = n + ' model' + (n === 1 ? '' : 's');
992
+ ask(n ? 'Remove the ' + x.name + ' endpoint and the ' + them + ' on the bench through it?\n\n'
993
+ + 'The ' + them + ' will be retired too. Their results and history are kept, and'
994
+ + ' Restore brings them back if an endpoint named ' + x.name + ' is added again.'
995
+ : 'Remove the ' + x.name + ' endpoint?', 'Remove', true).then(function (yes) {
996
+ if (!yes) return;
997
+ post('/api/endpoint/remove', {name: x.name, with_models: n > 0}).then(function (r) {
998
+ if (!r.ok) { listErrors(e.listErrors, r.errors); return; }
999
+ reloadSoon('Removed ' + x.name + '; ' + r.note + '.');
1000
+ });
1001
+ });
1002
+ }
1003
+ };
1004
+ $('tab-endpoints').addEventListener('click', function (ev) {
1005
+ var b = ev.target.closest('.ep button[data-ep-act]');
1006
+ if (b) rowActs[b.dataset.epAct](JSON.parse(b.closest('.ep').dataset.ep));
1007
+ });
1008
+ function loadLimits() {
1009
+ var list = $('l-list');
1010
+ fetch('/api/connections').then(function (r) { return r.json(); }).then(function (res) {
1011
+ list.textContent = '';
1012
+ res.connections.forEach(function (c) {
1013
+ var row = el('div', {class: 'endpoint'});
1014
+ var left = el('div');
1015
+ left.appendChild(el('div', {class: 'what'}, c.name));
1016
+ left.appendChild(el('div', {class: 'base'}, c.models.join(', ')));
1017
+ var inputs = el('div', {class: 'limits'});
1018
+ res.keys.forEach(function (k) {
1019
+ var lab = el('label', {}, k + ' ');
1020
+ var box = el('input', {type: 'number', min: '1', 'data-k': k, placeholder: '—'});
1021
+ if (c.limits[k]) box.value = c.limits[k];
1022
+ lab.appendChild(box); inputs.appendChild(lab);
1023
+ });
1024
+ left.appendChild(inputs);
1025
+ row.appendChild(left);
1026
+ var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
1027
+ save.addEventListener('click', function () {
1028
+ var limits = {};
1029
+ inputs.querySelectorAll('input').forEach(function (b) { if (b.value) limits[b.dataset.k] = parseInt(b.value, 10); });
1030
+ $('l-errors').textContent = ''; $('l-done').textContent = '';
1031
+ post('/api/connection-limits', {connection: c.name, limits: limits}).then(function (r) {
1032
+ if (!r.ok) { listErrors($('l-errors'), r.errors); return; }
1033
+ $('l-done').textContent = 'Saved limits for ' + c.name + '.';
1034
+ });
1035
+ });
1036
+ var acts = el('div', {class: 'acts'}); acts.appendChild(save); row.appendChild(acts);
1037
+ list.appendChild(row);
1038
+ });
1039
+ });
1040
+ }
1041
+ onShow.endpoints = function () { // its own-app and rate-limit sections
1042
+ $('cn-app-errors').textContent = $('cn-app-done').textContent = '';
1043
+ loadProviders(); loadLimits();
1044
+ };
1045
+ // The dialog, in one of three modes. add: the choices, API key first, then
1046
+ // each provider's sign-in; change: one endpoint's key form; again: a
1047
+ // sign-in under way for an endpoint that has one.
1048
+ function openEndpoint(mode, x) {
1049
+ cn.run++;
1050
+ e.adding = mode === 'add'; // Add refuses a name in use; Change key edits that one
1051
+ $('e-title').textContent = mode === 'change' ? 'Change the key for ' + x.name
1052
+ : mode === 'again' ? 'Sign in to ' + x.connect.label + ' again' : 'Add an endpoint';
1053
+ $('e-blurb').hidden = cn.list.hidden = $('e-kinds').hidden = !e.adding;
1054
+ e.key.placeholder = e.adding ? 'blank if it needs none, as a local runtime' : 'leave blank to keep the current one';
1055
+ cn.wait.hidden = true; $('e-after').hidden = true;
1056
+ resetKeyForm();
1057
+ choose(mode === 'again' ? null : 'key');
1058
+ if (mode === 'change') {
1059
+ e.name.value = x.name; e.base.value = x.base; e.name.readOnly = true;
1060
+ $('e-admin-row').hidden = !x.hub; // a hub's admin key changes here too
1061
+ }
1062
+ if (e.adding) loadProviders();
1063
+ if (!ed.open) ed.showModal();
1064
+ (mode === 'change' ? e.key : e.name).focus();
1065
+ }
1066
+ // Which form shows: 'key' (paste an API key), a provider (sign in), or none.
1067
+ function choose(p) {
1068
+ cn.pick = p && p !== 'key' ? p : null;
1069
+ $('e-form').hidden = p !== 'key';
1070
+ $('cn-form').hidden = !cn.pick;
1071
+ if (cn.pick) {
1072
+ $('cn-name').value = cn.pick.suggest;
1073
+ $('cn-go').textContent = (cn.pick.flow === 'local' ? 'Add ' : 'Sign in to ') + cn.pick.label;
1074
+ }
1075
+ e.errors.textContent = ''; e.done.textContent = '';
1076
+ press();
1077
+ }
1078
+ function press() {
1079
+ var on = cn.pick ? cn.pick.id : $('e-form').hidden ? '' : 'key';
1080
+ cn.list.querySelectorAll('button').forEach(function (b) { b.setAttribute('aria-pressed', String(b.dataset.choice === on)); });
1081
+ }
1082
+ function resetKeyForm() {
1083
+ e.name.value = e.base.value = e.key.value = e.admin.value = ''; e.name.readOnly = false;
1084
+ e.kind.value = 'provider'; e.hub.selectedIndex = 0; $('e-hub-row').hidden = $('e-admin-row').hidden = true;
1085
+ untested();
1086
+ }
1087
+ // Save opens only after Test passes, and closes again when anything in the
1088
+ // form changes. The server tests again on Save: this is the form's manners,
1089
+ // not the guarantee.
1090
+ function untested() { e.save.disabled = true; e.save.title = 'Test it first'; }
1091
+ $('e-form').addEventListener('input', function () { untested(); e.done.textContent = ''; });
1092
+ function keyBody() {
1093
+ var body = {name: e.name.value, base: e.base.value, key: e.key.value, new: e.adding,
1094
+ admin: $('e-admin-row').hidden ? '' : e.admin.value};
1095
+ if (e.adding) { body.kind = e.kind.value; body.hub = e.kind.value === 'hub' ? e.hub.value : ''; } // Change key keeps its kind
1096
+ return body;
1097
+ }
1098
+ $('e-add').addEventListener('click', function () { openEndpoint('add'); });
1099
+ e.test.addEventListener('click', function () {
1100
+ e.errors.textContent = ''; e.done.textContent = ''; untested();
1101
+ e.test.disabled = true; e.test.textContent = 'Testing…';
1102
+ var body = keyBody(), sent = JSON.stringify(body), run = cn.run; body.test = true;
1103
+ function end() { e.test.disabled = false; e.test.textContent = 'Test'; }
1104
+ post('/api/endpoint', body).then(function (r) {
1105
+ end();
1106
+ if (run !== cn.run || JSON.stringify(keyBody()) !== sent) return; // opened afresh, or edited, meanwhile
1107
+ if (!r.ok) { listErrors(e.errors, r.errors); return; }
1108
+ e.done.textContent = 'It answers: ' + r.served + ' models listed. ' + (r.note || '');
1109
+ e.save.disabled = false; e.save.title = '';
1110
+ }).catch(function (err) { end(); e.errors.textContent = String(err); });
1111
+ });
1112
+ e.save.addEventListener('click', function () {
1113
+ e.errors.textContent = ''; e.save.disabled = e.test.disabled = true; e.save.textContent = 'Saving…';
1114
+ function end() { e.save.textContent = 'Save'; e.test.disabled = false; }
1115
+ post('/api/endpoint', keyBody()).then(function (r) {
1116
+ end();
1117
+ if (!r.ok) { listErrors(e.errors, r.errors); return; } // Save stays shut: test again
1118
+ var name = e.name.value.trim().toLowerCase();
1119
+ endpointDone('Saved. ' + name + ' lists ' + r.served + ' models; they appear in the onboarding picker. '
1120
+ + (r.note || ''), e.adding ? name : '');
1121
+ }).catch(function (err) { end(); e.errors.textContent = String(err); });
1122
+ });
1123
+ // Saved or signed in: the forms fold, the message stays, with the next step.
1124
+ function endpointDone(message, endpoint) {
1125
+ resetKeyForm(); choose(null);
1126
+ cn.list.hidden = $('e-blurb').hidden = cn.wait.hidden = true; $('e-after').hidden = false;
1127
+ e.done.textContent = message;
1128
+ if (endpoint) e.done.appendChild(onboardFrom(endpoint));
1129
+ e.changed = true; // the page reloads when the dialog closes
1130
+ }
1131
+
1132
+ // ---- connect: sign in on the provider's page (metrics/connect.py) --------
1133
+ // `run` counts the dialog's openings: a sign-in still finishing after the
1134
+ // dialog was closed or opened again reports on the Endpoints page instead,
1135
+ // and reloads it once nothing is open over it.
1136
+ var cn = {list: $('cn-list'), wait: $('cn-wait'), msg: $('cn-msg'), timer: null, run: 0};
1137
+ // Reached from another machine (a hosted install): a callback to this
1138
+ // machine's 127.0.0.1 could not arrive, so sign-ins go by code instead.
1139
+ var remote = !/^(localhost|127\.0\.0\.1|\[::1\])$/.test(location.hostname);
1140
+ function connectDone(r, run) {
1141
+ clearInterval(cn.timer); cn.timer = null;
1142
+ var said = r.ok && r.label + ' is connected as ' + r.endpoint + ': it lists ' + r.served + ' models. ' + (r.note || '');
1143
+ if (!ed.open || run !== cn.run) {
1144
+ if (!r.ok) listErrors(e.listErrors, r.errors);
1145
+ else if (ed.open) { e.listDone.textContent = said; e.changed = true; }
1146
+ else reloadSoon(said);
1147
+ return;
1148
+ }
1149
+ cn.wait.hidden = true;
1150
+ if (!r.ok) { // back to the provider's form to try again, or just Close
1151
+ $('cn-form').hidden = !cn.pick; $('e-after').hidden = !!cn.pick;
1152
+ listErrors(e.errors, r.errors); return;
1153
+ }
1154
+ endpointDone(said, r.endpoint);
1155
+ }
1156
+ // The next step for a new endpoint, signed in or pasted: onboard what it serves.
1157
+ function onboardFrom(endpoint) {
1158
+ var go = el('button', {class: 'btn small', type: 'button', style: 'margin-left:.6rem'}, 'Onboard a model from it');
1159
+ go.addEventListener('click', function () {
1160
+ // No reload on the way: onboarding opens over the page, and the page's
1161
+ // fresh-results button offers the reload once the files have changed.
1162
+ e.changed = false; ed.close(); $('onboard-open').click(); o.find.value = endpoint; // the picker filters on it once loaded
1163
+ });
1164
+ return go;
1165
+ }
1166
+ function connectStart(p, endpoint) {
1167
+ e.errors.textContent = ''; e.done.textContent = '';
1168
+ var run = cn.run;
1169
+ // Opened now, while this is still the click: a window opened after the
1170
+ // request returns is a popup the browser blocks.
1171
+ var byCode = remote && p.flow !== 'local';
1172
+ var tab = !byCode && (p.flow === 'key' || p.flow === 'oauth') ? window.open('', '_blank') : null;
1173
+ if (tab) tab.opener = null;
1174
+ post('/api/connect/start', {provider: p.id, endpoint: endpoint, code: byCode}).then(function (r) {
1175
+ if (!r.ok || r.state) { if (tab) tab.close(); connectDone(r, run); return; } // refused, or local: saved at once
1176
+ cn.wait.hidden = false; cn.msg.textContent = ''; $('e-after').hidden = false;
1177
+ if (r.url) {
1178
+ if (tab) tab.location = r.url; else window.open(r.url, '_blank', 'noopener');
1179
+ cn.msg.textContent = 'Sign in to ' + p.label + ' in the tab that opened. This updates when it is done.';
1180
+ } else {
1181
+ cn.msg.appendChild(document.createTextNode('Open '));
1182
+ cn.msg.appendChild(el('a', {href: r.verify, target: '_blank', rel: 'noopener'}, r.verify));
1183
+ cn.msg.appendChild(document.createTextNode(' and enter the code '));
1184
+ cn.msg.appendChild(el('strong', {class: 'mono'}, r.code));
1185
+ }
1186
+ var tries = 0;
1187
+ clearInterval(cn.timer);
1188
+ cn.timer = setInterval(function () {
1189
+ if (++tries > 450) { connectDone({ok: false, errors: ['No sign-in after fifteen minutes; start again.']}, run); return; }
1190
+ post('/api/connect/poll', {id: r.id}).then(function (s) { if (s.state !== 'pending') connectDone(s, run); });
1191
+ }, 2000);
1192
+ }).catch(function (err) { if (tab) tab.close(); e.errors.textContent = String(err); });
1193
+ }
1194
+ // The dialog's choices (API key, then each provider) and the Manage page's
1195
+ // own-app rows, from one listing.
1196
+ function loadProviders() {
1197
+ fetch('/api/connect').then(function (r) { return r.json(); }).then(function (res) {
1198
+ cn.list.textContent = ''; $('cn-apps').textContent = '';
1199
+ choice('key', 'API key', '').addEventListener('click', function () { choose('key'); e.name.focus(); });
1200
+ res.providers.forEach(function (p) {
1201
+ // A new endpoint, named first (a provider can be several: a pasted key
1202
+ // and a sign-in, or two accounts). Signing an existing one in again is
1203
+ // its row's button.
1204
+ var why = p.missing || (remote && !p.code ? p.label + ' signs in only through a page on the machine'
1205
+ + ' Benchwarmers runs on; from here, choose API key' : '');
1206
+ choice(p.id, p.label, why).addEventListener('click', function () { choose(p); $('cn-name').select(); });
1207
+ if (p.register) $('cn-apps').appendChild(appRow(p));
1208
+ });
1209
+ press();
1210
+ });
1211
+ }
1212
+ function choice(id, label, why) {
1213
+ var b = el('button', {class: 'btn', type: 'button', 'data-choice': id, 'aria-pressed': 'false'}, label);
1214
+ if (why) { b.disabled = true; b.title = why; }
1215
+ cn.list.appendChild(b);
1216
+ return b;
1217
+ }
1218
+ // The Sign in click itself opens the provider's tab (connectStart), so the
1219
+ // browser does not block it as a popup.
1220
+ $('cn-go').addEventListener('click', function () { $('cn-form').hidden = true; connectStart(cn.pick, $('cn-name').value); });
1221
+ // Use your own app: a provider's own client id, instead of the shipped one.
1222
+ function appRow(p) {
1223
+ var row = el('div', {class: 'endpoint'});
1224
+ var left = el('div');
1225
+ left.appendChild(el('div', {class: 'what'}, p.label));
1226
+ left.appendChild(el('div', {class: 'base'}, 'Register ' + p.register));
1227
+ var box = el('input', {spellcheck: 'false', autocomplete: 'off', 'aria-label': p.label + ' client id',
1228
+ placeholder: p.shipped ? "Wendell Research's app (leave blank)" : 'client id, needed to sign in'});
1229
+ box.value = p.own_id || '';
1230
+ var line = el('div', {class: 'row-input'}); // the box and its Save on one line
1231
+ line.appendChild(box);
1232
+ left.appendChild(line);
1233
+ row.appendChild(left);
1234
+ var save = el('button', {class: 'btn small', type: 'button'}, 'Save');
1235
+ save.addEventListener('click', function () {
1236
+ $('cn-app-errors').textContent = ''; $('cn-app-done').textContent = '';
1237
+ post('/api/connect/app', {provider: p.id, client_id: box.value}).then(function (r) {
1238
+ if (!r.ok) { listErrors($('cn-app-errors'), r.errors); return; }
1239
+ $('cn-app-done').textContent = box.value.trim() ? p.label + ' signs in with your app now.'
1240
+ : p.label + (p.shipped ? " is back to Wendell Research's app." : ' has no app until one is added.');
1241
+ loadProviders();
1242
+ });
1243
+ });
1244
+ line.appendChild(save);
1245
+ return row;
1246
+ }
1247
+
1248
+ if ($('tab-endpoints').classList.contains('active')) onShow.endpoints(); // the page opened on it
1249
+
1250
+ // ---- first run: no endpoint yet, no model, no collection (serve.first_run) -
1251
+ var first = $('first-run');
1252
+ function firstStep() {
1253
+ if (first.dataset.step === 'onboard') { $('onboard-open').click(); return; }
1254
+ if (first.dataset.step === 'collection') { show('collections'); $('c-new').click(); return; }
1255
+ show('endpoints'); openEndpoint('add');
1256
+ }
1257
+ if (first) {
1258
+ $('first-go').addEventListener('click', firstStep);
1259
+ if (first.dataset.step === 'endpoint') firstStep(); // straight into Add: an empty bench offers nothing else
1260
+ }
1261
+
1262
+ // ---- collections: what the bench gateway runs together (its own tab) -----
1263
+ var c = {list: $('c-list'), name: $('c-name'), members: $('c-members'), count: $('c-count'),
1264
+ shape: $('c-shape'), desc: $('c-desc'), tags: $('c-tags'), errors: $('c-errors'), done: $('c-done'),
1265
+ head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
1266
+ gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
1267
+ del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter')};
1268
+ var C = null, open = null; // the collection whose details are showing
1269
+ function explorersFor(n) {
1270
+ return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
1271
+ }
1272
+ function picked() {
1273
+ return Array.prototype.filter.call(c.members.querySelectorAll('input'), function (x) { return x.checked; })
1274
+ .map(function (x) { return x.value; });
1275
+ }
1276
+ function recount() {
1277
+ var n = picked().length, ex = explorersFor(n);
1278
+ var rows = c.members.querySelectorAll('label.check'), shown = c.members.querySelectorAll('label.check:not([hidden])');
1279
+ var greyed = c.members.querySelectorAll('label.check.blocked:not(.clash):not([hidden])').length;
1280
+ var clash = c.members.querySelectorAll('label.check.clash').length;
1281
+ var paid = c.members.querySelectorAll('label.check[data-paid="1"][hidden]').length;
1282
+ c.count.textContent = n + ' member' + (n === 1 ? '' : 's') + ' + ' + ex + ' explorer' + (ex === 1 ? '' : 's') +
1283
+ ' = ' + (n + ex) + ' calls per request' + (n && (n < 2 || n > 4) ? ' · two to four members is recommended' : '') +
1284
+ (shown.length < rows.length ? ' · showing ' + shown.length + ' of ' + rows.length + ' models' : '') +
1285
+ (paid ? ' (' + paid + ' paid hidden by the free tag)' : '') +
1286
+ (greyed ? ' · ' + greyed + ' greyed out by the tags (hover one for why)' : '') +
1287
+ (clash ? ' · ' + clash + ' ticked member' + (clash === 1 ? ' does' : 's do') + ' not fit the tags' : '');
1288
+ c.members.querySelectorAll('input').forEach(function (x) {
1289
+ x.disabled = !x.checked && (n >= C.settings.max_calls || !!x.dataset.blocked);
1290
+ });
1291
+ }
1292
+ // Tags that say what every member must do: why a model does not fit them
1293
+ // (none: it fits). From collections_view's facts; an unknown context is
1294
+ // let through, an unknown capability is not, as the gateway's vision seats.
1295
+ var CAP_TAGS = {vision: ['vision', 'take images'], tools: ['tools', 'call tools'], json: ['structured_output', 'give structured output']};
1296
+ function tagBlocks(alias) {
1297
+ var f = (C.facts || {})[alias], why = [];
1298
+ if (!f) return why;
1299
+ if (f.role === 'image') why.push('an image generator: it never answers a panel');
1300
+ tagList().forEach(function (t) {
1301
+ var cap = CAP_TAGS[t], roles = (C.shape_roles || {})[t];
1302
+ if (cap && !f.caps[cap[0]]) {
1303
+ why.push(f.caps[cap[0]] === false ? 'does not ' + cap[1]
1304
+ : 'the catalog does not say it can ' + cap[1] + '; tick it in its model popup if it does');
1305
+ }
1306
+ else if (t === 'long-context' && f.context && f.context < C.long_context) {
1307
+ why.push('holds ' + Math.round(f.context / 1000) + 'k tokens, under ' + Math.round(C.long_context / 1000) + 'k');
1308
+ }
1309
+ // general admits only the general role, but as a tag it means "for anything".
1310
+ else if (roles && t !== 'general' && roles.indexOf(f.role) === -1) {
1311
+ why.push('role ' + f.role + ': not for ' + t + ' tasks');
1312
+ }
1313
+ });
1314
+ return why;
1315
+ }
1316
+ // Greyed out, not hidden, so you see what the tags rule out; a ticked one
1317
+ // stays tickable (marked) so it can be taken off. `free` hides the paid ones.
1318
+ function applyTags() {
1319
+ var free = tagList().indexOf('free') !== -1;
1320
+ c.members.querySelectorAll('label.check').forEach(function (lab) {
1321
+ var box = lab.querySelector('input'), why = tagBlocks(box.value);
1322
+ var paid = free && ((C.facts || {})[box.value] || {}).price === 'paid';
1323
+ if (paid) why.push('paid, and the collection is tagged free');
1324
+ lab.classList.toggle('blocked', why.length > 0);
1325
+ lab.classList.toggle('clash', why.length > 0 && box.checked);
1326
+ lab.dataset.paid = paid ? '1' : '';
1327
+ if (why.length) box.dataset.blocked = '1'; else delete box.dataset.blocked;
1328
+ var served = lab.dataset.served;
1329
+ lab.title = (served && served !== box.value ? served : '') + (why.length ? (served && served !== box.value ? '\n' : '') + why.join('\n') : '');
1330
+ });
1331
+ filterMembers();
1332
+ }
1333
+ // The member search: the filter boxes' match (any text, case-insensitive),
1334
+ // over alias and served name. It only hides rows, and never a ticked one, so
1335
+ // what is picked is always in view and every count still includes it.
1336
+ function filterMembers() {
1337
+ var q = c.find.value.trim().toLowerCase();
1338
+ c.members.querySelectorAll('label.check').forEach(function (lab) {
1339
+ var box = lab.querySelector('input');
1340
+ lab.hidden = !box.checked && (!!lab.dataset.paid ||
1341
+ !!q && (box.value + ' ' + (lab.dataset.served || '')).toLowerCase().indexOf(q) === -1);
1342
+ });
1343
+ recount();
1344
+ }
1345
+ c.find.addEventListener('input', filterMembers);
1346
+ // Escape in a box with text clears it; only an empty box lets Escape close the popup.
1347
+ c.find.addEventListener('keydown', function (e) {
1348
+ if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
1349
+ });
1350
+ // Suggested tags under the field: a click adds or removes one, and the
1351
+ // field stays free for any other.
1352
+ function tagList() {
1353
+ return c.tags.value.split(',').map(function (t) { return t.trim().toLowerCase(); }).filter(Boolean);
1354
+ }
1355
+ function drawTagPicks() {
1356
+ var box = $('c-tag-picks'), have = tagList();
1357
+ box.textContent = '';
1358
+ (C.tag_suggestions || []).forEach(function (t) {
1359
+ var on = have.indexOf(t) !== -1;
1360
+ var b = el('button', {type: 'button', class: 'chip' + (on ? ' ok' : ''), 'aria-pressed': String(on)}, (on ? '✓ ' : '+ ') + t);
1361
+ b.addEventListener('click', function () {
1362
+ var now = tagList(), i = now.indexOf(t);
1363
+ if (i === -1) now.push(t); else now.splice(i, 1);
1364
+ c.tags.value = now.join(', '); drawTagPicks(); applyTags();
1365
+ });
1366
+ box.appendChild(b);
1367
+ });
1368
+ }
1369
+ c.tags.addEventListener('input', function () { drawTagPicks(); applyTags(); });
1370
+ function showMerger() { $('c-merger-row').hidden = c.reply.value !== 'merged'; }
1371
+ c.reply.addEventListener('change', showMerger);
1372
+ // A collection opens in a popup, as every other edit on the dashboard does:
1373
+ // the one clicked, or a new one from New collection.
1374
+ function closeForm() {
1375
+ if (c.form.open) c.form.close();
1376
+ open = null;
1377
+ }
1378
+ c.form.addEventListener('close', function () { open = null; });
1379
+ function fillForm(row) {
1380
+ c.errors.textContent = ''; c.done.textContent = ''; c.status.textContent = '';
1381
+ open = row ? row.name : null; c.del.hidden = !row;
1382
+ c.head.textContent = row ? 'bench/' + row.name : 'New collection';
1383
+ if (!c.form.open) c.form.showModal();
1384
+ c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
1385
+ c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
1386
+ drawTagPicks();
1387
+ c.members.textContent = ''; c.find.value = '';
1388
+ C.models.forEach(function (alias) {
1389
+ var box = el('input', {type: 'checkbox', value: alias});
1390
+ box.checked = !!row && row.members.indexOf(alias) !== -1;
1391
+ box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
1392
+ var served = (C.served || {})[alias] || '';
1393
+ var lab = el('label', {class: 'check', 'data-served': served});
1394
+ lab.appendChild(box); lab.appendChild(el('span', {class: 'mono'}, alias));
1395
+ c.members.appendChild(lab);
1396
+ });
1397
+ c.shape.textContent = '';
1398
+ c.shape.appendChild(el('option', {value: ''}, 'automatic'));
1399
+ C.shapes.forEach(function (s) {
1400
+ var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
1401
+ });
1402
+ c.reply.value = row ? row.reply : 'all';
1403
+ c.merger.textContent = '';
1404
+ c.merger.appendChild(el('option', {value: ''}, 'automatic'));
1405
+ C.models.forEach(function (alias) {
1406
+ var o = el('option', {value: alias}, alias); if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
1407
+ });
1408
+ showMerger(); applyTags();
1409
+ }
1410
+ function loadCollections(keep) {
1411
+ fetch('/api/collections').then(function (r) { return r.json(); }).then(function (res) {
1412
+ C = res; c.list.textContent = '';
1413
+ res.collections.forEach(function (row) {
1414
+ // The whole row opens the collection: no tight button pair on it.
1415
+ var box = el('button', {class: 'endpoint c-row', type: 'button', 'data-name': row.name,
1416
+ 'aria-label': 'Open bench/' + row.name});
1417
+ var left = el('div');
1418
+ var title = el('div', {class: 'what'});
1419
+ title.appendChild(el('span', {class: 'alias'}, 'bench/' + row.name + ' '));
1420
+ row.tags.forEach(function (t) { title.appendChild(el('span', {class: 'chip', style: 'margin-left:.3rem'}, t)); });
1421
+ left.appendChild(title);
1422
+ if (row.description) left.appendChild(el('div', {}, row.description));
1423
+ left.appendChild(el('div', {class: 'base'}, row.members.join(', ') + ' + ' + row.explorers + ' explorer' +
1424
+ (row.explorers === 1 ? '' : 's') + (row.shape ? ' (' + row.shape + ')' : '') +
1425
+ (row.reply === 'merged' ? ' · merged by ' + (row.merger || 'verdicts') : '')));
1426
+ var t = row.together;
1427
+ if (t) {
1428
+ var lim = Object.keys(t.limits || {}).filter(function (k) { return t.limits[k]; })
1429
+ .map(function (k) { return t.limits[k].toLocaleString() + ' ' + k; });
1430
+ left.appendChild(el('div', {class: 'base'}, 'Together: ' +
1431
+ (t.context_window ? 'prompts up to ' + Math.floor(t.context_window / 1000).toLocaleString() + 'k tokens' : 'context unknown') +
1432
+ (t.context_unknown.length ? ' (unknown for ' + t.context_unknown.join(', ') + ')' : '') +
1433
+ ' · all can: ' + (t.capabilities.join(', ') || 'nothing known in common') +
1434
+ ' · ' + t.connections.length + ' connection' + (t.connections.length === 1 ? '' : 's') +
1435
+ (lim.length ? ' · tightest ' + lim.join(', ') : '')));
1436
+ }
1437
+ box.appendChild(left);
1438
+ box.appendChild(el('span', {class: 'open-hint', 'aria-hidden': 'true'}, 'Open ›'));
1439
+ box.addEventListener('click', function () { fillForm(row); });
1440
+ c.list.appendChild(box);
1441
+ });
1442
+ if (!res.collections.length) c.list.appendChild(el('p', {class: 'empty'}, 'No collections yet.'));
1443
+ c.maxCalls.value = res.settings.max_calls; c.maxExp.value = res.settings.max_explorers;
1444
+ if (!keep) closeForm();
1445
+ });
1446
+ }
1447
+ loadCollections();
1448
+ $('c-new').addEventListener('click', function () { fillForm(null); c.name.focus(); });
1449
+ $('c-close').addEventListener('click', closeForm);
1450
+ // Done: say so in the popup, close it a moment later, and keep the message
1451
+ // above the list, as onboarding and retiring do.
1452
+ function finished(message) {
1453
+ c.done.textContent = message; c.status.textContent = message; loadCollections(true);
1454
+ setTimeout(closeForm, 1200);
1455
+ }
1456
+ c.del.addEventListener('click', function () {
1457
+ var name = open;
1458
+ if (!name) return;
1459
+ ask('Delete bench/' + name + '? Callers asking for it will get an error.', 'Delete', true).then(function (yes) {
1460
+ if (!yes) return;
1461
+ post('/api/collection/remove', {name: name}).then(function (r) {
1462
+ if (!r.ok) { listErrors(c.errors, r.errors); return; }
1463
+ finished('Deleted bench/' + name + '.');
1464
+ });
1465
+ });
1466
+ });
1467
+ $('c-save').addEventListener('click', function () {
1468
+ c.errors.textContent = ''; c.done.textContent = '';
1469
+ var name = c.name.value.trim().replace(/^bench\//, '');
1470
+ post('/api/collection', {name: name, members: picked(), shape: c.shape.value, description: c.desc.value,
1471
+ tags: c.tags.value.split(','), reply: c.reply.value,
1472
+ merger: c.merger.value}).then(function (r) {
1473
+ if (!r.ok) { listErrors(c.errors, r.errors); return; }
1474
+ finished('Saved bench/' + name + '.');
1475
+ }).catch(function (err) { c.errors.textContent = String(err); });
1476
+ });
1477
+ $('g-save').addEventListener('click', function () {
1478
+ c.gErrors.textContent = ''; c.gDone.textContent = '';
1479
+ post('/api/gateway-settings', {max_calls: parseInt(c.maxCalls.value, 10),
1480
+ max_explorers: parseInt(c.maxExp.value, 10)}).then(function (r) {
1481
+ if (!r.ok) { listErrors(c.gErrors, r.errors); return; }
1482
+ c.gDone.textContent = 'Settings saved.'; loadCollections(true);
1483
+ }).catch(function (err) { c.gErrors.textContent = String(err); });
1484
+ });
1485
+
1486
+ // ---- a click on the backdrop closes a popup, as its Close button does ----
1487
+ // Pressed AND released outside: a text selection dragged out of an input
1488
+ // must not throw the form away.
1489
+ document.querySelectorAll('dialog').forEach(function (d) {
1490
+ var downOutside = false;
1491
+ function outside(e) {
1492
+ var r = d.getBoundingClientRect();
1493
+ return e.target === d && (e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom);
1494
+ }
1495
+ d.addEventListener('pointerdown', function (e) { downOutside = outside(e); });
1496
+ d.addEventListener('click', function (e) { if (downOutside && outside(e)) d.close(); downOutside = false; });
1497
+ });
1498
+
1499
+ // ---- verdict on one gateway seat --------------------------------------
1500
+ var vd = $('verdict-dialog'), vcall = null;
1501
+ function lines(id) { return $(id).value.split('\n').map(function (s) { return s.trim(); }).filter(Boolean); }
1502
+ document.addEventListener('click', function (e) {
1503
+ var b = e.target.closest('button.seat');
1504
+ if (!b) return;
1505
+ vcall = b.dataset.call;
1506
+ $('v-errors').textContent = ''; $('v-done').textContent = '';
1507
+ fetch('/api/gateway-seat?call=' + encodeURIComponent(vcall)).then(function (r) { return r.json(); }).then(function (s) {
1508
+ if (!s.ok) { listErrors($('v-errors'), s.errors); vd.showModal(); return; }
1509
+ $('v-title').textContent = s.alias + ' · ' + s.seat;
1510
+ $('v-meta').textContent = s.model + ' · ' + s.status + ' in ' + s.seconds + 's · ' + s.at.slice(0, 16).replace('T', ' ') + ' · ' + s.call;
1511
+ $('v-prompt').textContent = s.prompt;
1512
+ $('v-answer').textContent = s.answer || (s.error ? 'Error: ' + s.error : 'Empty reply: usually a token budget, not incapacity.');
1513
+ document.querySelectorAll('input[name=v-useful]').forEach(function (r) {
1514
+ r.checked = s.useful !== null && r.value === (s.useful ? 'yes' : 'no');
1515
+ });
1516
+ $('v-note').value = s.note; $('v-real').value = s.real.join('\n'); $('v-false').value = s.false.join('\n');
1517
+ vd.showModal();
1518
+ });
1519
+ });
1520
+ $('v-close').addEventListener('click', function () { vd.close(); });
1521
+ $('v-save').addEventListener('click', function () {
1522
+ var pick = document.querySelector('input[name=v-useful]:checked');
1523
+ $('v-errors').textContent = ''; $('v-done').textContent = '';
1524
+ post('/api/verdict', {call: vcall, useful: pick ? pick.value === 'yes' : null, note: $('v-note').value,
1525
+ real: lines('v-real'), false: lines('v-false')}).then(function (r) {
1526
+ if (!r.ok) { listErrors($('v-errors'), r.errors); return; }
1527
+ $('v-done').textContent = 'Recorded: ' + r.changed.join(', ') + '. Reloading…';
1528
+ setTimeout(function () { location.reload(); }, 700);
1529
+ }).catch(function (err) { $('v-errors').textContent = String(err); });
1530
+ });
1531
+
1532
+ // ---- Benchmarks charts: Observable Plot, data from serve.charts_data ------
1533
+ // A function, not run once: a theme switched on Settings redraws them in its colours.
1534
+ function drawCharts() {
1535
+ var holder = $('charts-data');
1536
+ if (!holder) return;
1537
+ var C = JSON.parse(holder.textContent);
1538
+ var css = getComputedStyle(document.documentElement);
1539
+ function tok(name, fallback) { return (css.getPropertyValue(name) || '').trim() || fallback; }
1540
+ // `good` is the theme's OK colour, not its brand: in a theme whose brand is
1541
+ // red (Hardwood), a found bug drawn in the brand colour would read as bad.
1542
+ var ink = tok('--ink', '#24331f'), good = tok('--ok', '#327a16'), err = tok('--err', '#a12b2b'),
1543
+ warn = tok('--warn', '#7a4d00'),
1544
+ signal = tok('--signal', '#fae66c'), sky = tok('--sky', '#69bdeb'), muted = tok('--muted', '#6b685a'),
1545
+ line = tok('--line', '#e4dfd2'), halo = tok('--paper-2', '#ffffff');
1546
+ var style = {background: 'transparent', color: ink, fontSize: '13px', overflow: 'visible'};
1547
+ // Tabs other than the open one have no width yet, so size from the column
1548
+ // the pages share. It was the whole page, which made every chart wider than
1549
+ // its column beside the sidebar, scaled down, with smaller text.
1550
+ function width() {
1551
+ var col = document.querySelector('.panes') || document.querySelector('main');
1552
+ return Math.max(520, (col.clientWidth || 900) - 50);
1553
+ }
1554
+ function put(id, make) {
1555
+ var box = $(id);
1556
+ if (!box) return;
1557
+ if (!window.Plot) {
1558
+ box.className += ' muted';
1559
+ box.textContent = 'Charts need Observable Plot from cdn.jsdelivr.net, which did not load; the table below has the same numbers.';
1560
+ return;
1561
+ }
1562
+ box.textContent = ''; // a redraw replaces the chart
1563
+ try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
1564
+ }
1565
+ // Pixel offsets that keep scatter labels apart: each label, in order, moves
1566
+ // up or down until it clears the ones already placed.
1567
+ function spread(points, xpx, ypx) {
1568
+ var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
1569
+ return points.map(function (d) {
1570
+ var cx = xpx(d), half = d.model.length * 2.9;
1571
+ for (var k = 0; k < tries.length; k++) {
1572
+ var cy = ypx(d) + tries[k];
1573
+ if (!boxes.some(function (b) { return Math.abs(b.x - cx) < b.half + half + 4 && Math.abs(b.y - cy) < 12; })) {
1574
+ boxes.push({x: cx, y: cy, half: half});
1575
+ return tries[k];
1576
+ }
1577
+ }
1578
+ return tries[0];
1579
+ });
1580
+ }
1581
+ function rows(n) { return 20 * n + 80; }
1582
+ // Who to trust: finds real bugs (right) vs makes things up (up = less).
1583
+ put('chart-overview', function (w) {
1584
+ if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
1585
+ var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
1586
+ var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
1587
+ var pxW = w - 110, pxH = 370;
1588
+ var pts = C.overview.slice().sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
1589
+ var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
1590
+ function (d) { return d.wrong / worst * pxH; });
1591
+ var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; });
1592
+ return Plot.plot({
1593
+ width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
1594
+ x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
1595
+ y: {label: '↑ Makes fewer things up', domain: [0, worst], reverse: true, tickFormat: '.0%'},
1596
+ marks: [
1597
+ // The zone a reviewer should be in: outlined, so it reads on a dark ground too.
1598
+ Plot.rect([{x1: x0 + (1 - x0) / 2, x2: 1, y1: 0, y2: worst / 4}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
1599
+ fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
1600
+ Plot.dot(C.overview, {x: 'recall', y: 'wrong', r: 6, fill: good, stroke: ink, strokeWidth: 0.5}),
1601
+ // A halo in the panel's colour keeps each name legible over dots, grid lines and the zone.
1602
+ Plot.text(labels, {x: 'recall', y: 'ty', text: 'model', fontSize: 11, fill: ink,
1603
+ stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
1604
+ Plot.tip(C.overview, Plot.pointer({x: 'recall', y: 'wrong',
1605
+ title: function (d) { return d.model + '\nfinds ' + d.found + ' real findings\nconfidently wrong in ' + d.wrongN + ' answers'; }}))
1606
+ ]
1607
+ });
1608
+ });
1609
+
1610
+ // Bug hunt: right of zero the bug tasks (found, wrong line, missed); left of
1611
+ // it the same files clean, where any bar is a false alarm.
1612
+ put('chart-hunt', function (w) {
1613
+ var segs = [], score = {}, tasks = {};
1614
+ C.hunt.forEach(function (h) {
1615
+ var bugs = h.found + h['wrong line'] + h.missed, clean = h.clean + h['false alarm'];
1616
+ tasks[h.model] = bugs + clean;
1617
+ var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
1618
+ var fa = clean ? h['false alarm'] / clean : 0;
1619
+ score[h.model] = f - fa;
1620
+ segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
1621
+ segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
1622
+ segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
1623
+ segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
1624
+ });
1625
+ var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
1626
+ return Plot.plot({
1627
+ width: w, height: rows(models.length), style: style, marginLeft: 250,
1628
+ x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
1629
+ tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
1630
+ y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
1631
+ color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
1632
+ marks: [
1633
+ Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
1634
+ title: function (d) { return d.model + ': ' + d.part + ', ' + d.n; }}),
1635
+ Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
1636
+ ]
1637
+ });
1638
+ });
1639
+
1640
+ // Real work: the share of judged offloads that helped, sample size beside the name.
1641
+ put('chart-work', function (w) {
1642
+ var R = C.work.slice().sort(function (a, b) { return b.useful / b.judged - a.useful / a.judged || b.judged - a.judged; });
1643
+ if (!R.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
1644
+ var by = {};
1645
+ R.forEach(function (r) { by[r.model] = r; });
1646
+ function share(r) { return r.useful / r.judged; }
1647
+ return Plot.plot({
1648
+ width: w, height: rows(R.length), style: style, marginLeft: 250, marginRight: 50,
1649
+ x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
1650
+ y: {domain: R.map(function (r) { return r.model; }), label: null,
1651
+ tickFormat: function (m) { return m + ' (' + by[m].judged + ')'; }},
1652
+ marks: [
1653
+ Plot.barX(R, {x: share, y: 'model', fill: function (r) { return share(r) >= 0.5 ? good : err; },
1654
+ title: function (r) { return r.model + ': helped ' + r.useful + ' of ' + r.judged + ' judged (' + r.offloads + ' offloads)'
1655
+ + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : ''); }}),
1656
+ Plot.text(R, {x: share, y: 'model', text: function (r) { return r.useful + '/' + r.judged; }, dx: 18, fill: ink}),
1657
+ Plot.ruleX([0], {stroke: ink})
1658
+ ]
1659
+ });
1660
+ });
1661
+
1662
+ // Did it answer: each probe of the last 14 days, by what came back.
1663
+ put('chart-reliability', function (w) {
1664
+ if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
1665
+ var total = {}, ok = {};
1666
+ C.reliability.forEach(function (d) {
1667
+ total[d.model] = (total[d.model] || 0) + d.count;
1668
+ if (d.outcome === 'answered') ok[d.model] = d.count;
1669
+ });
1670
+ var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
1671
+ var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
1672
+ entitlement: '#c0563f', other: muted, error: ink};
1673
+ var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
1674
+ return Plot.plot({
1675
+ width: w, height: rows(models.length), style: style, marginLeft: 250,
1676
+ x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
1677
+ y: {domain: models, label: null},
1678
+ color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
1679
+ marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
1680
+ title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
1681
+ });
1682
+ });
1683
+
1684
+ // Speed vs quality: median seconds on our own tasks against findings located.
1685
+ put('chart-speed', function (w) {
1686
+ if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
1687
+ var s = C.speed.map(function (d) { return d.seconds; });
1688
+ var lo = Math.max(1, Math.min.apply(null, s) * 0.8), hi = Math.max.apply(null, s) * 1.25;
1689
+ var y0 = Math.max(0, Math.floor((Math.min.apply(null, C.speed.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
1690
+ var pxW = w - 110, pxH = 340;
1691
+ var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
1692
+ var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
1693
+ function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
1694
+ var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; });
1695
+ return Plot.plot({
1696
+ width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
1697
+ x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
1698
+ y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
1699
+ marks: [
1700
+ Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
1701
+ Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink}),
1702
+ Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
1703
+ title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
1704
+ ]
1705
+ });
1706
+ });
1707
+
1708
+ // Our code: one square per task and model, grouped by kind of task.
1709
+ put('chart-code', function (w) {
1710
+ var fam = {}, first = {};
1711
+ C.heat.forEach(function (d) { fam[d.task] = d.family; });
1712
+ var tasks = Object.keys(fam).sort(function (a, b) {
1713
+ return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
1714
+ });
1715
+ tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
1716
+ var label = {};
1717
+ Object.keys(first).forEach(function (f) { label[first[f]] = f; });
1718
+ // By total found, so a model that played three tasks does not top one that played forty.
1719
+ var sum = {};
1720
+ C.heat.forEach(function (d) { sum[d.model] = (sum[d.model] || 0) + (d.rate || 0); });
1721
+ var models = Object.keys(sum).sort(function (a, b) { return sum[b] - sum[a]; });
1722
+ function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
1723
+ return Plot.plot({
1724
+ width: w, height: 22 * models.length + 70, style: style, marginLeft: 230, marginTop: 30, padding: 0.08,
1725
+ x: {domain: tasks, axis: 'top', label: null, tickSize: 0, tickFormat: function (t) { return label[t] || ''; }},
1726
+ y: {domain: models, label: null, tickSize: 0},
1727
+ color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
1728
+ marks: [
1729
+ Plot.cell(C.heat.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
1730
+ Plot.cell(C.heat.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
1731
+ Plot.dot(C.heat.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
1732
+ Plot.ruleX(Object.keys(label), {x: function (t) { return t; }, stroke: ink, strokeOpacity: 0.25, dx: -2})
1733
+ ]
1734
+ });
1735
+ });
1736
+
1737
+ // Confidently wrong: stacked by kind, worst at the top.
1738
+ put('chart-wrong', function (w) {
1739
+ var T = C.wrongTotals;
1740
+ var models = Object.keys(T).sort(function (a, b) { return T[b].rate - T[a].rate || T[b].answers - T[a].answers; });
1741
+ return Plot.plot({
1742
+ width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
1743
+ x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
1744
+ // The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
1745
+ y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
1746
+ color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
1747
+ marks: [
1748
+ Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
1749
+ Plot.ruleX([0], {stroke: ink})
1750
+ ]
1751
+ });
1752
+ });
1753
+
1754
+ }
1755
+ drawCharts();
1756
+ })();