@wendellresearch-ai/benchwarmers 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -1
- package/BENCHMARKS.md +1 -1
- package/README.md +2 -0
- package/USAGE.md +4 -6
- package/VERSION +1 -1
- package/package.json +1 -1
- package/scripts/check.py +1 -0
- package/scripts/clients_doc.py +98 -0
- package/scripts/provider_tags.py +19 -3
- package/scripts/test_no_owui.py +8 -0
- package/serve.py +130 -248
- package/ui/bench.css +37 -11
- package/ui/bench.js +355 -118
package/ui/bench.js
CHANGED
|
@@ -213,7 +213,7 @@
|
|
|
213
213
|
var rendered = document.querySelector('.tab:not([data-lazy])');
|
|
214
214
|
Promise.resolve().then(function () { if (rendered) setUp(rendered.id.slice(4)); });
|
|
215
215
|
var wanted = store('bench.tab'); // else the one rendered: storage blocked or cleared, the cookie kept
|
|
216
|
-
show(wanted && $('tab-' + wanted) ? wanted : rendered ? rendered.id.slice(4) : '
|
|
216
|
+
show(wanted && $('tab-' + wanted) ? wanted : rendered ? rendered.id.slice(4) : 'suite');
|
|
217
217
|
|
|
218
218
|
// ---- sortable tables -------------------------------------------------
|
|
219
219
|
function key(cell) {
|
|
@@ -253,13 +253,21 @@
|
|
|
253
253
|
var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
|
|
254
254
|
var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
|
|
255
255
|
scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
|
|
256
|
-
|
|
256
|
+
// A group's name (data-conn: Models' connection) counts as text of each of its rows.
|
|
257
|
+
var g = r.closest('[data-conn]'), text = r.textContent + (g ? ' ' + g.dataset.conn : '');
|
|
258
|
+
r.hidden = (q && text.toLowerCase().indexOf(q) === -1)
|
|
257
259
|
|| facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
|
|
258
260
|
});
|
|
259
261
|
// A match inside a folded list (an endpoint's models) opens it.
|
|
260
262
|
if (q) scope.querySelectorAll('details.models').forEach(function (d) {
|
|
261
263
|
if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
|
|
262
264
|
});
|
|
265
|
+
// A group with no row left is hidden; one with a match opens.
|
|
266
|
+
scope.querySelectorAll('[data-conn]').forEach(function (g) {
|
|
267
|
+
var any = !!g.querySelector('tbody tr:not([hidden])');
|
|
268
|
+
g.hidden = !!q && !any;
|
|
269
|
+
if (q && any && g.tagName === 'DETAILS') g.open = true;
|
|
270
|
+
});
|
|
263
271
|
}
|
|
264
272
|
function scopeOf(c) {
|
|
265
273
|
var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
|
|
@@ -268,7 +276,75 @@
|
|
|
268
276
|
document.addEventListener('input', function (ev) { if (ev.target.matches('input[data-filter]')) applyFilter(scopeOf(ev.target)); });
|
|
269
277
|
document.addEventListener('change', function (ev) { if (ev.target.matches('select[data-facet]')) applyFilter(scopeOf(ev.target)); });
|
|
270
278
|
|
|
271
|
-
// ----
|
|
279
|
+
// ---- Expand all / Collapse all: every fold of the page's groups (Models' connections).
|
|
280
|
+
// One labelled toggle, as accordions have it: Collapse all while every one is open,
|
|
281
|
+
// else Expand all. Kept right after any fold opens or shuts, by hand, by it or by a live swap.
|
|
282
|
+
function foldButtons(tab) {
|
|
283
|
+
var folds = tab.querySelectorAll('details.model-card'), open = 0;
|
|
284
|
+
folds.forEach(function (d) { if (d.open) open++; });
|
|
285
|
+
var all = open === folds.length;
|
|
286
|
+
tab.querySelectorAll('[data-fold-all]').forEach(function (b) {
|
|
287
|
+
b.dataset.foldAll = all ? 'close' : 'open';
|
|
288
|
+
b.setAttribute('aria-expanded', String(all));
|
|
289
|
+
b.lastChild.textContent = all ? 'Collapse all' : 'Expand all';
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
document.addEventListener('click', function (e) {
|
|
293
|
+
var b = e.target.closest('[data-fold-all]');
|
|
294
|
+
if (!b) return;
|
|
295
|
+
var tab = b.closest('.tab');
|
|
296
|
+
tab.querySelectorAll('details.model-card').forEach(function (d) { d.open = b.dataset.foldAll === 'open'; });
|
|
297
|
+
foldButtons(tab);
|
|
298
|
+
});
|
|
299
|
+
document.addEventListener('toggle', function (e) { // toggle does not bubble: caught on its way down
|
|
300
|
+
if (e.target.matches && e.target.matches('details.model-card')) foldButtons(e.target.closest('.tab'));
|
|
301
|
+
}, true);
|
|
302
|
+
|
|
303
|
+
// ---- Logs: Export as text (every log in one .txt, to read) and CSV (one log, its
|
|
304
|
+
// own columns, for a spreadsheet). Both as the page shows them, the filter applied.
|
|
305
|
+
function stampNow() { // local time, as the page shows
|
|
306
|
+
var now = new Date(), two = function (n) { return String(n).padStart(2, '0'); };
|
|
307
|
+
return now.getFullYear() + '-' + two(now.getMonth() + 1) + '-' + two(now.getDate()) + '-' + two(now.getHours()) + two(now.getMinutes());
|
|
308
|
+
}
|
|
309
|
+
// The file itself: Share's saveFile(blob, name) and csv(rows) (RFC 4180, with a BOM).
|
|
310
|
+
document.addEventListener('click', function (e) {
|
|
311
|
+
var b = e.target.closest('[data-csv]');
|
|
312
|
+
if (!b) return;
|
|
313
|
+
e.preventDefault(); // in the summary: no fold
|
|
314
|
+
var d = b.closest('details'), rows = [];
|
|
315
|
+
var cells = function (tr) {
|
|
316
|
+
return Array.prototype.map.call(tr.children, function (c) { return c.textContent.trim().replace(/\s+/g, ' '); });
|
|
317
|
+
};
|
|
318
|
+
var head = d.querySelector('thead tr');
|
|
319
|
+
if (head) {
|
|
320
|
+
rows.push(cells(head));
|
|
321
|
+
d.querySelectorAll('tbody tr').forEach(function (r) { if (!r.hidden) rows.push(cells(r)); });
|
|
322
|
+
} else {
|
|
323
|
+
rows.push(['Line']);
|
|
324
|
+
d.querySelectorAll('.log-line').forEach(function (r) { if (!r.hidden) rows.push([r.textContent.trim()]); });
|
|
325
|
+
}
|
|
326
|
+
var slug = d.dataset.log.toLowerCase().replace(/[^a-z0-9]+/g, '-');
|
|
327
|
+
saveFile(new Blob([csv(rows)], {type: 'text/csv;charset=utf-8'}), 'benchwarmers-' + slug + '-' + stampNow() + '.csv');
|
|
328
|
+
});
|
|
329
|
+
on('click', 'logs-export', function () {
|
|
330
|
+
var tab = $('tab-logs'), out = ['Benchwarmers logs, exported ' + new Date().toString()];
|
|
331
|
+
var q = (tab.querySelector('input[data-filter]') || {}).value;
|
|
332
|
+
if (q) out.push('Filtered by: ' + q);
|
|
333
|
+
tab.querySelectorAll('details.panel').forEach(function (d) {
|
|
334
|
+
out.push('', '== ' + d.dataset.log + ' ==');
|
|
335
|
+
var cells = function (tr) {
|
|
336
|
+
return Array.prototype.map.call(tr.children, function (c) { return c.textContent.trim().replace(/\s+/g, ' '); }).join('\t');
|
|
337
|
+
};
|
|
338
|
+
var head = d.querySelector('thead tr');
|
|
339
|
+
if (head) out.push(cells(head));
|
|
340
|
+
d.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
|
|
341
|
+
if (!r.hidden) out.push(r.matches('tr') ? cells(r) : r.textContent.trim());
|
|
342
|
+
});
|
|
343
|
+
});
|
|
344
|
+
saveFile(new Blob([out.join('\n') + '\n'], {type: 'text/plain;charset=utf-8'}), 'benchwarmers-logs-' + stampNow() + '.txt');
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
// ---- a link to another page ----
|
|
272
348
|
document.addEventListener('click', function (e) {
|
|
273
349
|
var b = e.target.closest('[data-go]');
|
|
274
350
|
if (b) { show(b.dataset.go); scrollTo(0, 0); }
|
|
@@ -759,8 +835,8 @@
|
|
|
759
835
|
});
|
|
760
836
|
|
|
761
837
|
// ---- after an action: the part it changed, not the whole page ----------
|
|
762
|
-
// Action needed's page
|
|
763
|
-
//
|
|
838
|
+
// Action needed's page and its count in the sidebar, from the server
|
|
839
|
+
// (GET /api/needs, a few ms): a page reload
|
|
764
840
|
// renders every tab, seconds.
|
|
765
841
|
var needsAsked = 0;
|
|
766
842
|
function refreshNeeds(live) {
|
|
@@ -775,10 +851,6 @@
|
|
|
775
851
|
count.textContent = n.count;
|
|
776
852
|
tab.setAttribute('aria-label', tab.firstChild.textContent + ', ' + n.count + ' open');
|
|
777
853
|
}
|
|
778
|
-
var line = document.querySelector('#tab-glance > .a-line');
|
|
779
|
-
if (line) line.remove();
|
|
780
|
-
// Not into a page not fetched yet: it says Loading… while empty, and comes with its line.
|
|
781
|
-
if (n.line && $('tab-glance').firstChild) $('tab-glance').insertAdjacentHTML('afterbegin', n.line);
|
|
782
854
|
}).catch(function () { location.reload(); }); // could not ask: the old way
|
|
783
855
|
}
|
|
784
856
|
// A retired model's rows grey out and hide behind Show retired, as the
|
|
@@ -1880,7 +1952,7 @@
|
|
|
1880
1952
|
pick: $('g-pick'), pickConf: $('g-pick-conf'),
|
|
1881
1953
|
gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
|
|
1882
1954
|
del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
|
|
1883
|
-
seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
|
|
1955
|
+
seats: $('c-seats'), prov: $('c-member-provider'), show: $('c-member-show'), adHoc: $('c-ad-hoc'),
|
|
1884
1956
|
ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
|
|
1885
1957
|
return loadCollections();
|
|
1886
1958
|
};
|
|
@@ -1994,7 +2066,7 @@
|
|
|
1994
2066
|
// the seats above the list always show what is picked, each with its ×, and
|
|
1995
2067
|
// every count still includes it. A row goes when all its chips do.
|
|
1996
2068
|
function filterMembers() {
|
|
1997
|
-
var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
|
|
2069
|
+
var q = c.find.value.trim().toLowerCase(), prov = c.prov.value, show = c.show.value;
|
|
1998
2070
|
c.members.querySelectorAll('.m-row').forEach(function (row) {
|
|
1999
2071
|
var named = !q || row.dataset.name.toLowerCase().indexOf(q) !== -1, any = false;
|
|
2000
2072
|
row.querySelectorAll('label.check').forEach(function (lab) {
|
|
@@ -2002,7 +2074,8 @@
|
|
|
2002
2074
|
var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
|
|
2003
2075
|
lab.textContent).toLowerCase();
|
|
2004
2076
|
lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
|
|
2005
|
-
|| !!prov && (fact(box.value).provider || '') !== prov
|
|
2077
|
+
|| !!prov && (fact(box.value).provider || '') !== prov
|
|
2078
|
+
|| show === 'ticked' && !box.checked || show === 'fits' && !!box.dataset.blocked;
|
|
2006
2079
|
any = any || !lab.hidden;
|
|
2007
2080
|
});
|
|
2008
2081
|
row.hidden = !any;
|
|
@@ -2011,6 +2084,23 @@
|
|
|
2011
2084
|
}
|
|
2012
2085
|
on('input', 'c-member-filter', filterMembers);
|
|
2013
2086
|
on('change', 'c-member-provider', filterMembers);
|
|
2087
|
+
on('change', 'c-member-show', filterMembers);
|
|
2088
|
+
// Refresh model details: the models and their facts fetched again, the list drawn
|
|
2089
|
+
// anew with the same ticks, search and choices. The model popup's directory too.
|
|
2090
|
+
function refreshMembers() {
|
|
2091
|
+
var chosen = picked(), out = $('c-refresh-out'), btn = $('c-refresh');
|
|
2092
|
+
btn.disabled = true; out.textContent = 'Refreshing…';
|
|
2093
|
+
return Promise.all([get('/api/collections'), directory()]).then(function (res) {
|
|
2094
|
+
C = res[0];
|
|
2095
|
+
drawMembers(chosen);
|
|
2096
|
+
applyTags();
|
|
2097
|
+
out.textContent = 'Up to date.';
|
|
2098
|
+
}).catch(function (err) { out.textContent = String(err); })
|
|
2099
|
+
.then(function () { btn.disabled = false; });
|
|
2100
|
+
}
|
|
2101
|
+
on('click', 'c-refresh', refreshMembers);
|
|
2102
|
+
// A model opened from the list may have been changed in its popup: what it now is, on close.
|
|
2103
|
+
md.addEventListener('close', function () { if (c.form && c.form.open) refreshMembers(); });
|
|
2014
2104
|
// Escape in a box with text clears it; only an empty box lets Escape close the popup.
|
|
2015
2105
|
on('keydown', 'c-member-filter', function (e) {
|
|
2016
2106
|
if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
|
|
@@ -2059,13 +2149,37 @@
|
|
|
2059
2149
|
c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
|
|
2060
2150
|
c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
|
|
2061
2151
|
drawTagPicks();
|
|
2062
|
-
c.
|
|
2152
|
+
c.find.value = ''; c.show.value = ''; c.prov.value = '';
|
|
2153
|
+
$('c-refresh-out').textContent = '';
|
|
2154
|
+
drawMembers(row ? row.members : []);
|
|
2155
|
+
c.shape.textContent = '';
|
|
2156
|
+
c.shape.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2157
|
+
C.shapes.forEach(function (s) {
|
|
2158
|
+
var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
|
|
2159
|
+
});
|
|
2160
|
+
c.reply.value = row ? row.reply : 'all';
|
|
2161
|
+
c.merger.textContent = '';
|
|
2162
|
+
c.merger.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2163
|
+
C.models.forEach(function (alias) {
|
|
2164
|
+
var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
|
|
2165
|
+
if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
|
|
2166
|
+
});
|
|
2167
|
+
showMerger(); applyTags();
|
|
2168
|
+
}
|
|
2169
|
+
// The member list: a table to scan down, one row per model (named by its shortest
|
|
2170
|
+
// alias; the name opens its popup, as everywhere), its context and what it can do
|
|
2171
|
+
// in columns of their own, then a chip per provider it is on, each ticking that
|
|
2172
|
+
// alias. A chip whose window differs from the row's says its own. `chosen`: ticked.
|
|
2173
|
+
function drawMembers(chosen) {
|
|
2174
|
+
var keepProv = c.prov.value;
|
|
2175
|
+
c.members.textContent = '';
|
|
2063
2176
|
c.prov.length = 1; // "Every provider", then each one a model is on
|
|
2064
2177
|
Array.from(new Set(C.models.map(function (a) { return fact(a).provider; }).filter(Boolean))).sort()
|
|
2065
2178
|
.forEach(function (p) { c.prov.appendChild(el('option', {value: p}, p)); });
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2179
|
+
c.prov.value = keepProv;
|
|
2180
|
+
var head = el('div', {class: 'm-head', 'aria-hidden': 'true'});
|
|
2181
|
+
['Model', 'Context', 'Can', 'On, tick one'].forEach(function (h) { head.appendChild(el('span', null, h)); });
|
|
2182
|
+
c.members.appendChild(head);
|
|
2069
2183
|
var groups = {};
|
|
2070
2184
|
C.models.forEach(function (a) { var f = fact(a).family || a; (groups[f] = groups[f] || []).push(a); });
|
|
2071
2185
|
Object.keys(groups).map(function (f) {
|
|
@@ -2073,18 +2187,25 @@
|
|
|
2073
2187
|
return {family: f, name: routes[0], routes: routes};
|
|
2074
2188
|
}).sort(function (x, y) { return x.name.toLowerCase() < y.name.toLowerCase() ? -1 : 1; }).forEach(function (g) {
|
|
2075
2189
|
var first = fact(g.name), caps = first.caps || {};
|
|
2076
|
-
var can = ['vision', 'tools'].filter(function (k) { return caps[k]; });
|
|
2077
2190
|
var line = el('div', {class: 'm-row', role: 'group', 'aria-label': g.name, 'data-family': g.family, 'data-name': g.name});
|
|
2078
2191
|
var name = el('div', {class: 'm-name'});
|
|
2079
|
-
name.
|
|
2080
|
-
|
|
2081
|
-
|
|
2192
|
+
// The same link as every model name elsewhere (a.model > .alias: the highlight on hover).
|
|
2193
|
+
var link = el('a', {href: '#', class: 'model', 'data-alias': g.name, title: 'Open ' + g.name});
|
|
2194
|
+
link.appendChild(el('span', {class: 'alias'}, g.name));
|
|
2195
|
+
name.appendChild(link);
|
|
2082
2196
|
line.appendChild(name);
|
|
2197
|
+
line.appendChild(el('span', {class: 'm-ctx'}, first.context ? tokens(first.context) : '—'));
|
|
2198
|
+
var can = el('span', {class: 'm-can'});
|
|
2199
|
+
[['vision', 'vision'], ['tools', 'tools'], ['reasoning', 'reasons']].forEach(function (k) {
|
|
2200
|
+
if (caps[k[0]]) can.appendChild(el('span', {class: 'chip'}, k[1]));
|
|
2201
|
+
});
|
|
2202
|
+
if (first.price === 'free') can.appendChild(el('span', {class: 'chip ok'}, 'free'));
|
|
2203
|
+
line.appendChild(can);
|
|
2083
2204
|
var chips = el('div', {class: 'm-routes'});
|
|
2084
2205
|
g.routes.forEach(function (alias) {
|
|
2085
2206
|
var f = fact(alias);
|
|
2086
2207
|
var box = el('input', {type: 'checkbox', value: alias, 'aria-label': alias + ' on ' + (f.provider || 'its endpoint')});
|
|
2087
|
-
box.checked =
|
|
2208
|
+
box.checked = chosen.indexOf(alias) !== -1;
|
|
2088
2209
|
box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
|
|
2089
2210
|
var lab = el('label', {class: 'check', 'data-served': (C.served || {})[alias] || ''});
|
|
2090
2211
|
lab.appendChild(box);
|
|
@@ -2095,19 +2216,6 @@
|
|
|
2095
2216
|
line.appendChild(chips);
|
|
2096
2217
|
c.members.appendChild(line);
|
|
2097
2218
|
});
|
|
2098
|
-
c.shape.textContent = '';
|
|
2099
|
-
c.shape.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2100
|
-
C.shapes.forEach(function (s) {
|
|
2101
|
-
var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
|
|
2102
|
-
});
|
|
2103
|
-
c.reply.value = row ? row.reply : 'all';
|
|
2104
|
-
c.merger.textContent = '';
|
|
2105
|
-
c.merger.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2106
|
-
C.models.forEach(function (alias) {
|
|
2107
|
-
var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
|
|
2108
|
-
if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
|
|
2109
|
-
});
|
|
2110
|
-
showMerger(); applyTags();
|
|
2111
2219
|
}
|
|
2112
2220
|
// What a collection's members can all do, in the words the tag checks use.
|
|
2113
2221
|
var CAN = {reasoning: 'reason'};
|
|
@@ -2660,7 +2768,8 @@
|
|
|
2660
2768
|
try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
|
|
2661
2769
|
}
|
|
2662
2770
|
// Pixel offsets that keep scatter labels apart: each label, in order, moves
|
|
2663
|
-
// up or down until it clears the ones already placed
|
|
2771
|
+
// up or down until it clears the ones already placed; null when no place is
|
|
2772
|
+
// free (that one is left to its hover, not drawn over another).
|
|
2664
2773
|
function spread(points, xpx, ypx) {
|
|
2665
2774
|
var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
|
|
2666
2775
|
return points.map(function (d) {
|
|
@@ -2672,7 +2781,7 @@
|
|
|
2672
2781
|
return tries[k];
|
|
2673
2782
|
}
|
|
2674
2783
|
}
|
|
2675
|
-
return
|
|
2784
|
+
return null;
|
|
2676
2785
|
});
|
|
2677
2786
|
}
|
|
2678
2787
|
function rows(n) { return 20 * n + 80; }
|
|
@@ -2682,6 +2791,16 @@
|
|
|
2682
2791
|
var keep = s.length - over - 1;
|
|
2683
2792
|
return over <= 0 ? s : s.slice(0, Math.ceil(keep / 2)) + '…' + s.slice(s.length - Math.floor(keep / 2));
|
|
2684
2793
|
}
|
|
2794
|
+
function fit(s, max) { return middle(s, s.length - max); }
|
|
2795
|
+
var FEW = 10; // under this many tasks or answers, a row is drawn faint
|
|
2796
|
+
// The 95% Wilson interval of k successes in n: [low, high]. Honest at small n and at
|
|
2797
|
+
// 0 or n of n, where k/n ± 1.96·se says nothing (3 of 3 is not "100% ± 0").
|
|
2798
|
+
function wilson(k, n) {
|
|
2799
|
+
if (!n) return [0, 1];
|
|
2800
|
+
var z = 1.96, p = k / n, z2 = z * z, den = 1 + z2 / n;
|
|
2801
|
+
var mid = p + z2 / (2 * n), half = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
|
|
2802
|
+
return [Math.max(0, (mid - half) / den), Math.min(1, (mid + half) / den)];
|
|
2803
|
+
}
|
|
2685
2804
|
|
|
2686
2805
|
// Trend: who moved, which way, by how much and how sure (serve.trend_data),
|
|
2687
2806
|
// in the window the card's select says. A row per model with a task to
|
|
@@ -2728,7 +2847,9 @@
|
|
|
2728
2847
|
var nameW = Math.min(260, Math.max(150, 7 * Math.max.apply(null, M.map(function (r) { return r.key.length; })) + 24));
|
|
2729
2848
|
var evW = 320;
|
|
2730
2849
|
w = $('chart-trend').clientWidth || w; // its own box: a phone's is narrower than width()'s 520
|
|
2731
|
-
|
|
2850
|
+
// Without the evidence column the names take what the bars do not need (260 px), not a
|
|
2851
|
+
// fixed 130 px: at the dashboard's width that cut "muse-glimmer-30b" to "muse-gl…mer-30b".
|
|
2852
|
+
if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, Math.max(130, w - 280)); }
|
|
2732
2853
|
var fits = Math.floor((nameW - 24) / 7), right = evW || 20, wide = w < nameW + right + 140;
|
|
2733
2854
|
w = Math.max(w, nameW + right + 140);
|
|
2734
2855
|
// Symmetric, to the biggest bar and room for its number; 20 pts at least.
|
|
@@ -2830,16 +2951,74 @@
|
|
|
2830
2951
|
// the time; under it, it is surer than it should be. Panels stack on a phone.
|
|
2831
2952
|
var DC = C.decide;
|
|
2832
2953
|
if (DC) put('chart-decide', function (w) {
|
|
2833
|
-
if (!DC.
|
|
2954
|
+
if (!(DC.scores || []).length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
|
|
2834
2955
|
var names = {};
|
|
2835
2956
|
DC.sets.forEach(function (s) { names[s[0]] = s[1]; });
|
|
2836
2957
|
var panels = DC.sets.map(function (s) { return s[1]; });
|
|
2958
|
+
// One colour per model across both charts, by name, never by rank.
|
|
2959
|
+
var models = DC.scores.map(function (p) { return p.model; })
|
|
2960
|
+
.filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
|
|
2961
|
+
var colors = {domain: models, range: [sky, good, signal, warn, muted, ink, err]};
|
|
2962
|
+
var box = el('div');
|
|
2963
|
+
box.appendChild(el('h3', {class: 'chart-head'}, 'Right of asked, per set'));
|
|
2964
|
+
box.appendChild(decideBars(w, names, panels, models, colors));
|
|
2965
|
+
// Calibration only says something once a model's answers spread over two or more
|
|
2966
|
+
// levels of sureness; while each sits in one top bin it is one dot in a corner.
|
|
2967
|
+
var levels = DC.points.some(function (p, i, all) {
|
|
2968
|
+
return all.some(function (q) { return q !== p && q.model === p.model && q.set === p.set; });
|
|
2969
|
+
});
|
|
2970
|
+
if (levels) {
|
|
2971
|
+
box.appendChild(el('h3', {class: 'chart-head'}, 'Calibration: said against right'));
|
|
2972
|
+
box.appendChild(decideCalibration(w, names, panels, models, colors));
|
|
2973
|
+
} else {
|
|
2974
|
+
box.appendChild(el('p', {class: 'hint'}, 'Calibration shows once a model gives answers at two or more levels'
|
|
2975
|
+
+ ' of sureness on a set; so far each answers at one, so the table\'s Brier and calibration gap say it.'));
|
|
2976
|
+
}
|
|
2977
|
+
return box;
|
|
2978
|
+
});
|
|
2979
|
+
// Right of asked as bars from 0: a small chart per set, a row per model asked on it,
|
|
2980
|
+
// best first. A model under the ranking floor (DC.low items) is drawn faint: shown,
|
|
2981
|
+
// not yet ranked.
|
|
2982
|
+
function decideBars(w, names, panels, models, colors) {
|
|
2983
|
+
var out = el('div');
|
|
2984
|
+
DC.sets.forEach(function (s) {
|
|
2985
|
+
var rows = DC.scores.filter(function (r) { return r.set === s[0]; }).map(function (r) {
|
|
2986
|
+
return Object.assign({share: r.correct / r.asked, ranked: r.asked >= DC.low}, r);
|
|
2987
|
+
}).sort(function (a, b) { return b.share - a.share || b.asked - a.asked || (a.model < b.model ? -1 : 1); });
|
|
2988
|
+
out.appendChild(el('p', {class: 'chart-label'}, s[1]));
|
|
2989
|
+
if (!rows.length) { out.appendChild(el('p', {class: 'hint'}, 'No model asked on this set yet.')); return; }
|
|
2990
|
+
out.appendChild(Plot.plot({
|
|
2991
|
+
width: w, height: rows.length * 26 + 40, style: style,
|
|
2992
|
+
marginLeft: w < 560 ? 120 : 160, marginRight: 50, marginTop: 4, marginBottom: 30,
|
|
2993
|
+
x: {domain: [0, 1], label: null, tickFormat: '.0%', ticks: 5, grid: true},
|
|
2994
|
+
y: {domain: rows.map(function (r) { return r.model; }), label: null},
|
|
2995
|
+
color: colors,
|
|
2996
|
+
marks: [
|
|
2997
|
+
Plot.barX(rows, {x: 'share', y: 'model', fill: 'model', insetTop: 4, insetBottom: 4, rx: 2,
|
|
2998
|
+
fillOpacity: function (r) { return r.ranked ? 1 : 0.4; },
|
|
2999
|
+
title: function (r) {
|
|
3000
|
+
return r.model + ' · ' + s[1] + '\n' + r.correct + ' right of ' + r.asked + ' asked ('
|
|
3001
|
+
+ Math.round(r.share * 100) + '%)' + (r.brier != null ? '\nBrier ' + r.brier.toFixed(3) : '')
|
|
3002
|
+
+ (r.ranked ? '' : '\nnot ranked yet: under ' + DC.low + ' items');
|
|
3003
|
+
}}),
|
|
3004
|
+
Plot.text(rows, {x: 'share', y: 'model', dx: 6, textAnchor: 'start', fill: ink,
|
|
3005
|
+
text: function (r) { return r.correct + '/' + r.asked; }}),
|
|
3006
|
+
Plot.ruleX([0], {stroke: ink})
|
|
3007
|
+
]
|
|
3008
|
+
}));
|
|
3009
|
+
});
|
|
3010
|
+
return out;
|
|
3011
|
+
}
|
|
3012
|
+
function decideCalibration(w, names, panels, models, colors) {
|
|
2837
3013
|
var pts = DC.points.map(function (p) { return Object.assign({panel: names[p.set]}, p); })
|
|
2838
3014
|
.sort(function (a, b) { return a.said - b.said; });
|
|
2839
|
-
|
|
2840
|
-
|
|
3015
|
+
// Both axes start just under the lowest answer, not at 0%: a decision model is
|
|
3016
|
+
// rarely under 50% sure of its own answer, so a 0-100% frame left every dot in
|
|
3017
|
+
// one corner under a long empty diagonal. Same floor on both, so it stays 45°.
|
|
3018
|
+
var low = Math.min.apply(null, pts.map(function (p) { return Math.min(p.said, p.right); }));
|
|
3019
|
+
var lo = Math.max(0, Math.min(0.9, Math.floor((low - 0.05) * 20) / 20));
|
|
2841
3020
|
var diag = [];
|
|
2842
|
-
panels.forEach(function (s) { diag.push({panel: s, x:
|
|
3021
|
+
panels.forEach(function (s) { diag.push({panel: s, x: lo, y: lo}, {panel: s, x: 1, y: 1}); });
|
|
2843
3022
|
function tip(p) {
|
|
2844
3023
|
return p.model + ' · ' + names[p.set] + '\n' + plural(p.n, 'answer') + ' it was about '
|
|
2845
3024
|
+ Math.round(p.said * 100) + '% sure of\n' + Math.round(p.right * 100) + '% of them right';
|
|
@@ -2850,10 +3029,11 @@
|
|
|
2850
3029
|
return Plot.plot(Object.assign(opts, {
|
|
2851
3030
|
width: narrow ? Math.min(w, 520) : w, height: narrow ? 3 * 230 + 70 : 320, style: style,
|
|
2852
3031
|
marginLeft: 52, marginRight: narrow ? 90 : 20, marginBottom: 44, marginTop: 28,
|
|
2853
|
-
|
|
2854
|
-
|
|
3032
|
+
inset: 14, // a dot at 100% clear of the frame
|
|
3033
|
+
x: {domain: [lo, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
|
|
3034
|
+
y: {domain: [lo, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
|
|
2855
3035
|
r: {range: [3, 13]},
|
|
2856
|
-
color: {legend: true,
|
|
3036
|
+
color: Object.assign({legend: true}, colors),
|
|
2857
3037
|
marks: [
|
|
2858
3038
|
Plot.frame({stroke: line}),
|
|
2859
3039
|
Plot.line(diag, Object.assign({x: 'x', y: 'y', z: 'panel', stroke: muted, strokeDasharray: '4 3'}, by)),
|
|
@@ -2862,17 +3042,24 @@
|
|
|
2862
3042
|
title: tip}, by))
|
|
2863
3043
|
]
|
|
2864
3044
|
}));
|
|
2865
|
-
}
|
|
3045
|
+
}
|
|
2866
3046
|
// Who to trust: finds real bugs (right) vs makes things up (up = less).
|
|
2867
3047
|
put('chart-overview', function (w) {
|
|
2868
3048
|
if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
|
|
2869
3049
|
var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
|
|
2870
3050
|
var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
|
|
2871
3051
|
var pxW = w - 110, pxH = 370;
|
|
2872
|
-
|
|
3052
|
+
// Named: the models in or next to the zone (the ones to pick from), then the clearest
|
|
3053
|
+
// warnings (most made up). The rest are on hover: named all at once they ran together.
|
|
3054
|
+
var zoneX = x0 + (1 - x0) / 2, zoneY = worst / 4;
|
|
3055
|
+
var near = function (d) { return d.recall >= zoneX - (1 - x0) * 0.08 && d.wrong <= zoneY * 1.4; };
|
|
3056
|
+
var worstFew = C.overview.slice().sort(function (a, b) { return b.wrong - a.wrong; }).slice(0, 3);
|
|
3057
|
+
var pts = C.overview.filter(function (d) { return near(d) || worstFew.indexOf(d) !== -1; })
|
|
3058
|
+
.sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
|
|
2873
3059
|
var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
|
|
2874
3060
|
function (d) { return d.wrong / worst * pxH; });
|
|
2875
|
-
var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
|
|
3061
|
+
var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
|
|
3062
|
+
.filter(Boolean);
|
|
2876
3063
|
return Plot.plot({
|
|
2877
3064
|
width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
2878
3065
|
x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
|
|
@@ -2900,71 +3087,72 @@
|
|
|
2900
3087
|
tasks[h.model] = bugs + clean;
|
|
2901
3088
|
var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
|
|
2902
3089
|
var fa = clean ? h['false alarm'] / clean : 0;
|
|
2903
|
-
|
|
3090
|
+
// Ranked on what is proven: the low end of the found share's 95% range less the HIGH
|
|
3091
|
+
// end of the false alarms' (never shown clean files: up to 100%), so a few tasks, or
|
|
3092
|
+
// none on clean code, cannot top the chart.
|
|
3093
|
+
score[h.model] = wilson(h.found, bugs)[0] - wilson(h['false alarm'], clean)[1];
|
|
2904
3094
|
segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
|
|
2905
3095
|
segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
|
|
2906
3096
|
segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
|
|
2907
3097
|
segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
|
|
2908
3098
|
});
|
|
2909
|
-
var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
|
|
3099
|
+
var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a] || tasks[b] - tasks[a]; });
|
|
2910
3100
|
return Plot.plot({
|
|
2911
|
-
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
3101
|
+
width: w, height: rows(models.length), style: style, marginLeft: w < 560 ? 150 : 250,
|
|
2912
3102
|
x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
|
|
2913
3103
|
tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
|
|
2914
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
|
|
3104
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + tasks[m] + ')'; }},
|
|
2915
3105
|
color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
|
|
2916
3106
|
marks: [
|
|
3107
|
+
// Under FEW tasks a model's bars are faint: shown, not much to go on.
|
|
2917
3108
|
Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
|
|
2918
|
-
|
|
3109
|
+
fillOpacity: function (d) { return tasks[d.model] < FEW ? 0.4 : 1; },
|
|
3110
|
+
title: function (d) { return d.model + ': ' + d.part + ', ' + d.n + (tasks[d.model] < FEW ? '\nfew tasks: little to go on' : ''); }}),
|
|
2919
3111
|
Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
|
|
2920
3112
|
]
|
|
2921
3113
|
});
|
|
2922
3114
|
});
|
|
2923
3115
|
|
|
2924
|
-
// Real work:
|
|
3116
|
+
// Real work: how often each model helped, with how sure that is. The dot is the share
|
|
3117
|
+
// helped, its size how many were judged, the line the range the true share likely sits
|
|
3118
|
+
// in (95% Wilson interval: wide for 3 of 3, narrow for 40 of 50). Ranked by the low end
|
|
3119
|
+
// of that range, so volume counts: 3 of 3 no longer outranks 40 of 50. The dashed line
|
|
3120
|
+
// is every model's offloads pooled.
|
|
2925
3121
|
put('chart-work', function (w) {
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
3122
|
+
if (!C.work.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
|
|
3123
|
+
var R = C.work.map(function (r) {
|
|
3124
|
+
var b = wilson(r.useful, r.judged), p = r.useful / r.judged;
|
|
3125
|
+
var verdict = b[0] >= 0.5 ? 'helps' : b[1] < 0.5 ? 'does not help' : 'too few to say';
|
|
3126
|
+
return Object.assign({p: p, lo: b[0], hi: b[1], verdict: verdict}, r);
|
|
3127
|
+
}).sort(function (a, b) { return b.lo - a.lo || b.judged - a.judged; });
|
|
3128
|
+
var all = R.reduce(function (s, r) { return [s[0] + r.useful, s[1] + r.judged]; }, [0, 0]);
|
|
3129
|
+
var pooled = all[0] / all[1];
|
|
3130
|
+
var narrowW = w < 560, nameW = narrowW ? 130 : 210;
|
|
2931
3131
|
return Plot.plot({
|
|
2932
|
-
width: w, height:
|
|
2933
|
-
x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
|
|
3132
|
+
width: w, height: 24 * R.length + 90, style: style, marginLeft: nameW, marginRight: 64, marginTop: 34,
|
|
3133
|
+
x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', ticks: 5, grid: true},
|
|
2934
3134
|
y: {domain: R.map(function (r) { return r.model; }), label: null,
|
|
2935
|
-
tickFormat: function (m) { return m
|
|
3135
|
+
tickFormat: function (m) { return fit(m, narrowW ? 16 : 26); }},
|
|
3136
|
+
r: {domain: [0, Math.max.apply(null, R.map(function (r) { return r.judged; }))], range: [3, 11]},
|
|
3137
|
+
color: {domain: ['helps', 'too few to say', 'does not help'], range: [good, muted, err], legend: true},
|
|
2936
3138
|
marks: [
|
|
2937
|
-
Plot.
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
Plot.
|
|
3139
|
+
Plot.ruleX([0.5], {stroke: line}),
|
|
3140
|
+
Plot.ruleX([pooled], {stroke: ink, strokeDasharray: '4 3', strokeOpacity: 0.6}),
|
|
3141
|
+
Plot.text([pooled], {x: function (d) { return d; }, frameAnchor: 'top', dy: -20, fill: muted, fontSize: 11,
|
|
3142
|
+
text: function () { return 'all models ' + Math.round(pooled * 100) + '%'; }}),
|
|
3143
|
+
Plot.ruleY(R, {y: 'model', x1: 'lo', x2: 'hi', stroke: 'verdict', strokeWidth: 2, strokeOpacity: 0.55}),
|
|
3144
|
+
Plot.dot(R, {x: 'p', y: 'model', r: 'judged', fill: 'verdict', stroke: halo, strokeWidth: 1.5,
|
|
3145
|
+
title: function (r) {
|
|
3146
|
+
return r.model + '\nhelped ' + r.useful + ' of ' + r.judged + ' judged (' + Math.round(r.p * 100) + '%; '
|
|
3147
|
+
+ r.offloads + ' offloads)\nlikely between ' + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%'
|
|
3148
|
+
+ (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : '');
|
|
3149
|
+
}}),
|
|
3150
|
+
Plot.text(R, {x: 'hi', y: 'model', dx: 8, textAnchor: 'start', fill: ink, fontSize: 11,
|
|
3151
|
+
text: function (r) { return r.useful + '/' + r.judged; }})
|
|
2942
3152
|
]
|
|
2943
3153
|
});
|
|
2944
3154
|
});
|
|
2945
3155
|
|
|
2946
|
-
// Did it answer: each probe of the last 14 days, by what came back.
|
|
2947
|
-
put('chart-reliability', function (w) {
|
|
2948
|
-
if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
|
|
2949
|
-
var total = {}, ok = {};
|
|
2950
|
-
C.reliability.forEach(function (d) {
|
|
2951
|
-
total[d.model] = (total[d.model] || 0) + d.count;
|
|
2952
|
-
if (d.outcome === 'answered') ok[d.model] = d.count;
|
|
2953
|
-
});
|
|
2954
|
-
var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
|
|
2955
|
-
var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
|
|
2956
|
-
entitlement: '#c0563f', other: muted, error: ink};
|
|
2957
|
-
var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
|
|
2958
|
-
return Plot.plot({
|
|
2959
|
-
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
2960
|
-
x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
|
|
2961
|
-
y: {domain: models, label: null},
|
|
2962
|
-
color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
|
|
2963
|
-
marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
|
|
2964
|
-
title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
|
|
2965
|
-
});
|
|
2966
|
-
});
|
|
2967
|
-
|
|
2968
3156
|
// Speed vs quality: median seconds on our own tasks against findings located.
|
|
2969
3157
|
put('chart-speed', function (w) {
|
|
2970
3158
|
if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
|
|
@@ -2975,14 +3163,20 @@
|
|
|
2975
3163
|
var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
|
|
2976
3164
|
var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
|
|
2977
3165
|
function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
|
|
2978
|
-
var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
|
|
3166
|
+
var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
|
|
3167
|
+
.filter(Boolean);
|
|
2979
3168
|
return Plot.plot({
|
|
2980
3169
|
width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
2981
3170
|
x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
|
|
2982
3171
|
y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
|
|
2983
3172
|
marks: [
|
|
3173
|
+
// The zone to be in, as on Who to trust: the faster half (on the log scale) and the
|
|
3174
|
+
// top quarter of findings. Outlined, so it reads on a dark ground too.
|
|
3175
|
+
Plot.rect([{x1: lo, x2: Math.sqrt(lo * hi), y1: y0 + (1 - y0) * 3 / 4, y2: 1}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
|
|
3176
|
+
fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
|
|
2984
3177
|
Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
|
|
2985
|
-
Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink
|
|
3178
|
+
Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink,
|
|
3179
|
+
stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
|
|
2986
3180
|
Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
|
|
2987
3181
|
title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
|
|
2988
3182
|
]
|
|
@@ -2997,25 +3191,41 @@
|
|
|
2997
3191
|
return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
|
|
2998
3192
|
});
|
|
2999
3193
|
tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
|
|
3000
|
-
|
|
3001
|
-
|
|
3194
|
+
// Each group's name over its first task, on two staggered rows: side by side they
|
|
3195
|
+
// ran into one another wherever a group is only a few tasks wide.
|
|
3196
|
+
var heads = C.families.filter(function (f) { return f in first; })
|
|
3197
|
+
.map(function (f, i) { return {task: first[f], name: f, row: i % 2}; });
|
|
3002
3198
|
// By total found, so a model that played three tasks does not top one that played forty.
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3199
|
+
// A model with no scored task (only grey) gets no row: it is named under the chart.
|
|
3200
|
+
var sum = {}, scored = {};
|
|
3201
|
+
C.heat.forEach(function (d) {
|
|
3202
|
+
sum[d.model] = (sum[d.model] || 0) + (d.rate || 0);
|
|
3203
|
+
if (d.rate != null) scored[d.model] = true;
|
|
3204
|
+
});
|
|
3205
|
+
var models = Object.keys(sum).filter(function (m) { return scored[m]; }).sort(function (a, b) { return sum[b] - sum[a]; });
|
|
3206
|
+
var idle = Object.keys(sum).filter(function (m) { return !scored[m]; }).sort();
|
|
3207
|
+
var shown = C.heat.filter(function (d) { return scored[d.model]; });
|
|
3208
|
+
var nameW = w < 560 ? 130 : 230;
|
|
3006
3209
|
function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3210
|
+
var box = el('div');
|
|
3211
|
+
box.appendChild(Plot.plot({
|
|
3212
|
+
width: w, height: 22 * models.length + 84, style: style, marginLeft: nameW, marginTop: 44, padding: 0.08,
|
|
3213
|
+
x: {domain: tasks, axis: null},
|
|
3214
|
+
y: {domain: models, label: null, tickSize: 0, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 30); }},
|
|
3011
3215
|
color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
|
|
3012
3216
|
marks: [
|
|
3013
|
-
Plot.cell(
|
|
3014
|
-
Plot.cell(
|
|
3015
|
-
Plot.dot(
|
|
3016
|
-
Plot.ruleX(
|
|
3217
|
+
Plot.cell(shown.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
|
|
3218
|
+
Plot.cell(shown.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
|
|
3219
|
+
Plot.dot(shown.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
|
|
3220
|
+
Plot.ruleX(heads, {x: 'task', stroke: ink, strokeOpacity: 0.25, dx: -2}),
|
|
3221
|
+
Plot.text(heads.filter(function (h) { return !h.row; }), {x: 'task', frameAnchor: 'top', dy: -30, dx: -2,
|
|
3222
|
+
text: 'name', textAnchor: 'start', fontSize: 11, fill: ink}),
|
|
3223
|
+
Plot.text(heads.filter(function (h) { return h.row; }), {x: 'task', frameAnchor: 'top', dy: -14, dx: -2,
|
|
3224
|
+
text: 'name', textAnchor: 'start', fontSize: 11, fill: ink})
|
|
3017
3225
|
]
|
|
3018
|
-
});
|
|
3226
|
+
}));
|
|
3227
|
+
if (idle.length) box.appendChild(el('p', {class: 'hint'}, 'No scored task yet, so no row: ' + idle.join(', ') + '.'));
|
|
3228
|
+
return box;
|
|
3019
3229
|
});
|
|
3020
3230
|
|
|
3021
3231
|
// Vision tasks: one square per field and model, the task's name over its first field.
|
|
@@ -3062,39 +3272,66 @@
|
|
|
3062
3272
|
kinds.forEach(function (k) { if (W[m][k]) data.push({model: m, kind: k, count: W[m][k]}); total[m] = (total[m] || 0) + W[m][k]; });
|
|
3063
3273
|
});
|
|
3064
3274
|
if (!data.length) return el('p', {class: 'hint'}, 'No vision panels through the gateway yet.');
|
|
3065
|
-
// The models doing the most image work first: their bars are the evidence.
|
|
3066
|
-
|
|
3275
|
+
// The models doing the most image work first: their bars are the evidence. Under FEW
|
|
3276
|
+
// seats a row was mostly "no answer" and said little: those are listed under the chart.
|
|
3277
|
+
var all = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
|
|
3278
|
+
var models = all.filter(function (m) { return total[m] >= FEW; });
|
|
3279
|
+
var few = all.filter(function (m) { return total[m] < FEW; });
|
|
3280
|
+
if (!models.length) { models = all; few = []; }
|
|
3281
|
+
data = data.filter(function (d) { return models.indexOf(d.model) !== -1; });
|
|
3067
3282
|
var colour = {'checked, helped': good, 'checked, did not help': err, 'useful, no claims': sky,
|
|
3068
3283
|
'no use, no claims': warn, 'not judged': line, 'no answer': muted};
|
|
3069
3284
|
// The claims checked by hand, beside the bar: the only right-or-wrong count real work has.
|
|
3070
3285
|
var claims = models.filter(function (m) { return W[m].real + W[m].false; }).map(function (m) {
|
|
3071
3286
|
return {model: m, text: W[m].real + ' of ' + (W[m].real + W[m].false) + ' claims right'};
|
|
3072
3287
|
});
|
|
3073
|
-
|
|
3074
|
-
|
|
3288
|
+
var narrowV = w < 560, box = el('div');
|
|
3289
|
+
box.appendChild(Plot.plot({
|
|
3290
|
+
width: w, height: rows(models.length), style: style, marginLeft: narrowV ? 150 : 260, marginRight: narrowV ? 20 : 150,
|
|
3075
3291
|
x: {label: 'Share of its seats on image panels →', tickFormat: '.0%', grid: true, domain: [0, 1]},
|
|
3076
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + total[m] + ')'; }},
|
|
3292
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, narrowV ? 16 : 30) + ' (' + total[m] + ')'; }},
|
|
3077
3293
|
color: {legend: true, domain: kinds, range: kinds.map(function (k) { return colour[k]; })},
|
|
3078
3294
|
marks: [
|
|
3079
3295
|
Plot.barX(data, Plot.stackX({x: 'count', y: 'model', fill: 'kind', offset: 'normalize', order: kinds,
|
|
3080
3296
|
title: function (d) { return d.model + ': ' + d.kind + ', ' + d.count + ' of ' + total[d.model] + ' seats'; }})),
|
|
3081
|
-
Plot.text(
|
|
3297
|
+
Plot.text(narrowV ? [] : claims.filter(function (c) { return models.indexOf(c.model) !== -1; }),
|
|
3298
|
+
{x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
|
|
3082
3299
|
]
|
|
3083
|
-
});
|
|
3300
|
+
}));
|
|
3301
|
+
if (few.length) box.appendChild(el('p', {class: 'hint'}, 'Under ' + FEW + ' seats, so no bar: '
|
|
3302
|
+
+ few.map(function (m) { return m + ' (' + total[m] + ')'; }).join(', ') + '. The table has them all.'));
|
|
3303
|
+
return box;
|
|
3084
3304
|
});
|
|
3085
3305
|
|
|
3086
|
-
// Confidently wrong: stacked by kind
|
|
3306
|
+
// Confidently wrong: stacked by kind. A thin line runs over each bar for the range the
|
|
3307
|
+
// true share likely sits in (95% Wilson), so 0 of 2 shows how bad it could be, not a
|
|
3308
|
+
// clean record. Worst proven at the top: by the low end of that range. Few answers: faint.
|
|
3087
3309
|
put('chart-wrong', function (w) {
|
|
3088
|
-
var T = C.wrongTotals;
|
|
3089
|
-
|
|
3310
|
+
var T = C.wrongTotals, R = {};
|
|
3311
|
+
Object.keys(T).forEach(function (m) {
|
|
3312
|
+
var b = wilson(T[m].wrong, T[m].answers);
|
|
3313
|
+
R[m] = {model: m, lo: b[0], hi: b[1], few: T[m].answers < FEW};
|
|
3314
|
+
});
|
|
3315
|
+
var models = Object.keys(T).sort(function (a, b) {
|
|
3316
|
+
return R[b].lo - R[a].lo || T[b].rate - T[a].rate || T[b].answers - T[a].answers;
|
|
3317
|
+
});
|
|
3318
|
+
var ranges = models.map(function (m) { return R[m]; });
|
|
3319
|
+
var top = Math.min(1, Math.max.apply(null, ranges.map(function (r) { return r.hi; })) + 0.02);
|
|
3090
3320
|
return Plot.plot({
|
|
3091
|
-
width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
|
|
3092
|
-
x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
|
|
3321
|
+
width: w, height: 20 * models.length + 70, style: style, marginLeft: w < 560 ? 150 : 250,
|
|
3322
|
+
x: {domain: [0, top], label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
|
|
3093
3323
|
// The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
|
|
3094
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
|
|
3324
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + T[m].answers + ')'; }},
|
|
3095
3325
|
color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
|
|
3096
3326
|
marks: [
|
|
3097
|
-
Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind',
|
|
3327
|
+
Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', insetTop: 3, insetBottom: 3,
|
|
3328
|
+
fillOpacity: function (d) { return R[d.model].few ? 0.4 : 1; },
|
|
3329
|
+
title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
|
|
3330
|
+
Plot.ruleY(ranges, {y: 'model', x1: 'lo', x2: 'hi', stroke: ink, strokeOpacity: 0.45, strokeWidth: 1.5,
|
|
3331
|
+
title: function (r) {
|
|
3332
|
+
return r.model + ': ' + T[r.model].wrong + ' of ' + T[r.model].answers + ' confidently wrong\nlikely between '
|
|
3333
|
+
+ Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%' + (r.few ? '\nfew answers: little to go on' : '');
|
|
3334
|
+
}}),
|
|
3098
3335
|
Plot.ruleX([0], {stroke: ink})
|
|
3099
3336
|
]
|
|
3100
3337
|
});
|