@wendellresearch-ai/benchwarmers 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/VERSION +1 -1
- package/metrics/route.py +6 -4
- package/package.json +1 -1
- package/scripts/provider_tags.py +19 -3
- package/scripts/test_no_owui.py +13 -0
- package/serve.py +126 -55
- package/ui/bench.css +30 -8
- package/ui/bench.js +349 -154
package/ui/bench.js
CHANGED
|
@@ -55,30 +55,33 @@
|
|
|
55
55
|
askD.addEventListener('close', function () { answer(false); }); // Escape
|
|
56
56
|
|
|
57
57
|
// ---- restart (serve.restart): the page reloads once the new one answers ----
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
// One button on Settings (restart) and one on Info (restart-info), each with its own status line.
|
|
59
|
+
function restart(id, force) {
|
|
60
|
+
var rs = $(id), out = $(id + '-out');
|
|
60
61
|
rs.disabled = true; out.textContent = 'Restarting…';
|
|
61
62
|
post('/api/restart', {force: force}).then(function (res) {
|
|
62
|
-
if (res.ok) return back(Date.now());
|
|
63
|
+
if (res.ok) return back(out, Date.now());
|
|
63
64
|
rs.disabled = false; out.textContent = '';
|
|
64
65
|
if (!res.busy) { listErrors(out, res.errors); return; }
|
|
65
66
|
ask('Still in progress: ' + res.errors.join('; ') + '. Restart anyway and cut them off?', 'Restart anyway', true)
|
|
66
|
-
.then(function (yes) { if (yes) restart(true); });
|
|
67
|
+
.then(function (yes) { if (yes) restart(id, true); });
|
|
67
68
|
}).catch(function (err) { rs.disabled = false; out.textContent = String(err); });
|
|
68
69
|
}
|
|
69
70
|
// The old one stops half a second after answering; /health answers again once the new one is up.
|
|
70
|
-
function back(since) {
|
|
71
|
+
function back(out, since) {
|
|
71
72
|
setTimeout(function () {
|
|
72
73
|
fetch('/health', {cache: 'no-store'}).then(function (r) { if (!r.ok) throw r.status; location.reload(); })
|
|
73
74
|
.catch(function () {
|
|
74
|
-
if (Date.now() - since < 120000) back(since);
|
|
75
|
-
else
|
|
75
|
+
if (Date.now() - since < 120000) back(out, since);
|
|
76
|
+
else out.textContent = 'Not back after two minutes: look at whatever runs it.';
|
|
76
77
|
});
|
|
77
78
|
}, 1500);
|
|
78
79
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
['restart', 'restart-info'].forEach(function (id) {
|
|
81
|
+
on('click', id, function () {
|
|
82
|
+
ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
|
|
83
|
+
.then(function (yes) { if (yes) restart(id, false); });
|
|
84
|
+
});
|
|
82
85
|
});
|
|
83
86
|
|
|
84
87
|
// ---- Export and import (serve.export_zip, import_zip) ----------------
|
|
@@ -250,13 +253,21 @@
|
|
|
250
253
|
var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
|
|
251
254
|
var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
|
|
252
255
|
scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
|
|
253
|
-
|
|
256
|
+
// A group's name (data-conn: Models' connection) counts as text of each of its rows.
|
|
257
|
+
var g = r.closest('[data-conn]'), text = r.textContent + (g ? ' ' + g.dataset.conn : '');
|
|
258
|
+
r.hidden = (q && text.toLowerCase().indexOf(q) === -1)
|
|
254
259
|
|| facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
|
|
255
260
|
});
|
|
256
261
|
// A match inside a folded list (an endpoint's models) opens it.
|
|
257
262
|
if (q) scope.querySelectorAll('details.models').forEach(function (d) {
|
|
258
263
|
if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
|
|
259
264
|
});
|
|
265
|
+
// A group with no row left is hidden; one with a match opens.
|
|
266
|
+
scope.querySelectorAll('[data-conn]').forEach(function (g) {
|
|
267
|
+
var any = !!g.querySelector('tbody tr:not([hidden])');
|
|
268
|
+
g.hidden = !!q && !any;
|
|
269
|
+
if (q && any && g.tagName === 'DETAILS') g.open = true;
|
|
270
|
+
});
|
|
260
271
|
}
|
|
261
272
|
function scopeOf(c) {
|
|
262
273
|
var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
|
|
@@ -490,7 +501,7 @@
|
|
|
490
501
|
reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
|
|
491
502
|
qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why'), roleHint: $('m-role-hint')};
|
|
492
503
|
m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
|
|
493
|
-
var editing = null;
|
|
504
|
+
var editing = null, shown = {};
|
|
494
505
|
|
|
495
506
|
function openModel(alias) {
|
|
496
507
|
var model = D.models[alias], gone = (D.retired || {})[alias];
|
|
@@ -539,6 +550,8 @@
|
|
|
539
550
|
+ (hint.at || '').slice(0, 10) + '). Only a suggestion: the role is the one you pick above.' : '';
|
|
540
551
|
m.save.disabled = false; m.save.textContent = 'Save';
|
|
541
552
|
m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
|
|
553
|
+
// What the popup opened with: Save sends only the parts that differ from it.
|
|
554
|
+
shown = {model: JSON.stringify(modelBody()), facts: factsBody(), opts: optsBody()};
|
|
542
555
|
}
|
|
543
556
|
if (!md.open) md.showModal(); // open already: shown again as it now is (shownAgain)
|
|
544
557
|
}
|
|
@@ -577,7 +590,7 @@
|
|
|
577
590
|
var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
|
|
578
591
|
box.hidden = profile.hidden = decision || !f;
|
|
579
592
|
profile.textContent = '';
|
|
580
|
-
$('f-errors').textContent = '';
|
|
593
|
+
$('f-errors').textContent = '';
|
|
581
594
|
if (box.hidden) return;
|
|
582
595
|
var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
|
|
583
596
|
// At a glance, as models.dev shows a model: what goes in, what comes out.
|
|
@@ -621,40 +634,25 @@
|
|
|
621
634
|
var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
|
|
622
635
|
$('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
|
|
623
636
|
}
|
|
624
|
-
|
|
625
|
-
|
|
637
|
+
// The facts as typed, as /api/model-facts takes them; null when the box is not shown.
|
|
638
|
+
function factsBody() {
|
|
639
|
+
if ($('m-facts').hidden) return null;
|
|
626
640
|
var facts = {limits: {}};
|
|
627
641
|
FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
|
|
628
642
|
FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
|
|
629
643
|
if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
|
|
630
644
|
document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
post('/api/model-facts', {alias: alias, facts: facts}).then(function (r) {
|
|
634
|
-
if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
|
|
635
|
-
// The tables that show facts (At a glance, Models, Quotas) follow once the popup closes (live).
|
|
636
|
-
shownAgain(alias, 'f-done', 'Saved. The tables show it once this closes.');
|
|
637
|
-
});
|
|
638
|
-
});
|
|
645
|
+
return JSON.stringify(facts);
|
|
646
|
+
}
|
|
639
647
|
// ---- call options: how the bench asks this model (serve.save_model_options)
|
|
640
648
|
function fillOptions(alias, decision) {
|
|
641
649
|
var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
|
|
642
650
|
$('m-opts').hidden = decision;
|
|
643
|
-
$('co-errors').textContent = '';
|
|
651
|
+
$('co-errors').textContent = '';
|
|
644
652
|
$('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
|
|
645
653
|
$('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
|
|
646
654
|
}
|
|
647
|
-
$('
|
|
648
|
-
if (!editing) return;
|
|
649
|
-
var v = $('co-max').value.trim();
|
|
650
|
-
$('co-errors').textContent = ''; $('co-done').textContent = 'Saving…'; // the llm CLI is told too: seconds
|
|
651
|
-
var alias = editing;
|
|
652
|
-
post('/api/model-options', {alias: alias, max_tokens: v ? Number(v) : null}).then(function (r) {
|
|
653
|
-
if (!r.ok) { $('co-done').textContent = ''; listErrors($('co-errors'), r.errors); return; }
|
|
654
|
-
ownChange(); // only this popup shows them
|
|
655
|
-
shownAgain(alias, 'co-done', 'Saved' + (r.note ? '; ' + r.note : '') + '.');
|
|
656
|
-
});
|
|
657
|
-
});
|
|
655
|
+
function optsBody() { return $('m-opts').hidden ? null : $('co-max').value.trim(); }
|
|
658
656
|
|
|
659
657
|
function refresh() {
|
|
660
658
|
m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
|
|
@@ -690,22 +688,57 @@
|
|
|
690
688
|
then(res);
|
|
691
689
|
};
|
|
692
690
|
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
|
|
691
|
+
function modelBody() {
|
|
692
|
+
var body = {new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
|
|
696
693
|
if (!m.qtypes.hidden) {
|
|
697
694
|
body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
|
|
698
695
|
function (c) { return c.checked; }).map(function (c) { return c.value; });
|
|
699
696
|
}
|
|
700
|
-
|
|
701
|
-
|
|
697
|
+
return body;
|
|
698
|
+
}
|
|
699
|
+
// The one Save: facts, then call options, then the model itself (alias, role, ad hoc,
|
|
700
|
+
// question types), each only if changed. The model goes last: a rename moves the
|
|
701
|
+
// facts and options saved under the old alias with it. The first refusal stops it.
|
|
702
|
+
m.save.addEventListener('click', function () {
|
|
703
|
+
if (!editing) return;
|
|
704
|
+
var alias = editing, body = modelBody(), facts = factsBody(), opts = optsBody(), said = [];
|
|
705
|
+
var steps = [];
|
|
706
|
+
if (facts !== shown.facts) steps.push(function () {
|
|
707
|
+
return post('/api/model-facts', {alias: alias, facts: JSON.parse(facts)}).then(function (r) {
|
|
708
|
+
if (!r.ok) { listErrors($('f-errors'), r.errors); $('m-facts').open = true; throw 'refused'; }
|
|
709
|
+
said.push('model facts');
|
|
710
|
+
});
|
|
711
|
+
});
|
|
712
|
+
if (opts !== shown.opts) steps.push(function () { // the llm CLI is told too: seconds
|
|
713
|
+
return post('/api/model-options', {alias: alias, max_tokens: opts ? Number(opts) : null}).then(function (r) {
|
|
714
|
+
if (!r.ok) { listErrors($('co-errors'), r.errors); $('m-opts').open = true; throw 'refused'; }
|
|
715
|
+
ownChange(); // only this popup shows them
|
|
716
|
+
said.push('call options' + (r.note ? ' (' + r.note + ')' : ''));
|
|
717
|
+
});
|
|
718
|
+
});
|
|
719
|
+
if (JSON.stringify(body) !== shown.model) steps.push(function () {
|
|
720
|
+
body.alias = alias;
|
|
721
|
+
return post('/api/model', body).then(function (r) {
|
|
722
|
+
if (!r.ok) { listErrors(m.errors, r.errors); throw 'refused'; }
|
|
723
|
+
said.push('the model (' + done(r).replace(/^Done: |\.$/g, '') + ')');
|
|
724
|
+
if (body.new_alias !== alias) { m.title.textContent = body.new_alias; refreshPanes(); }
|
|
725
|
+
});
|
|
726
|
+
});
|
|
727
|
+
m.errors.textContent = ''; m.done.textContent = ''; $('f-errors').textContent = ''; $('co-errors').textContent = '';
|
|
728
|
+
if (!steps.length) { m.done.textContent = 'Nothing changed.'; return; }
|
|
729
|
+
(body.new_alias === alias ? Promise.resolve(true)
|
|
730
|
+
: ask('Rename ' + alias + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
|
|
702
731
|
).then(function (yes) {
|
|
703
732
|
if (!yes) return;
|
|
704
|
-
m.save.disabled = true; m.save.textContent = 'Saving…';
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
shownAgain(body.new_alias, 'm-done',
|
|
708
|
-
})
|
|
733
|
+
m.save.disabled = true; m.save.textContent = 'Saving…';
|
|
734
|
+
steps.reduce(function (p, step) { return p.then(step); }, Promise.resolve()).then(function () {
|
|
735
|
+
// The tables that show facts follow once the popup closes (live).
|
|
736
|
+
shownAgain(body.new_alias, 'm-done', 'Saved: ' + said.join(' · ') + '.');
|
|
737
|
+
}).catch(function (err) {
|
|
738
|
+
if (err !== 'refused') m.errors.textContent = String(err);
|
|
739
|
+
if (said.length) m.done.textContent = 'Saved before that: ' + said.join(' · ') + '.';
|
|
740
|
+
m.save.disabled = false; m.save.textContent = 'Save';
|
|
741
|
+
});
|
|
709
742
|
});
|
|
710
743
|
});
|
|
711
744
|
// The confirm is the last word: the popup closes at once and the page
|
|
@@ -1855,7 +1888,7 @@
|
|
|
1855
1888
|
pick: $('g-pick'), pickConf: $('g-pick-conf'),
|
|
1856
1889
|
gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
|
|
1857
1890
|
del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
|
|
1858
|
-
seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
|
|
1891
|
+
seats: $('c-seats'), prov: $('c-member-provider'), show: $('c-member-show'), adHoc: $('c-ad-hoc'),
|
|
1859
1892
|
ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
|
|
1860
1893
|
return loadCollections();
|
|
1861
1894
|
};
|
|
@@ -1969,7 +2002,7 @@
|
|
|
1969
2002
|
// the seats above the list always show what is picked, each with its ×, and
|
|
1970
2003
|
// every count still includes it. A row goes when all its chips do.
|
|
1971
2004
|
function filterMembers() {
|
|
1972
|
-
var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
|
|
2005
|
+
var q = c.find.value.trim().toLowerCase(), prov = c.prov.value, show = c.show.value;
|
|
1973
2006
|
c.members.querySelectorAll('.m-row').forEach(function (row) {
|
|
1974
2007
|
var named = !q || row.dataset.name.toLowerCase().indexOf(q) !== -1, any = false;
|
|
1975
2008
|
row.querySelectorAll('label.check').forEach(function (lab) {
|
|
@@ -1977,7 +2010,8 @@
|
|
|
1977
2010
|
var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
|
|
1978
2011
|
lab.textContent).toLowerCase();
|
|
1979
2012
|
lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
|
|
1980
|
-
|| !!prov && (fact(box.value).provider || '') !== prov
|
|
2013
|
+
|| !!prov && (fact(box.value).provider || '') !== prov
|
|
2014
|
+
|| show === 'ticked' && !box.checked || show === 'fits' && !!box.dataset.blocked;
|
|
1981
2015
|
any = any || !lab.hidden;
|
|
1982
2016
|
});
|
|
1983
2017
|
row.hidden = !any;
|
|
@@ -1986,6 +2020,23 @@
|
|
|
1986
2020
|
}
|
|
1987
2021
|
on('input', 'c-member-filter', filterMembers);
|
|
1988
2022
|
on('change', 'c-member-provider', filterMembers);
|
|
2023
|
+
on('change', 'c-member-show', filterMembers);
|
|
2024
|
+
// Refresh model details: the models and their facts fetched again, the list drawn
|
|
2025
|
+
// anew with the same ticks, search and choices. The model popup's directory too.
|
|
2026
|
+
function refreshMembers() {
|
|
2027
|
+
var chosen = picked(), out = $('c-refresh-out'), btn = $('c-refresh');
|
|
2028
|
+
btn.disabled = true; out.textContent = 'Refreshing…';
|
|
2029
|
+
return Promise.all([get('/api/collections'), directory()]).then(function (res) {
|
|
2030
|
+
C = res[0];
|
|
2031
|
+
drawMembers(chosen);
|
|
2032
|
+
applyTags();
|
|
2033
|
+
out.textContent = 'Up to date.';
|
|
2034
|
+
}).catch(function (err) { out.textContent = String(err); })
|
|
2035
|
+
.then(function () { btn.disabled = false; });
|
|
2036
|
+
}
|
|
2037
|
+
on('click', 'c-refresh', refreshMembers);
|
|
2038
|
+
// A model opened from the list may have been changed in its popup: what it now is, on close.
|
|
2039
|
+
md.addEventListener('close', function () { if (c.form && c.form.open) refreshMembers(); });
|
|
1989
2040
|
// Escape in a box with text clears it; only an empty box lets Escape close the popup.
|
|
1990
2041
|
on('keydown', 'c-member-filter', function (e) {
|
|
1991
2042
|
if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
|
|
@@ -2034,13 +2085,37 @@
|
|
|
2034
2085
|
c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
|
|
2035
2086
|
c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
|
|
2036
2087
|
drawTagPicks();
|
|
2037
|
-
c.
|
|
2088
|
+
c.find.value = ''; c.show.value = ''; c.prov.value = '';
|
|
2089
|
+
$('c-refresh-out').textContent = '';
|
|
2090
|
+
drawMembers(row ? row.members : []);
|
|
2091
|
+
c.shape.textContent = '';
|
|
2092
|
+
c.shape.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2093
|
+
C.shapes.forEach(function (s) {
|
|
2094
|
+
var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
|
|
2095
|
+
});
|
|
2096
|
+
c.reply.value = row ? row.reply : 'all';
|
|
2097
|
+
c.merger.textContent = '';
|
|
2098
|
+
c.merger.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2099
|
+
C.models.forEach(function (alias) {
|
|
2100
|
+
var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
|
|
2101
|
+
if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
|
|
2102
|
+
});
|
|
2103
|
+
showMerger(); applyTags();
|
|
2104
|
+
}
|
|
2105
|
+
// The member list: a table to scan down, one row per model (named by its shortest
|
|
2106
|
+
// alias; the name opens its popup, as everywhere), its context and what it can do
|
|
2107
|
+
// in columns of their own, then a chip per provider it is on, each ticking that
|
|
2108
|
+
// alias. A chip whose window differs from the row's says its own. `chosen`: ticked.
|
|
2109
|
+
function drawMembers(chosen) {
|
|
2110
|
+
var keepProv = c.prov.value;
|
|
2111
|
+
c.members.textContent = '';
|
|
2038
2112
|
c.prov.length = 1; // "Every provider", then each one a model is on
|
|
2039
2113
|
Array.from(new Set(C.models.map(function (a) { return fact(a).provider; }).filter(Boolean))).sort()
|
|
2040
2114
|
.forEach(function (p) { c.prov.appendChild(el('option', {value: p}, p)); });
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2115
|
+
c.prov.value = keepProv;
|
|
2116
|
+
var head = el('div', {class: 'm-head', 'aria-hidden': 'true'});
|
|
2117
|
+
['Model', 'Context', 'Can', 'On, tick one'].forEach(function (h) { head.appendChild(el('span', null, h)); });
|
|
2118
|
+
c.members.appendChild(head);
|
|
2044
2119
|
var groups = {};
|
|
2045
2120
|
C.models.forEach(function (a) { var f = fact(a).family || a; (groups[f] = groups[f] || []).push(a); });
|
|
2046
2121
|
Object.keys(groups).map(function (f) {
|
|
@@ -2048,18 +2123,22 @@
|
|
|
2048
2123
|
return {family: f, name: routes[0], routes: routes};
|
|
2049
2124
|
}).sort(function (x, y) { return x.name.toLowerCase() < y.name.toLowerCase() ? -1 : 1; }).forEach(function (g) {
|
|
2050
2125
|
var first = fact(g.name), caps = first.caps || {};
|
|
2051
|
-
var can = ['vision', 'tools'].filter(function (k) { return caps[k]; });
|
|
2052
2126
|
var line = el('div', {class: 'm-row', role: 'group', 'aria-label': g.name, 'data-family': g.family, 'data-name': g.name});
|
|
2053
2127
|
var name = el('div', {class: 'm-name'});
|
|
2054
|
-
name.appendChild(el('
|
|
2055
|
-
name.appendChild(el('span', {class: 'm-facts'}, [first.context ? tokens(first.context) : '', can.join(', '),
|
|
2056
|
-
first.price === 'unknown' ? '' : first.price].filter(Boolean).join(' · ')));
|
|
2128
|
+
name.appendChild(el('a', {href: '#', class: 'model mono', 'data-alias': g.name, title: 'Open ' + g.name}, g.name));
|
|
2057
2129
|
line.appendChild(name);
|
|
2130
|
+
line.appendChild(el('span', {class: 'm-ctx'}, first.context ? tokens(first.context) : '—'));
|
|
2131
|
+
var can = el('span', {class: 'm-can'});
|
|
2132
|
+
[['vision', 'vision'], ['tools', 'tools'], ['reasoning', 'reasons']].forEach(function (k) {
|
|
2133
|
+
if (caps[k[0]]) can.appendChild(el('span', {class: 'chip'}, k[1]));
|
|
2134
|
+
});
|
|
2135
|
+
if (first.price === 'free') can.appendChild(el('span', {class: 'chip ok'}, 'free'));
|
|
2136
|
+
line.appendChild(can);
|
|
2058
2137
|
var chips = el('div', {class: 'm-routes'});
|
|
2059
2138
|
g.routes.forEach(function (alias) {
|
|
2060
2139
|
var f = fact(alias);
|
|
2061
2140
|
var box = el('input', {type: 'checkbox', value: alias, 'aria-label': alias + ' on ' + (f.provider || 'its endpoint')});
|
|
2062
|
-
box.checked =
|
|
2141
|
+
box.checked = chosen.indexOf(alias) !== -1;
|
|
2063
2142
|
box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
|
|
2064
2143
|
var lab = el('label', {class: 'check', 'data-served': (C.served || {})[alias] || ''});
|
|
2065
2144
|
lab.appendChild(box);
|
|
@@ -2070,19 +2149,6 @@
|
|
|
2070
2149
|
line.appendChild(chips);
|
|
2071
2150
|
c.members.appendChild(line);
|
|
2072
2151
|
});
|
|
2073
|
-
c.shape.textContent = '';
|
|
2074
|
-
c.shape.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2075
|
-
C.shapes.forEach(function (s) {
|
|
2076
|
-
var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
|
|
2077
|
-
});
|
|
2078
|
-
c.reply.value = row ? row.reply : 'all';
|
|
2079
|
-
c.merger.textContent = '';
|
|
2080
|
-
c.merger.appendChild(el('option', {value: ''}, 'automatic'));
|
|
2081
|
-
C.models.forEach(function (alias) {
|
|
2082
|
-
var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
|
|
2083
|
-
if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
|
|
2084
|
-
});
|
|
2085
|
-
showMerger(); applyTags();
|
|
2086
2152
|
}
|
|
2087
2153
|
// What a collection's members can all do, in the words the tag checks use.
|
|
2088
2154
|
var CAN = {reasoning: 'reason'};
|
|
@@ -2635,7 +2701,8 @@
|
|
|
2635
2701
|
try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
|
|
2636
2702
|
}
|
|
2637
2703
|
// Pixel offsets that keep scatter labels apart: each label, in order, moves
|
|
2638
|
-
// up or down until it clears the ones already placed
|
|
2704
|
+
// up or down until it clears the ones already placed; null when no place is
|
|
2705
|
+
// free (that one is left to its hover, not drawn over another).
|
|
2639
2706
|
function spread(points, xpx, ypx) {
|
|
2640
2707
|
var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
|
|
2641
2708
|
return points.map(function (d) {
|
|
@@ -2647,7 +2714,7 @@
|
|
|
2647
2714
|
return tries[k];
|
|
2648
2715
|
}
|
|
2649
2716
|
}
|
|
2650
|
-
return
|
|
2717
|
+
return null;
|
|
2651
2718
|
});
|
|
2652
2719
|
}
|
|
2653
2720
|
function rows(n) { return 20 * n + 80; }
|
|
@@ -2657,6 +2724,16 @@
|
|
|
2657
2724
|
var keep = s.length - over - 1;
|
|
2658
2725
|
return over <= 0 ? s : s.slice(0, Math.ceil(keep / 2)) + '…' + s.slice(s.length - Math.floor(keep / 2));
|
|
2659
2726
|
}
|
|
2727
|
+
function fit(s, max) { return middle(s, s.length - max); }
|
|
2728
|
+
var FEW = 10; // under this many tasks or answers, a row is drawn faint
|
|
2729
|
+
// The 95% Wilson interval of k successes in n: [low, high]. Honest at small n and at
|
|
2730
|
+
// 0 or n of n, where k/n ± 1.96·se says nothing (3 of 3 is not "100% ± 0").
|
|
2731
|
+
function wilson(k, n) {
|
|
2732
|
+
if (!n) return [0, 1];
|
|
2733
|
+
var z = 1.96, p = k / n, z2 = z * z, den = 1 + z2 / n;
|
|
2734
|
+
var mid = p + z2 / (2 * n), half = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
|
|
2735
|
+
return [Math.max(0, (mid - half) / den), Math.min(1, (mid + half) / den)];
|
|
2736
|
+
}
|
|
2660
2737
|
|
|
2661
2738
|
// Trend: who moved, which way, by how much and how sure (serve.trend_data),
|
|
2662
2739
|
// in the window the card's select says. A row per model with a task to
|
|
@@ -2703,7 +2780,9 @@
|
|
|
2703
2780
|
var nameW = Math.min(260, Math.max(150, 7 * Math.max.apply(null, M.map(function (r) { return r.key.length; })) + 24));
|
|
2704
2781
|
var evW = 320;
|
|
2705
2782
|
w = $('chart-trend').clientWidth || w; // its own box: a phone's is narrower than width()'s 520
|
|
2706
|
-
|
|
2783
|
+
// Without the evidence column the names take what the bars do not need (260 px), not a
|
|
2784
|
+
// fixed 130 px: at the dashboard's width that cut "muse-glimmer-30b" to "muse-gl…mer-30b".
|
|
2785
|
+
if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, Math.max(130, w - 280)); }
|
|
2707
2786
|
var fits = Math.floor((nameW - 24) / 7), right = evW || 20, wide = w < nameW + right + 140;
|
|
2708
2787
|
w = Math.max(w, nameW + right + 140);
|
|
2709
2788
|
// Symmetric, to the biggest bar and room for its number; 20 pts at least.
|
|
@@ -2805,16 +2884,74 @@
|
|
|
2805
2884
|
// the time; under it, it is surer than it should be. Panels stack on a phone.
|
|
2806
2885
|
var DC = C.decide;
|
|
2807
2886
|
if (DC) put('chart-decide', function (w) {
|
|
2808
|
-
if (!DC.
|
|
2887
|
+
if (!(DC.scores || []).length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
|
|
2809
2888
|
var names = {};
|
|
2810
2889
|
DC.sets.forEach(function (s) { names[s[0]] = s[1]; });
|
|
2811
2890
|
var panels = DC.sets.map(function (s) { return s[1]; });
|
|
2891
|
+
// One colour per model across both charts, by name, never by rank.
|
|
2892
|
+
var models = DC.scores.map(function (p) { return p.model; })
|
|
2893
|
+
.filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
|
|
2894
|
+
var colors = {domain: models, range: [sky, good, signal, warn, muted, ink, err]};
|
|
2895
|
+
var box = el('div');
|
|
2896
|
+
box.appendChild(el('h3', {class: 'chart-head'}, 'Right of asked, per set'));
|
|
2897
|
+
box.appendChild(decideBars(w, names, panels, models, colors));
|
|
2898
|
+
// Calibration only says something once a model's answers spread over two or more
|
|
2899
|
+
// levels of sureness; while each sits in one top bin it is one dot in a corner.
|
|
2900
|
+
var levels = DC.points.some(function (p, i, all) {
|
|
2901
|
+
return all.some(function (q) { return q !== p && q.model === p.model && q.set === p.set; });
|
|
2902
|
+
});
|
|
2903
|
+
if (levels) {
|
|
2904
|
+
box.appendChild(el('h3', {class: 'chart-head'}, 'Calibration: said against right'));
|
|
2905
|
+
box.appendChild(decideCalibration(w, names, panels, models, colors));
|
|
2906
|
+
} else {
|
|
2907
|
+
box.appendChild(el('p', {class: 'hint'}, 'Calibration shows once a model gives answers at two or more levels'
|
|
2908
|
+
+ ' of sureness on a set; so far each answers at one, so the table\'s Brier and calibration gap say it.'));
|
|
2909
|
+
}
|
|
2910
|
+
return box;
|
|
2911
|
+
});
|
|
2912
|
+
// Right of asked as bars from 0: a small chart per set, a row per model asked on it,
|
|
2913
|
+
// best first. A model under the ranking floor (DC.low items) is drawn faint: shown,
|
|
2914
|
+
// not yet ranked.
|
|
2915
|
+
function decideBars(w, names, panels, models, colors) {
|
|
2916
|
+
var out = el('div');
|
|
2917
|
+
DC.sets.forEach(function (s) {
|
|
2918
|
+
var rows = DC.scores.filter(function (r) { return r.set === s[0]; }).map(function (r) {
|
|
2919
|
+
return Object.assign({share: r.correct / r.asked, ranked: r.asked >= DC.low}, r);
|
|
2920
|
+
}).sort(function (a, b) { return b.share - a.share || b.asked - a.asked || (a.model < b.model ? -1 : 1); });
|
|
2921
|
+
out.appendChild(el('p', {class: 'chart-label'}, s[1]));
|
|
2922
|
+
if (!rows.length) { out.appendChild(el('p', {class: 'hint'}, 'No model asked on this set yet.')); return; }
|
|
2923
|
+
out.appendChild(Plot.plot({
|
|
2924
|
+
width: w, height: rows.length * 26 + 40, style: style,
|
|
2925
|
+
marginLeft: w < 560 ? 120 : 160, marginRight: 50, marginTop: 4, marginBottom: 30,
|
|
2926
|
+
x: {domain: [0, 1], label: null, tickFormat: '.0%', ticks: 5, grid: true},
|
|
2927
|
+
y: {domain: rows.map(function (r) { return r.model; }), label: null},
|
|
2928
|
+
color: colors,
|
|
2929
|
+
marks: [
|
|
2930
|
+
Plot.barX(rows, {x: 'share', y: 'model', fill: 'model', insetTop: 4, insetBottom: 4, rx: 2,
|
|
2931
|
+
fillOpacity: function (r) { return r.ranked ? 1 : 0.4; },
|
|
2932
|
+
title: function (r) {
|
|
2933
|
+
return r.model + ' · ' + s[1] + '\n' + r.correct + ' right of ' + r.asked + ' asked ('
|
|
2934
|
+
+ Math.round(r.share * 100) + '%)' + (r.brier != null ? '\nBrier ' + r.brier.toFixed(3) : '')
|
|
2935
|
+
+ (r.ranked ? '' : '\nnot ranked yet: under ' + DC.low + ' items');
|
|
2936
|
+
}}),
|
|
2937
|
+
Plot.text(rows, {x: 'share', y: 'model', dx: 6, textAnchor: 'start', fill: ink,
|
|
2938
|
+
text: function (r) { return r.correct + '/' + r.asked; }}),
|
|
2939
|
+
Plot.ruleX([0], {stroke: ink})
|
|
2940
|
+
]
|
|
2941
|
+
}));
|
|
2942
|
+
});
|
|
2943
|
+
return out;
|
|
2944
|
+
}
|
|
2945
|
+
function decideCalibration(w, names, panels, models, colors) {
|
|
2812
2946
|
var pts = DC.points.map(function (p) { return Object.assign({panel: names[p.set]}, p); })
|
|
2813
2947
|
.sort(function (a, b) { return a.said - b.said; });
|
|
2814
|
-
|
|
2815
|
-
|
|
2948
|
+
// Both axes start just under the lowest answer, not at 0%: a decision model is
|
|
2949
|
+
// rarely under 50% sure of its own answer, so a 0-100% frame left every dot in
|
|
2950
|
+
// one corner under a long empty diagonal. Same floor on both, so it stays 45°.
|
|
2951
|
+
var low = Math.min.apply(null, pts.map(function (p) { return Math.min(p.said, p.right); }));
|
|
2952
|
+
var lo = Math.max(0, Math.min(0.9, Math.floor((low - 0.05) * 20) / 20));
|
|
2816
2953
|
var diag = [];
|
|
2817
|
-
panels.forEach(function (s) { diag.push({panel: s, x:
|
|
2954
|
+
panels.forEach(function (s) { diag.push({panel: s, x: lo, y: lo}, {panel: s, x: 1, y: 1}); });
|
|
2818
2955
|
function tip(p) {
|
|
2819
2956
|
return p.model + ' · ' + names[p.set] + '\n' + plural(p.n, 'answer') + ' it was about '
|
|
2820
2957
|
+ Math.round(p.said * 100) + '% sure of\n' + Math.round(p.right * 100) + '% of them right';
|
|
@@ -2825,10 +2962,11 @@
|
|
|
2825
2962
|
return Plot.plot(Object.assign(opts, {
|
|
2826
2963
|
width: narrow ? Math.min(w, 520) : w, height: narrow ? 3 * 230 + 70 : 320, style: style,
|
|
2827
2964
|
marginLeft: 52, marginRight: narrow ? 90 : 20, marginBottom: 44, marginTop: 28,
|
|
2828
|
-
|
|
2829
|
-
|
|
2965
|
+
inset: 14, // a dot at 100% clear of the frame
|
|
2966
|
+
x: {domain: [lo, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
|
|
2967
|
+
y: {domain: [lo, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
|
|
2830
2968
|
r: {range: [3, 13]},
|
|
2831
|
-
color: {legend: true,
|
|
2969
|
+
color: Object.assign({legend: true}, colors),
|
|
2832
2970
|
marks: [
|
|
2833
2971
|
Plot.frame({stroke: line}),
|
|
2834
2972
|
Plot.line(diag, Object.assign({x: 'x', y: 'y', z: 'panel', stroke: muted, strokeDasharray: '4 3'}, by)),
|
|
@@ -2837,17 +2975,24 @@
|
|
|
2837
2975
|
title: tip}, by))
|
|
2838
2976
|
]
|
|
2839
2977
|
}));
|
|
2840
|
-
}
|
|
2978
|
+
}
|
|
2841
2979
|
// Who to trust: finds real bugs (right) vs makes things up (up = less).
|
|
2842
2980
|
put('chart-overview', function (w) {
|
|
2843
2981
|
if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
|
|
2844
2982
|
var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
|
|
2845
2983
|
var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
|
|
2846
2984
|
var pxW = w - 110, pxH = 370;
|
|
2847
|
-
|
|
2985
|
+
// Named: the models in or next to the zone (the ones to pick from), then the clearest
|
|
2986
|
+
// warnings (most made up). The rest are on hover: named all at once they ran together.
|
|
2987
|
+
var zoneX = x0 + (1 - x0) / 2, zoneY = worst / 4;
|
|
2988
|
+
var near = function (d) { return d.recall >= zoneX - (1 - x0) * 0.08 && d.wrong <= zoneY * 1.4; };
|
|
2989
|
+
var worstFew = C.overview.slice().sort(function (a, b) { return b.wrong - a.wrong; }).slice(0, 3);
|
|
2990
|
+
var pts = C.overview.filter(function (d) { return near(d) || worstFew.indexOf(d) !== -1; })
|
|
2991
|
+
.sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
|
|
2848
2992
|
var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
|
|
2849
2993
|
function (d) { return d.wrong / worst * pxH; });
|
|
2850
|
-
var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
|
|
2994
|
+
var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
|
|
2995
|
+
.filter(Boolean);
|
|
2851
2996
|
return Plot.plot({
|
|
2852
2997
|
width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
2853
2998
|
x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
|
|
@@ -2875,71 +3020,72 @@
|
|
|
2875
3020
|
tasks[h.model] = bugs + clean;
|
|
2876
3021
|
var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
|
|
2877
3022
|
var fa = clean ? h['false alarm'] / clean : 0;
|
|
2878
|
-
|
|
3023
|
+
// Ranked on what is proven: the low end of the found share's 95% range less the HIGH
|
|
3024
|
+
// end of the false alarms' (never shown clean files: up to 100%), so a few tasks, or
|
|
3025
|
+
// none on clean code, cannot top the chart.
|
|
3026
|
+
score[h.model] = wilson(h.found, bugs)[0] - wilson(h['false alarm'], clean)[1];
|
|
2879
3027
|
segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
|
|
2880
3028
|
segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
|
|
2881
3029
|
segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
|
|
2882
3030
|
segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
|
|
2883
3031
|
});
|
|
2884
|
-
var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
|
|
3032
|
+
var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a] || tasks[b] - tasks[a]; });
|
|
2885
3033
|
return Plot.plot({
|
|
2886
|
-
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
3034
|
+
width: w, height: rows(models.length), style: style, marginLeft: w < 560 ? 150 : 250,
|
|
2887
3035
|
x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
|
|
2888
3036
|
tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
|
|
2889
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
|
|
3037
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + tasks[m] + ')'; }},
|
|
2890
3038
|
color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
|
|
2891
3039
|
marks: [
|
|
3040
|
+
// Under FEW tasks a model's bars are faint: shown, not much to go on.
|
|
2892
3041
|
Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
|
|
2893
|
-
|
|
3042
|
+
fillOpacity: function (d) { return tasks[d.model] < FEW ? 0.4 : 1; },
|
|
3043
|
+
title: function (d) { return d.model + ': ' + d.part + ', ' + d.n + (tasks[d.model] < FEW ? '\nfew tasks: little to go on' : ''); }}),
|
|
2894
3044
|
Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
|
|
2895
3045
|
]
|
|
2896
3046
|
});
|
|
2897
3047
|
});
|
|
2898
3048
|
|
|
2899
|
-
// Real work:
|
|
3049
|
+
// Real work: how often each model helped, with how sure that is. The dot is the share
|
|
3050
|
+
// helped, its size how many were judged, the line the range the true share likely sits
|
|
3051
|
+
// in (95% Wilson interval: wide for 3 of 3, narrow for 40 of 50). Ranked by the low end
|
|
3052
|
+
// of that range, so volume counts: 3 of 3 no longer outranks 40 of 50. The dashed line
|
|
3053
|
+
// is every model's offloads pooled.
|
|
2900
3054
|
put('chart-work', function (w) {
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
3055
|
+
if (!C.work.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
|
|
3056
|
+
var R = C.work.map(function (r) {
|
|
3057
|
+
var b = wilson(r.useful, r.judged), p = r.useful / r.judged;
|
|
3058
|
+
var verdict = b[0] >= 0.5 ? 'helps' : b[1] < 0.5 ? 'does not help' : 'too few to say';
|
|
3059
|
+
return Object.assign({p: p, lo: b[0], hi: b[1], verdict: verdict}, r);
|
|
3060
|
+
}).sort(function (a, b) { return b.lo - a.lo || b.judged - a.judged; });
|
|
3061
|
+
var all = R.reduce(function (s, r) { return [s[0] + r.useful, s[1] + r.judged]; }, [0, 0]);
|
|
3062
|
+
var pooled = all[0] / all[1];
|
|
3063
|
+
var narrowW = w < 560, nameW = narrowW ? 130 : 210;
|
|
2906
3064
|
return Plot.plot({
|
|
2907
|
-
width: w, height:
|
|
2908
|
-
x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
|
|
3065
|
+
width: w, height: 24 * R.length + 90, style: style, marginLeft: nameW, marginRight: 64, marginTop: 34,
|
|
3066
|
+
x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', ticks: 5, grid: true},
|
|
2909
3067
|
y: {domain: R.map(function (r) { return r.model; }), label: null,
|
|
2910
|
-
tickFormat: function (m) { return m
|
|
3068
|
+
tickFormat: function (m) { return fit(m, narrowW ? 16 : 26); }},
|
|
3069
|
+
r: {domain: [0, Math.max.apply(null, R.map(function (r) { return r.judged; }))], range: [3, 11]},
|
|
3070
|
+
color: {domain: ['helps', 'too few to say', 'does not help'], range: [good, muted, err], legend: true},
|
|
2911
3071
|
marks: [
|
|
2912
|
-
Plot.
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
Plot.
|
|
3072
|
+
Plot.ruleX([0.5], {stroke: line}),
|
|
3073
|
+
Plot.ruleX([pooled], {stroke: ink, strokeDasharray: '4 3', strokeOpacity: 0.6}),
|
|
3074
|
+
Plot.text([pooled], {x: function (d) { return d; }, frameAnchor: 'top', dy: -20, fill: muted, fontSize: 11,
|
|
3075
|
+
text: function () { return 'all models ' + Math.round(pooled * 100) + '%'; }}),
|
|
3076
|
+
Plot.ruleY(R, {y: 'model', x1: 'lo', x2: 'hi', stroke: 'verdict', strokeWidth: 2, strokeOpacity: 0.55}),
|
|
3077
|
+
Plot.dot(R, {x: 'p', y: 'model', r: 'judged', fill: 'verdict', stroke: halo, strokeWidth: 1.5,
|
|
3078
|
+
title: function (r) {
|
|
3079
|
+
return r.model + '\nhelped ' + r.useful + ' of ' + r.judged + ' judged (' + Math.round(r.p * 100) + '%; '
|
|
3080
|
+
+ r.offloads + ' offloads)\nlikely between ' + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%'
|
|
3081
|
+
+ (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : '');
|
|
3082
|
+
}}),
|
|
3083
|
+
Plot.text(R, {x: 'hi', y: 'model', dx: 8, textAnchor: 'start', fill: ink, fontSize: 11,
|
|
3084
|
+
text: function (r) { return r.useful + '/' + r.judged; }})
|
|
2917
3085
|
]
|
|
2918
3086
|
});
|
|
2919
3087
|
});
|
|
2920
3088
|
|
|
2921
|
-
// Did it answer: each probe of the last 14 days, by what came back.
|
|
2922
|
-
put('chart-reliability', function (w) {
|
|
2923
|
-
if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
|
|
2924
|
-
var total = {}, ok = {};
|
|
2925
|
-
C.reliability.forEach(function (d) {
|
|
2926
|
-
total[d.model] = (total[d.model] || 0) + d.count;
|
|
2927
|
-
if (d.outcome === 'answered') ok[d.model] = d.count;
|
|
2928
|
-
});
|
|
2929
|
-
var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
|
|
2930
|
-
var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
|
|
2931
|
-
entitlement: '#c0563f', other: muted, error: ink};
|
|
2932
|
-
var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
|
|
2933
|
-
return Plot.plot({
|
|
2934
|
-
width: w, height: rows(models.length), style: style, marginLeft: 250,
|
|
2935
|
-
x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
|
|
2936
|
-
y: {domain: models, label: null},
|
|
2937
|
-
color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
|
|
2938
|
-
marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
|
|
2939
|
-
title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
|
|
2940
|
-
});
|
|
2941
|
-
});
|
|
2942
|
-
|
|
2943
3089
|
// Speed vs quality: median seconds on our own tasks against findings located.
|
|
2944
3090
|
put('chart-speed', function (w) {
|
|
2945
3091
|
if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
|
|
@@ -2950,14 +3096,20 @@
|
|
|
2950
3096
|
var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
|
|
2951
3097
|
var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
|
|
2952
3098
|
function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
|
|
2953
|
-
var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
|
|
3099
|
+
var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
|
|
3100
|
+
.filter(Boolean);
|
|
2954
3101
|
return Plot.plot({
|
|
2955
3102
|
width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
|
|
2956
3103
|
x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
|
|
2957
3104
|
y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
|
|
2958
3105
|
marks: [
|
|
3106
|
+
// The zone to be in, as on Who to trust: the faster half (on the log scale) and the
|
|
3107
|
+
// top quarter of findings. Outlined, so it reads on a dark ground too.
|
|
3108
|
+
Plot.rect([{x1: lo, x2: Math.sqrt(lo * hi), y1: y0 + (1 - y0) * 3 / 4, y2: 1}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
|
|
3109
|
+
fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
|
|
2959
3110
|
Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
|
|
2960
|
-
Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink
|
|
3111
|
+
Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink,
|
|
3112
|
+
stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
|
|
2961
3113
|
Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
|
|
2962
3114
|
title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
|
|
2963
3115
|
]
|
|
@@ -2972,25 +3124,41 @@
|
|
|
2972
3124
|
return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
|
|
2973
3125
|
});
|
|
2974
3126
|
tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
|
|
2975
|
-
|
|
2976
|
-
|
|
3127
|
+
// Each group's name over its first task, on two staggered rows: side by side they
|
|
3128
|
+
// ran into one another wherever a group is only a few tasks wide.
|
|
3129
|
+
var heads = C.families.filter(function (f) { return f in first; })
|
|
3130
|
+
.map(function (f, i) { return {task: first[f], name: f, row: i % 2}; });
|
|
2977
3131
|
// By total found, so a model that played three tasks does not top one that played forty.
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
3132
|
+
// A model with no scored task (only grey) gets no row: it is named under the chart.
|
|
3133
|
+
var sum = {}, scored = {};
|
|
3134
|
+
C.heat.forEach(function (d) {
|
|
3135
|
+
sum[d.model] = (sum[d.model] || 0) + (d.rate || 0);
|
|
3136
|
+
if (d.rate != null) scored[d.model] = true;
|
|
3137
|
+
});
|
|
3138
|
+
var models = Object.keys(sum).filter(function (m) { return scored[m]; }).sort(function (a, b) { return sum[b] - sum[a]; });
|
|
3139
|
+
var idle = Object.keys(sum).filter(function (m) { return !scored[m]; }).sort();
|
|
3140
|
+
var shown = C.heat.filter(function (d) { return scored[d.model]; });
|
|
3141
|
+
var nameW = w < 560 ? 130 : 230;
|
|
2981
3142
|
function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
3143
|
+
var box = el('div');
|
|
3144
|
+
box.appendChild(Plot.plot({
|
|
3145
|
+
width: w, height: 22 * models.length + 84, style: style, marginLeft: nameW, marginTop: 44, padding: 0.08,
|
|
3146
|
+
x: {domain: tasks, axis: null},
|
|
3147
|
+
y: {domain: models, label: null, tickSize: 0, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 30); }},
|
|
2986
3148
|
color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
|
|
2987
3149
|
marks: [
|
|
2988
|
-
Plot.cell(
|
|
2989
|
-
Plot.cell(
|
|
2990
|
-
Plot.dot(
|
|
2991
|
-
Plot.ruleX(
|
|
3150
|
+
Plot.cell(shown.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
|
|
3151
|
+
Plot.cell(shown.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
|
|
3152
|
+
Plot.dot(shown.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
|
|
3153
|
+
Plot.ruleX(heads, {x: 'task', stroke: ink, strokeOpacity: 0.25, dx: -2}),
|
|
3154
|
+
Plot.text(heads.filter(function (h) { return !h.row; }), {x: 'task', frameAnchor: 'top', dy: -30, dx: -2,
|
|
3155
|
+
text: 'name', textAnchor: 'start', fontSize: 11, fill: ink}),
|
|
3156
|
+
Plot.text(heads.filter(function (h) { return h.row; }), {x: 'task', frameAnchor: 'top', dy: -14, dx: -2,
|
|
3157
|
+
text: 'name', textAnchor: 'start', fontSize: 11, fill: ink})
|
|
2992
3158
|
]
|
|
2993
|
-
});
|
|
3159
|
+
}));
|
|
3160
|
+
if (idle.length) box.appendChild(el('p', {class: 'hint'}, 'No scored task yet, so no row: ' + idle.join(', ') + '.'));
|
|
3161
|
+
return box;
|
|
2994
3162
|
});
|
|
2995
3163
|
|
|
2996
3164
|
// Vision tasks: one square per field and model, the task's name over its first field.
|
|
@@ -3037,39 +3205,66 @@
|
|
|
3037
3205
|
kinds.forEach(function (k) { if (W[m][k]) data.push({model: m, kind: k, count: W[m][k]}); total[m] = (total[m] || 0) + W[m][k]; });
|
|
3038
3206
|
});
|
|
3039
3207
|
if (!data.length) return el('p', {class: 'hint'}, 'No vision panels through the gateway yet.');
|
|
3040
|
-
// The models doing the most image work first: their bars are the evidence.
|
|
3041
|
-
|
|
3208
|
+
// The models doing the most image work first: their bars are the evidence. Under FEW
|
|
3209
|
+
// seats a row was mostly "no answer" and said little: those are listed under the chart.
|
|
3210
|
+
var all = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
|
|
3211
|
+
var models = all.filter(function (m) { return total[m] >= FEW; });
|
|
3212
|
+
var few = all.filter(function (m) { return total[m] < FEW; });
|
|
3213
|
+
if (!models.length) { models = all; few = []; }
|
|
3214
|
+
data = data.filter(function (d) { return models.indexOf(d.model) !== -1; });
|
|
3042
3215
|
var colour = {'checked, helped': good, 'checked, did not help': err, 'useful, no claims': sky,
|
|
3043
3216
|
'no use, no claims': warn, 'not judged': line, 'no answer': muted};
|
|
3044
3217
|
// The claims checked by hand, beside the bar: the only right-or-wrong count real work has.
|
|
3045
3218
|
var claims = models.filter(function (m) { return W[m].real + W[m].false; }).map(function (m) {
|
|
3046
3219
|
return {model: m, text: W[m].real + ' of ' + (W[m].real + W[m].false) + ' claims right'};
|
|
3047
3220
|
});
|
|
3048
|
-
|
|
3049
|
-
|
|
3221
|
+
var narrowV = w < 560, box = el('div');
|
|
3222
|
+
box.appendChild(Plot.plot({
|
|
3223
|
+
width: w, height: rows(models.length), style: style, marginLeft: narrowV ? 150 : 260, marginRight: narrowV ? 20 : 150,
|
|
3050
3224
|
x: {label: 'Share of its seats on image panels →', tickFormat: '.0%', grid: true, domain: [0, 1]},
|
|
3051
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + total[m] + ')'; }},
|
|
3225
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, narrowV ? 16 : 30) + ' (' + total[m] + ')'; }},
|
|
3052
3226
|
color: {legend: true, domain: kinds, range: kinds.map(function (k) { return colour[k]; })},
|
|
3053
3227
|
marks: [
|
|
3054
3228
|
Plot.barX(data, Plot.stackX({x: 'count', y: 'model', fill: 'kind', offset: 'normalize', order: kinds,
|
|
3055
3229
|
title: function (d) { return d.model + ': ' + d.kind + ', ' + d.count + ' of ' + total[d.model] + ' seats'; }})),
|
|
3056
|
-
Plot.text(
|
|
3230
|
+
Plot.text(narrowV ? [] : claims.filter(function (c) { return models.indexOf(c.model) !== -1; }),
|
|
3231
|
+
{x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
|
|
3057
3232
|
]
|
|
3058
|
-
});
|
|
3233
|
+
}));
|
|
3234
|
+
if (few.length) box.appendChild(el('p', {class: 'hint'}, 'Under ' + FEW + ' seats, so no bar: '
|
|
3235
|
+
+ few.map(function (m) { return m + ' (' + total[m] + ')'; }).join(', ') + '. The table has them all.'));
|
|
3236
|
+
return box;
|
|
3059
3237
|
});
|
|
3060
3238
|
|
|
3061
|
-
// Confidently wrong: stacked by kind
|
|
3239
|
+
// Confidently wrong: stacked by kind. A thin line runs over each bar for the range the
|
|
3240
|
+
// true share likely sits in (95% Wilson), so 0 of 2 shows how bad it could be, not a
|
|
3241
|
+
// clean record. Worst proven at the top: by the low end of that range. Few answers: faint.
|
|
3062
3242
|
put('chart-wrong', function (w) {
|
|
3063
|
-
var T = C.wrongTotals;
|
|
3064
|
-
|
|
3243
|
+
var T = C.wrongTotals, R = {};
|
|
3244
|
+
Object.keys(T).forEach(function (m) {
|
|
3245
|
+
var b = wilson(T[m].wrong, T[m].answers);
|
|
3246
|
+
R[m] = {model: m, lo: b[0], hi: b[1], few: T[m].answers < FEW};
|
|
3247
|
+
});
|
|
3248
|
+
var models = Object.keys(T).sort(function (a, b) {
|
|
3249
|
+
return R[b].lo - R[a].lo || T[b].rate - T[a].rate || T[b].answers - T[a].answers;
|
|
3250
|
+
});
|
|
3251
|
+
var ranges = models.map(function (m) { return R[m]; });
|
|
3252
|
+
var top = Math.min(1, Math.max.apply(null, ranges.map(function (r) { return r.hi; })) + 0.02);
|
|
3065
3253
|
return Plot.plot({
|
|
3066
|
-
width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
|
|
3067
|
-
x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
|
|
3254
|
+
width: w, height: 20 * models.length + 70, style: style, marginLeft: w < 560 ? 150 : 250,
|
|
3255
|
+
x: {domain: [0, top], label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
|
|
3068
3256
|
// The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
|
|
3069
|
-
y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
|
|
3257
|
+
y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + T[m].answers + ')'; }},
|
|
3070
3258
|
color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
|
|
3071
3259
|
marks: [
|
|
3072
|
-
Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind',
|
|
3260
|
+
Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', insetTop: 3, insetBottom: 3,
|
|
3261
|
+
fillOpacity: function (d) { return R[d.model].few ? 0.4 : 1; },
|
|
3262
|
+
title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
|
|
3263
|
+
Plot.ruleY(ranges, {y: 'model', x1: 'lo', x2: 'hi', stroke: ink, strokeOpacity: 0.45, strokeWidth: 1.5,
|
|
3264
|
+
title: function (r) {
|
|
3265
|
+
return r.model + ': ' + T[r.model].wrong + ' of ' + T[r.model].answers + ' confidently wrong\nlikely between '
|
|
3266
|
+
+ Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%' + (r.few ? '\nfew answers: little to go on' : '');
|
|
3267
|
+
}}),
|
|
3073
3268
|
Plot.ruleX([0], {stroke: ink})
|
|
3074
3269
|
]
|
|
3075
3270
|
});
|