@wendellresearch-ai/benchwarmers 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ui/bench.js CHANGED
@@ -55,30 +55,33 @@
55
55
  askD.addEventListener('close', function () { answer(false); }); // Escape
56
56
 
57
57
  // ---- restart (serve.restart): the page reloads once the new one answers ----
58
- function restart(force) {
59
- var rs = $('restart'), out = $('restart-out');
58
+ // One button on Settings (restart) and one on Info (restart-info), each with its own status line.
59
+ function restart(id, force) {
60
+ var rs = $(id), out = $(id + '-out');
60
61
  rs.disabled = true; out.textContent = 'Restarting…';
61
62
  post('/api/restart', {force: force}).then(function (res) {
62
- if (res.ok) return back(Date.now());
63
+ if (res.ok) return back(out, Date.now());
63
64
  rs.disabled = false; out.textContent = '';
64
65
  if (!res.busy) { listErrors(out, res.errors); return; }
65
66
  ask('Still in progress: ' + res.errors.join('; ') + '. Restart anyway and cut them off?', 'Restart anyway', true)
66
- .then(function (yes) { if (yes) restart(true); });
67
+ .then(function (yes) { if (yes) restart(id, true); });
67
68
  }).catch(function (err) { rs.disabled = false; out.textContent = String(err); });
68
69
  }
69
70
  // The old one stops half a second after answering; /health answers again once the new one is up.
70
- function back(since) {
71
+ function back(out, since) {
71
72
  setTimeout(function () {
72
73
  fetch('/health', {cache: 'no-store'}).then(function (r) { if (!r.ok) throw r.status; location.reload(); })
73
74
  .catch(function () {
74
- if (Date.now() - since < 120000) back(since);
75
- else $('restart-out').textContent = 'Not back after two minutes: look at whatever runs it.';
75
+ if (Date.now() - since < 120000) back(out, since);
76
+ else out.textContent = 'Not back after two minutes: look at whatever runs it.';
76
77
  });
77
78
  }, 1500);
78
79
  }
79
- on('click', 'restart', function () {
80
- ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
81
- .then(function (yes) { if (yes) restart(false); });
80
+ ['restart', 'restart-info'].forEach(function (id) {
81
+ on('click', id, function () {
82
+ ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
83
+ .then(function (yes) { if (yes) restart(id, false); });
84
+ });
82
85
  });
83
86
 
84
87
  // ---- Export and import (serve.export_zip, import_zip) ----------------
@@ -250,13 +253,21 @@
250
253
  var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
251
254
  var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
252
255
  scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
253
- r.hidden = (q && r.textContent.toLowerCase().indexOf(q) === -1)
256
+ // A group's name (data-conn: Models' connection) counts as text of each of its rows.
257
+ var g = r.closest('[data-conn]'), text = r.textContent + (g ? ' ' + g.dataset.conn : '');
258
+ r.hidden = (q && text.toLowerCase().indexOf(q) === -1)
254
259
  || facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
255
260
  });
256
261
  // A match inside a folded list (an endpoint's models) opens it.
257
262
  if (q) scope.querySelectorAll('details.models').forEach(function (d) {
258
263
  if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
259
264
  });
265
+ // A group with no row left is hidden; one with a match opens.
266
+ scope.querySelectorAll('[data-conn]').forEach(function (g) {
267
+ var any = !!g.querySelector('tbody tr:not([hidden])');
268
+ g.hidden = !!q && !any;
269
+ if (q && any && g.tagName === 'DETAILS') g.open = true;
270
+ });
260
271
  }
261
272
  function scopeOf(c) {
262
273
  var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
@@ -490,7 +501,7 @@
490
501
  reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
491
502
  qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why'), roleHint: $('m-role-hint')};
492
503
  m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
493
- var editing = null;
504
+ var editing = null, shown = {};
494
505
 
495
506
  function openModel(alias) {
496
507
  var model = D.models[alias], gone = (D.retired || {})[alias];
@@ -539,6 +550,8 @@
539
550
  + (hint.at || '').slice(0, 10) + '). Only a suggestion: the role is the one you pick above.' : '';
540
551
  m.save.disabled = false; m.save.textContent = 'Save';
541
552
  m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
553
+ // What the popup opened with: Save sends only the parts that differ from it.
554
+ shown = {model: JSON.stringify(modelBody()), facts: factsBody(), opts: optsBody()};
542
555
  }
543
556
  if (!md.open) md.showModal(); // open already: shown again as it now is (shownAgain)
544
557
  }
@@ -577,7 +590,7 @@
577
590
  var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
578
591
  box.hidden = profile.hidden = decision || !f;
579
592
  profile.textContent = '';
580
- $('f-errors').textContent = ''; $('f-done').textContent = '';
593
+ $('f-errors').textContent = '';
581
594
  if (box.hidden) return;
582
595
  var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
583
596
  // At a glance, as models.dev shows a model: what goes in, what comes out.
@@ -621,40 +634,25 @@
621
634
  var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
622
635
  $('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
623
636
  }
624
- $('f-save').addEventListener('click', function () {
625
- if (!editing) return;
637
+ // The facts as typed, as /api/model-facts takes them; null when the box is not shown.
638
+ function factsBody() {
639
+ if ($('m-facts').hidden) return null;
626
640
  var facts = {limits: {}};
627
641
  FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
628
642
  FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
629
643
  if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
630
644
  document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
631
- $('f-errors').textContent = ''; $('f-done').textContent = '';
632
- var alias = editing;
633
- post('/api/model-facts', {alias: alias, facts: facts}).then(function (r) {
634
- if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
635
- // The tables that show facts (At a glance, Models, Quotas) follow once the popup closes (live).
636
- shownAgain(alias, 'f-done', 'Saved. The tables show it once this closes.');
637
- });
638
- });
645
+ return JSON.stringify(facts);
646
+ }
639
647
  // ---- call options: how the bench asks this model (serve.save_model_options)
640
648
  function fillOptions(alias, decision) {
641
649
  var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
642
650
  $('m-opts').hidden = decision;
643
- $('co-errors').textContent = ''; $('co-done').textContent = '';
651
+ $('co-errors').textContent = '';
644
652
  $('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
645
653
  $('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
646
654
  }
647
- $('co-save').addEventListener('click', function () {
648
- if (!editing) return;
649
- var v = $('co-max').value.trim();
650
- $('co-errors').textContent = ''; $('co-done').textContent = 'Saving…'; // the llm CLI is told too: seconds
651
- var alias = editing;
652
- post('/api/model-options', {alias: alias, max_tokens: v ? Number(v) : null}).then(function (r) {
653
- if (!r.ok) { $('co-done').textContent = ''; listErrors($('co-errors'), r.errors); return; }
654
- ownChange(); // only this popup shows them
655
- shownAgain(alias, 'co-done', 'Saved' + (r.note ? '; ' + r.note : '') + '.');
656
- });
657
- });
655
+ function optsBody() { return $('m-opts').hidden ? null : $('co-max').value.trim(); }
658
656
 
659
657
  function refresh() {
660
658
  m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
@@ -690,22 +688,57 @@
690
688
  then(res);
691
689
  };
692
690
  }
693
- m.save.addEventListener('click', function () {
694
- if (!editing) return;
695
- var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
691
+ function modelBody() {
692
+ var body = {new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
696
693
  if (!m.qtypes.hidden) {
697
694
  body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
698
695
  function (c) { return c.checked; }).map(function (c) { return c.value; });
699
696
  }
700
- (body.new_alias === editing ? Promise.resolve(true)
701
- : ask('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
697
+ return body;
698
+ }
699
+ // The one Save: facts, then call options, then the model itself (alias, role, ad hoc,
700
+ // question types), each only if changed. The model goes last: a rename moves the
701
+ // facts and options saved under the old alias with it. The first refusal stops it.
702
+ m.save.addEventListener('click', function () {
703
+ if (!editing) return;
704
+ var alias = editing, body = modelBody(), facts = factsBody(), opts = optsBody(), said = [];
705
+ var steps = [];
706
+ if (facts !== shown.facts) steps.push(function () {
707
+ return post('/api/model-facts', {alias: alias, facts: JSON.parse(facts)}).then(function (r) {
708
+ if (!r.ok) { listErrors($('f-errors'), r.errors); $('m-facts').open = true; throw 'refused'; }
709
+ said.push('model facts');
710
+ });
711
+ });
712
+ if (opts !== shown.opts) steps.push(function () { // the llm CLI is told too: seconds
713
+ return post('/api/model-options', {alias: alias, max_tokens: opts ? Number(opts) : null}).then(function (r) {
714
+ if (!r.ok) { listErrors($('co-errors'), r.errors); $('m-opts').open = true; throw 'refused'; }
715
+ ownChange(); // only this popup shows them
716
+ said.push('call options' + (r.note ? ' (' + r.note + ')' : ''));
717
+ });
718
+ });
719
+ if (JSON.stringify(body) !== shown.model) steps.push(function () {
720
+ body.alias = alias;
721
+ return post('/api/model', body).then(function (r) {
722
+ if (!r.ok) { listErrors(m.errors, r.errors); throw 'refused'; }
723
+ said.push('the model (' + done(r).replace(/^Done: |\.$/g, '') + ')');
724
+ if (body.new_alias !== alias) { m.title.textContent = body.new_alias; refreshPanes(); }
725
+ });
726
+ });
727
+ m.errors.textContent = ''; m.done.textContent = ''; $('f-errors').textContent = ''; $('co-errors').textContent = '';
728
+ if (!steps.length) { m.done.textContent = 'Nothing changed.'; return; }
729
+ (body.new_alias === alias ? Promise.resolve(true)
730
+ : ask('Rename ' + alias + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
702
731
  ).then(function (yes) {
703
732
  if (!yes) return;
704
- m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
705
- post('/api/model', body).then(after(m.save, 'Save', function (res) {
706
- if (body.new_alias !== body.alias) { m.title.textContent = body.new_alias; refreshPanes(); }
707
- shownAgain(body.new_alias, 'm-done', done(res));
708
- })).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
733
+ m.save.disabled = true; m.save.textContent = 'Saving…';
734
+ steps.reduce(function (p, step) { return p.then(step); }, Promise.resolve()).then(function () {
735
+ // The tables that show facts follow once the popup closes (live).
736
+ shownAgain(body.new_alias, 'm-done', 'Saved: ' + said.join(' · ') + '.');
737
+ }).catch(function (err) {
738
+ if (err !== 'refused') m.errors.textContent = String(err);
739
+ if (said.length) m.done.textContent = 'Saved before that: ' + said.join(' · ') + '.';
740
+ m.save.disabled = false; m.save.textContent = 'Save';
741
+ });
709
742
  });
710
743
  });
711
744
  // The confirm is the last word: the popup closes at once and the page
@@ -1855,7 +1888,7 @@
1855
1888
  pick: $('g-pick'), pickConf: $('g-pick-conf'),
1856
1889
  gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
1857
1890
  del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
1858
- seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
1891
+ seats: $('c-seats'), prov: $('c-member-provider'), show: $('c-member-show'), adHoc: $('c-ad-hoc'),
1859
1892
  ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
1860
1893
  return loadCollections();
1861
1894
  };
@@ -1969,7 +2002,7 @@
1969
2002
  // the seats above the list always show what is picked, each with its ×, and
1970
2003
  // every count still includes it. A row goes when all its chips do.
1971
2004
  function filterMembers() {
1972
- var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
2005
+ var q = c.find.value.trim().toLowerCase(), prov = c.prov.value, show = c.show.value;
1973
2006
  c.members.querySelectorAll('.m-row').forEach(function (row) {
1974
2007
  var named = !q || row.dataset.name.toLowerCase().indexOf(q) !== -1, any = false;
1975
2008
  row.querySelectorAll('label.check').forEach(function (lab) {
@@ -1977,7 +2010,8 @@
1977
2010
  var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
1978
2011
  lab.textContent).toLowerCase();
1979
2012
  lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
1980
- || !!prov && (fact(box.value).provider || '') !== prov;
2013
+ || !!prov && (fact(box.value).provider || '') !== prov
2014
+ || show === 'ticked' && !box.checked || show === 'fits' && !!box.dataset.blocked;
1981
2015
  any = any || !lab.hidden;
1982
2016
  });
1983
2017
  row.hidden = !any;
@@ -1986,6 +2020,23 @@
1986
2020
  }
1987
2021
  on('input', 'c-member-filter', filterMembers);
1988
2022
  on('change', 'c-member-provider', filterMembers);
2023
+ on('change', 'c-member-show', filterMembers);
2024
+ // Refresh model details: the models and their facts fetched again, the list drawn
2025
+ // anew with the same ticks, search and choices. The model popup's directory too.
2026
+ function refreshMembers() {
2027
+ var chosen = picked(), out = $('c-refresh-out'), btn = $('c-refresh');
2028
+ btn.disabled = true; out.textContent = 'Refreshing…';
2029
+ return Promise.all([get('/api/collections'), directory()]).then(function (res) {
2030
+ C = res[0];
2031
+ drawMembers(chosen);
2032
+ applyTags();
2033
+ out.textContent = 'Up to date.';
2034
+ }).catch(function (err) { out.textContent = String(err); })
2035
+ .then(function () { btn.disabled = false; });
2036
+ }
2037
+ on('click', 'c-refresh', refreshMembers);
2038
+ // A model opened from the list may have been changed in its popup: what it now is, on close.
2039
+ md.addEventListener('close', function () { if (c.form && c.form.open) refreshMembers(); });
1989
2040
  // Escape in a box with text clears it; only an empty box lets Escape close the popup.
1990
2041
  on('keydown', 'c-member-filter', function (e) {
1991
2042
  if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
@@ -2034,13 +2085,37 @@
2034
2085
  c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
2035
2086
  c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
2036
2087
  drawTagPicks();
2037
- c.members.textContent = ''; c.find.value = '';
2088
+ c.find.value = ''; c.show.value = ''; c.prov.value = '';
2089
+ $('c-refresh-out').textContent = '';
2090
+ drawMembers(row ? row.members : []);
2091
+ c.shape.textContent = '';
2092
+ c.shape.appendChild(el('option', {value: ''}, 'automatic'));
2093
+ C.shapes.forEach(function (s) {
2094
+ var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
2095
+ });
2096
+ c.reply.value = row ? row.reply : 'all';
2097
+ c.merger.textContent = '';
2098
+ c.merger.appendChild(el('option', {value: ''}, 'automatic'));
2099
+ C.models.forEach(function (alias) {
2100
+ var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
2101
+ if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
2102
+ });
2103
+ showMerger(); applyTags();
2104
+ }
2105
+ // The member list: a table to scan down, one row per model (named by its shortest
2106
+ // alias; the name opens its popup, as everywhere), its context and what it can do
2107
+ // in columns of their own, then a chip per provider it is on, each ticking that
2108
+ // alias. A chip whose window differs from the row's says its own. `chosen`: ticked.
2109
+ function drawMembers(chosen) {
2110
+ var keepProv = c.prov.value;
2111
+ c.members.textContent = '';
2038
2112
  c.prov.length = 1; // "Every provider", then each one a model is on
2039
2113
  Array.from(new Set(C.models.map(function (a) { return fact(a).provider; }).filter(Boolean))).sort()
2040
2114
  .forEach(function (p) { c.prov.appendChild(el('option', {value: p}, p)); });
2041
- // One row per model, named by its shortest alias, with what the first
2042
- // route says it holds and can do; each provider it is on is a chip that
2043
- // ticks that alias. A chip whose window differs from the row's says its own.
2115
+ c.prov.value = keepProv;
2116
+ var head = el('div', {class: 'm-head', 'aria-hidden': 'true'});
2117
+ ['Model', 'Context', 'Can', 'On, tick one'].forEach(function (h) { head.appendChild(el('span', null, h)); });
2118
+ c.members.appendChild(head);
2044
2119
  var groups = {};
2045
2120
  C.models.forEach(function (a) { var f = fact(a).family || a; (groups[f] = groups[f] || []).push(a); });
2046
2121
  Object.keys(groups).map(function (f) {
@@ -2048,18 +2123,22 @@
2048
2123
  return {family: f, name: routes[0], routes: routes};
2049
2124
  }).sort(function (x, y) { return x.name.toLowerCase() < y.name.toLowerCase() ? -1 : 1; }).forEach(function (g) {
2050
2125
  var first = fact(g.name), caps = first.caps || {};
2051
- var can = ['vision', 'tools'].filter(function (k) { return caps[k]; });
2052
2126
  var line = el('div', {class: 'm-row', role: 'group', 'aria-label': g.name, 'data-family': g.family, 'data-name': g.name});
2053
2127
  var name = el('div', {class: 'm-name'});
2054
- name.appendChild(el('span', {class: 'mono'}, g.name));
2055
- name.appendChild(el('span', {class: 'm-facts'}, [first.context ? tokens(first.context) : '', can.join(', '),
2056
- first.price === 'unknown' ? '' : first.price].filter(Boolean).join(' · ')));
2128
+ name.appendChild(el('a', {href: '#', class: 'model mono', 'data-alias': g.name, title: 'Open ' + g.name}, g.name));
2057
2129
  line.appendChild(name);
2130
+ line.appendChild(el('span', {class: 'm-ctx'}, first.context ? tokens(first.context) : '—'));
2131
+ var can = el('span', {class: 'm-can'});
2132
+ [['vision', 'vision'], ['tools', 'tools'], ['reasoning', 'reasons']].forEach(function (k) {
2133
+ if (caps[k[0]]) can.appendChild(el('span', {class: 'chip'}, k[1]));
2134
+ });
2135
+ if (first.price === 'free') can.appendChild(el('span', {class: 'chip ok'}, 'free'));
2136
+ line.appendChild(can);
2058
2137
  var chips = el('div', {class: 'm-routes'});
2059
2138
  g.routes.forEach(function (alias) {
2060
2139
  var f = fact(alias);
2061
2140
  var box = el('input', {type: 'checkbox', value: alias, 'aria-label': alias + ' on ' + (f.provider || 'its endpoint')});
2062
- box.checked = !!row && row.members.indexOf(alias) !== -1;
2141
+ box.checked = chosen.indexOf(alias) !== -1;
2063
2142
  box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
2064
2143
  var lab = el('label', {class: 'check', 'data-served': (C.served || {})[alias] || ''});
2065
2144
  lab.appendChild(box);
@@ -2070,19 +2149,6 @@
2070
2149
  line.appendChild(chips);
2071
2150
  c.members.appendChild(line);
2072
2151
  });
2073
- c.shape.textContent = '';
2074
- c.shape.appendChild(el('option', {value: ''}, 'automatic'));
2075
- C.shapes.forEach(function (s) {
2076
- var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
2077
- });
2078
- c.reply.value = row ? row.reply : 'all';
2079
- c.merger.textContent = '';
2080
- c.merger.appendChild(el('option', {value: ''}, 'automatic'));
2081
- C.models.forEach(function (alias) {
2082
- var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
2083
- if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
2084
- });
2085
- showMerger(); applyTags();
2086
2152
  }
2087
2153
  // What a collection's members can all do, in the words the tag checks use.
2088
2154
  var CAN = {reasoning: 'reason'};
@@ -2635,7 +2701,8 @@
2635
2701
  try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
2636
2702
  }
2637
2703
  // Pixel offsets that keep scatter labels apart: each label, in order, moves
2638
- // up or down until it clears the ones already placed.
2704
+ // up or down until it clears the ones already placed; null when no place is
2705
+ // free (that one is left to its hover, not drawn over another).
2639
2706
  function spread(points, xpx, ypx) {
2640
2707
  var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
2641
2708
  return points.map(function (d) {
@@ -2647,7 +2714,7 @@
2647
2714
  return tries[k];
2648
2715
  }
2649
2716
  }
2650
- return tries[0];
2717
+ return null;
2651
2718
  });
2652
2719
  }
2653
2720
  function rows(n) { return 20 * n + 80; }
@@ -2657,6 +2724,16 @@
2657
2724
  var keep = s.length - over - 1;
2658
2725
  return over <= 0 ? s : s.slice(0, Math.ceil(keep / 2)) + '…' + s.slice(s.length - Math.floor(keep / 2));
2659
2726
  }
2727
+ function fit(s, max) { return middle(s, s.length - max); }
2728
+ var FEW = 10; // under this many tasks or answers, a row is drawn faint
2729
+ // The 95% Wilson interval of k successes in n: [low, high]. Honest at small n and at
2730
+ // 0 or n of n, where k/n ± 1.96·se says nothing (3 of 3 is not "100% ± 0").
2731
+ function wilson(k, n) {
2732
+ if (!n) return [0, 1];
2733
+ var z = 1.96, p = k / n, z2 = z * z, den = 1 + z2 / n;
2734
+ var mid = p + z2 / (2 * n), half = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
2735
+ return [Math.max(0, (mid - half) / den), Math.min(1, (mid + half) / den)];
2736
+ }
2660
2737
 
2661
2738
  // Trend: who moved, which way, by how much and how sure (serve.trend_data),
2662
2739
  // in the window the card's select says. A row per model with a task to
@@ -2703,7 +2780,9 @@
2703
2780
  var nameW = Math.min(260, Math.max(150, 7 * Math.max.apply(null, M.map(function (r) { return r.key.length; })) + 24));
2704
2781
  var evW = 320;
2705
2782
  w = $('chart-trend').clientWidth || w; // its own box: a phone's is narrower than width()'s 520
2706
- if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, 130); }
2783
+ // Without the evidence column the names take what the bars do not need (260 px), not a
2784
+ // fixed 130 px: at the dashboard's width that cut "muse-glimmer-30b" to "muse-gl…mer-30b".
2785
+ if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, Math.max(130, w - 280)); }
2707
2786
  var fits = Math.floor((nameW - 24) / 7), right = evW || 20, wide = w < nameW + right + 140;
2708
2787
  w = Math.max(w, nameW + right + 140);
2709
2788
  // Symmetric, to the biggest bar and room for its number; 20 pts at least.
@@ -2805,16 +2884,74 @@
2805
2884
  // the time; under it, it is surer than it should be. Panels stack on a phone.
2806
2885
  var DC = C.decide;
2807
2886
  if (DC) put('chart-decide', function (w) {
2808
- if (!DC.points.length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
2887
+ if (!(DC.scores || []).length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
2809
2888
  var names = {};
2810
2889
  DC.sets.forEach(function (s) { names[s[0]] = s[1]; });
2811
2890
  var panels = DC.sets.map(function (s) { return s[1]; });
2891
+ // One colour per model across both charts, by name, never by rank.
2892
+ var models = DC.scores.map(function (p) { return p.model; })
2893
+ .filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
2894
+ var colors = {domain: models, range: [sky, good, signal, warn, muted, ink, err]};
2895
+ var box = el('div');
2896
+ box.appendChild(el('h3', {class: 'chart-head'}, 'Right of asked, per set'));
2897
+ box.appendChild(decideBars(w, names, panels, models, colors));
2898
+ // Calibration only says something once a model's answers spread over two or more
2899
+ // levels of sureness; while each sits in one top bin it is one dot in a corner.
2900
+ var levels = DC.points.some(function (p, i, all) {
2901
+ return all.some(function (q) { return q !== p && q.model === p.model && q.set === p.set; });
2902
+ });
2903
+ if (levels) {
2904
+ box.appendChild(el('h3', {class: 'chart-head'}, 'Calibration: said against right'));
2905
+ box.appendChild(decideCalibration(w, names, panels, models, colors));
2906
+ } else {
2907
+ box.appendChild(el('p', {class: 'hint'}, 'Calibration shows once a model gives answers at two or more levels'
2908
+ + ' of sureness on a set; so far each answers at one, so the table\'s Brier and calibration gap say it.'));
2909
+ }
2910
+ return box;
2911
+ });
2912
+ // Right of asked as bars from 0: a small chart per set, a row per model asked on it,
2913
+ // best first. A model under the ranking floor (DC.low items) is drawn faint: shown,
2914
+ // not yet ranked.
2915
+ function decideBars(w, names, panels, models, colors) {
2916
+ var out = el('div');
2917
+ DC.sets.forEach(function (s) {
2918
+ var rows = DC.scores.filter(function (r) { return r.set === s[0]; }).map(function (r) {
2919
+ return Object.assign({share: r.correct / r.asked, ranked: r.asked >= DC.low}, r);
2920
+ }).sort(function (a, b) { return b.share - a.share || b.asked - a.asked || (a.model < b.model ? -1 : 1); });
2921
+ out.appendChild(el('p', {class: 'chart-label'}, s[1]));
2922
+ if (!rows.length) { out.appendChild(el('p', {class: 'hint'}, 'No model asked on this set yet.')); return; }
2923
+ out.appendChild(Plot.plot({
2924
+ width: w, height: rows.length * 26 + 40, style: style,
2925
+ marginLeft: w < 560 ? 120 : 160, marginRight: 50, marginTop: 4, marginBottom: 30,
2926
+ x: {domain: [0, 1], label: null, tickFormat: '.0%', ticks: 5, grid: true},
2927
+ y: {domain: rows.map(function (r) { return r.model; }), label: null},
2928
+ color: colors,
2929
+ marks: [
2930
+ Plot.barX(rows, {x: 'share', y: 'model', fill: 'model', insetTop: 4, insetBottom: 4, rx: 2,
2931
+ fillOpacity: function (r) { return r.ranked ? 1 : 0.4; },
2932
+ title: function (r) {
2933
+ return r.model + ' · ' + s[1] + '\n' + r.correct + ' right of ' + r.asked + ' asked ('
2934
+ + Math.round(r.share * 100) + '%)' + (r.brier != null ? '\nBrier ' + r.brier.toFixed(3) : '')
2935
+ + (r.ranked ? '' : '\nnot ranked yet: under ' + DC.low + ' items');
2936
+ }}),
2937
+ Plot.text(rows, {x: 'share', y: 'model', dx: 6, textAnchor: 'start', fill: ink,
2938
+ text: function (r) { return r.correct + '/' + r.asked; }}),
2939
+ Plot.ruleX([0], {stroke: ink})
2940
+ ]
2941
+ }));
2942
+ });
2943
+ return out;
2944
+ }
2945
+ function decideCalibration(w, names, panels, models, colors) {
2812
2946
  var pts = DC.points.map(function (p) { return Object.assign({panel: names[p.set]}, p); })
2813
2947
  .sort(function (a, b) { return a.said - b.said; });
2814
- var models = pts.map(function (p) { return p.model; })
2815
- .filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
2948
+ // Both axes start just under the lowest answer, not at 0%: a decision model is
2949
+ // rarely under 50% sure of its own answer, so a 0-100% frame left every dot in
2950
+ // one corner under a long empty diagonal. Same floor on both, so it stays 45°.
2951
+ var low = Math.min.apply(null, pts.map(function (p) { return Math.min(p.said, p.right); }));
2952
+ var lo = Math.max(0, Math.min(0.9, Math.floor((low - 0.05) * 20) / 20));
2816
2953
  var diag = [];
2817
- panels.forEach(function (s) { diag.push({panel: s, x: 0, y: 0}, {panel: s, x: 1, y: 1}); });
2954
+ panels.forEach(function (s) { diag.push({panel: s, x: lo, y: lo}, {panel: s, x: 1, y: 1}); });
2818
2955
  function tip(p) {
2819
2956
  return p.model + ' · ' + names[p.set] + '\n' + plural(p.n, 'answer') + ' it was about '
2820
2957
  + Math.round(p.said * 100) + '% sure of\n' + Math.round(p.right * 100) + '% of them right';
@@ -2825,10 +2962,11 @@
2825
2962
  return Plot.plot(Object.assign(opts, {
2826
2963
  width: narrow ? Math.min(w, 520) : w, height: narrow ? 3 * 230 + 70 : 320, style: style,
2827
2964
  marginLeft: 52, marginRight: narrow ? 90 : 20, marginBottom: 44, marginTop: 28,
2828
- x: {domain: [0, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
2829
- y: {domain: [0, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
2965
+ inset: 14, // a dot at 100% clear of the frame
2966
+ x: {domain: [lo, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
2967
+ y: {domain: [lo, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
2830
2968
  r: {range: [3, 13]},
2831
- color: {legend: true, domain: models, range: [sky, good, signal, warn, muted, ink, err]},
2969
+ color: Object.assign({legend: true}, colors),
2832
2970
  marks: [
2833
2971
  Plot.frame({stroke: line}),
2834
2972
  Plot.line(diag, Object.assign({x: 'x', y: 'y', z: 'panel', stroke: muted, strokeDasharray: '4 3'}, by)),
@@ -2837,17 +2975,24 @@
2837
2975
  title: tip}, by))
2838
2976
  ]
2839
2977
  }));
2840
- });
2978
+ }
2841
2979
  // Who to trust: finds real bugs (right) vs makes things up (up = less).
2842
2980
  put('chart-overview', function (w) {
2843
2981
  if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
2844
2982
  var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
2845
2983
  var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
2846
2984
  var pxW = w - 110, pxH = 370;
2847
- var pts = C.overview.slice().sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
2985
+ // Named: the models in or next to the zone (the ones to pick from), then the clearest
2986
+ // warnings (most made up). The rest are on hover: named all at once they ran together.
2987
+ var zoneX = x0 + (1 - x0) / 2, zoneY = worst / 4;
2988
+ var near = function (d) { return d.recall >= zoneX - (1 - x0) * 0.08 && d.wrong <= zoneY * 1.4; };
2989
+ var worstFew = C.overview.slice().sort(function (a, b) { return b.wrong - a.wrong; }).slice(0, 3);
2990
+ var pts = C.overview.filter(function (d) { return near(d) || worstFew.indexOf(d) !== -1; })
2991
+ .sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
2848
2992
  var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
2849
2993
  function (d) { return d.wrong / worst * pxH; });
2850
- var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; });
2994
+ var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
2995
+ .filter(Boolean);
2851
2996
  return Plot.plot({
2852
2997
  width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
2853
2998
  x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
@@ -2875,71 +3020,72 @@
2875
3020
  tasks[h.model] = bugs + clean;
2876
3021
  var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
2877
3022
  var fa = clean ? h['false alarm'] / clean : 0;
2878
- score[h.model] = f - fa;
3023
+ // Ranked on what is proven: the low end of the found share's 95% range less the HIGH
3024
+ // end of the false alarms' (never shown clean files: up to 100%), so a few tasks, or
3025
+ // none on clean code, cannot top the chart.
3026
+ score[h.model] = wilson(h.found, bugs)[0] - wilson(h['false alarm'], clean)[1];
2879
3027
  segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
2880
3028
  segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
2881
3029
  segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
2882
3030
  segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
2883
3031
  });
2884
- var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
3032
+ var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a] || tasks[b] - tasks[a]; });
2885
3033
  return Plot.plot({
2886
- width: w, height: rows(models.length), style: style, marginLeft: 250,
3034
+ width: w, height: rows(models.length), style: style, marginLeft: w < 560 ? 150 : 250,
2887
3035
  x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
2888
3036
  tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
2889
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
3037
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + tasks[m] + ')'; }},
2890
3038
  color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
2891
3039
  marks: [
3040
+ // Under FEW tasks a model's bars are faint: shown, not much to go on.
2892
3041
  Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
2893
- title: function (d) { return d.model + ': ' + d.part + ', ' + d.n; }}),
3042
+ fillOpacity: function (d) { return tasks[d.model] < FEW ? 0.4 : 1; },
3043
+ title: function (d) { return d.model + ': ' + d.part + ', ' + d.n + (tasks[d.model] < FEW ? '\nfew tasks: little to go on' : ''); }}),
2894
3044
  Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
2895
3045
  ]
2896
3046
  });
2897
3047
  });
2898
3048
 
2899
- // Real work: the share of judged offloads that helped, sample size beside the name.
3049
+ // Real work: how often each model helped, with how sure that is. The dot is the share
3050
+ // helped, its size how many were judged, the line the range the true share likely sits
3051
+ // in (95% Wilson interval: wide for 3 of 3, narrow for 40 of 50). Ranked by the low end
3052
+ // of that range, so volume counts: 3 of 3 no longer outranks 40 of 50. The dashed line
3053
+ // is every model's offloads pooled.
2900
3054
  put('chart-work', function (w) {
2901
- var R = C.work.slice().sort(function (a, b) { return b.useful / b.judged - a.useful / a.judged || b.judged - a.judged; });
2902
- if (!R.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
2903
- var by = {};
2904
- R.forEach(function (r) { by[r.model] = r; });
2905
- function share(r) { return r.useful / r.judged; }
3055
+ if (!C.work.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
3056
+ var R = C.work.map(function (r) {
3057
+ var b = wilson(r.useful, r.judged), p = r.useful / r.judged;
3058
+ var verdict = b[0] >= 0.5 ? 'helps' : b[1] < 0.5 ? 'does not help' : 'too few to say';
3059
+ return Object.assign({p: p, lo: b[0], hi: b[1], verdict: verdict}, r);
3060
+ }).sort(function (a, b) { return b.lo - a.lo || b.judged - a.judged; });
3061
+ var all = R.reduce(function (s, r) { return [s[0] + r.useful, s[1] + r.judged]; }, [0, 0]);
3062
+ var pooled = all[0] / all[1];
3063
+ var narrowW = w < 560, nameW = narrowW ? 130 : 210;
2906
3064
  return Plot.plot({
2907
- width: w, height: rows(R.length), style: style, marginLeft: 250, marginRight: 50,
2908
- x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
3065
+ width: w, height: 24 * R.length + 90, style: style, marginLeft: nameW, marginRight: 64, marginTop: 34,
3066
+ x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', ticks: 5, grid: true},
2909
3067
  y: {domain: R.map(function (r) { return r.model; }), label: null,
2910
- tickFormat: function (m) { return m + ' (' + by[m].judged + ')'; }},
3068
+ tickFormat: function (m) { return fit(m, narrowW ? 16 : 26); }},
3069
+ r: {domain: [0, Math.max.apply(null, R.map(function (r) { return r.judged; }))], range: [3, 11]},
3070
+ color: {domain: ['helps', 'too few to say', 'does not help'], range: [good, muted, err], legend: true},
2911
3071
  marks: [
2912
- Plot.barX(R, {x: share, y: 'model', fill: function (r) { return share(r) >= 0.5 ? good : err; },
2913
- title: function (r) { return r.model + ': helped ' + r.useful + ' of ' + r.judged + ' judged (' + r.offloads + ' offloads)'
2914
- + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : ''); }}),
2915
- Plot.text(R, {x: share, y: 'model', text: function (r) { return r.useful + '/' + r.judged; }, dx: 18, fill: ink}),
2916
- Plot.ruleX([0], {stroke: ink})
3072
+ Plot.ruleX([0.5], {stroke: line}),
3073
+ Plot.ruleX([pooled], {stroke: ink, strokeDasharray: '4 3', strokeOpacity: 0.6}),
3074
+ Plot.text([pooled], {x: function (d) { return d; }, frameAnchor: 'top', dy: -20, fill: muted, fontSize: 11,
3075
+ text: function () { return 'all models ' + Math.round(pooled * 100) + '%'; }}),
3076
+ Plot.ruleY(R, {y: 'model', x1: 'lo', x2: 'hi', stroke: 'verdict', strokeWidth: 2, strokeOpacity: 0.55}),
3077
+ Plot.dot(R, {x: 'p', y: 'model', r: 'judged', fill: 'verdict', stroke: halo, strokeWidth: 1.5,
3078
+ title: function (r) {
3079
+ return r.model + '\nhelped ' + r.useful + ' of ' + r.judged + ' judged (' + Math.round(r.p * 100) + '%; '
3080
+ + r.offloads + ' offloads)\nlikely between ' + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%'
3081
+ + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : '');
3082
+ }}),
3083
+ Plot.text(R, {x: 'hi', y: 'model', dx: 8, textAnchor: 'start', fill: ink, fontSize: 11,
3084
+ text: function (r) { return r.useful + '/' + r.judged; }})
2917
3085
  ]
2918
3086
  });
2919
3087
  });
2920
3088
 
2921
- // Did it answer: each probe of the last 14 days, by what came back.
2922
- put('chart-reliability', function (w) {
2923
- if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
2924
- var total = {}, ok = {};
2925
- C.reliability.forEach(function (d) {
2926
- total[d.model] = (total[d.model] || 0) + d.count;
2927
- if (d.outcome === 'answered') ok[d.model] = d.count;
2928
- });
2929
- var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
2930
- var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
2931
- entitlement: '#c0563f', other: muted, error: ink};
2932
- var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
2933
- return Plot.plot({
2934
- width: w, height: rows(models.length), style: style, marginLeft: 250,
2935
- x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
2936
- y: {domain: models, label: null},
2937
- color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
2938
- marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
2939
- title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
2940
- });
2941
- });
2942
-
2943
3089
  // Speed vs quality: median seconds on our own tasks against findings located.
2944
3090
  put('chart-speed', function (w) {
2945
3091
  if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
@@ -2950,14 +3096,20 @@
2950
3096
  var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
2951
3097
  var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
2952
3098
  function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
2953
- var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; });
3099
+ var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
3100
+ .filter(Boolean);
2954
3101
  return Plot.plot({
2955
3102
  width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
2956
3103
  x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
2957
3104
  y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
2958
3105
  marks: [
3106
+ // The zone to be in, as on Who to trust: the faster half (on the log scale) and the
3107
+ // top quarter of findings. Outlined, so it reads on a dark ground too.
3108
+ Plot.rect([{x1: lo, x2: Math.sqrt(lo * hi), y1: y0 + (1 - y0) * 3 / 4, y2: 1}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
3109
+ fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
2959
3110
  Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
2960
- Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink}),
3111
+ Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink,
3112
+ stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
2961
3113
  Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
2962
3114
  title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
2963
3115
  ]
@@ -2972,25 +3124,41 @@
2972
3124
  return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
2973
3125
  });
2974
3126
  tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
2975
- var label = {};
2976
- Object.keys(first).forEach(function (f) { label[first[f]] = f; });
3127
+ // Each group's name over its first task, on two staggered rows: side by side they
3128
+ // ran into one another wherever a group is only a few tasks wide.
3129
+ var heads = C.families.filter(function (f) { return f in first; })
3130
+ .map(function (f, i) { return {task: first[f], name: f, row: i % 2}; });
2977
3131
  // By total found, so a model that played three tasks does not top one that played forty.
2978
- var sum = {};
2979
- C.heat.forEach(function (d) { sum[d.model] = (sum[d.model] || 0) + (d.rate || 0); });
2980
- var models = Object.keys(sum).sort(function (a, b) { return sum[b] - sum[a]; });
3132
+ // A model with no scored task (only grey) gets no row: it is named under the chart.
3133
+ var sum = {}, scored = {};
3134
+ C.heat.forEach(function (d) {
3135
+ sum[d.model] = (sum[d.model] || 0) + (d.rate || 0);
3136
+ if (d.rate != null) scored[d.model] = true;
3137
+ });
3138
+ var models = Object.keys(sum).filter(function (m) { return scored[m]; }).sort(function (a, b) { return sum[b] - sum[a]; });
3139
+ var idle = Object.keys(sum).filter(function (m) { return !scored[m]; }).sort();
3140
+ var shown = C.heat.filter(function (d) { return scored[d.model]; });
3141
+ var nameW = w < 560 ? 130 : 230;
2981
3142
  function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
2982
- return Plot.plot({
2983
- width: w, height: 22 * models.length + 70, style: style, marginLeft: 230, marginTop: 30, padding: 0.08,
2984
- x: {domain: tasks, axis: 'top', label: null, tickSize: 0, tickFormat: function (t) { return label[t] || ''; }},
2985
- y: {domain: models, label: null, tickSize: 0},
3143
+ var box = el('div');
3144
+ box.appendChild(Plot.plot({
3145
+ width: w, height: 22 * models.length + 84, style: style, marginLeft: nameW, marginTop: 44, padding: 0.08,
3146
+ x: {domain: tasks, axis: null},
3147
+ y: {domain: models, label: null, tickSize: 0, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 30); }},
2986
3148
  color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
2987
3149
  marks: [
2988
- Plot.cell(C.heat.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
2989
- Plot.cell(C.heat.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
2990
- Plot.dot(C.heat.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
2991
- Plot.ruleX(Object.keys(label), {x: function (t) { return t; }, stroke: ink, strokeOpacity: 0.25, dx: -2})
3150
+ Plot.cell(shown.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
3151
+ Plot.cell(shown.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
3152
+ Plot.dot(shown.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
3153
+ Plot.ruleX(heads, {x: 'task', stroke: ink, strokeOpacity: 0.25, dx: -2}),
3154
+ Plot.text(heads.filter(function (h) { return !h.row; }), {x: 'task', frameAnchor: 'top', dy: -30, dx: -2,
3155
+ text: 'name', textAnchor: 'start', fontSize: 11, fill: ink}),
3156
+ Plot.text(heads.filter(function (h) { return h.row; }), {x: 'task', frameAnchor: 'top', dy: -14, dx: -2,
3157
+ text: 'name', textAnchor: 'start', fontSize: 11, fill: ink})
2992
3158
  ]
2993
- });
3159
+ }));
3160
+ if (idle.length) box.appendChild(el('p', {class: 'hint'}, 'No scored task yet, so no row: ' + idle.join(', ') + '.'));
3161
+ return box;
2994
3162
  });
2995
3163
 
2996
3164
  // Vision tasks: one square per field and model, the task's name over its first field.
@@ -3037,39 +3205,66 @@
3037
3205
  kinds.forEach(function (k) { if (W[m][k]) data.push({model: m, kind: k, count: W[m][k]}); total[m] = (total[m] || 0) + W[m][k]; });
3038
3206
  });
3039
3207
  if (!data.length) return el('p', {class: 'hint'}, 'No vision panels through the gateway yet.');
3040
- // The models doing the most image work first: their bars are the evidence.
3041
- var models = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
3208
+ // The models doing the most image work first: their bars are the evidence. Under FEW
3209
+ // seats a row was mostly "no answer" and said little: those are listed under the chart.
3210
+ var all = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
3211
+ var models = all.filter(function (m) { return total[m] >= FEW; });
3212
+ var few = all.filter(function (m) { return total[m] < FEW; });
3213
+ if (!models.length) { models = all; few = []; }
3214
+ data = data.filter(function (d) { return models.indexOf(d.model) !== -1; });
3042
3215
  var colour = {'checked, helped': good, 'checked, did not help': err, 'useful, no claims': sky,
3043
3216
  'no use, no claims': warn, 'not judged': line, 'no answer': muted};
3044
3217
  // The claims checked by hand, beside the bar: the only right-or-wrong count real work has.
3045
3218
  var claims = models.filter(function (m) { return W[m].real + W[m].false; }).map(function (m) {
3046
3219
  return {model: m, text: W[m].real + ' of ' + (W[m].real + W[m].false) + ' claims right'};
3047
3220
  });
3048
- return Plot.plot({
3049
- width: w, height: rows(models.length), style: style, marginLeft: 300, marginRight: 150,
3221
+ var narrowV = w < 560, box = el('div');
3222
+ box.appendChild(Plot.plot({
3223
+ width: w, height: rows(models.length), style: style, marginLeft: narrowV ? 150 : 260, marginRight: narrowV ? 20 : 150,
3050
3224
  x: {label: 'Share of its seats on image panels →', tickFormat: '.0%', grid: true, domain: [0, 1]},
3051
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + total[m] + ')'; }},
3225
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, narrowV ? 16 : 30) + ' (' + total[m] + ')'; }},
3052
3226
  color: {legend: true, domain: kinds, range: kinds.map(function (k) { return colour[k]; })},
3053
3227
  marks: [
3054
3228
  Plot.barX(data, Plot.stackX({x: 'count', y: 'model', fill: 'kind', offset: 'normalize', order: kinds,
3055
3229
  title: function (d) { return d.model + ': ' + d.kind + ', ' + d.count + ' of ' + total[d.model] + ' seats'; }})),
3056
- Plot.text(claims, {x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
3230
+ Plot.text(narrowV ? [] : claims.filter(function (c) { return models.indexOf(c.model) !== -1; }),
3231
+ {x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
3057
3232
  ]
3058
- });
3233
+ }));
3234
+ if (few.length) box.appendChild(el('p', {class: 'hint'}, 'Under ' + FEW + ' seats, so no bar: '
3235
+ + few.map(function (m) { return m + ' (' + total[m] + ')'; }).join(', ') + '. The table has them all.'));
3236
+ return box;
3059
3237
  });
3060
3238
 
3061
- // Confidently wrong: stacked by kind, worst at the top.
3239
+ // Confidently wrong: stacked by kind. A thin line runs over each bar for the range the
3240
+ // true share likely sits in (95% Wilson), so 0 of 2 shows how bad it could be, not a
3241
+ // clean record. Worst proven at the top: by the low end of that range. Few answers: faint.
3062
3242
  put('chart-wrong', function (w) {
3063
- var T = C.wrongTotals;
3064
- var models = Object.keys(T).sort(function (a, b) { return T[b].rate - T[a].rate || T[b].answers - T[a].answers; });
3243
+ var T = C.wrongTotals, R = {};
3244
+ Object.keys(T).forEach(function (m) {
3245
+ var b = wilson(T[m].wrong, T[m].answers);
3246
+ R[m] = {model: m, lo: b[0], hi: b[1], few: T[m].answers < FEW};
3247
+ });
3248
+ var models = Object.keys(T).sort(function (a, b) {
3249
+ return R[b].lo - R[a].lo || T[b].rate - T[a].rate || T[b].answers - T[a].answers;
3250
+ });
3251
+ var ranges = models.map(function (m) { return R[m]; });
3252
+ var top = Math.min(1, Math.max.apply(null, ranges.map(function (r) { return r.hi; })) + 0.02);
3065
3253
  return Plot.plot({
3066
- width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
3067
- x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
3254
+ width: w, height: 20 * models.length + 70, style: style, marginLeft: w < 560 ? 150 : 250,
3255
+ x: {domain: [0, top], label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
3068
3256
  // The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
3069
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
3257
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + T[m].answers + ')'; }},
3070
3258
  color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
3071
3259
  marks: [
3072
- Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
3260
+ Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', insetTop: 3, insetBottom: 3,
3261
+ fillOpacity: function (d) { return R[d.model].few ? 0.4 : 1; },
3262
+ title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
3263
+ Plot.ruleY(ranges, {y: 'model', x1: 'lo', x2: 'hi', stroke: ink, strokeOpacity: 0.45, strokeWidth: 1.5,
3264
+ title: function (r) {
3265
+ return r.model + ': ' + T[r.model].wrong + ' of ' + T[r.model].answers + ' confidently wrong\nlikely between '
3266
+ + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%' + (r.few ? '\nfew answers: little to go on' : '');
3267
+ }}),
3073
3268
  Plot.ruleX([0], {stroke: ink})
3074
3269
  ]
3075
3270
  });