@wendellresearch-ai/benchwarmers 0.6.2 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ui/bench.js CHANGED
@@ -213,7 +213,7 @@
213
213
  var rendered = document.querySelector('.tab:not([data-lazy])');
214
214
  Promise.resolve().then(function () { if (rendered) setUp(rendered.id.slice(4)); });
215
215
  var wanted = store('bench.tab'); // else the one rendered: storage blocked or cleared, the cookie kept
216
- show(wanted && $('tab-' + wanted) ? wanted : rendered ? rendered.id.slice(4) : 'glance');
216
+ show(wanted && $('tab-' + wanted) ? wanted : rendered ? rendered.id.slice(4) : 'suite');
217
217
 
218
218
  // ---- sortable tables -------------------------------------------------
219
219
  function key(cell) {
@@ -253,13 +253,21 @@
253
253
  var box = scope.querySelector('input[data-filter]'), q = box ? box.value.trim().toLowerCase() : '';
254
254
  var facets = Array.prototype.filter.call(scope.querySelectorAll('select[data-facet]'), function (s) { return s.value; });
255
255
  scope.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
256
- r.hidden = (q && r.textContent.toLowerCase().indexOf(q) === -1)
256
+ // A group's name (data-conn: Models' connection) counts as text of each of its rows.
257
+ var g = r.closest('[data-conn]'), text = r.textContent + (g ? ' ' + g.dataset.conn : '');
258
+ r.hidden = (q && text.toLowerCase().indexOf(q) === -1)
257
259
  || facets.some(function (s) { return r.dataset[s.dataset.facet] !== s.value; });
258
260
  });
259
261
  // A match inside a folded list (an endpoint's models) opens it.
260
262
  if (q) scope.querySelectorAll('details.models').forEach(function (d) {
261
263
  if (d.querySelector('tbody tr:not([hidden])')) d.open = true;
262
264
  });
265
+ // A group with no row left is hidden; one with a match opens.
266
+ scope.querySelectorAll('[data-conn]').forEach(function (g) {
267
+ var any = !!g.querySelector('tbody tr:not([hidden])');
268
+ g.hidden = !!q && !any;
269
+ if (q && any && g.tagName === 'DETAILS') g.open = true;
270
+ });
263
271
  }
264
272
  function scopeOf(c) {
265
273
  var box = c.closest('.toolbar') ? c.closest('.toolbar').querySelector('input[data-filter]') : c;
@@ -268,7 +276,75 @@
268
276
  document.addEventListener('input', function (ev) { if (ev.target.matches('input[data-filter]')) applyFilter(scopeOf(ev.target)); });
269
277
  document.addEventListener('change', function (ev) { if (ev.target.matches('select[data-facet]')) applyFilter(scopeOf(ev.target)); });
270
278
 
271
- // ---- a link to another page (At a glance's line to Action needed) ----
279
+ // ---- Expand all / Collapse all: every fold of the page's groups (Models' connections).
280
+ // One labelled toggle, as accordions have it: Collapse all while every one is open,
281
+ // else Expand all. Kept right after any fold opens or shuts, by hand, by it or by a live swap.
282
+ function foldButtons(tab) {
283
+ var folds = tab.querySelectorAll('details.model-card'), open = 0;
284
+ folds.forEach(function (d) { if (d.open) open++; });
285
+ var all = open === folds.length;
286
+ tab.querySelectorAll('[data-fold-all]').forEach(function (b) {
287
+ b.dataset.foldAll = all ? 'close' : 'open';
288
+ b.setAttribute('aria-expanded', String(all));
289
+ b.lastChild.textContent = all ? 'Collapse all' : 'Expand all';
290
+ });
291
+ }
292
+ document.addEventListener('click', function (e) {
293
+ var b = e.target.closest('[data-fold-all]');
294
+ if (!b) return;
295
+ var tab = b.closest('.tab');
296
+ tab.querySelectorAll('details.model-card').forEach(function (d) { d.open = b.dataset.foldAll === 'open'; });
297
+ foldButtons(tab);
298
+ });
299
+ document.addEventListener('toggle', function (e) { // toggle does not bubble: caught on its way down
300
+ if (e.target.matches && e.target.matches('details.model-card')) foldButtons(e.target.closest('.tab'));
301
+ }, true);
302
+
303
+ // ---- Logs: Export as text (every log in one .txt, to read) and CSV (one log, its
304
+ // own columns, for a spreadsheet). Both as the page shows them, the filter applied.
305
+ function stampNow() { // local time, as the page shows
306
+ var now = new Date(), two = function (n) { return String(n).padStart(2, '0'); };
307
+ return now.getFullYear() + '-' + two(now.getMonth() + 1) + '-' + two(now.getDate()) + '-' + two(now.getHours()) + two(now.getMinutes());
308
+ }
309
+ // The file itself: Share's saveFile(blob, name) and csv(rows) (RFC 4180, with a BOM).
310
+ document.addEventListener('click', function (e) {
311
+ var b = e.target.closest('[data-csv]');
312
+ if (!b) return;
313
+ e.preventDefault(); // in the summary: no fold
314
+ var d = b.closest('details'), rows = [];
315
+ var cells = function (tr) {
316
+ return Array.prototype.map.call(tr.children, function (c) { return c.textContent.trim().replace(/\s+/g, ' '); });
317
+ };
318
+ var head = d.querySelector('thead tr');
319
+ if (head) {
320
+ rows.push(cells(head));
321
+ d.querySelectorAll('tbody tr').forEach(function (r) { if (!r.hidden) rows.push(cells(r)); });
322
+ } else {
323
+ rows.push(['Line']);
324
+ d.querySelectorAll('.log-line').forEach(function (r) { if (!r.hidden) rows.push([r.textContent.trim()]); });
325
+ }
326
+ var slug = d.dataset.log.toLowerCase().replace(/[^a-z0-9]+/g, '-');
327
+ saveFile(new Blob([csv(rows)], {type: 'text/csv;charset=utf-8'}), 'benchwarmers-' + slug + '-' + stampNow() + '.csv');
328
+ });
329
+ on('click', 'logs-export', function () {
330
+ var tab = $('tab-logs'), out = ['Benchwarmers logs, exported ' + new Date().toString()];
331
+ var q = (tab.querySelector('input[data-filter]') || {}).value;
332
+ if (q) out.push('Filtered by: ' + q);
333
+ tab.querySelectorAll('details.panel').forEach(function (d) {
334
+ out.push('', '== ' + d.dataset.log + ' ==');
335
+ var cells = function (tr) {
336
+ return Array.prototype.map.call(tr.children, function (c) { return c.textContent.trim().replace(/\s+/g, ' '); }).join('\t');
337
+ };
338
+ var head = d.querySelector('thead tr');
339
+ if (head) out.push(cells(head));
340
+ d.querySelectorAll('tbody tr, .log-line').forEach(function (r) {
341
+ if (!r.hidden) out.push(r.matches('tr') ? cells(r) : r.textContent.trim());
342
+ });
343
+ });
344
+ saveFile(new Blob([out.join('\n') + '\n'], {type: 'text/plain;charset=utf-8'}), 'benchwarmers-logs-' + stampNow() + '.txt');
345
+ });
346
+
347
+ // ---- a link to another page ----
272
348
  document.addEventListener('click', function (e) {
273
349
  var b = e.target.closest('[data-go]');
274
350
  if (b) { show(b.dataset.go); scrollTo(0, 0); }
@@ -759,8 +835,8 @@
759
835
  });
760
836
 
761
837
  // ---- after an action: the part it changed, not the whole page ----------
762
- // Action needed's page, its count in the sidebar and At a glance's line
763
- // about it, from the server (GET /api/needs, a few ms): a page reload
838
+ // Action needed's page and its count in the sidebar, from the server
839
+ // (GET /api/needs, a few ms): a page reload
764
840
  // renders every tab, seconds.
765
841
  var needsAsked = 0;
766
842
  function refreshNeeds(live) {
@@ -775,10 +851,6 @@
775
851
  count.textContent = n.count;
776
852
  tab.setAttribute('aria-label', tab.firstChild.textContent + ', ' + n.count + ' open');
777
853
  }
778
- var line = document.querySelector('#tab-glance > .a-line');
779
- if (line) line.remove();
780
- // Not into a page not fetched yet: it says Loading… while empty, and comes with its line.
781
- if (n.line && $('tab-glance').firstChild) $('tab-glance').insertAdjacentHTML('afterbegin', n.line);
782
854
  }).catch(function () { location.reload(); }); // could not ask: the old way
783
855
  }
784
856
  // A retired model's rows grey out and hide behind Show retired, as the
@@ -1880,7 +1952,7 @@
1880
1952
  pick: $('g-pick'), pickConf: $('g-pick-conf'),
1881
1953
  gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
1882
1954
  del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
1883
- seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
1955
+ seats: $('c-seats'), prov: $('c-member-provider'), show: $('c-member-show'), adHoc: $('c-ad-hoc'),
1884
1956
  ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
1885
1957
  return loadCollections();
1886
1958
  };
@@ -1994,7 +2066,7 @@
1994
2066
  // the seats above the list always show what is picked, each with its ×, and
1995
2067
  // every count still includes it. A row goes when all its chips do.
1996
2068
  function filterMembers() {
1997
- var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
2069
+ var q = c.find.value.trim().toLowerCase(), prov = c.prov.value, show = c.show.value;
1998
2070
  c.members.querySelectorAll('.m-row').forEach(function (row) {
1999
2071
  var named = !q || row.dataset.name.toLowerCase().indexOf(q) !== -1, any = false;
2000
2072
  row.querySelectorAll('label.check').forEach(function (lab) {
@@ -2002,7 +2074,8 @@
2002
2074
  var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
2003
2075
  lab.textContent).toLowerCase();
2004
2076
  lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
2005
- || !!prov && (fact(box.value).provider || '') !== prov;
2077
+ || !!prov && (fact(box.value).provider || '') !== prov
2078
+ || show === 'ticked' && !box.checked || show === 'fits' && !!box.dataset.blocked;
2006
2079
  any = any || !lab.hidden;
2007
2080
  });
2008
2081
  row.hidden = !any;
@@ -2011,6 +2084,23 @@
2011
2084
  }
2012
2085
  on('input', 'c-member-filter', filterMembers);
2013
2086
  on('change', 'c-member-provider', filterMembers);
2087
+ on('change', 'c-member-show', filterMembers);
2088
+ // Refresh model details: the models and their facts fetched again, the list drawn
2089
+ // anew with the same ticks, search and choices. The model popup's directory too.
2090
+ function refreshMembers() {
2091
+ var chosen = picked(), out = $('c-refresh-out'), btn = $('c-refresh');
2092
+ btn.disabled = true; out.textContent = 'Refreshing…';
2093
+ return Promise.all([get('/api/collections'), directory()]).then(function (res) {
2094
+ C = res[0];
2095
+ drawMembers(chosen);
2096
+ applyTags();
2097
+ out.textContent = 'Up to date.';
2098
+ }).catch(function (err) { out.textContent = String(err); })
2099
+ .then(function () { btn.disabled = false; });
2100
+ }
2101
+ on('click', 'c-refresh', refreshMembers);
2102
+ // A model opened from the list may have been changed in its popup: what it now is, on close.
2103
+ md.addEventListener('close', function () { if (c.form && c.form.open) refreshMembers(); });
2014
2104
  // Escape in a box with text clears it; only an empty box lets Escape close the popup.
2015
2105
  on('keydown', 'c-member-filter', function (e) {
2016
2106
  if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
@@ -2059,13 +2149,37 @@
2059
2149
  c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
2060
2150
  c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
2061
2151
  drawTagPicks();
2062
- c.members.textContent = ''; c.find.value = '';
2152
+ c.find.value = ''; c.show.value = ''; c.prov.value = '';
2153
+ $('c-refresh-out').textContent = '';
2154
+ drawMembers(row ? row.members : []);
2155
+ c.shape.textContent = '';
2156
+ c.shape.appendChild(el('option', {value: ''}, 'automatic'));
2157
+ C.shapes.forEach(function (s) {
2158
+ var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
2159
+ });
2160
+ c.reply.value = row ? row.reply : 'all';
2161
+ c.merger.textContent = '';
2162
+ c.merger.appendChild(el('option', {value: ''}, 'automatic'));
2163
+ C.models.forEach(function (alias) {
2164
+ var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
2165
+ if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
2166
+ });
2167
+ showMerger(); applyTags();
2168
+ }
2169
+ // The member list: a table to scan down, one row per model (named by its shortest
2170
+ // alias; the name opens its popup, as everywhere), its context and what it can do
2171
+ // in columns of their own, then a chip per provider it is on, each ticking that
2172
+ // alias. A chip whose window differs from the row's says its own. `chosen`: ticked.
2173
+ function drawMembers(chosen) {
2174
+ var keepProv = c.prov.value;
2175
+ c.members.textContent = '';
2063
2176
  c.prov.length = 1; // "Every provider", then each one a model is on
2064
2177
  Array.from(new Set(C.models.map(function (a) { return fact(a).provider; }).filter(Boolean))).sort()
2065
2178
  .forEach(function (p) { c.prov.appendChild(el('option', {value: p}, p)); });
2066
- // One row per model, named by its shortest alias, with what the first
2067
- // route says it holds and can do; each provider it is on is a chip that
2068
- // ticks that alias. A chip whose window differs from the row's says its own.
2179
+ c.prov.value = keepProv;
2180
+ var head = el('div', {class: 'm-head', 'aria-hidden': 'true'});
2181
+ ['Model', 'Context', 'Can', 'On, tick one'].forEach(function (h) { head.appendChild(el('span', null, h)); });
2182
+ c.members.appendChild(head);
2069
2183
  var groups = {};
2070
2184
  C.models.forEach(function (a) { var f = fact(a).family || a; (groups[f] = groups[f] || []).push(a); });
2071
2185
  Object.keys(groups).map(function (f) {
@@ -2073,18 +2187,25 @@
2073
2187
  return {family: f, name: routes[0], routes: routes};
2074
2188
  }).sort(function (x, y) { return x.name.toLowerCase() < y.name.toLowerCase() ? -1 : 1; }).forEach(function (g) {
2075
2189
  var first = fact(g.name), caps = first.caps || {};
2076
- var can = ['vision', 'tools'].filter(function (k) { return caps[k]; });
2077
2190
  var line = el('div', {class: 'm-row', role: 'group', 'aria-label': g.name, 'data-family': g.family, 'data-name': g.name});
2078
2191
  var name = el('div', {class: 'm-name'});
2079
- name.appendChild(el('span', {class: 'mono'}, g.name));
2080
- name.appendChild(el('span', {class: 'm-facts'}, [first.context ? tokens(first.context) : '', can.join(', '),
2081
- first.price === 'unknown' ? '' : first.price].filter(Boolean).join(' · ')));
2192
+ // The same link as every model name elsewhere (a.model > .alias: the highlight on hover).
2193
+ var link = el('a', {href: '#', class: 'model', 'data-alias': g.name, title: 'Open ' + g.name});
2194
+ link.appendChild(el('span', {class: 'alias'}, g.name));
2195
+ name.appendChild(link);
2082
2196
  line.appendChild(name);
2197
+ line.appendChild(el('span', {class: 'm-ctx'}, first.context ? tokens(first.context) : '—'));
2198
+ var can = el('span', {class: 'm-can'});
2199
+ [['vision', 'vision'], ['tools', 'tools'], ['reasoning', 'reasons']].forEach(function (k) {
2200
+ if (caps[k[0]]) can.appendChild(el('span', {class: 'chip'}, k[1]));
2201
+ });
2202
+ if (first.price === 'free') can.appendChild(el('span', {class: 'chip ok'}, 'free'));
2203
+ line.appendChild(can);
2083
2204
  var chips = el('div', {class: 'm-routes'});
2084
2205
  g.routes.forEach(function (alias) {
2085
2206
  var f = fact(alias);
2086
2207
  var box = el('input', {type: 'checkbox', value: alias, 'aria-label': alias + ' on ' + (f.provider || 'its endpoint')});
2087
- box.checked = !!row && row.members.indexOf(alias) !== -1;
2208
+ box.checked = chosen.indexOf(alias) !== -1;
2088
2209
  box.addEventListener('change', applyTags); // ticked or not changes what a misfit shows as
2089
2210
  var lab = el('label', {class: 'check', 'data-served': (C.served || {})[alias] || ''});
2090
2211
  lab.appendChild(box);
@@ -2095,19 +2216,6 @@
2095
2216
  line.appendChild(chips);
2096
2217
  c.members.appendChild(line);
2097
2218
  });
2098
- c.shape.textContent = '';
2099
- c.shape.appendChild(el('option', {value: ''}, 'automatic'));
2100
- C.shapes.forEach(function (s) {
2101
- var o = el('option', {value: s}, s); if (row && row.shape === s) o.selected = true; c.shape.appendChild(o);
2102
- });
2103
- c.reply.value = row ? row.reply : 'all';
2104
- c.merger.textContent = '';
2105
- c.merger.appendChild(el('option', {value: ''}, 'automatic'));
2106
- C.models.forEach(function (alias) {
2107
- var o = el('option', {value: alias}, alias + (fact(alias).provider ? ' · ' + fact(alias).provider : ''));
2108
- if (row && row.merger === alias) o.selected = true; c.merger.appendChild(o);
2109
- });
2110
- showMerger(); applyTags();
2111
2219
  }
2112
2220
  // What a collection's members can all do, in the words the tag checks use.
2113
2221
  var CAN = {reasoning: 'reason'};
@@ -2660,7 +2768,8 @@
2660
2768
  try { box.appendChild(make(width())); } catch (e) { box.textContent = 'Chart failed: ' + e; }
2661
2769
  }
2662
2770
  // Pixel offsets that keep scatter labels apart: each label, in order, moves
2663
- // up or down until it clears the ones already placed.
2771
+ // up or down until it clears the ones already placed; null when no place is
2772
+ // free (that one is left to its hover, not drawn over another).
2664
2773
  function spread(points, xpx, ypx) {
2665
2774
  var boxes = [], tries = [-12, 14, -26, 28, -40, 42, -54, 56];
2666
2775
  return points.map(function (d) {
@@ -2672,7 +2781,7 @@
2672
2781
  return tries[k];
2673
2782
  }
2674
2783
  }
2675
- return tries[0];
2784
+ return null;
2676
2785
  });
2677
2786
  }
2678
2787
  function rows(n) { return 20 * n + 80; }
@@ -2682,6 +2791,16 @@
2682
2791
  var keep = s.length - over - 1;
2683
2792
  return over <= 0 ? s : s.slice(0, Math.ceil(keep / 2)) + '…' + s.slice(s.length - Math.floor(keep / 2));
2684
2793
  }
2794
+ function fit(s, max) { return middle(s, s.length - max); }
2795
+ var FEW = 10; // under this many tasks or answers, a row is drawn faint
2796
+ // The 95% Wilson interval of k successes in n: [low, high]. Honest at small n and at
2797
+ // 0 or n of n, where k/n ± 1.96·se says nothing (3 of 3 is not "100% ± 0").
2798
+ function wilson(k, n) {
2799
+ if (!n) return [0, 1];
2800
+ var z = 1.96, p = k / n, z2 = z * z, den = 1 + z2 / n;
2801
+ var mid = p + z2 / (2 * n), half = z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n));
2802
+ return [Math.max(0, (mid - half) / den), Math.min(1, (mid + half) / den)];
2803
+ }
2685
2804
 
2686
2805
  // Trend: who moved, which way, by how much and how sure (serve.trend_data),
2687
2806
  // in the window the card's select says. A row per model with a task to
@@ -2728,7 +2847,9 @@
2728
2847
  var nameW = Math.min(260, Math.max(150, 7 * Math.max.apply(null, M.map(function (r) { return r.key.length; })) + 24));
2729
2848
  var evW = 320;
2730
2849
  w = $('chart-trend').clientWidth || w; // its own box: a phone's is narrower than width()'s 520
2731
- if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, 130); }
2850
+ // Without the evidence column the names take what the bars do not need (260 px), not a
2851
+ // fixed 130 px: at the dashboard's width that cut "muse-glimmer-30b" to "muse-gl…mer-30b".
2852
+ if (w < nameW + evW + 260) { evW = 0; nameW = Math.min(nameW, Math.max(130, w - 280)); }
2732
2853
  var fits = Math.floor((nameW - 24) / 7), right = evW || 20, wide = w < nameW + right + 140;
2733
2854
  w = Math.max(w, nameW + right + 140);
2734
2855
  // Symmetric, to the biggest bar and room for its number; 20 pts at least.
@@ -2830,16 +2951,74 @@
2830
2951
  // the time; under it, it is surer than it should be. Panels stack on a phone.
2831
2952
  var DC = C.decide;
2832
2953
  if (DC) put('chart-decide', function (w) {
2833
- if (!DC.points.length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
2954
+ if (!(DC.scores || []).length) return el('p', {class: 'hint'}, 'No decision model has answered a benchmark question yet.');
2834
2955
  var names = {};
2835
2956
  DC.sets.forEach(function (s) { names[s[0]] = s[1]; });
2836
2957
  var panels = DC.sets.map(function (s) { return s[1]; });
2958
+ // One colour per model across both charts, by name, never by rank.
2959
+ var models = DC.scores.map(function (p) { return p.model; })
2960
+ .filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
2961
+ var colors = {domain: models, range: [sky, good, signal, warn, muted, ink, err]};
2962
+ var box = el('div');
2963
+ box.appendChild(el('h3', {class: 'chart-head'}, 'Right of asked, per set'));
2964
+ box.appendChild(decideBars(w, names, panels, models, colors));
2965
+ // Calibration only says something once a model's answers spread over two or more
2966
+ // levels of sureness; while each sits in one top bin it is one dot in a corner.
2967
+ var levels = DC.points.some(function (p, i, all) {
2968
+ return all.some(function (q) { return q !== p && q.model === p.model && q.set === p.set; });
2969
+ });
2970
+ if (levels) {
2971
+ box.appendChild(el('h3', {class: 'chart-head'}, 'Calibration: said against right'));
2972
+ box.appendChild(decideCalibration(w, names, panels, models, colors));
2973
+ } else {
2974
+ box.appendChild(el('p', {class: 'hint'}, 'Calibration shows once a model gives answers at two or more levels'
2975
+ + ' of sureness on a set; so far each answers at one, so the table\'s Brier and calibration gap say it.'));
2976
+ }
2977
+ return box;
2978
+ });
2979
+ // Right of asked as bars from 0: a small chart per set, a row per model asked on it,
2980
+ // best first. A model under the ranking floor (DC.low items) is drawn faint: shown,
2981
+ // not yet ranked.
2982
+ function decideBars(w, names, panels, models, colors) {
2983
+ var out = el('div');
2984
+ DC.sets.forEach(function (s) {
2985
+ var rows = DC.scores.filter(function (r) { return r.set === s[0]; }).map(function (r) {
2986
+ return Object.assign({share: r.correct / r.asked, ranked: r.asked >= DC.low}, r);
2987
+ }).sort(function (a, b) { return b.share - a.share || b.asked - a.asked || (a.model < b.model ? -1 : 1); });
2988
+ out.appendChild(el('p', {class: 'chart-label'}, s[1]));
2989
+ if (!rows.length) { out.appendChild(el('p', {class: 'hint'}, 'No model asked on this set yet.')); return; }
2990
+ out.appendChild(Plot.plot({
2991
+ width: w, height: rows.length * 26 + 40, style: style,
2992
+ marginLeft: w < 560 ? 120 : 160, marginRight: 50, marginTop: 4, marginBottom: 30,
2993
+ x: {domain: [0, 1], label: null, tickFormat: '.0%', ticks: 5, grid: true},
2994
+ y: {domain: rows.map(function (r) { return r.model; }), label: null},
2995
+ color: colors,
2996
+ marks: [
2997
+ Plot.barX(rows, {x: 'share', y: 'model', fill: 'model', insetTop: 4, insetBottom: 4, rx: 2,
2998
+ fillOpacity: function (r) { return r.ranked ? 1 : 0.4; },
2999
+ title: function (r) {
3000
+ return r.model + ' · ' + s[1] + '\n' + r.correct + ' right of ' + r.asked + ' asked ('
3001
+ + Math.round(r.share * 100) + '%)' + (r.brier != null ? '\nBrier ' + r.brier.toFixed(3) : '')
3002
+ + (r.ranked ? '' : '\nnot ranked yet: under ' + DC.low + ' items');
3003
+ }}),
3004
+ Plot.text(rows, {x: 'share', y: 'model', dx: 6, textAnchor: 'start', fill: ink,
3005
+ text: function (r) { return r.correct + '/' + r.asked; }}),
3006
+ Plot.ruleX([0], {stroke: ink})
3007
+ ]
3008
+ }));
3009
+ });
3010
+ return out;
3011
+ }
3012
+ function decideCalibration(w, names, panels, models, colors) {
2837
3013
  var pts = DC.points.map(function (p) { return Object.assign({panel: names[p.set]}, p); })
2838
3014
  .sort(function (a, b) { return a.said - b.said; });
2839
- var models = pts.map(function (p) { return p.model; })
2840
- .filter(function (m, i, all) { return all.indexOf(m) === i; }).sort();
3015
+ // Both axes start just under the lowest answer, not at 0%: a decision model is
3016
+ // rarely under 50% sure of its own answer, so a 0-100% frame left every dot in
3017
+ // one corner under a long empty diagonal. Same floor on both, so it stays 45°.
3018
+ var low = Math.min.apply(null, pts.map(function (p) { return Math.min(p.said, p.right); }));
3019
+ var lo = Math.max(0, Math.min(0.9, Math.floor((low - 0.05) * 20) / 20));
2841
3020
  var diag = [];
2842
- panels.forEach(function (s) { diag.push({panel: s, x: 0, y: 0}, {panel: s, x: 1, y: 1}); });
3021
+ panels.forEach(function (s) { diag.push({panel: s, x: lo, y: lo}, {panel: s, x: 1, y: 1}); });
2843
3022
  function tip(p) {
2844
3023
  return p.model + ' · ' + names[p.set] + '\n' + plural(p.n, 'answer') + ' it was about '
2845
3024
  + Math.round(p.said * 100) + '% sure of\n' + Math.round(p.right * 100) + '% of them right';
@@ -2850,10 +3029,11 @@
2850
3029
  return Plot.plot(Object.assign(opts, {
2851
3030
  width: narrow ? Math.min(w, 520) : w, height: narrow ? 3 * 230 + 70 : 320, style: style,
2852
3031
  marginLeft: 52, marginRight: narrow ? 90 : 20, marginBottom: 44, marginTop: 28,
2853
- x: {domain: [0, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
2854
- y: {domain: [0, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
3032
+ inset: 14, // a dot at 100% clear of the frame
3033
+ x: {domain: [lo, 1], label: 'Probability it gave its answer →', tickFormat: '.0%', ticks: 5, grid: true},
3034
+ y: {domain: [lo, 1], label: '↑ Share right', tickFormat: '.0%', ticks: 5, grid: true},
2855
3035
  r: {range: [3, 13]},
2856
- color: {legend: true, domain: models, range: [sky, good, signal, warn, muted, ink, err]},
3036
+ color: Object.assign({legend: true}, colors),
2857
3037
  marks: [
2858
3038
  Plot.frame({stroke: line}),
2859
3039
  Plot.line(diag, Object.assign({x: 'x', y: 'y', z: 'panel', stroke: muted, strokeDasharray: '4 3'}, by)),
@@ -2862,17 +3042,24 @@
2862
3042
  title: tip}, by))
2863
3043
  ]
2864
3044
  }));
2865
- });
3045
+ }
2866
3046
  // Who to trust: finds real bugs (right) vs makes things up (up = less).
2867
3047
  put('chart-overview', function (w) {
2868
3048
  if (!C.overview.length) return el('p', {class: 'hint'}, 'No model has both suite scores and answers yet.');
2869
3049
  var worst = Math.max(0.1, Math.max.apply(null, C.overview.map(function (d) { return d.wrong; })));
2870
3050
  var x0 = Math.max(0, Math.floor((Math.min.apply(null, C.overview.map(function (d) { return d.recall; })) - 0.05) * 20) / 20);
2871
3051
  var pxW = w - 110, pxH = 370;
2872
- var pts = C.overview.slice().sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
3052
+ // Named: the models in or next to the zone (the ones to pick from), then the clearest
3053
+ // warnings (most made up). The rest are on hover: named all at once they ran together.
3054
+ var zoneX = x0 + (1 - x0) / 2, zoneY = worst / 4;
3055
+ var near = function (d) { return d.recall >= zoneX - (1 - x0) * 0.08 && d.wrong <= zoneY * 1.4; };
3056
+ var worstFew = C.overview.slice().sort(function (a, b) { return b.wrong - a.wrong; }).slice(0, 3);
3057
+ var pts = C.overview.filter(function (d) { return near(d) || worstFew.indexOf(d) !== -1; })
3058
+ .sort(function (a, b) { return a.wrong - b.wrong || b.recall - a.recall; });
2873
3059
  var offs = spread(pts, function (d) { return (d.recall - x0) / (1 - x0) * pxW; },
2874
3060
  function (d) { return d.wrong / worst * pxH; });
2875
- var labels = pts.map(function (d, i) { return {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; });
3061
+ var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, recall: d.recall, ty: d.wrong + offs[i] * worst / pxH}; })
3062
+ .filter(Boolean);
2876
3063
  return Plot.plot({
2877
3064
  width: w, height: 440, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
2878
3065
  x: {label: 'Finds real bugs in our code →', domain: [x0, 1], tickFormat: '.0%'},
@@ -2900,71 +3087,72 @@
2900
3087
  tasks[h.model] = bugs + clean;
2901
3088
  var f = bugs ? h.found / bugs : 0, wl = bugs ? h['wrong line'] / bugs : 0, ms = bugs ? h.missed / bugs : 0;
2902
3089
  var fa = clean ? h['false alarm'] / clean : 0;
2903
- score[h.model] = f - fa;
3090
+ // Ranked on what is proven: the low end of the found share's 95% range less the HIGH
3091
+ // end of the false alarms' (never shown clean files: up to 100%), so a few tasks, or
3092
+ // none on clean code, cannot top the chart.
3093
+ score[h.model] = wilson(h.found, bugs)[0] - wilson(h['false alarm'], clean)[1];
2904
3094
  segs.push({model: h.model, part: 'false alarm', x1: -fa, x2: 0, n: h['false alarm'] + ' of ' + clean + ' clean files'});
2905
3095
  segs.push({model: h.model, part: 'found', x1: 0, x2: f, n: h.found + ' of ' + bugs + ' bugs'});
2906
3096
  segs.push({model: h.model, part: 'wrong line', x1: f, x2: f + wl, n: h['wrong line'] + ' of ' + bugs + ' bugs'});
2907
3097
  segs.push({model: h.model, part: 'missed', x1: f + wl, x2: f + wl + ms, n: h.missed + ' of ' + bugs + ' bugs'});
2908
3098
  });
2909
- var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a]; });
3099
+ var models = Object.keys(score).sort(function (a, b) { return score[b] - score[a] || tasks[b] - tasks[a]; });
2910
3100
  return Plot.plot({
2911
- width: w, height: rows(models.length), style: style, marginLeft: 250,
3101
+ width: w, height: rows(models.length), style: style, marginLeft: w < 560 ? 150 : 250,
2912
3102
  x: {domain: [-1, 1], label: '← false alarms on clean files · bugs found →', grid: true,
2913
3103
  tickFormat: function (d) { return Math.abs(Math.round(d * 100)) + '%'; }},
2914
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + tasks[m] + ')'; }},
3104
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + tasks[m] + ')'; }},
2915
3105
  color: {legend: true, domain: ['found', 'wrong line', 'missed', 'false alarm'], range: [good, signal, line, err]},
2916
3106
  marks: [
3107
+ // Under FEW tasks a model's bars are faint: shown, not much to go on.
2917
3108
  Plot.barX(segs.filter(function (s) { return s.x2 > s.x1; }), {x1: 'x1', x2: 'x2', y: 'model', fill: 'part',
2918
- title: function (d) { return d.model + ': ' + d.part + ', ' + d.n; }}),
3109
+ fillOpacity: function (d) { return tasks[d.model] < FEW ? 0.4 : 1; },
3110
+ title: function (d) { return d.model + ': ' + d.part + ', ' + d.n + (tasks[d.model] < FEW ? '\nfew tasks: little to go on' : ''); }}),
2919
3111
  Plot.ruleX([0], {stroke: ink, strokeWidth: 1.5})
2920
3112
  ]
2921
3113
  });
2922
3114
  });
2923
3115
 
2924
- // Real work: the share of judged offloads that helped, sample size beside the name.
3116
+ // Real work: how often each model helped, with how sure that is. The dot is the share
3117
+ // helped, its size how many were judged, the line the range the true share likely sits
3118
+ // in (95% Wilson interval: wide for 3 of 3, narrow for 40 of 50). Ranked by the low end
3119
+ // of that range, so volume counts: 3 of 3 no longer outranks 40 of 50. The dashed line
3120
+ // is every model's offloads pooled.
2925
3121
  put('chart-work', function (w) {
2926
- var R = C.work.slice().sort(function (a, b) { return b.useful / b.judged - a.useful / a.judged || b.judged - a.judged; });
2927
- if (!R.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
2928
- var by = {};
2929
- R.forEach(function (r) { by[r.model] = r; });
2930
- function share(r) { return r.useful / r.judged; }
3122
+ if (!C.work.length) return el('p', {class: 'hint'}, 'No judged offloads yet.');
3123
+ var R = C.work.map(function (r) {
3124
+ var b = wilson(r.useful, r.judged), p = r.useful / r.judged;
3125
+ var verdict = b[0] >= 0.5 ? 'helps' : b[1] < 0.5 ? 'does not help' : 'too few to say';
3126
+ return Object.assign({p: p, lo: b[0], hi: b[1], verdict: verdict}, r);
3127
+ }).sort(function (a, b) { return b.lo - a.lo || b.judged - a.judged; });
3128
+ var all = R.reduce(function (s, r) { return [s[0] + r.useful, s[1] + r.judged]; }, [0, 0]);
3129
+ var pooled = all[0] / all[1];
3130
+ var narrowW = w < 560, nameW = narrowW ? 130 : 210;
2931
3131
  return Plot.plot({
2932
- width: w, height: rows(R.length), style: style, marginLeft: 250, marginRight: 50,
2933
- x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', grid: true},
3132
+ width: w, height: 24 * R.length + 90, style: style, marginLeft: nameW, marginRight: 64, marginTop: 34,
3133
+ x: {domain: [0, 1], label: 'Helped, share of judged offloads →', tickFormat: '.0%', ticks: 5, grid: true},
2934
3134
  y: {domain: R.map(function (r) { return r.model; }), label: null,
2935
- tickFormat: function (m) { return m + ' (' + by[m].judged + ')'; }},
3135
+ tickFormat: function (m) { return fit(m, narrowW ? 16 : 26); }},
3136
+ r: {domain: [0, Math.max.apply(null, R.map(function (r) { return r.judged; }))], range: [3, 11]},
3137
+ color: {domain: ['helps', 'too few to say', 'does not help'], range: [good, muted, err], legend: true},
2936
3138
  marks: [
2937
- Plot.barX(R, {x: share, y: 'model', fill: function (r) { return share(r) >= 0.5 ? good : err; },
2938
- title: function (r) { return r.model + ': helped ' + r.useful + ' of ' + r.judged + ' judged (' + r.offloads + ' offloads)'
2939
- + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : ''); }}),
2940
- Plot.text(R, {x: share, y: 'model', text: function (r) { return r.useful + '/' + r.judged; }, dx: 18, fill: ink}),
2941
- Plot.ruleX([0], {stroke: ink})
3139
+ Plot.ruleX([0.5], {stroke: line}),
3140
+ Plot.ruleX([pooled], {stroke: ink, strokeDasharray: '4 3', strokeOpacity: 0.6}),
3141
+ Plot.text([pooled], {x: function (d) { return d; }, frameAnchor: 'top', dy: -20, fill: muted, fontSize: 11,
3142
+ text: function () { return 'all models ' + Math.round(pooled * 100) + '%'; }}),
3143
+ Plot.ruleY(R, {y: 'model', x1: 'lo', x2: 'hi', stroke: 'verdict', strokeWidth: 2, strokeOpacity: 0.55}),
3144
+ Plot.dot(R, {x: 'p', y: 'model', r: 'judged', fill: 'verdict', stroke: halo, strokeWidth: 1.5,
3145
+ title: function (r) {
3146
+ return r.model + '\nhelped ' + r.useful + ' of ' + r.judged + ' judged (' + Math.round(r.p * 100) + '%; '
3147
+ + r.offloads + ' offloads)\nlikely between ' + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%'
3148
+ + (r.real || r.false ? '\nverified findings ' + r.real + ', false claims ' + r.false : '');
3149
+ }}),
3150
+ Plot.text(R, {x: 'hi', y: 'model', dx: 8, textAnchor: 'start', fill: ink, fontSize: 11,
3151
+ text: function (r) { return r.useful + '/' + r.judged; }})
2942
3152
  ]
2943
3153
  });
2944
3154
  });
2945
3155
 
2946
- // Did it answer: each probe of the last 14 days, by what came back.
2947
- put('chart-reliability', function (w) {
2948
- if (!C.reliability.length) return el('p', {class: 'hint'}, 'No probes in the last 14 days.');
2949
- var total = {}, ok = {};
2950
- C.reliability.forEach(function (d) {
2951
- total[d.model] = (total[d.model] || 0) + d.count;
2952
- if (d.outcome === 'answered') ok[d.model] = d.count;
2953
- });
2954
- var models = Object.keys(total).sort(function (a, b) { return (ok[b] || 0) / total[b] - (ok[a] || 0) / total[a]; });
2955
- var colour = {answered: good, capacity: signal, timeout: sky, structural: err, paywalled: warn,
2956
- entitlement: '#c0563f', other: muted, error: ink};
2957
- var order = Object.keys(colour).filter(function (o) { return C.reliability.some(function (d) { return d.outcome === o; }); });
2958
- return Plot.plot({
2959
- width: w, height: rows(models.length), style: style, marginLeft: 250,
2960
- x: {label: 'Share of probes →', tickFormat: '.0%', grid: true},
2961
- y: {domain: models, label: null},
2962
- color: {legend: true, domain: order, range: order.map(function (o) { return colour[o]; })},
2963
- marks: [Plot.barX(C.reliability, Plot.stackX({x: 'count', y: 'model', fill: 'outcome', offset: 'normalize', order: order,
2964
- title: function (d) { return d.model + ': ' + d.outcome + ' ' + d.count + ' of ' + total[d.model] + ' probes'; }}))]
2965
- });
2966
- });
2967
-
2968
3156
  // Speed vs quality: median seconds on our own tasks against findings located.
2969
3157
  put('chart-speed', function (w) {
2970
3158
  if (!C.speed.length) return el('p', {class: 'hint'}, 'No suite timings yet.');
@@ -2975,14 +3163,20 @@
2975
3163
  var pts = C.speed.slice().sort(function (a, b) { return b.recall - a.recall; });
2976
3164
  var offs = spread(pts, function (d) { return (Math.log(d.seconds) - Math.log(lo)) / (Math.log(hi) - Math.log(lo)) * pxW; },
2977
3165
  function (d) { return (1 - d.recall) / (1 - y0) * pxH; });
2978
- var labels = pts.map(function (d, i) { return {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; });
3166
+ var labels = pts.map(function (d, i) { return offs[i] == null ? null : {model: d.model, seconds: d.seconds, ty: d.recall - offs[i] * (1 - y0) / pxH}; })
3167
+ .filter(Boolean);
2979
3168
  return Plot.plot({
2980
3169
  width: w, height: 420, style: style, grid: true, inset: 16, marginRight: 50, marginTop: 30,
2981
3170
  x: {type: 'log', domain: [lo, hi], label: 'Median seconds per answer on our tasks →'},
2982
3171
  y: {domain: [y0, 1], label: '↑ Findings located', tickFormat: '.0%'},
2983
3172
  marks: [
3173
+ // The zone to be in, as on Who to trust: the faster half (on the log scale) and the
3174
+ // top quarter of findings. Outlined, so it reads on a dark ground too.
3175
+ Plot.rect([{x1: lo, x2: Math.sqrt(lo * hi), y1: y0 + (1 - y0) * 3 / 4, y2: 1}], {x1: 'x1', x2: 'x2', y1: 'y1', y2: 'y2',
3176
+ fill: good, fillOpacity: 0.12, stroke: good, strokeOpacity: 0.5, strokeDasharray: '4 3'}),
2984
3177
  Plot.dot(C.speed, {x: 'seconds', y: 'recall', r: 6, fill: sky, stroke: ink, strokeWidth: 0.5}),
2985
- Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink}),
3178
+ Plot.text(labels, {x: 'seconds', y: 'ty', text: 'model', fontSize: 11, fill: ink,
3179
+ stroke: halo, strokeWidth: 3.5, strokeOpacity: 0.9, paintOrder: 'stroke'}),
2986
3180
  Plot.tip(C.speed, Plot.pointer({x: 'seconds', y: 'recall',
2987
3181
  title: function (d) { return d.model + '\nmedian ' + d.seconds.toFixed(0) + 's over ' + d.tasks + ' tasks\nfinds ' + Math.round(d.recall * 100) + '%'; }}))
2988
3182
  ]
@@ -2997,25 +3191,41 @@
2997
3191
  return C.families.indexOf(fam[a]) - C.families.indexOf(fam[b]) || (a < b ? -1 : 1);
2998
3192
  });
2999
3193
  tasks.forEach(function (t) { if (!(fam[t] in first)) first[fam[t]] = t; });
3000
- var label = {};
3001
- Object.keys(first).forEach(function (f) { label[first[f]] = f; });
3194
+ // Each group's name over its first task, on two staggered rows: side by side they
3195
+ // ran into one another wherever a group is only a few tasks wide.
3196
+ var heads = C.families.filter(function (f) { return f in first; })
3197
+ .map(function (f, i) { return {task: first[f], name: f, row: i % 2}; });
3002
3198
  // By total found, so a model that played three tasks does not top one that played forty.
3003
- var sum = {};
3004
- C.heat.forEach(function (d) { sum[d.model] = (sum[d.model] || 0) + (d.rate || 0); });
3005
- var models = Object.keys(sum).sort(function (a, b) { return sum[b] - sum[a]; });
3199
+ // A model with no scored task (only grey) gets no row: it is named under the chart.
3200
+ var sum = {}, scored = {};
3201
+ C.heat.forEach(function (d) {
3202
+ sum[d.model] = (sum[d.model] || 0) + (d.rate || 0);
3203
+ if (d.rate != null) scored[d.model] = true;
3204
+ });
3205
+ var models = Object.keys(sum).filter(function (m) { return scored[m]; }).sort(function (a, b) { return sum[b] - sum[a]; });
3206
+ var idle = Object.keys(sum).filter(function (m) { return !scored[m]; }).sort();
3207
+ var shown = C.heat.filter(function (d) { return scored[d.model]; });
3208
+ var nameW = w < 560 ? 130 : 230;
3006
3209
  function tip(d) { return d.model + '\n' + d.task + ' (' + d.family + ')\n' + d.label + (d.invented ? '\ninvented something' : ''); }
3007
- return Plot.plot({
3008
- width: w, height: 22 * models.length + 70, style: style, marginLeft: 230, marginTop: 30, padding: 0.08,
3009
- x: {domain: tasks, axis: 'top', label: null, tickSize: 0, tickFormat: function (t) { return label[t] || ''; }},
3010
- y: {domain: models, label: null, tickSize: 0},
3210
+ var box = el('div');
3211
+ box.appendChild(Plot.plot({
3212
+ width: w, height: 22 * models.length + 84, style: style, marginLeft: nameW, marginTop: 44, padding: 0.08,
3213
+ x: {domain: tasks, axis: null},
3214
+ y: {domain: models, label: null, tickSize: 0, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 30); }},
3011
3215
  color: {type: 'linear', domain: [0, 0.5, 1], range: [err, signal, good]},
3012
3216
  marks: [
3013
- Plot.cell(C.heat.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
3014
- Plot.cell(C.heat.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
3015
- Plot.dot(C.heat.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
3016
- Plot.ruleX(Object.keys(label), {x: function (t) { return t; }, stroke: ink, strokeOpacity: 0.25, dx: -2})
3217
+ Plot.cell(shown.filter(function (d) { return d.rate == null; }), {x: 'task', y: 'model', fill: line, title: tip}),
3218
+ Plot.cell(shown.filter(function (d) { return d.rate != null; }), {x: 'task', y: 'model', fill: 'rate', title: tip}),
3219
+ Plot.dot(shown.filter(function (d) { return d.invented; }), {x: 'task', y: 'model', r: 2.4, fill: ink}),
3220
+ Plot.ruleX(heads, {x: 'task', stroke: ink, strokeOpacity: 0.25, dx: -2}),
3221
+ Plot.text(heads.filter(function (h) { return !h.row; }), {x: 'task', frameAnchor: 'top', dy: -30, dx: -2,
3222
+ text: 'name', textAnchor: 'start', fontSize: 11, fill: ink}),
3223
+ Plot.text(heads.filter(function (h) { return h.row; }), {x: 'task', frameAnchor: 'top', dy: -14, dx: -2,
3224
+ text: 'name', textAnchor: 'start', fontSize: 11, fill: ink})
3017
3225
  ]
3018
- });
3226
+ }));
3227
+ if (idle.length) box.appendChild(el('p', {class: 'hint'}, 'No scored task yet, so no row: ' + idle.join(', ') + '.'));
3228
+ return box;
3019
3229
  });
3020
3230
 
3021
3231
  // Vision tasks: one square per field and model, the task's name over its first field.
@@ -3062,39 +3272,66 @@
3062
3272
  kinds.forEach(function (k) { if (W[m][k]) data.push({model: m, kind: k, count: W[m][k]}); total[m] = (total[m] || 0) + W[m][k]; });
3063
3273
  });
3064
3274
  if (!data.length) return el('p', {class: 'hint'}, 'No vision panels through the gateway yet.');
3065
- // The models doing the most image work first: their bars are the evidence.
3066
- var models = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
3275
+ // The models doing the most image work first: their bars are the evidence. Under FEW
3276
+ // seats a row was mostly "no answer" and said little: those are listed under the chart.
3277
+ var all = Object.keys(total).sort(function (a, b) { return total[b] - total[a] || (a < b ? -1 : 1); });
3278
+ var models = all.filter(function (m) { return total[m] >= FEW; });
3279
+ var few = all.filter(function (m) { return total[m] < FEW; });
3280
+ if (!models.length) { models = all; few = []; }
3281
+ data = data.filter(function (d) { return models.indexOf(d.model) !== -1; });
3067
3282
  var colour = {'checked, helped': good, 'checked, did not help': err, 'useful, no claims': sky,
3068
3283
  'no use, no claims': warn, 'not judged': line, 'no answer': muted};
3069
3284
  // The claims checked by hand, beside the bar: the only right-or-wrong count real work has.
3070
3285
  var claims = models.filter(function (m) { return W[m].real + W[m].false; }).map(function (m) {
3071
3286
  return {model: m, text: W[m].real + ' of ' + (W[m].real + W[m].false) + ' claims right'};
3072
3287
  });
3073
- return Plot.plot({
3074
- width: w, height: rows(models.length), style: style, marginLeft: 300, marginRight: 150,
3288
+ var narrowV = w < 560, box = el('div');
3289
+ box.appendChild(Plot.plot({
3290
+ width: w, height: rows(models.length), style: style, marginLeft: narrowV ? 150 : 260, marginRight: narrowV ? 20 : 150,
3075
3291
  x: {label: 'Share of its seats on image panels →', tickFormat: '.0%', grid: true, domain: [0, 1]},
3076
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + total[m] + ')'; }},
3292
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, narrowV ? 16 : 30) + ' (' + total[m] + ')'; }},
3077
3293
  color: {legend: true, domain: kinds, range: kinds.map(function (k) { return colour[k]; })},
3078
3294
  marks: [
3079
3295
  Plot.barX(data, Plot.stackX({x: 'count', y: 'model', fill: 'kind', offset: 'normalize', order: kinds,
3080
3296
  title: function (d) { return d.model + ': ' + d.kind + ', ' + d.count + ' of ' + total[d.model] + ' seats'; }})),
3081
- Plot.text(claims, {x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
3297
+ Plot.text(narrowV ? [] : claims.filter(function (c) { return models.indexOf(c.model) !== -1; }),
3298
+ {x: 1, y: 'model', text: 'text', dx: 8, textAnchor: 'start', fill: ink})
3082
3299
  ]
3083
- });
3300
+ }));
3301
+ if (few.length) box.appendChild(el('p', {class: 'hint'}, 'Under ' + FEW + ' seats, so no bar: '
3302
+ + few.map(function (m) { return m + ' (' + total[m] + ')'; }).join(', ') + '. The table has them all.'));
3303
+ return box;
3084
3304
  });
3085
3305
 
3086
- // Confidently wrong: stacked by kind, worst at the top.
3306
+ // Confidently wrong: stacked by kind. A thin line runs over each bar for the range the
3307
+ // true share likely sits in (95% Wilson), so 0 of 2 shows how bad it could be, not a
3308
+ // clean record. Worst proven at the top: by the low end of that range. Few answers: faint.
3087
3309
  put('chart-wrong', function (w) {
3088
- var T = C.wrongTotals;
3089
- var models = Object.keys(T).sort(function (a, b) { return T[b].rate - T[a].rate || T[b].answers - T[a].answers; });
3310
+ var T = C.wrongTotals, R = {};
3311
+ Object.keys(T).forEach(function (m) {
3312
+ var b = wilson(T[m].wrong, T[m].answers);
3313
+ R[m] = {model: m, lo: b[0], hi: b[1], few: T[m].answers < FEW};
3314
+ });
3315
+ var models = Object.keys(T).sort(function (a, b) {
3316
+ return R[b].lo - R[a].lo || T[b].rate - T[a].rate || T[b].answers - T[a].answers;
3317
+ });
3318
+ var ranges = models.map(function (m) { return R[m]; });
3319
+ var top = Math.min(1, Math.max.apply(null, ranges.map(function (r) { return r.hi; })) + 0.02);
3090
3320
  return Plot.plot({
3091
- width: w, height: 20 * models.length + 70, style: style, marginLeft: 250,
3092
- x: {label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
3321
+ width: w, height: 20 * models.length + 70, style: style, marginLeft: w < 560 ? 150 : 250,
3322
+ x: {domain: [0, top], label: 'Confidently wrong, share of answers that could have been →', tickFormat: '.0%', grid: true},
3093
3323
  // The count beside each name: 1 of 2 is not the same evidence as 17 of 26.
3094
- y: {domain: models, label: null, tickFormat: function (m) { return m + ' (' + T[m].answers + ')'; }},
3324
+ y: {domain: models, label: null, tickFormat: function (m) { return fit(m, w < 560 ? 16 : 28) + ' (' + T[m].answers + ')'; }},
3095
3325
  color: {legend: true, domain: C.kinds, range: [err, signal, sky, muted]},
3096
3326
  marks: [
3097
- Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
3327
+ Plot.barX(C.wrong, {x: 'rate', y: 'model', fill: 'kind', insetTop: 3, insetBottom: 3,
3328
+ fillOpacity: function (d) { return R[d.model].few ? 0.4 : 1; },
3329
+ title: function (d) { return d.model + ': ' + d.kind + ' ' + d.n; }}),
3330
+ Plot.ruleY(ranges, {y: 'model', x1: 'lo', x2: 'hi', stroke: ink, strokeOpacity: 0.45, strokeWidth: 1.5,
3331
+ title: function (r) {
3332
+ return r.model + ': ' + T[r.model].wrong + ' of ' + T[r.model].answers + ' confidently wrong\nlikely between '
3333
+ + Math.round(r.lo * 100) + '% and ' + Math.round(r.hi * 100) + '%' + (r.few ? '\nfew answers: little to go on' : '');
3334
+ }}),
3098
3335
  Plot.ruleX([0], {stroke: ink})
3099
3336
  ]
3100
3337
  });