@setsquare/review-sdk 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -7
- package/dist/axe.d.ts.map +1 -1
- package/dist/axe.js +25 -3
- package/dist/axe.js.map +1 -1
- package/dist/checks/context-change.d.ts.map +1 -1
- package/dist/checks/context-change.js +3 -0
- package/dist/checks/context-change.js.map +1 -1
- package/dist/checks/dynamic.d.ts.map +1 -1
- package/dist/checks/dynamic.js +3 -0
- package/dist/checks/dynamic.js.map +1 -1
- package/dist/checks/focus.d.ts +25 -5
- package/dist/checks/focus.d.ts.map +1 -1
- package/dist/checks/focus.js +203 -15
- package/dist/checks/focus.js.map +1 -1
- package/dist/checks/heuristics.js +6 -0
- package/dist/checks/heuristics.js.map +1 -1
- package/dist/checks/keyboard.js +12 -0
- package/dist/checks/keyboard.js.map +1 -1
- package/dist/checks/reflow.d.ts.map +1 -1
- package/dist/checks/reflow.js +4 -0
- package/dist/checks/reflow.js.map +1 -1
- package/dist/checks/shortcuts.d.ts.map +1 -1
- package/dist/checks/shortcuts.js +3 -0
- package/dist/checks/shortcuts.js.map +1 -1
- package/dist/colour-shim.d.ts +77 -0
- package/dist/colour-shim.d.ts.map +1 -0
- package/dist/colour-shim.js +192 -0
- package/dist/colour-shim.js.map +1 -0
- package/dist/findings.d.ts.map +1 -1
- package/dist/findings.js +57 -0
- package/dist/findings.js.map +1 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +54 -12
- package/dist/html-report.js.map +1 -1
- package/dist/options.d.ts +20 -9
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +23 -1
- package/dist/options.js.map +1 -1
- package/dist/reporter.js +2 -2
- package/dist/reporter.js.map +1 -1
- package/dist/screens.d.ts.map +1 -1
- package/dist/screens.js +34 -1
- package/dist/screens.js.map +1 -1
- package/dist/summary.d.ts +20 -1
- package/dist/summary.d.ts.map +1 -1
- package/dist/summary.js +220 -39
- package/dist/summary.js.map +1 -1
- package/package.json +4 -4
package/dist/summary.d.ts
CHANGED
|
@@ -31,7 +31,15 @@ export interface SummaryInput {
|
|
|
31
31
|
envelopePath: string;
|
|
32
32
|
/** Surfaces the suite visited that nothing scanned. Reported, never hidden. */
|
|
33
33
|
warnings: string[];
|
|
34
|
-
|
|
34
|
+
/**
|
|
35
|
+
* Print every section, rather than the part a person reads.
|
|
36
|
+
*
|
|
37
|
+
* Default (false) ends at the criteria table and points at the report. The
|
|
38
|
+
* blocks that QUALIFY the figures above — what could not be checked, what is
|
|
39
|
+
* worth knowing — stay in both, because a shorter summary that is also a more
|
|
40
|
+
* confident one would be the wrong trade in this product.
|
|
41
|
+
*/
|
|
42
|
+
verbose?: boolean;
|
|
35
43
|
/**
|
|
36
44
|
* Playwright's own HTML report, when one was generated.
|
|
37
45
|
*
|
|
@@ -118,6 +126,8 @@ interface RuleGroup {
|
|
|
118
126
|
ruleId: string;
|
|
119
127
|
impact: string;
|
|
120
128
|
status: AxeFinding['status'];
|
|
129
|
+
/** On an incomplete: whose work it is. Absent means the run did not state it. */
|
|
130
|
+
undecidable?: AxeFinding['undecidable'];
|
|
121
131
|
nodes: number;
|
|
122
132
|
contexts: string[];
|
|
123
133
|
/** The same contexts with their element counts, for ranking worst-first. */
|
|
@@ -250,6 +260,15 @@ export interface RunTotals {
|
|
|
250
260
|
scans: number;
|
|
251
261
|
violations: number;
|
|
252
262
|
incomplete: number;
|
|
263
|
+
/**
|
|
264
|
+
* The two halves of `incomplete`, which was one word doing two jobs.
|
|
265
|
+
*
|
|
266
|
+
* `judgement` is the reader's work and includes rows the run did not classify —
|
|
267
|
+
* absence is not "measurement", and reading it as such would inflate the one
|
|
268
|
+
* figure that is meant to fall. `unmeasured` is ours.
|
|
269
|
+
*/
|
|
270
|
+
judgement: number;
|
|
271
|
+
unmeasured: number;
|
|
253
272
|
passes: number;
|
|
254
273
|
clean: number;
|
|
255
274
|
}
|
package/dist/summary.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"summary.d.ts","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAoBtE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,
|
|
1
|
+
{"version":3,"file":"summary.d.ts","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAoBtE,OAAO,EAA6B,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,IAAI,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;IACzB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD;;;;;;;;OAQG;IACH,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,2DAA2D;IAC3D,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;CAC9C;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,EAAE,CA+CxD;AAID,UAAU,SAAS;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC7B,iFAAiF;IACjF,WAAW,CAAC,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,4EAA4E;IAC5E,YAAY,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjD;;;;;OAKG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AA+CD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,gBAAgB,GAAG,SAAS,EAAE,CAmErE;AASD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;;;;;;;;OAQG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9C;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,UAAU,EAAE,CA+CpE;AAED,MAAM,WAAW,eAAe;IAC9B,kEAAkE;IAClE,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;OAUG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,gBAAgB,GAAG,eAAe,CAmC3E;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAeD;;;;;;;;;GASG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,EAAE,CAwB9D;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,gBAAgB,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,YAAY,CA4C5F;AAED,wBAAgB,MAAM,CAAC,QAAQ,EAAE,gBAAgB,GAAG,SAAS,CAwC5D;AAED,kEAAkE;AAClE,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAEjE;AAgCD;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAukBxE;AA03BD,oEAAoE;AACpE,wBAAgB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CA6J1D;AAED,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;AAEzD,wBAAgB,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAsC3F"}
|
package/dist/summary.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { conformanceGrid, conformanceSummary, ruleSurfaceShare, stateCoverage, } from '@setsquare/schema/conformance';
|
|
2
|
-
import {
|
|
2
|
+
import { ruleHelpUrl } from './links.js';
|
|
3
3
|
import { continuationLines, createStyle, figureBlock, figureLine, heading, } from './style.js';
|
|
4
4
|
import { flakeNotes, insightBlocks } from './insights.js';
|
|
5
5
|
/**
|
|
@@ -120,6 +120,7 @@ export function groupFindings(envelope) {
|
|
|
120
120
|
ruleId: finding.ruleId,
|
|
121
121
|
impact: finding.impact ?? 'unknown',
|
|
122
122
|
status: finding.status,
|
|
123
|
+
...(finding.undecidable ? { undecidable: finding.undecidable } : {}),
|
|
123
124
|
nodes: finding.targets.nodeCount,
|
|
124
125
|
contexts: [label],
|
|
125
126
|
contextNodes: [{ label, nodes: finding.targets.nodeCount }],
|
|
@@ -373,6 +374,8 @@ export function totals(envelope) {
|
|
|
373
374
|
const themes = new Set();
|
|
374
375
|
let violations = 0;
|
|
375
376
|
let incomplete = 0;
|
|
377
|
+
let judgement = 0;
|
|
378
|
+
let unmeasured = 0;
|
|
376
379
|
let passes = 0;
|
|
377
380
|
let clean = 0;
|
|
378
381
|
for (const scan of envelope.scans) {
|
|
@@ -381,9 +384,17 @@ export function totals(envelope) {
|
|
|
381
384
|
themes.add(scan.context.theme);
|
|
382
385
|
const scanViolations = scan.findings.filter((f) => f.status === 'violation');
|
|
383
386
|
violations += scanViolations.reduce((sum, f) => sum + f.targets.nodeCount, 0);
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
+
for (const f of scan.findings) {
|
|
388
|
+
if (f.status !== 'incomplete')
|
|
389
|
+
continue;
|
|
390
|
+
incomplete += f.targets.nodeCount;
|
|
391
|
+
// Unclassified rows count as a judgement: absence is not "measurement",
|
|
392
|
+
// and inflating our own debt figure would make it useless as a measure.
|
|
393
|
+
if (f.undecidable === 'measurement')
|
|
394
|
+
unmeasured += f.targets.nodeCount;
|
|
395
|
+
else
|
|
396
|
+
judgement += f.targets.nodeCount;
|
|
397
|
+
}
|
|
387
398
|
passes += scan.passes.reduce((sum, p) => sum + p.nodeCount, 0);
|
|
388
399
|
if (scanViolations.length === 0)
|
|
389
400
|
clean += 1;
|
|
@@ -394,6 +405,8 @@ export function totals(envelope) {
|
|
|
394
405
|
scans: envelope.scans.length,
|
|
395
406
|
violations,
|
|
396
407
|
incomplete,
|
|
408
|
+
judgement,
|
|
409
|
+
unmeasured,
|
|
397
410
|
passes,
|
|
398
411
|
clean,
|
|
399
412
|
};
|
|
@@ -455,14 +468,31 @@ export function renderText(input, columns) {
|
|
|
455
468
|
* strip before a word is read.
|
|
456
469
|
*/
|
|
457
470
|
/**
|
|
458
|
-
* Where the run's own artefacts are, and
|
|
471
|
+
* Where the run's own artefacts are, and the closing block.
|
|
472
|
+
*
|
|
473
|
+
* There used to be a five-line pitch here, on by default, switchable off with
|
|
474
|
+
* `promo: false`. It is gone, and the reasoning is worth keeping because it
|
|
475
|
+
* will be proposed again.
|
|
476
|
+
*
|
|
477
|
+
* The design answered two questions at once and could not be right about both.
|
|
478
|
+
* It read as an advert — a product name, a benefit list, a URL — but every
|
|
479
|
+
* line of it was TRUE ABOUT THEIR RUN, and it was switchable, which only makes
|
|
480
|
+
* sense for an advert. A disclosure is not optional; you cannot turn off a
|
|
481
|
+
* limitation.
|
|
459
482
|
*
|
|
460
|
-
*
|
|
461
|
-
*
|
|
462
|
-
*
|
|
463
|
-
*
|
|
483
|
+
* This repository's own rule about `doctor`'s exit code settled it: *we do not
|
|
484
|
+
* fail a customer's build, and the promise is weaker if it has an opt-out*.
|
|
485
|
+
* "We do not advertise in your build output" is a stronger promise than "we
|
|
486
|
+
* advertise unless you ask us not to" — and the flag was a fiction anyway,
|
|
487
|
+
* governing one of three places the pitch appeared while `verify` and the
|
|
488
|
+
* SARIF caveat carried it regardless, the second of those into the customer's
|
|
489
|
+
* own code-scanning UI.
|
|
490
|
+
*
|
|
491
|
+
* What survives is the half that was a fact about their run, as a caveat
|
|
492
|
+
* rather than a pitch: no product name, no URL, and not switchable, because
|
|
493
|
+
* "this run only" is a real limit on what the reader is holding.
|
|
464
494
|
*/
|
|
465
|
-
const trailer = (
|
|
495
|
+
const trailer = (scanned = true) => [
|
|
466
496
|
...(input.warnings.length > 0 ? [''] : []),
|
|
467
497
|
...warningLines(input.warnings, width, evidenceQuality(envelope), envelope),
|
|
468
498
|
'',
|
|
@@ -472,7 +502,21 @@ export function renderText(input, columns) {
|
|
|
472
502
|
? [figureLine(style, '', `${input.htmlReportPath} ${style.dim('(open in a browser)')}`)]
|
|
473
503
|
: []),
|
|
474
504
|
...(input.screensPath
|
|
475
|
-
? [
|
|
505
|
+
? [
|
|
506
|
+
figureLine(style, '', `${input.screensPath} ${style.dim('(screenshots, one page)')}`),
|
|
507
|
+
/*
|
|
508
|
+
* Said where the path is, because that is where someone decides
|
|
509
|
+
* whether to keep it — and measured rather than assumed: Playwright
|
|
510
|
+
* clears its whole output directory at the start of a run, including
|
|
511
|
+
* captures belonging to tests that run does not execute.
|
|
512
|
+
*
|
|
513
|
+
* The sheet indexes those images rather than copying them, which is a
|
|
514
|
+
* privacy guarantee and not tidiness: Playwright's own captures are
|
|
515
|
+
* unmasked, and this package writes no images at all.
|
|
516
|
+
*/
|
|
517
|
+
...continuationLines(style, "That page links to Playwright's images rather than copying them, and Playwright " +
|
|
518
|
+
'clears its output directory on its next run — so copy it now if this run matters.', width),
|
|
519
|
+
]
|
|
476
520
|
: []),
|
|
477
521
|
...(input.reportPath
|
|
478
522
|
? [figureLine(style, '', `${input.reportPath} ${style.dim('(Playwright report)')}`)]
|
|
@@ -484,7 +528,13 @@ export function renderText(input, columns) {
|
|
|
484
528
|
...groupLines(input.group).flatMap((line) => wrap(line, Math.max(40, width - 8), '').map((wrapped) => figureLine(style, '', wrapped))),
|
|
485
529
|
]
|
|
486
530
|
: []),
|
|
487
|
-
|
|
531
|
+
/*
|
|
532
|
+
* A run that scanned nothing gets no caveat, for the reason the pitch used
|
|
533
|
+
* to be suppressed there: the figures are all zero, and a line about what
|
|
534
|
+
* this run does not cover reads as noise bolted onto a failure. The block
|
|
535
|
+
* above already says nothing was looked at, which is the stronger statement.
|
|
536
|
+
*/
|
|
537
|
+
...(scanned ? historyCaveat(input, style, width) : ['']),
|
|
488
538
|
];
|
|
489
539
|
lines.push('');
|
|
490
540
|
lines.push(` ${style.accent('Setsquare')} — accessibility evidence`);
|
|
@@ -607,10 +657,54 @@ export function renderText(input, columns) {
|
|
|
607
657
|
lines.push(figureLine(style, failingRules, `rule${failingRules === 1 ? '' : 's'} failing, on ${n(count.violations)} ` +
|
|
608
658
|
`element${count.violations === 1 ? '' : 's'}`));
|
|
609
659
|
if (count.incomplete > 0) {
|
|
610
|
-
|
|
660
|
+
/*
|
|
661
|
+
* Two figures, because two different things were sharing one word — and the
|
|
662
|
+
* old line credited axe for both, which stopped being true when most of the
|
|
663
|
+
* undecided rows came from our own checks.
|
|
664
|
+
*/
|
|
665
|
+
if (count.judgement > 0) {
|
|
666
|
+
lines.push(figureLine(style, count.judgement, 'needing a judgement — a person decides'));
|
|
667
|
+
}
|
|
668
|
+
if (count.unmeasured > 0) {
|
|
669
|
+
lines.push(figureLine(style, count.unmeasured, 'we could not measure — our debt, not yours'));
|
|
670
|
+
}
|
|
611
671
|
}
|
|
612
672
|
lines.push(figureLine(style, count.passes, style.dim(`checks passed, across ${quality.rulesAssessed} distinct rules`)));
|
|
673
|
+
/*
|
|
674
|
+
* The element figure holds still only in the sense that the rule count above it
|
|
675
|
+
* does. Said once, here, where someone is about to quote a number.
|
|
676
|
+
*/
|
|
677
|
+
if (count.violations > 0) {
|
|
678
|
+
lines.push(...continuationLines(style, 'Rule and surface counts are the stable pair. Element counts follow how much data ' +
|
|
679
|
+
'is on the page, so they move between runs of unchanged code — measured at 1,712 ' +
|
|
680
|
+
'and then 1,626 on two runs of one commit, while rules × surfaces held at 230.', width));
|
|
681
|
+
}
|
|
613
682
|
lines.push(...conformanceLines(input, style, width));
|
|
683
|
+
/*
|
|
684
|
+
* THE CUT. Everything below is the report's job unless asked for.
|
|
685
|
+
*
|
|
686
|
+
* Friction record run 4 measured 1,080 lines on Eos, and said the useful part
|
|
687
|
+
* was the first forty. Both halves of that matter: a summary nobody reads is
|
|
688
|
+
* not merely long, it takes the part that WAS worth reading down with it,
|
|
689
|
+
* because by the end of a run those lines have scrolled away.
|
|
690
|
+
*
|
|
691
|
+
* Measured on the fixture suite at 26 surfaces — where the enumerations are
|
|
692
|
+
* the bulk, and the criteria table is the last thing a person reads:
|
|
693
|
+
*
|
|
694
|
+
* RULES FAILING 152 · NEEDS A JUDGEMENT 82 · CONFORMANCE 62 · SURFACES 36
|
|
695
|
+
*
|
|
696
|
+
* What still prints below the cut is anything that QUALIFIES the figures
|
|
697
|
+
* above — what could not be checked, what is worth knowing. Dropping those
|
|
698
|
+
* would make the short summary a MORE confident one than the long summary,
|
|
699
|
+
* which is exactly the wrong direction for an evidence product: the reader
|
|
700
|
+
* would be left with the numbers and without the reasons to distrust them.
|
|
701
|
+
*/
|
|
702
|
+
if (!input.verbose) {
|
|
703
|
+
lines.push(...qualifyingBlocks(input, style, width));
|
|
704
|
+
lines.push(...moreInTheReport(input, style, width));
|
|
705
|
+
lines.push(...trailer());
|
|
706
|
+
return join(lines);
|
|
707
|
+
}
|
|
614
708
|
lines.push(...gridLines(input, style, width));
|
|
615
709
|
lines.push('');
|
|
616
710
|
/*
|
|
@@ -737,24 +831,46 @@ export function renderText(input, columns) {
|
|
|
737
831
|
for (const group of failingRuleGroups)
|
|
738
832
|
lines.push(...ruleEntry(group));
|
|
739
833
|
}
|
|
740
|
-
|
|
834
|
+
/*
|
|
835
|
+
* TWO sections, because two different things were sharing one word.
|
|
836
|
+
*
|
|
837
|
+
* Friction record run 4 measured what that cost: roughly 273 of Eos's 374
|
|
838
|
+
* needs-review rows were measurements we had not taken rather than judgements
|
|
839
|
+
* a person must make. A reader asked to work through a bucket where most rows
|
|
840
|
+
* cannot be acted on without a colour picker learns to skip the bucket — and
|
|
841
|
+
* the rows they skip include the ones an auditor actually wants.
|
|
842
|
+
*
|
|
843
|
+
* So the judgements come first and keep the section a person is meant to read,
|
|
844
|
+
* and our own debt is stated separately, as a figure, under our own name.
|
|
845
|
+
*/
|
|
846
|
+
const judgement = review.filter((group) => group.undecidable !== 'measurement');
|
|
847
|
+
const measurement = review.filter((group) => group.undecidable === 'measurement');
|
|
848
|
+
if (judgement.length > 0) {
|
|
741
849
|
/*
|
|
742
|
-
* A section of its own, not a parenthesis on the failure list.
|
|
743
|
-
*
|
|
744
|
-
*
|
|
745
|
-
*
|
|
746
|
-
* uncomputable from the DOM. These are neither passes nor failures, and
|
|
747
|
-
* listing them beside violations invites both of the available mistakes:
|
|
748
|
-
* treating them as broken and panicking, or treating them as fine and
|
|
749
|
-
* shipping. They are the one category that REQUIRES a person, which is worth
|
|
750
|
-
* saying plainly given we sell evidence.
|
|
850
|
+
* A section of its own, not a parenthesis on the failure list. These are
|
|
851
|
+
* neither passes nor failures, and listing them beside violations invites
|
|
852
|
+
* both available mistakes: treating them as broken and panicking, or
|
|
853
|
+
* treating them as fine and shipping.
|
|
751
854
|
*/
|
|
752
855
|
lines.push('');
|
|
753
|
-
lines.push(heading(style, 'Needs
|
|
754
|
-
lines.push(...continuationLines(style, '
|
|
755
|
-
'
|
|
856
|
+
lines.push(heading(style, 'Needs a judgement'));
|
|
857
|
+
lines.push(...continuationLines(style, 'Measured as far as measuring goes, and what is left is not mechanical — ' +
|
|
858
|
+
'does this heading describe what it heads, is this error announced. A person ' +
|
|
859
|
+
'settles these by looking, with no tools.', width));
|
|
860
|
+
lines.push('');
|
|
861
|
+
for (const group of judgement)
|
|
862
|
+
lines.push(...ruleEntry(group));
|
|
863
|
+
}
|
|
864
|
+
if (measurement.length > 0) {
|
|
865
|
+
const elements = measurement.reduce((sum, group) => sum + group.nodes, 0);
|
|
866
|
+
lines.push('');
|
|
867
|
+
lines.push(heading(style, 'Could not measure'));
|
|
868
|
+
lines.push(...continuationLines(style, `${elements} element${elements === 1 ? '' : 's'} where we did not get a number — a ` +
|
|
869
|
+
'gradient with no single colour behind it, a value we could not resolve. ' +
|
|
870
|
+
'This is our debt rather than your work, and it is here so it can be seen ' +
|
|
871
|
+
'falling rather than sitting inside a bucket labelled as yours.', width));
|
|
756
872
|
lines.push('');
|
|
757
|
-
for (const group of
|
|
873
|
+
for (const group of measurement)
|
|
758
874
|
lines.push(...ruleEntry(group));
|
|
759
875
|
}
|
|
760
876
|
/*
|
|
@@ -781,6 +897,52 @@ export function renderText(input, columns) {
|
|
|
781
897
|
lines.push(...trailer());
|
|
782
898
|
return join(lines);
|
|
783
899
|
}
|
|
900
|
+
/**
|
|
901
|
+
* The blocks that qualify the figures, and print above the cut as well as below.
|
|
902
|
+
*
|
|
903
|
+
* Deliberately a short list rather than "the insight blocks". `Could not be
|
|
904
|
+
* checked` is the surfaces the run reached and could not scan, and `Worth
|
|
905
|
+
* knowing` carries the scans the settle ceiling cut short — both are reasons to
|
|
906
|
+
* trust the numbers above LESS, and a concise summary that dropped them would be
|
|
907
|
+
* more confident than the full one. `Page runtime`, `Across surfaces`,
|
|
908
|
+
* `Coverage` and `Suite timing` are additional findings rather than
|
|
909
|
+
* qualifications, so they wait for the report or `--verbose`.
|
|
910
|
+
*/
|
|
911
|
+
const QUALIFYING = new Set(['Could not be checked', 'Worth knowing']);
|
|
912
|
+
function qualifyingBlocks(input, style, width) {
|
|
913
|
+
const lines = [];
|
|
914
|
+
for (const block of insightBlocks(input.envelope, {
|
|
915
|
+
...(input.declaredAxes ? { declaredAxes: input.declaredAxes } : {}),
|
|
916
|
+
...(input.unmeasuredAxes ? { unmeasuredAxes: input.unmeasuredAxes } : {}),
|
|
917
|
+
...(input.suite ? { suite: input.suite } : {}),
|
|
918
|
+
...(input.overhead ? { overhead: input.overhead } : {}),
|
|
919
|
+
})) {
|
|
920
|
+
if (!QUALIFYING.has(block.heading))
|
|
921
|
+
continue;
|
|
922
|
+
lines.push(...figureBlock(style, block.heading, block.figures, block.notes, width));
|
|
923
|
+
}
|
|
924
|
+
return lines;
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* One line naming what was left out, and how to see it.
|
|
928
|
+
*
|
|
929
|
+
* "See the report" on its own is a dead end — a reader cannot tell whether the
|
|
930
|
+
* thing they want is in there. So the sections are named, and so is the way to
|
|
931
|
+
* get them back in a terminal, because a CI log is a place where scrollback IS
|
|
932
|
+
* the artifact and there is no report to open.
|
|
933
|
+
*/
|
|
934
|
+
function moreInTheReport(input, style, width) {
|
|
935
|
+
const count = totals(input.envelope);
|
|
936
|
+
const rules = new Set(input.envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
|
|
937
|
+
if (rules === 0 && count.incomplete === 0)
|
|
938
|
+
return [];
|
|
939
|
+
return [
|
|
940
|
+
'',
|
|
941
|
+
...continuationLines(style, `Which rules, on which surfaces, with the selectors and the criteria grid: in the ` +
|
|
942
|
+
`report below. SETSQUARE_VERBOSE=1 prints all of it here instead, for a CI log with ` +
|
|
943
|
+
`no report to open.`, width),
|
|
944
|
+
];
|
|
945
|
+
}
|
|
784
946
|
/**
|
|
785
947
|
* Join the assembled lines, collapsing any run of blank lines to one.
|
|
786
948
|
*
|
|
@@ -1022,7 +1184,19 @@ function conformanceLines(input, style, width) {
|
|
|
1022
1184
|
lines.push(figureLine(style, rulesPassed, 'criteria: every automated rule passed — evidence, not conformance'));
|
|
1023
1185
|
}
|
|
1024
1186
|
if (cantTell > 0) {
|
|
1025
|
-
|
|
1187
|
+
/*
|
|
1188
|
+
* A criterion, not a row — so it stays one state.
|
|
1189
|
+
*
|
|
1190
|
+
* A cell rolls up many findings and can hold both kinds at once, and there is
|
|
1191
|
+
* no defensible precedence between them: a criterion with one unmeasured row
|
|
1192
|
+
* and one genuine judgement is not more one than the other. "Automation could
|
|
1193
|
+
* not decide" is true of the cell whichever it holds, and the two sections
|
|
1194
|
+
* above carry the why at the level where a row is the unit.
|
|
1195
|
+
*
|
|
1196
|
+
* The old wording — "criteria need a person" — was the one that stopped being
|
|
1197
|
+
* true, because a cell can now be undecided entirely because of our debt.
|
|
1198
|
+
*/
|
|
1199
|
+
lines.push(figureLine(style, cantTell, 'criteria automation could not decide — see the two sections above'));
|
|
1026
1200
|
}
|
|
1027
1201
|
if (inapplicable > 0) {
|
|
1028
1202
|
// EARL's inapplicable, spelled out: the checks RAN and the pages contain
|
|
@@ -1185,7 +1359,7 @@ function gridLines(input, style, width) {
|
|
|
1185
1359
|
lines.push((indent + label + cells).trimEnd());
|
|
1186
1360
|
}
|
|
1187
1361
|
lines.push('');
|
|
1188
|
-
lines.push(...continuationLines(style, `${glyph.failed} failed ${glyph.cantTell}
|
|
1362
|
+
lines.push(...continuationLines(style, `${glyph.failed} failed ${glyph.cantTell} automation could not decide ` +
|
|
1189
1363
|
`${glyph.rulesPassed} every automated rule passed ` +
|
|
1190
1364
|
`${glyph.inapplicable} nothing to check here ` +
|
|
1191
1365
|
`${glyph.assertedBySuite} covered by your own test ` +
|
|
@@ -1277,22 +1451,29 @@ function warningLines(warnings, width, quality, envelope) {
|
|
|
1277
1451
|
* only things the free layer structurally cannot do. Nothing is removed from the
|
|
1278
1452
|
* local tool to create the paid one, and this block must never imply otherwise.
|
|
1279
1453
|
*/
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1454
|
+
/**
|
|
1455
|
+
* What this run is, and what it is not, in one line.
|
|
1456
|
+
*
|
|
1457
|
+
* Derived from the run's own numbers, which is the only thing that licensed the
|
|
1458
|
+
* block it replaces. No product name and no URL: the reader is running the tool,
|
|
1459
|
+
* so they know where it comes from, and a limit stated as a limit does not need
|
|
1460
|
+
* a destination attached to be useful.
|
|
1461
|
+
*
|
|
1462
|
+
* Not switchable. "Local runs keep no history, baselines or comparison" is a
|
|
1463
|
+
* fact about the artefact in front of them — the dev plan holds it as a
|
|
1464
|
+
* permanent property of the free layer rather than a gap — so it belongs beside
|
|
1465
|
+
* the figures whether or not anyone would prefer it quiet.
|
|
1466
|
+
*/
|
|
1467
|
+
function historyCaveat(input, style, width) {
|
|
1468
|
+
const count = totals(input.envelope);
|
|
1283
1469
|
const failingRules = new Set(input.envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
|
|
1284
1470
|
const subject = failingRules > 0
|
|
1285
1471
|
? `${failingRules} rule${failingRules === 1 ? '' : 's'} failing across ${count.surfaces} surface${count.surfaces === 1 ? '' : 's'}`
|
|
1286
1472
|
: `${count.surfaces} clean surface${count.surfaces === 1 ? '' : 's'}`;
|
|
1287
1473
|
return [
|
|
1288
1474
|
'',
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
...wrap(`${subject}, this run only. Setsquare Cloud adds what a local tool cannot:`, width - 4, '').map((line) => ` ${line}`),
|
|
1292
|
-
' · is this better or worse than last week? (local runs keep no history)',
|
|
1293
|
-
' · which change introduced each one, on the PR that did it',
|
|
1294
|
-
' · the root cause behind repeats, grouped instead of repeated',
|
|
1295
|
-
` ${PLATFORM_URL}`,
|
|
1475
|
+
...continuationLines(style, `${subject}, this run only — local runs keep no history, no baselines and no ` +
|
|
1476
|
+
'comparison with a previous run, by design.', width),
|
|
1296
1477
|
'',
|
|
1297
1478
|
];
|
|
1298
1479
|
}
|