@setsquare/review-sdk 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +57 -0
- package/README.md +338 -0
- package/THIRD-PARTY-NOTICES.md +61 -0
- package/dist/autoscan.d.ts +25 -0
- package/dist/autoscan.d.ts.map +1 -0
- package/dist/autoscan.js +332 -0
- package/dist/autoscan.js.map +1 -0
- package/dist/axe.d.ts +96 -0
- package/dist/axe.d.ts.map +1 -0
- package/dist/axe.js +210 -0
- package/dist/axe.js.map +1 -0
- package/dist/bin/setsquare.d.ts +3 -0
- package/dist/bin/setsquare.d.ts.map +1 -0
- package/dist/bin/setsquare.js +21 -0
- package/dist/bin/setsquare.js.map +1 -0
- package/dist/brand.d.ts +84 -0
- package/dist/brand.d.ts.map +1 -0
- package/dist/brand.js +335 -0
- package/dist/brand.js.map +1 -0
- package/dist/checks/dynamic.d.ts +57 -0
- package/dist/checks/dynamic.d.ts.map +1 -0
- package/dist/checks/dynamic.js +184 -0
- package/dist/checks/dynamic.js.map +1 -0
- package/dist/checks/rule-pack.d.ts +34 -0
- package/dist/checks/rule-pack.d.ts.map +1 -0
- package/dist/checks/rule-pack.js +270 -0
- package/dist/checks/rule-pack.js.map +1 -0
- package/dist/checks/versions.d.ts +28 -0
- package/dist/checks/versions.d.ts.map +1 -0
- package/dist/checks/versions.js +45 -0
- package/dist/checks/versions.js.map +1 -0
- package/dist/ci.d.ts +37 -0
- package/dist/ci.d.ts.map +1 -0
- package/dist/ci.js +199 -0
- package/dist/ci.js.map +1 -0
- package/dist/contract.d.ts +334 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +35 -0
- package/dist/contract.js.map +1 -0
- package/dist/dedupe.d.ts +79 -0
- package/dist/dedupe.d.ts.map +1 -0
- package/dist/dedupe.js +102 -0
- package/dist/dedupe.js.map +1 -0
- package/dist/document.d.ts +29 -0
- package/dist/document.d.ts.map +1 -0
- package/dist/document.js +94 -0
- package/dist/document.js.map +1 -0
- package/dist/envelope.d.ts +124 -0
- package/dist/envelope.d.ts.map +1 -0
- package/dist/envelope.js +337 -0
- package/dist/envelope.js.map +1 -0
- package/dist/escape.d.ts +14 -0
- package/dist/escape.d.ts.map +1 -0
- package/dist/escape.js +20 -0
- package/dist/escape.js.map +1 -0
- package/dist/findings.d.ts +47 -0
- package/dist/findings.d.ts.map +1 -0
- package/dist/findings.js +229 -0
- package/dist/findings.js.map +1 -0
- package/dist/fixture.d.ts +7 -0
- package/dist/fixture.d.ts.map +1 -0
- package/dist/fixture.js +237 -0
- package/dist/fixture.js.map +1 -0
- package/dist/group.d.ts +214 -0
- package/dist/group.d.ts.map +1 -0
- package/dist/group.js +709 -0
- package/dist/group.js.map +1 -0
- package/dist/html-report.d.ts +68 -0
- package/dist/html-report.d.ts.map +1 -0
- package/dist/html-report.js +649 -0
- package/dist/html-report.js.map +1 -0
- package/dist/human.d.ts +65 -0
- package/dist/human.d.ts.map +1 -0
- package/dist/human.js +111 -0
- package/dist/human.js.map +1 -0
- package/dist/index.d.ts +34 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +33 -0
- package/dist/index.js.map +1 -0
- package/dist/infer.d.ts +65 -0
- package/dist/infer.d.ts.map +1 -0
- package/dist/infer.js +67 -0
- package/dist/infer.js.map +1 -0
- package/dist/insights.d.ts +140 -0
- package/dist/insights.d.ts.map +1 -0
- package/dist/insights.js +483 -0
- package/dist/insights.js.map +1 -0
- package/dist/install.d.ts +8 -0
- package/dist/install.d.ts.map +1 -0
- package/dist/install.js +114 -0
- package/dist/install.js.map +1 -0
- package/dist/links.d.ts +45 -0
- package/dist/links.d.ts.map +1 -0
- package/dist/links.js +45 -0
- package/dist/links.js.map +1 -0
- package/dist/mask.d.ts +85 -0
- package/dist/mask.d.ts.map +1 -0
- package/dist/mask.js +98 -0
- package/dist/mask.js.map +1 -0
- package/dist/observe.d.ts +51 -0
- package/dist/observe.d.ts.map +1 -0
- package/dist/observe.js +233 -0
- package/dist/observe.js.map +1 -0
- package/dist/options.d.ts +249 -0
- package/dist/options.d.ts.map +1 -0
- package/dist/options.js +185 -0
- package/dist/options.js.map +1 -0
- package/dist/output.d.ts +113 -0
- package/dist/output.d.ts.map +1 -0
- package/dist/output.js +231 -0
- package/dist/output.js.map +1 -0
- package/dist/perf.d.ts +11 -0
- package/dist/perf.d.ts.map +1 -0
- package/dist/perf.js +150 -0
- package/dist/perf.js.map +1 -0
- package/dist/reading-level.d.ts +73 -0
- package/dist/reading-level.d.ts.map +1 -0
- package/dist/reading-level.js +189 -0
- package/dist/reading-level.js.map +1 -0
- package/dist/reporter.d.ts +94 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporter.js +709 -0
- package/dist/reporter.js.map +1 -0
- package/dist/rules.d.ts +55 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/rules.js +77 -0
- package/dist/rules.js.map +1 -0
- package/dist/runtime.d.ts +32 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +67 -0
- package/dist/runtime.js.map +1 -0
- package/dist/sanitise.d.ts +116 -0
- package/dist/sanitise.d.ts.map +1 -0
- package/dist/sanitise.js +247 -0
- package/dist/sanitise.js.map +1 -0
- package/dist/scan.d.ts +60 -0
- package/dist/scan.d.ts.map +1 -0
- package/dist/scan.js +246 -0
- package/dist/scan.js.map +1 -0
- package/dist/screens.d.ts +98 -0
- package/dist/screens.d.ts.map +1 -0
- package/dist/screens.js +301 -0
- package/dist/screens.js.map +1 -0
- package/dist/settle.d.ts +65 -0
- package/dist/settle.d.ts.map +1 -0
- package/dist/settle.js +162 -0
- package/dist/settle.js.map +1 -0
- package/dist/shots.d.ts +109 -0
- package/dist/shots.d.ts.map +1 -0
- package/dist/shots.js +113 -0
- package/dist/shots.js.map +1 -0
- package/dist/style.d.ts +47 -0
- package/dist/style.d.ts.map +1 -0
- package/dist/style.js +154 -0
- package/dist/style.js.map +1 -0
- package/dist/suite.d.ts +218 -0
- package/dist/suite.d.ts.map +1 -0
- package/dist/suite.js +219 -0
- package/dist/suite.js.map +1 -0
- package/dist/summary.d.ts +268 -0
- package/dist/summary.d.ts.map +1 -0
- package/dist/summary.js +1308 -0
- package/dist/summary.js.map +1 -0
- package/dist/unscannable.d.ts +75 -0
- package/dist/unscannable.d.ts.map +1 -0
- package/dist/unscannable.js +111 -0
- package/dist/unscannable.js.map +1 -0
- package/dist/versions.d.ts +13 -0
- package/dist/versions.d.ts.map +1 -0
- package/dist/versions.js +39 -0
- package/dist/versions.js.map +1 -0
- package/dist/wcag-map.d.ts +19 -0
- package/dist/wcag-map.d.ts.map +1 -0
- package/dist/wcag-map.js +52 -0
- package/dist/wcag-map.js.map +1 -0
- package/dist/with-setsquare.d.ts +26 -0
- package/dist/with-setsquare.d.ts.map +1 -0
- package/dist/with-setsquare.js +70 -0
- package/dist/with-setsquare.js.map +1 -0
- package/package.json +81 -0
package/dist/summary.js
ADDED
|
@@ -0,0 +1,1308 @@
|
|
|
1
|
+
import { conformanceGrid, conformanceSummary, ruleSurfaceShare, stateCoverage, } from '@setsquare/schema/conformance';
|
|
2
|
+
import { PLATFORM_URL, ruleHelpUrl } from './links.js';
|
|
3
|
+
import { continuationLines, createStyle, figureBlock, figureLine, heading, } from './style.js';
|
|
4
|
+
import { flakeNotes, insightBlocks } from './insights.js';
|
|
5
|
+
/**
|
|
6
|
+
* One wording for every renderer, because the three formats must never say
|
|
7
|
+
* different things about the same coverage.
|
|
8
|
+
*/
|
|
9
|
+
export function groupLines(group) {
|
|
10
|
+
const lines = [];
|
|
11
|
+
lines.push(`This invocation${group.lane ? ` (lane "${group.lane}")` : ''} is part of run group ` +
|
|
12
|
+
`"${group.id}". The summary above covers this invocation only; the evidence files ` +
|
|
13
|
+
`hold the whole group.`);
|
|
14
|
+
lines.push(`${group.mergedParts} invocation${group.mergedParts === 1 ? '' : 's'} merged so far` +
|
|
15
|
+
(group.labels.length > 0 ? ` (${group.labels.join(', ')})` : '') +
|
|
16
|
+
'.');
|
|
17
|
+
if (group.missing > 0) {
|
|
18
|
+
lines.push(`${group.missing} invocation${group.missing === 1 ? '' : 's'} began but ` +
|
|
19
|
+
`${group.missing === 1 ? 'was' : 'were'} never merged — that evidence is absent, ` +
|
|
20
|
+
'not clean. The merged run is partial.');
|
|
21
|
+
}
|
|
22
|
+
switch (group.coverage) {
|
|
23
|
+
case 'complete':
|
|
24
|
+
lines.push(`All ${group.expectedParts} declared invocation${group.expectedParts === 1 ? '' : 's'} ` +
|
|
25
|
+
'merged. The merged artifacts are final for this group.');
|
|
26
|
+
break;
|
|
27
|
+
case 'partial':
|
|
28
|
+
if (group.expectedParts !== null && group.missing === 0) {
|
|
29
|
+
lines.push(`${group.mergedParts} of ${group.expectedParts} declared invocations merged. ` +
|
|
30
|
+
'The merged artifacts are partial until the rest arrive.');
|
|
31
|
+
}
|
|
32
|
+
break;
|
|
33
|
+
case 'unknown':
|
|
34
|
+
lines.push('Whether more invocations are coming is not declared — set SETSQUARE_RUN_PARTS to say ' +
|
|
35
|
+
'how many to expect. Until then, treat the merged artifacts as current, not final.');
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
return lines;
|
|
39
|
+
}
|
|
40
|
+
const IMPACT_ORDER = ['critical', 'serious', 'moderate', 'minor'];
|
|
41
|
+
/** Selectors shown per rule entry in the terminal; the full set is in report.html. */
|
|
42
|
+
const MAX_ENTRY_SELECTORS = 5;
|
|
43
|
+
/** `job-editor · dark` — the coordinates a person can act on. */
|
|
44
|
+
function contextLabel(context) {
|
|
45
|
+
return [context.surface, context.theme, context.persona, context.state]
|
|
46
|
+
.filter(Boolean)
|
|
47
|
+
.join(' · ');
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Collapse findings to one row per (rule, status).
|
|
51
|
+
*
|
|
52
|
+
* Deliberately not per rule alone: a rule that fails on one surface and is
|
|
53
|
+
* *unresolvable* on another is telling you two different things, and merging them
|
|
54
|
+
* would report "we could not check this" as "this is broken".
|
|
55
|
+
*/
|
|
56
|
+
export function groupFindings(envelope) {
|
|
57
|
+
const groups = new Map();
|
|
58
|
+
for (const scan of envelope.scans) {
|
|
59
|
+
for (const finding of scan.findings) {
|
|
60
|
+
const key = `${finding.ruleId}\u0000${finding.status}`;
|
|
61
|
+
const existing = groups.get(key);
|
|
62
|
+
const label = contextLabel(scan.context);
|
|
63
|
+
if (existing) {
|
|
64
|
+
existing.nodes += finding.targets.nodeCount;
|
|
65
|
+
if (!existing.contexts.includes(label))
|
|
66
|
+
existing.contexts.push(label);
|
|
67
|
+
const slot = existing.contextNodes.find((c) => c.label === label);
|
|
68
|
+
if (slot)
|
|
69
|
+
slot.nodes += finding.targets.nodeCount;
|
|
70
|
+
else
|
|
71
|
+
existing.contextNodes.push({ label, nodes: finding.targets.nodeCount });
|
|
72
|
+
for (const selector of finding.targets.selectors) {
|
|
73
|
+
if (existing.selectors.length >= MAX_ENTRY_SELECTORS) {
|
|
74
|
+
existing.selectorsTruncated = true;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
if (!existing.selectors.includes(selector))
|
|
78
|
+
existing.selectors.push(selector);
|
|
79
|
+
}
|
|
80
|
+
if (finding.targets.selectorsTruncated)
|
|
81
|
+
existing.selectorsTruncated = true;
|
|
82
|
+
if (!existing.failureSummary && finding.evidence?.failureSummary) {
|
|
83
|
+
existing.failureSummary = finding.evidence.failureSummary;
|
|
84
|
+
}
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
groups.set(key, {
|
|
88
|
+
ruleId: finding.ruleId,
|
|
89
|
+
impact: finding.impact ?? 'unknown',
|
|
90
|
+
status: finding.status,
|
|
91
|
+
nodes: finding.targets.nodeCount,
|
|
92
|
+
contexts: [label],
|
|
93
|
+
contextNodes: [{ label, nodes: finding.targets.nodeCount }],
|
|
94
|
+
selectors: finding.targets.selectors.slice(0, MAX_ENTRY_SELECTORS),
|
|
95
|
+
selectorsTruncated: finding.targets.selectorsTruncated ||
|
|
96
|
+
finding.targets.selectors.length > MAX_ENTRY_SELECTORS,
|
|
97
|
+
...(finding.evidence?.failureSummary
|
|
98
|
+
? { failureSummary: finding.evidence.failureSummary }
|
|
99
|
+
: {}),
|
|
100
|
+
...(ruleHelpUrl(finding) ? { helpUrl: ruleHelpUrl(finding) } : {}),
|
|
101
|
+
criteria: finding.wcag?.criteria ?? [],
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/*
|
|
106
|
+
* Violations before incompletes, then by impact, then by size, then by rule id.
|
|
107
|
+
*
|
|
108
|
+
* Fully determined at every level so two runs of the same suite produce
|
|
109
|
+
* byte-identical output — a summary you can diff is worth more than one sorted
|
|
110
|
+
* by whichever rule axe happened to mention first.
|
|
111
|
+
*/
|
|
112
|
+
return [...groups.values()].sort((a, b) => {
|
|
113
|
+
if (a.status !== b.status)
|
|
114
|
+
return a.status === 'violation' ? -1 : 1;
|
|
115
|
+
const impactDelta = impactRank(a.impact) - impactRank(b.impact);
|
|
116
|
+
if (impactDelta !== 0)
|
|
117
|
+
return impactDelta;
|
|
118
|
+
if (a.nodes !== b.nodes)
|
|
119
|
+
return b.nodes - a.nodes;
|
|
120
|
+
return a.ruleId.localeCompare(b.ruleId);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function impactRank(impact) {
|
|
124
|
+
const index = IMPACT_ORDER.indexOf(impact);
|
|
125
|
+
// An unknown impact sorts last rather than first: axe leaves it off some
|
|
126
|
+
// incompletes, and "we do not know how bad this is" should not lead the report.
|
|
127
|
+
return index === -1 ? IMPACT_ORDER.length : index;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* One row per scanned context, worst first.
|
|
131
|
+
*
|
|
132
|
+
* The summary used to report only totals and rules, which answered "what is
|
|
133
|
+
* broken" without answering "where". Both matter, and the second is the one a
|
|
134
|
+
* person acts on: they open a surface, not a rule.
|
|
135
|
+
*/
|
|
136
|
+
export function surfaceRows(envelope) {
|
|
137
|
+
/*
|
|
138
|
+
* Merged by LABEL, not by scan.
|
|
139
|
+
*
|
|
140
|
+
* Dedupe keys on the route pattern, so two genuinely different routes a
|
|
141
|
+
* customer has deliberately named the same surface both survive it — correctly,
|
|
142
|
+
* since they are different pages. But they must not appear as two identical
|
|
143
|
+
* rows here, because a reader has no way to tell them apart and will read it as
|
|
144
|
+
* the tool repeating itself.
|
|
145
|
+
*/
|
|
146
|
+
const merged = new Map();
|
|
147
|
+
for (const scan of envelope.scans) {
|
|
148
|
+
const label = contextLabel(scan.context);
|
|
149
|
+
const failing = scan.findings.filter((finding) => finding.status === 'violation');
|
|
150
|
+
const unresolved = scan.findings.filter((finding) => finding.status === 'incomplete');
|
|
151
|
+
const row = merged.get(label) ?? { label, violations: 0, incomplete: 0, rules: [] };
|
|
152
|
+
row.violations += failing.reduce((sum, finding) => sum + finding.targets.nodeCount, 0);
|
|
153
|
+
row.incomplete += unresolved.reduce((sum, finding) => sum + finding.targets.nodeCount, 0);
|
|
154
|
+
row.rules = [...new Set([...row.rules, ...failing.map((finding) => finding.ruleId)])].sort();
|
|
155
|
+
merged.set(label, row);
|
|
156
|
+
}
|
|
157
|
+
const rows = [...merged.values()];
|
|
158
|
+
// Failing surfaces first and by severity of volume, then clean ones
|
|
159
|
+
// alphabetically. Determinate at every level so two runs stay diffable.
|
|
160
|
+
return rows.sort((a, b) => {
|
|
161
|
+
if (a.violations > 0 !== b.violations > 0)
|
|
162
|
+
return a.violations > 0 ? -1 : 1;
|
|
163
|
+
if (a.violations !== b.violations)
|
|
164
|
+
return b.violations - a.violations;
|
|
165
|
+
return a.label.localeCompare(b.label);
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* How much the findings are worth, as distinct from what they say.
|
|
170
|
+
*
|
|
171
|
+
* The worst failure available to an accessibility scanner is not a crash but a
|
|
172
|
+
* CONFIDENT FALSE PASS — audit a page before its content arrives, find nothing,
|
|
173
|
+
* report conformance nobody verified. That failure is invisible in the findings
|
|
174
|
+
* themselves, which is why the schema carries the conditions each scan fired
|
|
175
|
+
* under. Until now nothing read them back: every scan in a run looked equally
|
|
176
|
+
* trustworthy in the summary, including the ones we already knew were not.
|
|
177
|
+
*
|
|
178
|
+
* `rulesAssessed` is the other half of the same idea. "2,518 checks passed" is an
|
|
179
|
+
* impressive number with no denominator — 2,518 out of what? Naming how many
|
|
180
|
+
* distinct rules ran turns it into a statement someone could audit.
|
|
181
|
+
*/
|
|
182
|
+
export function evidenceQuality(envelope) {
|
|
183
|
+
const rules = new Set();
|
|
184
|
+
let quiet = 0;
|
|
185
|
+
let teardown = 0;
|
|
186
|
+
let ceiling = 0;
|
|
187
|
+
for (const scan of envelope.scans) {
|
|
188
|
+
for (const pass of scan.passes ?? [])
|
|
189
|
+
rules.add(pass.ruleId);
|
|
190
|
+
for (const finding of scan.findings)
|
|
191
|
+
rules.add(finding.ruleId);
|
|
192
|
+
// Absent on explicit scans, where the customer chose the moment and no
|
|
193
|
+
// heuristic was involved — so there is nothing to be suspicious of.
|
|
194
|
+
switch (scan.settle?.reason) {
|
|
195
|
+
case 'quiet':
|
|
196
|
+
quiet += 1;
|
|
197
|
+
break;
|
|
198
|
+
case 'teardown':
|
|
199
|
+
teardown += 1;
|
|
200
|
+
break;
|
|
201
|
+
case 'ceiling':
|
|
202
|
+
ceiling += 1;
|
|
203
|
+
break;
|
|
204
|
+
default:
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return { rulesAssessed: rules.size, quiet, teardown, ceiling };
|
|
209
|
+
}
|
|
210
|
+
/*
|
|
211
|
+
* Google's "good" thresholds for the two Core Web Vitals we can measure in a lab.
|
|
212
|
+
*
|
|
213
|
+
* Named constants with their source, because a bare 2500 in a comparison is a
|
|
214
|
+
* number nobody can check. These are the field thresholds, and applying them to
|
|
215
|
+
* CI-lab timings overstates how good the numbers are rather than how bad — a
|
|
216
|
+
* localhost page load on an idle runner beats a real phone on a real network
|
|
217
|
+
* every time. That direction matters: a threshold that flatters is safe to show
|
|
218
|
+
* as long as the report says what it is measuring, which is why the heading does.
|
|
219
|
+
*/
|
|
220
|
+
const GOOD_LCP_MS = 2500;
|
|
221
|
+
const GOOD_CLS = 0.1;
|
|
222
|
+
/**
|
|
223
|
+
* One row per page LOAD, not per scan.
|
|
224
|
+
*
|
|
225
|
+
* `setsquare.scan()` can run several times against one navigation — modal closed, then
|
|
226
|
+
* modal open — and every one of those scans carries the same `perf` numbers,
|
|
227
|
+
* because there was only ever one load. Averaging or ranking without collapsing
|
|
228
|
+
* them would weight a surface by how many states it happens to have, which is a
|
|
229
|
+
* property of how somebody wrote their test rather than of the page. `visitRef`
|
|
230
|
+
* exists in the schema precisely to make this collapse possible.
|
|
231
|
+
*/
|
|
232
|
+
export function perfRows(envelope) {
|
|
233
|
+
const byVisit = new Map();
|
|
234
|
+
for (const scan of envelope.scans) {
|
|
235
|
+
if (!scan.perf)
|
|
236
|
+
continue;
|
|
237
|
+
const key = scan.visitRef ?? `${scan.testId}:${contextLabel(scan.context)}`;
|
|
238
|
+
if (byVisit.has(key))
|
|
239
|
+
continue;
|
|
240
|
+
byVisit.set(key, {
|
|
241
|
+
label: contextLabel(scan.context),
|
|
242
|
+
...(scan.perf.lcpMs !== undefined ? { lcpMs: scan.perf.lcpMs } : {}),
|
|
243
|
+
...(scan.perf.cls !== undefined ? { cls: scan.perf.cls } : {}),
|
|
244
|
+
...(scan.perf.loadMs !== undefined ? { loadMs: scan.perf.loadMs } : {}),
|
|
245
|
+
...(scan.perf.ttfbMs !== undefined ? { ttfbMs: scan.perf.ttfbMs } : {}),
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
return [...byVisit.values()].sort((a, b) => (b.lcpMs ?? -1) - (a.lcpMs ?? -1) || a.label.localeCompare(b.label));
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* What Playwright itself reported — counted the way Playwright counts it.
|
|
252
|
+
*
|
|
253
|
+
* The first version of this disagreed with the run it was printed underneath. On
|
|
254
|
+
* Eos, Playwright said *26 passed, 2 failed, 2 flaky, 82 did not run* and we said
|
|
255
|
+
* *28 passed, 6 failed*. Both of our numbers were defensible and both were wrong
|
|
256
|
+
* to print: the 6 counted every failed ATTEMPT including retries, and the 28
|
|
257
|
+
* quietly folded the two flaky tests in with the healthy ones.
|
|
258
|
+
*
|
|
259
|
+
* A tool that runs inside somebody's suite and then contradicts that suite's own
|
|
260
|
+
* summary, on the same screen, a few lines further down, has undermined itself
|
|
261
|
+
* before it says anything about accessibility. So: one row per TEST, the last
|
|
262
|
+
* attempt decides, and a test that needed a retry is named as flaky rather than
|
|
263
|
+
* absorbed.
|
|
264
|
+
*
|
|
265
|
+
* `notRun` is the one that changes what the rest of the report means. Eos's run
|
|
266
|
+
* scanned 10 surfaces — which reads as coverage of the application until you know
|
|
267
|
+
* that 82 of its 112 tests never executed, because a failing setup project took
|
|
268
|
+
* their dependents down with it. Evidence from a quarter of a suite must not look
|
|
269
|
+
* like evidence from a suite.
|
|
270
|
+
*/
|
|
271
|
+
export function testOutcomes(envelope, plannedTests) {
|
|
272
|
+
const attempts = new Map();
|
|
273
|
+
for (const result of envelope.results) {
|
|
274
|
+
const rows = attempts.get(result.testId) ?? [];
|
|
275
|
+
rows.push({ retry: result.retry ?? 0, status: result.status });
|
|
276
|
+
attempts.set(result.testId, rows);
|
|
277
|
+
}
|
|
278
|
+
let passed = 0;
|
|
279
|
+
let failed = 0;
|
|
280
|
+
let flaky = 0;
|
|
281
|
+
let skipped = 0;
|
|
282
|
+
for (const rows of attempts.values()) {
|
|
283
|
+
// Highest retry number is the final attempt. Sorting rather than trusting
|
|
284
|
+
// arrival order: workers report concurrently and nothing guarantees a retry
|
|
285
|
+
// lands after the attempt it followed.
|
|
286
|
+
const final = [...rows].sort((a, b) => a.retry - b.retry).at(-1);
|
|
287
|
+
const everFailed = rows.some((row) => row.status !== 'passed' && row.status !== 'skipped');
|
|
288
|
+
if (final?.status === 'skipped')
|
|
289
|
+
skipped += 1;
|
|
290
|
+
else if (final?.status === 'passed') {
|
|
291
|
+
if (everFailed)
|
|
292
|
+
flaky += 1;
|
|
293
|
+
else
|
|
294
|
+
passed += 1;
|
|
295
|
+
}
|
|
296
|
+
else
|
|
297
|
+
failed += 1;
|
|
298
|
+
}
|
|
299
|
+
const reported = attempts.size;
|
|
300
|
+
return {
|
|
301
|
+
passed,
|
|
302
|
+
failed,
|
|
303
|
+
flaky,
|
|
304
|
+
skipped,
|
|
305
|
+
/*
|
|
306
|
+
* Only when we were told the size of the suite, and never negative. A
|
|
307
|
+
* reporter can be handed a filtered run (`--grep`, `--last-failed`, a single
|
|
308
|
+
* spec path), in which case the planned total IS the filtered total and this
|
|
309
|
+
* is correctly zero — the number describes what this run intended to do, not
|
|
310
|
+
* what the repository contains.
|
|
311
|
+
*/
|
|
312
|
+
...(plannedTests !== undefined ? { notRun: Math.max(0, plannedTests - reported) } : {}),
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
export function totals(envelope) {
|
|
316
|
+
const surfaces = new Set();
|
|
317
|
+
const themes = new Set();
|
|
318
|
+
let violations = 0;
|
|
319
|
+
let incomplete = 0;
|
|
320
|
+
let passes = 0;
|
|
321
|
+
let clean = 0;
|
|
322
|
+
for (const scan of envelope.scans) {
|
|
323
|
+
surfaces.add(scan.context.surface);
|
|
324
|
+
if (scan.context.theme)
|
|
325
|
+
themes.add(scan.context.theme);
|
|
326
|
+
const scanViolations = scan.findings.filter((f) => f.status === 'violation');
|
|
327
|
+
violations += scanViolations.reduce((sum, f) => sum + f.targets.nodeCount, 0);
|
|
328
|
+
incomplete += scan.findings
|
|
329
|
+
.filter((f) => f.status === 'incomplete')
|
|
330
|
+
.reduce((sum, f) => sum + f.targets.nodeCount, 0);
|
|
331
|
+
passes += scan.passes.reduce((sum, p) => sum + p.nodeCount, 0);
|
|
332
|
+
if (scanViolations.length === 0)
|
|
333
|
+
clean += 1;
|
|
334
|
+
}
|
|
335
|
+
return {
|
|
336
|
+
surfaces: surfaces.size,
|
|
337
|
+
themes: themes.size,
|
|
338
|
+
scans: envelope.scans.length,
|
|
339
|
+
violations,
|
|
340
|
+
incomplete,
|
|
341
|
+
passes,
|
|
342
|
+
clean,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
/** Never wider than the terminal, never narrower than legible. */
|
|
346
|
+
export function terminalWidth(columns) {
|
|
347
|
+
return Math.max(80, Math.min(columns ?? 80, 120));
|
|
348
|
+
}
|
|
349
|
+
/** 1259 -> 1,259. A four-digit count read as a price is a count read wrong. */
|
|
350
|
+
function n(value) {
|
|
351
|
+
return value.toLocaleString('en-GB');
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Wrap on word boundaries, continuation lines indented.
|
|
355
|
+
*
|
|
356
|
+
* Truncation was the first attempt and it cut the useful half off: a note saying
|
|
357
|
+
* WHICH surfaces went unscanned became "…the test moved on …", which is the part
|
|
358
|
+
* nobody needed. If a line is worth printing it is worth finishing.
|
|
359
|
+
*/
|
|
360
|
+
function wrap(value, width, indent) {
|
|
361
|
+
const words = value.split(/\s+/);
|
|
362
|
+
const lines = [];
|
|
363
|
+
let current = '';
|
|
364
|
+
for (const word of words) {
|
|
365
|
+
if (current.length > 0 && (current + ' ' + word).length > width) {
|
|
366
|
+
lines.push(current);
|
|
367
|
+
current = word;
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
current = current.length > 0 ? current + ' ' + word : word;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if (current.length > 0)
|
|
374
|
+
lines.push(current);
|
|
375
|
+
return lines.map((line, index) => (index === 0 ? line : indent + line));
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* The plain-text summary.
|
|
379
|
+
*
|
|
380
|
+
* Two spaces of indent throughout rather than aligned columns: alignment breaks
|
|
381
|
+
* the moment a surface name is long, and a broken column looks like a bug in the
|
|
382
|
+
* tool rather than a long name.
|
|
383
|
+
*/
|
|
384
|
+
export function renderText(input, columns) {
|
|
385
|
+
const { envelope } = input;
|
|
386
|
+
const width = terminalWidth(columns);
|
|
387
|
+
const style = createStyle();
|
|
388
|
+
const lines = [];
|
|
389
|
+
const count = totals(envelope);
|
|
390
|
+
const tests = testOutcomes(envelope, input.plannedTests);
|
|
391
|
+
/*
|
|
392
|
+
* Every figure in one right-aligned gutter, matching the coverage audit.
|
|
393
|
+
*
|
|
394
|
+
* The two commands are one product and had two voices — the audit scannable,
|
|
395
|
+
* this a wall of even-weight prose with the numbers buried mid-sentence. A
|
|
396
|
+
* reader arriving here has just watched their suite run and wants three facts
|
|
397
|
+
* before they decide whether to care: did the tests pass, how much is broken,
|
|
398
|
+
* and where. Those are the three figures, and they now land in the left-hand
|
|
399
|
+
* strip before a word is read.
|
|
400
|
+
*/
|
|
401
|
+
/**
|
|
402
|
+
* Where the run's own artefacts are, and — usually — the closing block.
|
|
403
|
+
*
|
|
404
|
+
* `promo` is a parameter because of one case: a run that scanned nothing must
|
|
405
|
+
* not be followed by a pitch. The numbers it would quote are all zero, so it
|
|
406
|
+
* would read as an advert bolted onto a failure, and the closing block's whole
|
|
407
|
+
* licence to exist is that it is derived from the run's own figures.
|
|
408
|
+
*/
|
|
409
|
+
const trailer = (promo = true) => [
|
|
410
|
+
...(input.warnings.length > 0 ? [''] : []),
|
|
411
|
+
...warningLines(input.warnings, width, evidenceQuality(envelope), envelope),
|
|
412
|
+
'',
|
|
413
|
+
heading(style, 'Evidence'),
|
|
414
|
+
figureLine(style, '', input.envelopePath),
|
|
415
|
+
...(input.htmlReportPath
|
|
416
|
+
? [figureLine(style, '', `${input.htmlReportPath} ${style.dim('(open in a browser)')}`)]
|
|
417
|
+
: []),
|
|
418
|
+
...(input.screensPath
|
|
419
|
+
? [figureLine(style, '', `${input.screensPath} ${style.dim('(screenshots, one page)')}`)]
|
|
420
|
+
: []),
|
|
421
|
+
...(input.reportPath
|
|
422
|
+
? [figureLine(style, '', `${input.reportPath} ${style.dim('(Playwright report)')}`)]
|
|
423
|
+
: []),
|
|
424
|
+
...(input.group
|
|
425
|
+
? [
|
|
426
|
+
'',
|
|
427
|
+
heading(style, 'Run group'),
|
|
428
|
+
...groupLines(input.group).flatMap((line) => wrap(line, Math.max(40, width - 8), '').map((wrapped) => figureLine(style, '', wrapped))),
|
|
429
|
+
]
|
|
430
|
+
: []),
|
|
431
|
+
...(promo ? promoLines(input, count, width) : ['']),
|
|
432
|
+
];
|
|
433
|
+
lines.push('');
|
|
434
|
+
lines.push(` ${style.accent('Setsquare')} — accessibility evidence`);
|
|
435
|
+
lines.push('');
|
|
436
|
+
/*
|
|
437
|
+
* Playwright's own verdict, first and always.
|
|
438
|
+
*
|
|
439
|
+
* This runs inside somebody's test suite and the suite's result is the thing
|
|
440
|
+
* they came for. Leading with our findings — or omitting theirs — would be a
|
|
441
|
+
* tool behaving as though it were the point of the run rather than a passenger
|
|
442
|
+
* on it. It also makes the independence obvious: their tests pass or fail on
|
|
443
|
+
* their own terms, and nothing here changes that.
|
|
444
|
+
*/
|
|
445
|
+
if (envelope.results.length > 0) {
|
|
446
|
+
lines.push(heading(style, 'Playwright'));
|
|
447
|
+
lines.push(figureLine(style, tests.passed, 'passed'));
|
|
448
|
+
if (tests.failed > 0)
|
|
449
|
+
lines.push(figureLine(style, tests.failed, 'failed'));
|
|
450
|
+
// Named, never folded into `passed`. A test that only goes green on a second
|
|
451
|
+
// attempt is not a passing test, and it is the category teams learn to ignore
|
|
452
|
+
// precisely because nothing puts a number on it.
|
|
453
|
+
if (tests.flaky > 0) {
|
|
454
|
+
lines.push(figureLine(style, tests.flaky, 'flaky — passed on retry'));
|
|
455
|
+
// WHICH tests, on which attempt, and where they first broke. The count alone
|
|
456
|
+
// is the number teams learn to ignore, precisely because it names nothing.
|
|
457
|
+
for (const note of flakeNotes(input.suite)) {
|
|
458
|
+
lines.push(...continuationLines(style, note, width));
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
if (tests.skipped > 0)
|
|
462
|
+
lines.push(figureLine(style, tests.skipped, 'skipped'));
|
|
463
|
+
if (tests.notRun) {
|
|
464
|
+
/*
|
|
465
|
+
* The line that changes what everything below it means.
|
|
466
|
+
*
|
|
467
|
+
* A run that scanned 10 surfaces reads as coverage of the application until
|
|
468
|
+
* you know that 82 of its 112 tests never executed. Evidence from a quarter
|
|
469
|
+
* of a suite must not look like evidence from a suite, so this is stated
|
|
470
|
+
* here rather than left for someone to work out from Playwright's output
|
|
471
|
+
* further up the screen.
|
|
472
|
+
*/
|
|
473
|
+
lines.push(figureLine(style, tests.notRun, 'never ran'));
|
|
474
|
+
lines.push(...continuationLines(style, 'Findings below cover only what did run.'));
|
|
475
|
+
}
|
|
476
|
+
lines.push('');
|
|
477
|
+
}
|
|
478
|
+
if (count.scans === 0) {
|
|
479
|
+
/*
|
|
480
|
+
* Nothing scanned is a real outcome, and "0 violations" here would be the
|
|
481
|
+
* single most dangerous sentence this tool could print: it reads as "no
|
|
482
|
+
* problems" when it means "nobody looked". So the figure is deliberately the
|
|
483
|
+
* count of surfaces LOOKED AT, which is zero, and never a count of findings.
|
|
484
|
+
*/
|
|
485
|
+
lines.push(heading(style, 'Accessibility'));
|
|
486
|
+
lines.push(figureLine(style, 0, 'surfaces scanned'));
|
|
487
|
+
lines.push(...continuationLines(style, 'This is not a clean result — it means nothing was looked at. Usually the ' +
|
|
488
|
+
'suite never opened a page, or every test finished before its page settled.', width));
|
|
489
|
+
lines.push(...trailer(false));
|
|
490
|
+
return join(lines);
|
|
491
|
+
}
|
|
492
|
+
const target = envelope.scans[0]?.wcagTarget ?? 'AA';
|
|
493
|
+
const quality = evidenceQuality(envelope);
|
|
494
|
+
lines.push(heading(style, `Accessibility — WCAG 2.2 ${target}`));
|
|
495
|
+
/*
|
|
496
|
+
* Rules first, elements second, and that ordering is the difference between a
|
|
497
|
+
* summary a person can act on and one that panics them.
|
|
498
|
+
*
|
|
499
|
+
* The fixture suite reports 626 failing elements, almost all of them one
|
|
500
|
+
* contrast rule on one deliberately absurd page. Leading with the element count
|
|
501
|
+
* reads as a fortnight of work; leading with the rule count reads as an
|
|
502
|
+
* afternoon, and is the truer description, because a person fixes a rule and its
|
|
503
|
+
* cause rather than an element.
|
|
504
|
+
*/
|
|
505
|
+
const failingRules = new Set(envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
|
|
506
|
+
lines.push(figureLine(style, count.surfaces, `surface${count.surfaces === 1 ? '' : 's'} scanned` +
|
|
507
|
+
(count.themes > 0 ? ` in ${count.themes} theme${count.themes === 1 ? '' : 's'}` : '')));
|
|
508
|
+
/*
|
|
509
|
+
* Changed-only mode, stated where nobody can miss it.
|
|
510
|
+
*
|
|
511
|
+
* A filtered run's conformance figures describe a SLICE, and the figures
|
|
512
|
+
* themselves look identical to a full run's. The unscannable block below counts
|
|
513
|
+
* the skips, but a reader who quotes "surfaces scanned" from the top of this
|
|
514
|
+
* output would be quoting a slice as the application. So the mode is named
|
|
515
|
+
* immediately under the figure it qualifies, not in a footnote.
|
|
516
|
+
*/
|
|
517
|
+
const changedOnly = (envelope.unscanned ?? []).filter((row) => row.reason === 'not-in-changed-set');
|
|
518
|
+
if (changedOnly.length > 0) {
|
|
519
|
+
const occurrences = changedOnly.reduce((total, row) => total + row.count, 0);
|
|
520
|
+
lines.push(...continuationLines(style, `Changed-only mode: ${changedOnly.length} ${changedOnly.length === 1 ? 'surface was' : 'surfaces were'} skipped (${occurrences} ${occurrences === 1 ? 'time' : 'times'}) because the change ` +
|
|
521
|
+
'does not reach them. Every figure below describes that slice, not the application.', width));
|
|
522
|
+
}
|
|
523
|
+
if (count.violations === 0 && count.incomplete === 0) {
|
|
524
|
+
lines.push(figureLine(style, 0, 'rules failing'));
|
|
525
|
+
lines.push(figureLine(style, count.passes, style.dim(`checks passed, across ${quality.rulesAssessed} distinct rules`)));
|
|
526
|
+
lines.push(...conformanceLines(input, style, width));
|
|
527
|
+
lines.push(...gridLines(input, style, width));
|
|
528
|
+
lines.push(...perfLines(envelope, style, width));
|
|
529
|
+
lines.push(...trailer());
|
|
530
|
+
return join(lines);
|
|
531
|
+
}
|
|
532
|
+
lines.push(figureLine(style, failingRules, `rule${failingRules === 1 ? '' : 's'} failing, on ${n(count.violations)} ` +
|
|
533
|
+
`element${count.violations === 1 ? '' : 's'}`));
|
|
534
|
+
if (count.incomplete > 0) {
|
|
535
|
+
lines.push(figureLine(style, count.incomplete, 'needing review — axe could not decide'));
|
|
536
|
+
}
|
|
537
|
+
lines.push(figureLine(style, count.passes, style.dim(`checks passed, across ${quality.rulesAssessed} distinct rules`)));
|
|
538
|
+
lines.push(...conformanceLines(input, style, width));
|
|
539
|
+
lines.push(...gridLines(input, style, width));
|
|
540
|
+
lines.push('');
|
|
541
|
+
/*
|
|
542
|
+
* Surfaces before rules.
|
|
543
|
+
*
|
|
544
|
+
* "Which pages are failing" is the question people actually arrive with, and it
|
|
545
|
+
* was answerable only by reading the rule entries and reassembling the list in
|
|
546
|
+
* your head. Failing surfaces are never truncated — a summary that hides a
|
|
547
|
+
* failure to stay tidy has chosen the wrong thing to optimise.
|
|
548
|
+
*/
|
|
549
|
+
const rows = surfaceRows(envelope);
|
|
550
|
+
const failing = rows.filter((row) => row.violations > 0);
|
|
551
|
+
/*
|
|
552
|
+
* Three states, not two.
|
|
553
|
+
*
|
|
554
|
+
* A surface with no violations but unresolved checks has NOT been verified, and
|
|
555
|
+
* labelling it `pass` beside a genuinely clean one is the confident false pass
|
|
556
|
+
* this product exists to avoid — quieter than a crash and far more damaging,
|
|
557
|
+
* because somebody signs a conformance statement on the strength of it.
|
|
558
|
+
*/
|
|
559
|
+
const needsReview = rows.filter((row) => row.violations === 0 && row.incomplete > 0);
|
|
560
|
+
const clean = rows.filter((row) => row.violations === 0 && row.incomplete === 0);
|
|
561
|
+
lines.push(heading(style, 'Surfaces'));
|
|
562
|
+
/*
|
|
563
|
+
* The violation count goes in the gutter and the verdict goes in the words.
|
|
564
|
+
*
|
|
565
|
+
* FAIL and pass are spelled out rather than shown by colour or by the number
|
|
566
|
+
* alone: a row has to state its own verdict when the escapes are stripped, and
|
|
567
|
+
* "0" is not a verdict — a surface with no violations might equally never have
|
|
568
|
+
* been looked at.
|
|
569
|
+
*/
|
|
570
|
+
const labelWidth = Math.min(Math.max(...rows.map((row) => row.label.length)), Math.max(20, width - 40));
|
|
571
|
+
for (const row of failing) {
|
|
572
|
+
const rules = row.rules.join(', ');
|
|
573
|
+
const room = width - labelWidth - 20;
|
|
574
|
+
/*
|
|
575
|
+
* The rule names wrap to their own line rather than being cut with an
|
|
576
|
+
* ellipsis, for the same reason the audit's route list is never capped: the
|
|
577
|
+
* rule id is the actionable half of the row. "color-contrast, target-…" tells
|
|
578
|
+
* a reader there is a second problem and refuses to say which, which is worse
|
|
579
|
+
* than either naming it or staying silent.
|
|
580
|
+
*/
|
|
581
|
+
if (rules.length <= room) {
|
|
582
|
+
lines.push(figureLine(style, row.violations, `FAIL ${row.label.padEnd(labelWidth)} ${style.dim(rules)}`));
|
|
583
|
+
}
|
|
584
|
+
else {
|
|
585
|
+
lines.push(figureLine(style, row.violations, `FAIL ${row.label}`));
|
|
586
|
+
lines.push(...continuationLines(style, rules, width));
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
/*
|
|
590
|
+
* Clean surfaces are listed too, and in full.
|
|
591
|
+
*
|
|
592
|
+
* They are the evidence half of "accessibility evidence": a surface looked at
|
|
593
|
+
* and found clean is a fact worth having, and it is the only thing separating a
|
|
594
|
+
* passing surface from one nothing ever visited. Trimming the list to keep the
|
|
595
|
+
* output short would delete exactly that distinction.
|
|
596
|
+
*/
|
|
597
|
+
for (const row of needsReview) {
|
|
598
|
+
lines.push(figureLine(style, row.incomplete, `CHECK ${row.label.padEnd(labelWidth)} ${style.dim('unresolved')}`));
|
|
599
|
+
}
|
|
600
|
+
for (const row of clean) {
|
|
601
|
+
lines.push(figureLine(style, '', `pass ${row.label}`));
|
|
602
|
+
}
|
|
603
|
+
lines.push(...perfLines(envelope, style, width));
|
|
604
|
+
const groups = groupFindings(envelope);
|
|
605
|
+
/*
|
|
606
|
+
* A rule entry reads like an instruction, not an inventory (reworked on
|
|
607
|
+
* feedback from the first real Eos run, 2026-08-19): what is wrong, what to
|
|
608
|
+
* change, where — then how widespread. The first cut led with an exhaustive
|
|
609
|
+
* surface list, which told a reader how big the problem was while refusing
|
|
610
|
+
* to say what to do about it, and duplicated SURFACES above line for line.
|
|
611
|
+
*/
|
|
612
|
+
const ruleEntry = (group) => {
|
|
613
|
+
const criteria = group.criteria.length > 0 ? ` — WCAG ${group.criteria.join(', ')}` : '';
|
|
614
|
+
const entry = [
|
|
615
|
+
figureLine(style, group.nodes, `${style.bold(group.ruleId)} ${style.dim(`${group.impact}${criteria}`)}`),
|
|
616
|
+
];
|
|
617
|
+
// What to change: axe's own fix text (or the setsquare check's), from a
|
|
618
|
+
// representative node — the ratios and names are that node's, the shape of
|
|
619
|
+
// the fix is every node's.
|
|
620
|
+
if (group.failureSummary) {
|
|
621
|
+
entry.push(...continuationLines(style, group.failureSummary, width));
|
|
622
|
+
}
|
|
623
|
+
// Where: a bounded sample of failing selectors. The full set survives in
|
|
624
|
+
// report.html and run.json; a terminal entry needs enough to open devtools.
|
|
625
|
+
if (group.selectors.length > 0) {
|
|
626
|
+
entry.push(...continuationLines(style, `where: ${group.selectors.join(' ')}` +
|
|
627
|
+
(group.selectorsTruncated ? ' … every selector is in report.html' : ''), width));
|
|
628
|
+
}
|
|
629
|
+
/*
|
|
630
|
+
* How widespread. "13 of 100 surfaces (13%)" is what keeps four thousand
|
|
631
|
+
* findings readable — and past a handful of surfaces the list is capped to
|
|
632
|
+
* the worst three by element count, because every failing surface is
|
|
633
|
+
* ALREADY listed under SURFACES with this rule against it. Repeating all
|
|
634
|
+
* sixteen here was information nobody could act on twice. Incomplete
|
|
635
|
+
* groups get no share: the share counts detected failures, and an
|
|
636
|
+
* unresolved check is not one.
|
|
637
|
+
*/
|
|
638
|
+
const share = group.status === 'violation' ? ruleSurfaceShare(envelope, group.ruleId) : undefined;
|
|
639
|
+
const lead = share
|
|
640
|
+
? `on ${surfaceShare(share.affected, share.total)}`
|
|
641
|
+
: `on ${group.contexts.length} surface${group.contexts.length === 1 ? '' : 's'}`;
|
|
642
|
+
const ranked = [...group.contextNodes].sort((a, b) => b.nodes - a.nodes || a.label.localeCompare(b.label));
|
|
643
|
+
const spread = ranked.length <= 6
|
|
644
|
+
? `${lead}: ${ranked.map((c) => c.label).join(', ')}`
|
|
645
|
+
: `${lead}, worst: ` +
|
|
646
|
+
ranked
|
|
647
|
+
.slice(0, 3)
|
|
648
|
+
.map((c) => `${c.label} (${n(c.nodes)})`)
|
|
649
|
+
.join(', ') +
|
|
650
|
+
` — all ${ranked.length} rows are marked under SURFACES above`;
|
|
651
|
+
entry.push(...continuationLines(style, spread, width));
|
|
652
|
+
if (group.helpUrl)
|
|
653
|
+
entry.push(...continuationLines(style, group.helpUrl, width));
|
|
654
|
+
entry.push('');
|
|
655
|
+
return entry;
|
|
656
|
+
};
|
|
657
|
+
const failingRuleGroups = groups.filter((group) => group.status === 'violation');
|
|
658
|
+
const review = groups.filter((group) => group.status === 'incomplete');
|
|
659
|
+
if (failingRuleGroups.length > 0) {
|
|
660
|
+
lines.push('');
|
|
661
|
+
lines.push(heading(style, 'Rules failing'));
|
|
662
|
+
for (const group of failingRuleGroups)
|
|
663
|
+
lines.push(...ruleEntry(group));
|
|
664
|
+
}
|
|
665
|
+
if (review.length > 0) {
|
|
666
|
+
/*
|
|
667
|
+
* A section of its own, not a parenthesis on the failure list.
|
|
668
|
+
*
|
|
669
|
+
* `incomplete` is axe saying "I could not decide" — most often contrast over
|
|
670
|
+
* a background image or a gradient, where the ratio is genuinely
|
|
671
|
+
* uncomputable from the DOM. These are neither passes nor failures, and
|
|
672
|
+
* listing them beside violations invites both of the available mistakes:
|
|
673
|
+
* treating them as broken and panicking, or treating them as fine and
|
|
674
|
+
* shipping. They are the one category that REQUIRES a person, which is worth
|
|
675
|
+
* saying plainly given we sell evidence.
|
|
676
|
+
*/
|
|
677
|
+
lines.push('');
|
|
678
|
+
lines.push(heading(style, 'Needs review'));
|
|
679
|
+
lines.push(...continuationLines(style, 'axe could not decide these by itself — usually contrast over an image or a ' +
|
|
680
|
+
'gradient. Not passes and not failures; a person has to look.', width));
|
|
681
|
+
lines.push('');
|
|
682
|
+
for (const group of review)
|
|
683
|
+
lines.push(...ruleEntry(group));
|
|
684
|
+
}
|
|
685
|
+
/*
|
|
686
|
+
* The P1b blocks, after the accessibility findings and before the trailer.
|
|
687
|
+
*
|
|
688
|
+
* Ordered by how surprising they are rather than by how new: page runtime first,
|
|
689
|
+
* because "a 500 happened and no test noticed" is the thing a reader most needs
|
|
690
|
+
* to not miss; then what could not be checked, which qualifies everything above
|
|
691
|
+
* it; then the cross-surface findings, coverage, and last the run's own
|
|
692
|
+
* mechanics.
|
|
693
|
+
*
|
|
694
|
+
* Every block is `undefined` when it has nothing to say, so a run without the
|
|
695
|
+
* sidecar — or an envelope written before the capability — renders exactly as it
|
|
696
|
+
* did before rather than growing empty headings.
|
|
697
|
+
*/
|
|
698
|
+
for (const block of insightBlocks(envelope, {
|
|
699
|
+
...(input.declaredAxes ? { declaredAxes: input.declaredAxes } : {}),
|
|
700
|
+
...(input.unmeasuredAxes ? { unmeasuredAxes: input.unmeasuredAxes } : {}),
|
|
701
|
+
...(input.suite ? { suite: input.suite } : {}),
|
|
702
|
+
...(input.overhead ? { overhead: input.overhead } : {}),
|
|
703
|
+
})) {
|
|
704
|
+
lines.push(...figureBlock(style, block.heading, block.figures, block.notes, width));
|
|
705
|
+
}
|
|
706
|
+
lines.push(...trailer());
|
|
707
|
+
return join(lines);
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Join the assembled lines, collapsing any run of blank lines to one.
|
|
711
|
+
*
|
|
712
|
+
* Sections are built independently and several of them end with a blank line, so
|
|
713
|
+
* the seams accumulate — three blank lines appeared between the last rule and the
|
|
714
|
+
* warnings. Fixing it at each seam means every future section has to remember the
|
|
715
|
+
* convention; fixing it here means none of them do, and the sections stay
|
|
716
|
+
* composable.
|
|
717
|
+
*/
|
|
718
|
+
function join(lines) {
|
|
719
|
+
const out = [];
|
|
720
|
+
for (const line of lines) {
|
|
721
|
+
if (line === '' && out[out.length - 1] === '')
|
|
722
|
+
continue;
|
|
723
|
+
out.push(line);
|
|
724
|
+
}
|
|
725
|
+
return out.join('\n');
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* Pillar 3, finally visible.
|
|
729
|
+
*
|
|
730
|
+
* The SDK has captured navigation timing, LCP and CLS on every scan since P1-4.4
|
|
731
|
+
* and the summary never mentioned it. A run of Eos's legal lane held timings for
|
|
732
|
+
* six page loads and said nothing — a whole pillar of the product, measured and
|
|
733
|
+
* discarded at the last step.
|
|
734
|
+
*
|
|
735
|
+
* Three rules, each an honesty constraint rather than a layout one:
|
|
736
|
+
*
|
|
737
|
+
* 1. **Say what these numbers are.** They are CI-lab timings from an idle
|
|
738
|
+
* runner, usually against localhost. They are not what a user on a phone
|
|
739
|
+
* experiences, and a report that let someone believe otherwise would be
|
|
740
|
+
* worse than one that stayed silent.
|
|
741
|
+
* 2. **Absent is not zero.** LCP and CLS reach a `PerformanceObserver` in
|
|
742
|
+
* Chromium and simply do not exist in Firefox or WebKit, so a run across
|
|
743
|
+
* three engines has loads with no vitals at all. Reporting those as 0ms
|
|
744
|
+
* would invent the best possible score out of missing data.
|
|
745
|
+
* 3. **The threshold is named, not implied.** "Slowest LCP 348ms" means nothing
|
|
746
|
+
* to a reader who does not carry 2,500ms in their head.
|
|
747
|
+
*/
|
|
748
|
+
function perfLines(envelope, style, width) {
|
|
749
|
+
const rows = perfRows(envelope);
|
|
750
|
+
if (rows.length === 0)
|
|
751
|
+
return [];
|
|
752
|
+
const lines = ['', heading(style, 'Performance')];
|
|
753
|
+
lines.push(figureLine(style, rows.length, `page load${rows.length === 1 ? '' : 's'} measured ${style.dim('(CI lab, not real users)')}`));
|
|
754
|
+
const withLcp = rows.filter((row) => row.lcpMs !== undefined);
|
|
755
|
+
const withCls = rows.filter((row) => row.cls !== undefined);
|
|
756
|
+
const slowLcp = withLcp.filter((row) => (row.lcpMs ?? 0) > GOOD_LCP_MS);
|
|
757
|
+
const highCls = withCls.filter((row) => (row.cls ?? 0) > GOOD_CLS);
|
|
758
|
+
const worstLcp = withLcp[0];
|
|
759
|
+
if (worstLcp?.lcpMs !== undefined) {
|
|
760
|
+
lines.push(figureLine(style, `${n(Math.round(worstLcp.lcpMs))}ms`, `slowest LCP — ${worstLcp.label}`));
|
|
761
|
+
}
|
|
762
|
+
const worstCls = [...withCls].sort((a, b) => (b.cls ?? 0) - (a.cls ?? 0))[0];
|
|
763
|
+
if (worstCls?.cls !== undefined) {
|
|
764
|
+
lines.push(figureLine(style, worstCls.cls.toFixed(2), `worst CLS — ${worstCls.label}`));
|
|
765
|
+
}
|
|
766
|
+
if (slowLcp.length > 0) {
|
|
767
|
+
lines.push('');
|
|
768
|
+
lines.push(figureLine(style, slowLcp.length, `over the ${n(GOOD_LCP_MS)}ms LCP threshold`));
|
|
769
|
+
for (const row of slowLcp) {
|
|
770
|
+
lines.push(figureLine(style, `${n(Math.round(row.lcpMs ?? 0))}ms`, style.dim(row.label)));
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
if (highCls.length > 0) {
|
|
774
|
+
lines.push('');
|
|
775
|
+
lines.push(figureLine(style, highCls.length, `over the ${GOOD_CLS.toFixed(2)} CLS threshold`));
|
|
776
|
+
for (const row of highCls) {
|
|
777
|
+
lines.push(figureLine(style, (row.cls ?? 0).toFixed(2), style.dim(row.label)));
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
if (slowLcp.length === 0 && highCls.length === 0 && withLcp.length > 0) {
|
|
781
|
+
lines.push(...continuationLines(style, `All within the ${n(GOOD_LCP_MS)}ms LCP and ${GOOD_CLS.toFixed(2)} CLS thresholds ` +
|
|
782
|
+
'that Google treats as good — measured in a lab, so treat them as a floor.', width));
|
|
783
|
+
}
|
|
784
|
+
const missing = rows.length - withLcp.length;
|
|
785
|
+
if (missing > 0) {
|
|
786
|
+
/*
|
|
787
|
+
* Named rather than left as a gap in the arithmetic. LCP and CLS reach a
|
|
788
|
+
* PerformanceObserver in Chromium and do not exist at all in Firefox or
|
|
789
|
+
* WebKit, so a three-engine run legitimately has loads with no vitals — and a
|
|
790
|
+
* reader who notices the numbers not adding up deserves the reason.
|
|
791
|
+
*/
|
|
792
|
+
lines.push(...continuationLines(style, `${missing} load${missing === 1 ? '' : 's'} reported no LCP: Firefox and WebKit do not ` +
|
|
793
|
+
'implement it, so it is absent rather than zero.', width));
|
|
794
|
+
}
|
|
795
|
+
return lines;
|
|
796
|
+
}
|
|
797
|
+
/** The `<1%` rule, matching `ruleSurfaceShare`: a real problem never rounds to nothing. */
|
|
798
|
+
function sharePercent(affected, total) {
|
|
799
|
+
if (total === 0 || affected === 0)
|
|
800
|
+
return '0%';
|
|
801
|
+
const rounded = Math.round((affected / total) * 100);
|
|
802
|
+
return rounded === 0 ? '<1%' : `${rounded}%`;
|
|
803
|
+
}
|
|
804
|
+
/** `13 of 100 surfaces (13%)` — count first, scale second, neither replacing the other. */
|
|
805
|
+
function surfaceShare(affected, total) {
|
|
806
|
+
return `${n(affected)} of ${n(total)} surface${total === 1 ? '' : 's'} (${sharePercent(affected, total)})`;
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* The conformance verdict, and the vocabulary discipline that governs it.
|
|
810
|
+
*
|
|
811
|
+
* Nothing here may say anyone "is" or "can claim" a level. WCAG conformance is
|
|
812
|
+
* defined per full page and complete process, includes requirements no scanner
|
|
813
|
+
* evaluates, and most criteria have failure modes no automated rule covers. So
|
|
814
|
+
* every sentence is about DETECTION: failures detected, no failures detected in
|
|
815
|
+
* what was scanned, checks that were not run. The moment this section says
|
|
816
|
+
* "conformant", somebody signs a statement on the strength of it — and the test
|
|
817
|
+
* suite greps for exactly that word to keep it out.
|
|
818
|
+
*/
|
|
819
|
+
function conformanceLines(input, style, width) {
|
|
820
|
+
const summary = conformanceSummary(input.envelope, {
|
|
821
|
+
...(input.criterionRules ? { criterionRules: input.criterionRules } : {}),
|
|
822
|
+
});
|
|
823
|
+
if (summary.surfacesScanned === 0)
|
|
824
|
+
return [];
|
|
825
|
+
const lines = ['', heading(style, 'Conformance — WCAG 2.2')];
|
|
826
|
+
lines.push(...continuationLines(style, 'Includes every 2.0 and 2.1 criterion. Automated checks cover only part of WCAG, ' +
|
|
827
|
+
'so this is evidence about what was scanned, never a conformance claim.', width));
|
|
828
|
+
lines.push('');
|
|
829
|
+
const criterionRow = (rollup) => [
|
|
830
|
+
figureLine(style, rollup.failedSurfaces.length, `${style.bold(`${rollup.criterion} ${rollup.name}`)} ${style.dim(`FAIL on ${surfaceShare(rollup.failedSurfaces.length, rollup.totalSurfaces)}`)}`),
|
|
831
|
+
...continuationLines(style, rollup.failingRules.join(', '), width),
|
|
832
|
+
];
|
|
833
|
+
for (const level of summary.levels) {
|
|
834
|
+
if (level.level === 'AAA' && !level.assessed) {
|
|
835
|
+
lines.push(...continuationLines(style, "Level AAA not assessed — AAA is opt-in; run with wcag: 'AAA' to include it.", width));
|
|
836
|
+
if (level.failing.length > 0) {
|
|
837
|
+
// Observed in passing on an AA-target run. Still evidence — hiding an
|
|
838
|
+
// observed failure would be the dishonest direction — but the verdict
|
|
839
|
+
// above never speaks to an unassessed level.
|
|
840
|
+
lines.push(...continuationLines(style, `${level.failing.length === 1 ? 'One AAA criterion' : `${level.failing.length} AAA criteria`} failed in passing:`, width));
|
|
841
|
+
for (const rollup of level.failing)
|
|
842
|
+
lines.push(...criterionRow(rollup));
|
|
843
|
+
}
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
846
|
+
if (level.failing.length > 0) {
|
|
847
|
+
lines.push(figureLine(style, level.failing.length, `Level ${level.level.padEnd(3)} failures detected — ` +
|
|
848
|
+
`${level.failing.length === 1 ? 'one criterion' : `${level.failing.length} criteria`} failing`));
|
|
849
|
+
for (const rollup of level.failing)
|
|
850
|
+
lines.push(...criterionRow(rollup));
|
|
851
|
+
}
|
|
852
|
+
else {
|
|
853
|
+
lines.push(figureLine(style, '', `Level ${level.level.padEnd(3)} no failures detected in what was scanned`));
|
|
854
|
+
}
|
|
855
|
+
if (level.assessedScans) {
|
|
856
|
+
lines.push(...continuationLines(style, `AAA was assessed on ${level.assessedScans.aaa} of ${level.assessedScans.total} scans — ` +
|
|
857
|
+
'the rest looked with AA rules only.', width));
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
if (summary.removed.length > 0) {
|
|
861
|
+
for (const rollup of summary.removed) {
|
|
862
|
+
lines.push(...continuationLines(style, `${rollup.criterion} ${rollup.name} failed on ${surfaceShare(rollup.failedSurfaces.length, rollup.totalSurfaces)} — ` +
|
|
863
|
+
'removed in WCAG 2.2, so it counts toward no level here.', width));
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
lines.push('');
|
|
867
|
+
/*
|
|
868
|
+
* The fix breakdown, fact-shaped. "To clear detected Level A failures" and
|
|
869
|
+
* never "to reach Level A": fixing what automation found removes what
|
|
870
|
+
* automation found, and nothing more.
|
|
871
|
+
*/
|
|
872
|
+
const { verdict } = summary;
|
|
873
|
+
if (verdict.failuresDetectedAt) {
|
|
874
|
+
const at = verdict.failuresDetectedAt;
|
|
875
|
+
const ids = (level) => summary.levels
|
|
876
|
+
.find((l) => l.level === level)
|
|
877
|
+
.failing.map((r) => r.criterion)
|
|
878
|
+
.join(', ');
|
|
879
|
+
let sentence = `To clear the detected Level ${at} failures, fix: ${verdict.nextFixes
|
|
880
|
+
.map((r) => r.criterion)
|
|
881
|
+
.join(', ')}.`;
|
|
882
|
+
const aa = summary.levels.find((l) => l.level === 'AA');
|
|
883
|
+
const aaa = summary.levels.find((l) => l.level === 'AAA');
|
|
884
|
+
if (at === 'A' && aa.failing.length > 0)
|
|
885
|
+
sentence += ` Then for AA: ${ids('AA')}.`;
|
|
886
|
+
if (at !== 'AAA' && aaa.assessed && aaa.failing.length > 0)
|
|
887
|
+
sentence += ` Then for AAA: ${ids('AAA')}.`;
|
|
888
|
+
lines.push(...continuationLines(style, sentence, width));
|
|
889
|
+
}
|
|
890
|
+
else if (verdict.noFailuresDetectedAt) {
|
|
891
|
+
lines.push(...continuationLines(style, `No Level ${verdict.noFailuresDetectedAt} violations detected across ` +
|
|
892
|
+
`${n(summary.surfacesScanned)} surface${summary.surfacesScanned === 1 ? '' : 's'} ` +
|
|
893
|
+
'(automated checks only).', width));
|
|
894
|
+
}
|
|
895
|
+
/*
|
|
896
|
+
* The coverage ledger. An auditor's first question about an untested
|
|
897
|
+
* criterion is "did you not check it, or can it not be checked?" — with the
|
|
898
|
+
* engine's rule map the two are split; without it the wording is combined,
|
|
899
|
+
* never wrong.
|
|
900
|
+
*/
|
|
901
|
+
const counted = summary.levels.filter((level) => level.level !== 'AAA' || level.assessed);
|
|
902
|
+
const rulesPassed = counted.reduce((sum, level) => sum + level.rulesPassed.length, 0);
|
|
903
|
+
const cantTell = counted.reduce((sum, level) => sum + level.cantTell.length, 0);
|
|
904
|
+
const inapplicable = counted.reduce((sum, level) => sum + level.inapplicable.length, 0);
|
|
905
|
+
const untested = counted.flatMap((level) => level.untested);
|
|
906
|
+
if (rulesPassed > 0) {
|
|
907
|
+
lines.push(figureLine(style, rulesPassed, 'criteria: every automated rule passed — evidence, not conformance'));
|
|
908
|
+
}
|
|
909
|
+
if (cantTell > 0) {
|
|
910
|
+
lines.push(figureLine(style, cantTell, 'criteria need a person — automation could not decide'));
|
|
911
|
+
}
|
|
912
|
+
if (inapplicable > 0) {
|
|
913
|
+
// EARL's inapplicable, spelled out: the checks RAN and the pages contain
|
|
914
|
+
// nothing these criteria regulate — coverage, not a gap.
|
|
915
|
+
lines.push(figureLine(style, inapplicable, 'criteria: checks ran, nothing to check on these pages (no such content)'));
|
|
916
|
+
}
|
|
917
|
+
if (untested.length > 0) {
|
|
918
|
+
if (input.criterionRules) {
|
|
919
|
+
const noCheck = untested.filter((r) => r.untestedReason === 'notAutomatable').length;
|
|
920
|
+
const notRun = untested.filter((r) => r.untestedReason === 'notRun').length;
|
|
921
|
+
if (noCheck > 0)
|
|
922
|
+
lines.push(figureLine(style, noCheck, 'criteria have no automated check in this engine'));
|
|
923
|
+
if (notRun > 0)
|
|
924
|
+
lines.push(figureLine(style, notRun, 'criteria have checks that did not run here'));
|
|
925
|
+
}
|
|
926
|
+
else {
|
|
927
|
+
lines.push(figureLine(style, untested.length, 'criteria not automatically checked in this run'));
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
const anyFailure = summary.levels.some((level) => level.failing.length > 0);
|
|
931
|
+
if (anyFailure) {
|
|
932
|
+
lines.push(...continuationLines(style, 'A surface counts as failing if it fails in any scanned state or theme.', width));
|
|
933
|
+
}
|
|
934
|
+
/*
|
|
935
|
+
* Envelopes written before pass rows carried WCAG identity cannot credit any
|
|
936
|
+
* criterion as passed, however clean the run was. Said out loud, because the
|
|
937
|
+
* alternative reading — "nothing passed" — is wrong in the damaging
|
|
938
|
+
* direction.
|
|
939
|
+
*/
|
|
940
|
+
const passRows = input.envelope.scans.flatMap((scan) => scan.passes);
|
|
941
|
+
if (passRows.length > 0 && passRows.every((pass) => !pass.wcag)) {
|
|
942
|
+
lines.push(...continuationLines(style, 'Rule passes in this run carry no WCAG mapping — it predates the mapping, so no ' +
|
|
943
|
+
'criterion can show its rules as passed. Re-run the suite to get full credit.', width));
|
|
944
|
+
}
|
|
945
|
+
return lines;
|
|
946
|
+
}
|
|
947
|
+
/**
|
|
948
|
+
* The criteria × surfaces matrix, drawn without box-drawing.
|
|
949
|
+
*
|
|
950
|
+
* Aligned single-glyph columns and a numbered legend instead of borders: it
|
|
951
|
+
* reads as a grid on screen and degrades to aligned text everywhere else. The
|
|
952
|
+
* glyph alone carries the meaning — colour never does — and the legend spells
|
|
953
|
+
* every glyph out in words, because ✓ at criterion level means "every
|
|
954
|
+
* automated rule passed", which is not the same claim as "passed".
|
|
955
|
+
*
|
|
956
|
+
* Steps aside by itself when the run has more surfaces than the width budget
|
|
957
|
+
* fits (the budget is the [80,120] clamp, not the raw terminal), and entirely
|
|
958
|
+
* when the customer chose `grid: 'rows'`.
|
|
959
|
+
*/
|
|
960
|
+
function gridLines(input, style, width) {
|
|
961
|
+
if ((input.grid ?? 'matrix') === 'rows')
|
|
962
|
+
return [];
|
|
963
|
+
const grid = conformanceGrid(input.envelope);
|
|
964
|
+
if (grid.rows.length === 0 || grid.surfaces.length === 0)
|
|
965
|
+
return [];
|
|
966
|
+
// ASCII glyphs whenever styling is off: the same environments that lose
|
|
967
|
+
// colour (CI, pipes, NO_COLOR) are the ones where glyph rendering is least
|
|
968
|
+
// trustworthy, and the fallback must be decided by something a test can set.
|
|
969
|
+
const glyph = style.on
|
|
970
|
+
? { failed: '✗', cantTell: '?', rulesPassed: '✓', inapplicable: '–', untested: '·' }
|
|
971
|
+
: { failed: 'F', cantTell: '?', rulesPassed: 'P', inapplicable: '-', untested: '.' };
|
|
972
|
+
const indent = ' ';
|
|
973
|
+
const idWidth = Math.max(...grid.rows.map((row) => row.criterion.length));
|
|
974
|
+
const nameWidth = Math.min(Math.max(...grid.rows.map((row) => row.name.length)), 28);
|
|
975
|
+
const colWidth = Math.max(2, String(grid.surfaces.length).length + 1);
|
|
976
|
+
const labelWidth = idWidth + 2 + nameWidth + 2 + 4;
|
|
977
|
+
const needed = indent.length + labelWidth + grid.surfaces.length * colWidth;
|
|
978
|
+
if (needed > width) {
|
|
979
|
+
return [
|
|
980
|
+
'',
|
|
981
|
+
...continuationLines(style, `The full criteria × surfaces grid needs more room than this summary has ` +
|
|
982
|
+
`(${grid.surfaces.length} surfaces) — see the HTML report for the matrix.`, width),
|
|
983
|
+
];
|
|
984
|
+
}
|
|
985
|
+
const lines = ['', heading(style, 'Conformance grid')];
|
|
986
|
+
// Column header: surface indices, resolved by the legend below the matrix.
|
|
987
|
+
const header = indent +
|
|
988
|
+
' '.repeat(labelWidth) +
|
|
989
|
+
grid.surfaces.map((_, i) => String(i + 1).padEnd(colWidth)).join('');
|
|
990
|
+
lines.push(style.dim(header.trimEnd()));
|
|
991
|
+
for (const row of grid.rows) {
|
|
992
|
+
const name = row.name.length > nameWidth ? `${row.name.slice(0, nameWidth - 1)}…` : row.name;
|
|
993
|
+
const marker = row.removedIn22 ? '—' : row.level;
|
|
994
|
+
const label = row.criterion.padEnd(idWidth) + ' ' + name.padEnd(nameWidth) + ' ' + marker.padEnd(4);
|
|
995
|
+
const cells = row.cells.map((cell) => glyph[cell.state].padEnd(colWidth)).join('');
|
|
996
|
+
lines.push((indent + label + cells).trimEnd());
|
|
997
|
+
}
|
|
998
|
+
lines.push('');
|
|
999
|
+
lines.push(...continuationLines(style, `${glyph.failed} failed ${glyph.cantTell} needs a person ` +
|
|
1000
|
+
`${glyph.rulesPassed} every automated rule passed ` +
|
|
1001
|
+
`${glyph.inapplicable} nothing to check here ${glyph.untested} untested`, width));
|
|
1002
|
+
lines.push(...continuationLines(style, grid.surfaces.map((surface, i) => `${i + 1} ${surface}`).join(' '), width));
|
|
1003
|
+
return lines;
|
|
1004
|
+
}
|
|
1005
|
+
function warningLines(warnings, width, quality, envelope) {
|
|
1006
|
+
const style = createStyle();
|
|
1007
|
+
const notes = [];
|
|
1008
|
+
/*
|
|
1009
|
+
* Coverage inequality, disclosed. A surface scanned in five states and one
|
|
1010
|
+
* scanned once look identical in every count above, and error/loading/empty
|
|
1011
|
+
* states are exactly where regressions hide — a form is usually fine until
|
|
1012
|
+
* it is showing validation errors. One line, only when it applies.
|
|
1013
|
+
*/
|
|
1014
|
+
if (envelope && envelope.scans.length > 0) {
|
|
1015
|
+
const coverage = stateCoverage(envelope);
|
|
1016
|
+
const defaultOnly = coverage.filter((row) => row.defaultOnly).length;
|
|
1017
|
+
if (defaultOnly > 0) {
|
|
1018
|
+
notes.push(`${defaultOnly} of ${coverage.length} surface${coverage.length === 1 ? ' has' : 's have'} evidence in ` +
|
|
1019
|
+
'their default state only — error, loading and empty states are where regressions hide. ' +
|
|
1020
|
+
'setsquare.scan(page, surface, { state }) captures them.');
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
/*
|
|
1024
|
+
* A scan the ceiling cut short is weaker evidence, and it must say so.
|
|
1025
|
+
*
|
|
1026
|
+
* The settle heuristic waits for a page to go quiet and gives up after five
|
|
1027
|
+
* seconds. Giving up is the right behaviour — scanning something is better than
|
|
1028
|
+
* scanning nothing, and skipping silently would be worse than either — but the
|
|
1029
|
+
* result is a scan of a page that was still moving. A clean result from one of
|
|
1030
|
+
* those is a result to be suspicious of, and the whole reason the schema carries
|
|
1031
|
+
* settle diagnostics is so that suspicion survives to the reader instead of
|
|
1032
|
+
* dying in the envelope.
|
|
1033
|
+
*/
|
|
1034
|
+
if (quality && quality.ceiling > 0) {
|
|
1035
|
+
notes.push(`${quality.ceiling} scan${quality.ceiling === 1 ? ' was' : 's were'} cut short after ` +
|
|
1036
|
+
'waiting 5s for the page to settle, so they describe a page that was still ' +
|
|
1037
|
+
'changing. Treat a clean result from those as weaker evidence.');
|
|
1038
|
+
}
|
|
1039
|
+
if (warnings.length === 0 && notes.length === 0)
|
|
1040
|
+
return [];
|
|
1041
|
+
const lines = ['', heading(style, 'Worth knowing')];
|
|
1042
|
+
for (const note of notes)
|
|
1043
|
+
lines.push(...continuationLines(style, note, width));
|
|
1044
|
+
/*
|
|
1045
|
+
* A section of its own rather than a footnote.
|
|
1046
|
+
*
|
|
1047
|
+
* These are the things the run could NOT tell you — surfaces visited but never
|
|
1048
|
+
* scanned, records that could not be read back. Missing evidence must never be
|
|
1049
|
+
* quieter than present evidence, because a reader who skims past it concludes
|
|
1050
|
+
* the run covered more than it did.
|
|
1051
|
+
*/
|
|
1052
|
+
for (const warning of warnings) {
|
|
1053
|
+
lines.push(...continuationLines(style, warning, width));
|
|
1054
|
+
}
|
|
1055
|
+
return lines;
|
|
1056
|
+
}
|
|
1057
|
+
/**
|
|
1058
|
+
* The closing block (decision 12, 2026-08-18).
|
|
1059
|
+
*
|
|
1060
|
+
* Replaces the single quiet info line the plan originally specified. Every
|
|
1061
|
+
* constraint that made that right survives — once per run, at the very end, never
|
|
1062
|
+
* per-surface, never interrupting findings, suppressible — and only the length
|
|
1063
|
+
* relaxes, because one line cannot carry *why* and a bare URL reads as an advert.
|
|
1064
|
+
*
|
|
1065
|
+
* Built from THIS run's numbers so it explains rather than sells, and it names
|
|
1066
|
+
* only things the free layer structurally cannot do. Nothing is removed from the
|
|
1067
|
+
* local tool to create the paid one, and this block must never imply otherwise.
|
|
1068
|
+
*/
|
|
1069
|
+
function promoLines(input, count, width) {
|
|
1070
|
+
if (!input.promo)
|
|
1071
|
+
return [];
|
|
1072
|
+
const failingRules = new Set(input.envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
|
|
1073
|
+
const subject = failingRules > 0
|
|
1074
|
+
? `${failingRules} rule${failingRules === 1 ? '' : 's'} failing across ${count.surfaces} surface${count.surfaces === 1 ? '' : 's'}`
|
|
1075
|
+
: `${count.surfaces} clean surface${count.surfaces === 1 ? '' : 's'}`;
|
|
1076
|
+
return [
|
|
1077
|
+
'',
|
|
1078
|
+
// Wrapped like everything else: a block that overflows the terminal is a
|
|
1079
|
+
// block that looks broken, and this one is read on every single run.
|
|
1080
|
+
...wrap(`${subject}, this run only. Setsquare Cloud adds what a local tool cannot:`, width - 4, '').map((line) => ` ${line}`),
|
|
1081
|
+
' · is this better or worse than last week? (local runs keep no history)',
|
|
1082
|
+
' · which change introduced each one, on the PR that did it',
|
|
1083
|
+
' · the root cause behind repeats, grouped instead of repeated',
|
|
1084
|
+
` ${PLATFORM_URL}`,
|
|
1085
|
+
'',
|
|
1086
|
+
];
|
|
1087
|
+
}
|
|
1088
|
+
/** The conformance block in markdown — same facts as the terminal, same vocabulary rules. */
|
|
1089
|
+
function markdownConformance(input) {
|
|
1090
|
+
const summary = conformanceSummary(input.envelope, {
|
|
1091
|
+
...(input.criterionRules ? { criterionRules: input.criterionRules } : {}),
|
|
1092
|
+
});
|
|
1093
|
+
if (summary.surfacesScanned === 0)
|
|
1094
|
+
return [];
|
|
1095
|
+
const lines = ['### Conformance — WCAG 2.2', ''];
|
|
1096
|
+
const { verdict } = summary;
|
|
1097
|
+
if (verdict.failuresDetectedAt) {
|
|
1098
|
+
lines.push(`**Failures detected at Level ${verdict.failuresDetectedAt}.** To clear them, fix: ` +
|
|
1099
|
+
`${verdict.nextFixes.map((r) => `${r.criterion} ${r.name}`).join(', ')}. ` +
|
|
1100
|
+
'Automated checks cover only part of WCAG.');
|
|
1101
|
+
}
|
|
1102
|
+
else if (verdict.noFailuresDetectedAt) {
|
|
1103
|
+
lines.push(`**No Level ${verdict.noFailuresDetectedAt} violations detected** across ` +
|
|
1104
|
+
`${summary.surfacesScanned} surfaces (automated checks only).`);
|
|
1105
|
+
}
|
|
1106
|
+
lines.push('');
|
|
1107
|
+
const withEvidence = summary.levels.flatMap((level) => level.level === 'AAA' && !level.assessed
|
|
1108
|
+
? level.failing
|
|
1109
|
+
: [...level.failing, ...level.cantTell, ...level.rulesPassed, ...level.inapplicable]);
|
|
1110
|
+
if (withEvidence.length > 0) {
|
|
1111
|
+
lines.push('| Criterion | Level | State | Surfaces |');
|
|
1112
|
+
lines.push('|---|---|---|---|');
|
|
1113
|
+
const stateWords = {
|
|
1114
|
+
failed: 'FAIL',
|
|
1115
|
+
cantTell: 'needs a person',
|
|
1116
|
+
rulesPassed: 'rules passed',
|
|
1117
|
+
inapplicable: 'nothing to check',
|
|
1118
|
+
untested: 'untested',
|
|
1119
|
+
};
|
|
1120
|
+
for (const rollup of [...withEvidence].sort((a, b) => a.criterion.localeCompare(b.criterion, 'en', { numeric: true }))) {
|
|
1121
|
+
const surfaces = rollup.state === 'failed'
|
|
1122
|
+
? `${rollup.failedSurfaces.length} of ${rollup.totalSurfaces} (${sharePercent(rollup.failedSurfaces.length, rollup.totalSurfaces)})`
|
|
1123
|
+
: `${rollup.totalSurfaces}`;
|
|
1124
|
+
lines.push(`| ${rollup.criterion} ${rollup.name} | ${rollup.level} | ${stateWords[rollup.state]} | ${surfaces} |`);
|
|
1125
|
+
}
|
|
1126
|
+
lines.push('');
|
|
1127
|
+
}
|
|
1128
|
+
const aaa = summary.levels.find((level) => level.level === 'AAA');
|
|
1129
|
+
if (aaa && !aaa.assessed) {
|
|
1130
|
+
lines.push("Level AAA not assessed — AAA is opt-in; run with `wcag: 'AAA'` to include it.");
|
|
1131
|
+
lines.push('');
|
|
1132
|
+
}
|
|
1133
|
+
return lines;
|
|
1134
|
+
}
|
|
1135
|
+
/** Markdown, for pasting into Slack, an issue, or an LLM prompt. */
|
|
1136
|
+
export function renderMarkdown(input) {
|
|
1137
|
+
const { envelope } = input;
|
|
1138
|
+
const count = totals(envelope);
|
|
1139
|
+
const lines = [];
|
|
1140
|
+
const tests = testOutcomes(envelope, input.plannedTests);
|
|
1141
|
+
lines.push('## Accessibility evidence');
|
|
1142
|
+
lines.push('');
|
|
1143
|
+
if (envelope.results.length > 0) {
|
|
1144
|
+
const parts = [`**${tests.passed}** passed`];
|
|
1145
|
+
if (tests.failed > 0)
|
|
1146
|
+
parts.push(`**${tests.failed}** failed`);
|
|
1147
|
+
if (tests.flaky > 0)
|
|
1148
|
+
parts.push(`**${tests.flaky}** flaky`);
|
|
1149
|
+
if (tests.skipped > 0)
|
|
1150
|
+
parts.push(`**${tests.skipped}** skipped`);
|
|
1151
|
+
if (tests.notRun)
|
|
1152
|
+
parts.push(`**${tests.notRun}** never ran`);
|
|
1153
|
+
lines.push(`Playwright: ${parts.join(', ')}.`);
|
|
1154
|
+
lines.push('');
|
|
1155
|
+
}
|
|
1156
|
+
if (count.scans === 0) {
|
|
1157
|
+
lines.push('**No surfaces were scanned.** This is not a clean result — nothing was looked at.');
|
|
1158
|
+
return lines.join('\n');
|
|
1159
|
+
}
|
|
1160
|
+
lines.push(`Scanned **${count.surfaces}** surfaces in **${count.themes}** themes against ` +
|
|
1161
|
+
`WCAG 2.2 ${envelope.scans[0]?.wcagTarget ?? 'AA'} — ` +
|
|
1162
|
+
`**${count.violations}** violations, **${count.incomplete}** needing review, ` +
|
|
1163
|
+
`**${count.passes}** passing checks.`);
|
|
1164
|
+
lines.push('');
|
|
1165
|
+
/*
|
|
1166
|
+
* The same conformance facts as the terminal, in markdown syntax. Always
|
|
1167
|
+
* criterion rows, never the matrix — a glyph grid pastes as soup.
|
|
1168
|
+
*/
|
|
1169
|
+
lines.push(...markdownConformance(input));
|
|
1170
|
+
const groups = groupFindings(envelope);
|
|
1171
|
+
if (groups.length > 0) {
|
|
1172
|
+
lines.push('| Impact | Rule | Elements | Surfaces | WCAG |');
|
|
1173
|
+
lines.push('|---|---|---|---|---|');
|
|
1174
|
+
for (const group of groups) {
|
|
1175
|
+
const rule = group.helpUrl ? `[${group.ruleId}](${group.helpUrl})` : group.ruleId;
|
|
1176
|
+
const status = group.status === 'incomplete' ? ' *(needs review)*' : '';
|
|
1177
|
+
lines.push(`| ${group.impact} | ${rule}${status} | ${group.nodes} | ` +
|
|
1178
|
+
`${group.contexts.length} | ${group.criteria.join(', ') || '—'} |`);
|
|
1179
|
+
}
|
|
1180
|
+
lines.push('');
|
|
1181
|
+
/*
|
|
1182
|
+
* Surfaces listed under the table rather than inside it. A context list can be
|
|
1183
|
+
* long, and a markdown table cell that wraps is a markdown table nobody can
|
|
1184
|
+
* read — including in the PR comments this format is rehearsing for.
|
|
1185
|
+
*/
|
|
1186
|
+
for (const group of groups) {
|
|
1187
|
+
lines.push(`- **${group.ruleId}**: ${group.contexts.join(', ')}`);
|
|
1188
|
+
if (group.failureSummary)
|
|
1189
|
+
lines.push(` - ${group.failureSummary}`);
|
|
1190
|
+
if (group.selectors.length > 0) {
|
|
1191
|
+
lines.push(` - where: ${group.selectors.map((sel) => `\`${sel}\``).join(', ')}` +
|
|
1192
|
+
(group.selectorsTruncated ? ' …' : ''));
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
lines.push('');
|
|
1196
|
+
}
|
|
1197
|
+
// Surfaces, in the same order and with the same completeness as the terminal
|
|
1198
|
+
// form. The two renderers differ in syntax and in nothing else — a pasted
|
|
1199
|
+
// summary that says less than the one on screen is a summary people stop
|
|
1200
|
+
// trusting.
|
|
1201
|
+
lines.push('### Surfaces');
|
|
1202
|
+
lines.push('');
|
|
1203
|
+
for (const row of surfaceRows(envelope)) {
|
|
1204
|
+
lines.push(row.violations > 0
|
|
1205
|
+
? `- **FAIL** ${row.label} — ${row.violations} elements: ${row.rules.join(', ')}`
|
|
1206
|
+
: `- pass ${row.label}`);
|
|
1207
|
+
}
|
|
1208
|
+
const perf = perfRows(envelope);
|
|
1209
|
+
if (perf.length > 0) {
|
|
1210
|
+
const withLcp = perf.filter((row) => row.lcpMs !== undefined);
|
|
1211
|
+
const slow = withLcp.filter((row) => (row.lcpMs ?? 0) > GOOD_LCP_MS);
|
|
1212
|
+
lines.push('');
|
|
1213
|
+
lines.push('### Performance');
|
|
1214
|
+
lines.push('');
|
|
1215
|
+
lines.push(`**${perf.length}** page loads measured in CI (lab timings, not real users). ` +
|
|
1216
|
+
(withLcp.length === 0
|
|
1217
|
+
? 'No LCP reported — Firefox and WebKit do not implement it.'
|
|
1218
|
+
: slow.length === 0
|
|
1219
|
+
? `All within the ${GOOD_LCP_MS}ms LCP threshold.`
|
|
1220
|
+
: `**${slow.length}** over the ${GOOD_LCP_MS}ms LCP threshold: ` +
|
|
1221
|
+
slow.map((row) => `${row.label} (${Math.round(row.lcpMs ?? 0)}ms)`).join(', ')));
|
|
1222
|
+
}
|
|
1223
|
+
/*
|
|
1224
|
+
* The same blocks as the terminal, in markdown. Same wording function, so the
|
|
1225
|
+
* two formats cannot come to say different things about the same run — which is
|
|
1226
|
+
* the rule `groupLines` established and this batch had to honour five more times.
|
|
1227
|
+
*/
|
|
1228
|
+
for (const block of insightBlocks(envelope, {
|
|
1229
|
+
...(input.declaredAxes ? { declaredAxes: input.declaredAxes } : {}),
|
|
1230
|
+
...(input.unmeasuredAxes ? { unmeasuredAxes: input.unmeasuredAxes } : {}),
|
|
1231
|
+
...(input.suite ? { suite: input.suite } : {}),
|
|
1232
|
+
...(input.overhead ? { overhead: input.overhead } : {}),
|
|
1233
|
+
})) {
|
|
1234
|
+
lines.push('');
|
|
1235
|
+
lines.push(`### ${block.heading}`);
|
|
1236
|
+
lines.push('');
|
|
1237
|
+
for (const figure of block.figures) {
|
|
1238
|
+
lines.push(`- **${figure.value}** ${figure.label}`);
|
|
1239
|
+
for (const note of figure.notes ?? [])
|
|
1240
|
+
lines.push(` - ${note}`);
|
|
1241
|
+
}
|
|
1242
|
+
for (const note of block.notes) {
|
|
1243
|
+
lines.push('');
|
|
1244
|
+
lines.push(note);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
if (tests.flaky > 0) {
|
|
1248
|
+
const flaky = flakeNotes(input.suite);
|
|
1249
|
+
if (flaky.length > 0) {
|
|
1250
|
+
lines.push('');
|
|
1251
|
+
lines.push('### Flaky');
|
|
1252
|
+
lines.push('');
|
|
1253
|
+
for (const note of flaky)
|
|
1254
|
+
lines.push(`- ${note}`);
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
lines.push('');
|
|
1258
|
+
lines.push(`Evidence: \`${input.envelopePath}\``);
|
|
1259
|
+
if (input.htmlReportPath)
|
|
1260
|
+
lines.push(`Report: \`${input.htmlReportPath}\``);
|
|
1261
|
+
if (input.screensPath)
|
|
1262
|
+
lines.push(`Screens: \`${input.screensPath}\``);
|
|
1263
|
+
if (input.reportPath)
|
|
1264
|
+
lines.push(`Playwright report: \`${input.reportPath}\``);
|
|
1265
|
+
if (input.group) {
|
|
1266
|
+
lines.push('');
|
|
1267
|
+
lines.push('### Run group');
|
|
1268
|
+
lines.push('');
|
|
1269
|
+
for (const line of groupLines(input.group))
|
|
1270
|
+
lines.push(`- ${line}`);
|
|
1271
|
+
}
|
|
1272
|
+
return lines.join('\n');
|
|
1273
|
+
}
|
|
1274
|
+
export function render(input, format, columns) {
|
|
1275
|
+
if (format === 'markdown')
|
|
1276
|
+
return renderMarkdown(input);
|
|
1277
|
+
if (format === 'json') {
|
|
1278
|
+
// Machine output stays machine output: no closing block, no notes, nothing a
|
|
1279
|
+
// parser has to skip.
|
|
1280
|
+
return JSON.stringify({
|
|
1281
|
+
totals: totals(input.envelope),
|
|
1282
|
+
findings: groupFindings(input.envelope),
|
|
1283
|
+
conformance: conformanceSummary(input.envelope, {
|
|
1284
|
+
...(input.criterionRules ? { criterionRules: input.criterionRules } : {}),
|
|
1285
|
+
}),
|
|
1286
|
+
// Machine consumers need the coverage verdict most of all: a parser
|
|
1287
|
+
// that cannot see "partial" will treat a half-merged group as a run.
|
|
1288
|
+
...(input.group ? { group: input.group } : {}),
|
|
1289
|
+
/*
|
|
1290
|
+
* The insight blocks as DATA, not as rendered lines.
|
|
1291
|
+
*
|
|
1292
|
+
* A machine consumer wants the figures, and the notes carry the caveats
|
|
1293
|
+
* that stop it drawing the wrong conclusion from them — particularly
|
|
1294
|
+
* "these counts are per engine and deliberately not totalled" and "this
|
|
1295
|
+
* is time spent, not time added". Dropping the notes here would leave a
|
|
1296
|
+
* parser holding numbers with none of the constraints attached.
|
|
1297
|
+
*/
|
|
1298
|
+
insights: insightBlocks(input.envelope, {
|
|
1299
|
+
...(input.declaredAxes ? { declaredAxes: input.declaredAxes } : {}),
|
|
1300
|
+
...(input.unmeasuredAxes ? { unmeasuredAxes: input.unmeasuredAxes } : {}),
|
|
1301
|
+
...(input.suite ? { suite: input.suite } : {}),
|
|
1302
|
+
...(input.overhead ? { overhead: input.overhead } : {}),
|
|
1303
|
+
}),
|
|
1304
|
+
}, null, 2);
|
|
1305
|
+
}
|
|
1306
|
+
return renderText(input, columns);
|
|
1307
|
+
}
|
|
1308
|
+
//# sourceMappingURL=summary.js.map
|