@qaguardian/playwright-score 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/METHODOLOGY.md +41 -0
- package/README.md +1 -1
- package/dist/cli.js +61 -5
- package/dist/confidence.d.ts +30 -0
- package/dist/confidence.js +67 -0
- package/dist/eslint-runner.js +18 -0
- package/dist/formatters/json.js +4 -1
- package/dist/formatters/markdown.d.ts +2 -0
- package/dist/formatters/markdown.js +15 -4
- package/dist/formatters/report-css.d.ts +2 -0
- package/dist/formatters/report-css.js +68 -0
- package/dist/formatters/report.d.ts +20 -0
- package/dist/formatters/report.js +279 -0
- package/dist/formatters/text.d.ts +6 -1
- package/dist/formatters/text.js +73 -28
- package/dist/index.js +17 -6
- package/dist/profiles.js +12 -0
- package/dist/report-model.d.ts +52 -0
- package/dist/report-model.js +162 -0
- package/dist/report-source.d.ts +11 -0
- package/dist/report-source.js +96 -0
- package/dist/rule-copy.d.ts +14 -0
- package/dist/rule-copy.js +215 -0
- package/dist/rules/ast-util.d.ts +60 -0
- package/dist/rules/ast-util.js +140 -0
- package/dist/rules/base-plugin.d.ts +19 -5
- package/dist/rules/base-plugin.js +75 -7
- package/dist/rules/no-brittle-selector.d.ts +3 -0
- package/dist/rules/no-brittle-selector.js +150 -0
- package/dist/rules/no-excessive-timeout.d.ts +2 -0
- package/dist/rules/no-excessive-timeout.js +87 -0
- package/dist/rules/no-fixme-test.d.ts +14 -0
- package/dist/rules/no-fixme-test.js +53 -0
- package/dist/rules/no-self-fulfilling-fallback.d.ts +29 -0
- package/dist/rules/no-self-fulfilling-fallback.js +221 -0
- package/dist/rules/no-shared-mutable-state.d.ts +4 -0
- package/dist/rules/no-shared-mutable-state.js +186 -0
- package/dist/rules/no-swallowed-assertion.d.ts +13 -0
- package/dist/rules/no-swallowed-assertion.js +113 -0
- package/dist/rules/no-unawaited-action.d.ts +2 -0
- package/dist/rules/no-unawaited-action.js +77 -0
- package/dist/rules/no-unawaited-expect-value.d.ts +20 -0
- package/dist/rules/no-unawaited-expect-value.js +46 -0
- package/dist/rules/no-weak-assertion.d.ts +19 -0
- package/dist/rules/no-weak-assertion.js +82 -0
- package/dist/rules/rule-util.d.ts +56 -0
- package/dist/rules/rule-util.js +253 -0
- package/dist/score-engine.d.ts +5 -0
- package/dist/score-engine.js +9 -9
- package/dist/suite-name.d.ts +11 -0
- package/dist/suite-name.js +36 -0
- package/dist/test-aliases.js +144 -3
- package/dist/types.d.ts +14 -0
- package/package.json +1 -1
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import { PROFILE_WEIGHTS } from '../profiles.js';
|
|
2
|
+
import { CONFIDENCE_LABEL, compareFindings, isNeedsCheck } from '../confidence.js';
|
|
3
|
+
import { copyFor } from '../rule-copy.js';
|
|
4
|
+
import { REPORT_CSS } from './report-css.js';
|
|
5
|
+
import { DIMENSIONS, barText, cleanCheckCalls, gradeBandText, mostAffectedFile, notScoredHeadline, notScoredState, plural, rankGroups, summaryTop, INCOMPLETE_NOTE, } from '../report-model.js';
|
|
6
|
+
export { rankTopFindings, compareGroups } from '../report-model.js';
|
|
7
|
+
/** Rules that make a test flaky rather than merely untidy. */
|
|
8
|
+
export const FLAKE_RULES = [
|
|
9
|
+
'playwright/no-wait-for-timeout',
|
|
10
|
+
'pwscore/no-timer-sleep',
|
|
11
|
+
'playwright/no-networkidle',
|
|
12
|
+
'playwright/no-force-option',
|
|
13
|
+
'playwright/no-wait-for-selector',
|
|
14
|
+
'playwright/missing-playwright-await',
|
|
15
|
+
];
|
|
16
|
+
const DETAIL_CARDS = 5;
|
|
17
|
+
const LEGEND = 'Definite: the code does this for certain. Likely: usually a problem, but there are legitimate uses. None of these are confirmed bugs in your product; they are patterns that make bugs easier to miss or tests less reliable.';
|
|
18
|
+
export function escapeHtml(s) {
|
|
19
|
+
return s
|
|
20
|
+
.replace(/&/g, '&')
|
|
21
|
+
.replace(/</g, '<')
|
|
22
|
+
.replace(/>/g, '>')
|
|
23
|
+
.replace(/"/g, '"')
|
|
24
|
+
.replace(/'/g, ''');
|
|
25
|
+
}
|
|
26
|
+
const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
|
27
|
+
function longDate(iso) {
|
|
28
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(iso);
|
|
29
|
+
const mon = m ? MONTHS[Number(m[2]) - 1] : undefined;
|
|
30
|
+
return m && mon ? `${Number(m[3])} ${mon} ${m[1]}` : iso;
|
|
31
|
+
}
|
|
32
|
+
function excerpt(source, line) {
|
|
33
|
+
const lines = source.split(/\r?\n/);
|
|
34
|
+
if (line < 1 || line > lines.length)
|
|
35
|
+
return [];
|
|
36
|
+
const from = Math.max(1, line - 3);
|
|
37
|
+
const to = Math.min(lines.length, line + 3);
|
|
38
|
+
const out = [];
|
|
39
|
+
for (let n = from; n <= to; n++) {
|
|
40
|
+
let text = (lines[n - 1] ?? '').replace(/\t/g, ' ');
|
|
41
|
+
if (text.length > 140)
|
|
42
|
+
text = text.slice(0, 137) + '...';
|
|
43
|
+
out.push({ n, text, hit: n === line });
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
function renderExcerpt(f, readSource) {
|
|
48
|
+
if (!f.line || !readSource)
|
|
49
|
+
return '';
|
|
50
|
+
const src = readSource(f.file);
|
|
51
|
+
if (src === undefined)
|
|
52
|
+
return '';
|
|
53
|
+
const rows = excerpt(src, f.line);
|
|
54
|
+
if (!rows.length)
|
|
55
|
+
return '';
|
|
56
|
+
const body = rows
|
|
57
|
+
.map((r) => r.hit
|
|
58
|
+
? `<span class="ln hit"><i>▸${r.n}</i><span class="sr-only">flagged line: </span>${escapeHtml(r.text) || ' '}</span>`
|
|
59
|
+
: `<span class="ln"><i>${r.n}</i>${escapeHtml(r.text) || ' '}</span>`)
|
|
60
|
+
.join('');
|
|
61
|
+
return `<pre class="code" aria-label="Code excerpt">${body}</pre>`;
|
|
62
|
+
}
|
|
63
|
+
function loc(f) {
|
|
64
|
+
return f.line ? `${f.file}:${f.line}` : f.file;
|
|
65
|
+
}
|
|
66
|
+
function baseName(p) {
|
|
67
|
+
return p.split(/[/\\]/).pop() || p;
|
|
68
|
+
}
|
|
69
|
+
function tag(g) {
|
|
70
|
+
return `<span class="tag">${CONFIDENCE_LABEL[g.confidence]}</span>`;
|
|
71
|
+
}
|
|
72
|
+
function renderTop3(groups) {
|
|
73
|
+
const items = groups
|
|
74
|
+
.map((g, i) => {
|
|
75
|
+
const c = copyFor(g.rule);
|
|
76
|
+
const most = mostAffectedFile(g);
|
|
77
|
+
const places = `${plural(g.findings.length, 'place')} in ${plural(g.testsAffected, 'test')}`;
|
|
78
|
+
const where = `<code>${escapeHtml(baseName(most.file))}</code>${most.line ? ` (line ${most.line})` : ''}`;
|
|
79
|
+
const sameFile = most.count === g.findings.length;
|
|
80
|
+
const count = g.findings.length === 1
|
|
81
|
+
? `${places}: ${where}.`
|
|
82
|
+
: `${places}. ${sameFile ? 'All in' : 'Most in'} ${where}.`;
|
|
83
|
+
return `<li class="top"><span class="rank" aria-hidden="true">${i + 1}</span>
|
|
84
|
+
<div><h3>${escapeHtml(c.title)} ${tag(g)}</h3>
|
|
85
|
+
<p class="count">${count}</p>
|
|
86
|
+
<p class="why">${escapeHtml(c.why)}</p></div></li>`;
|
|
87
|
+
})
|
|
88
|
+
.join('\n');
|
|
89
|
+
return `<ol class="top3" role="list">\n${items}\n</ol>`;
|
|
90
|
+
}
|
|
91
|
+
function renderDetailCards(groups, readSource) {
|
|
92
|
+
return groups
|
|
93
|
+
.map((g, i) => {
|
|
94
|
+
const c = copyFor(g.rule);
|
|
95
|
+
const others = g.findings.length - 1;
|
|
96
|
+
return `<article class="finding sev-${g.severity}">
|
|
97
|
+
<h3><span class="rank" aria-hidden="true">${i + 1}</span>${escapeHtml(c.title)} ${tag(g)}</h3>
|
|
98
|
+
<p class="where"><code>${escapeHtml(loc(g.example))}</code></p>
|
|
99
|
+
${renderExcerpt(g.example, readSource)}
|
|
100
|
+
<p class="why">${escapeHtml(c.why)}</p>
|
|
101
|
+
<p class="muted">${others > 0 ? `${plural(g.findings.length, 'occurrence')} across ${plural(g.testsAffected, 'test')}.` : 'One occurrence.'}</p>
|
|
102
|
+
</article>`;
|
|
103
|
+
})
|
|
104
|
+
.join('\n');
|
|
105
|
+
}
|
|
106
|
+
function flakeFilesLine(findings) {
|
|
107
|
+
const files = new Map();
|
|
108
|
+
for (const f of findings) {
|
|
109
|
+
if (FLAKE_RULES.includes(f.rule) && !isNeedsCheck(f))
|
|
110
|
+
files.set(f.file, (files.get(f.file) ?? 0) + 1);
|
|
111
|
+
}
|
|
112
|
+
if (!files.size)
|
|
113
|
+
return '';
|
|
114
|
+
const worst = [...files.entries()]
|
|
115
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
116
|
+
.slice(0, 3)
|
|
117
|
+
.map(([file, n]) => `<code>${escapeHtml(file)}</code> (${n})`)
|
|
118
|
+
.join(', ');
|
|
119
|
+
return `<p class="muted">Most flake-prone files: ${worst}.</p>`;
|
|
120
|
+
}
|
|
121
|
+
function renderNeedsCheck(findings) {
|
|
122
|
+
const items = findings.filter(isNeedsCheck).sort(compareFindings);
|
|
123
|
+
if (!items.length)
|
|
124
|
+
return '';
|
|
125
|
+
const byFile = new Map();
|
|
126
|
+
for (const f of items) {
|
|
127
|
+
const list = byFile.get(f.file);
|
|
128
|
+
if (list)
|
|
129
|
+
list.push(f);
|
|
130
|
+
else
|
|
131
|
+
byFile.set(f.file, [f]);
|
|
132
|
+
}
|
|
133
|
+
const files = [...byFile.entries()];
|
|
134
|
+
const shown = files.slice(0, 6);
|
|
135
|
+
const rows = shown
|
|
136
|
+
.map(([file, list]) => {
|
|
137
|
+
const lines = list.filter((f) => f.line).map((f) => f.line).join(', ');
|
|
138
|
+
const calls = cleanCheckCalls(list[0].message);
|
|
139
|
+
const what = calls
|
|
140
|
+
? `Calls ${calls} from a helper we could not read.`
|
|
141
|
+
: 'Its checks could not be seen.';
|
|
142
|
+
return `<li><code>${escapeHtml(file)}</code> <span class="muted">${plural(list.length, 'test')}${lines ? `, line${list.length === 1 ? '' : 's'} ${escapeHtml(lines)}` : ''}</span><br><span class="muted">${escapeHtml(what)}</span></li>`;
|
|
143
|
+
})
|
|
144
|
+
.join('');
|
|
145
|
+
const hidden = items.length - shown.reduce((n, [, l]) => n + l.length, 0);
|
|
146
|
+
const more = hidden > 0 ? `<p class="muted">and ${plural(hidden, 'more test')} in ${plural(files.length - shown.length, 'other file')}.</p>` : '';
|
|
147
|
+
return `<h3 id="human-check">Needs a human check</h3>
|
|
148
|
+
<p class="note">${plural(items.length, 'test')} ${items.length === 1 ? 'relies' : 'rely'} on helpers we could not read (page objects, fixtures or imported functions). The checks probably live there, but the score cannot see them, so ${items.length === 1 ? 'this test is' : 'these tests are'} left out of the score. Ask someone to confirm each helper really checks the page.</p><ul class="plain">${rows}</ul>${more}`;
|
|
149
|
+
}
|
|
150
|
+
const GENERIC_STEPS = [
|
|
151
|
+
'Re-run the score in CI with --threshold so the number cannot slip back unnoticed.',
|
|
152
|
+
'Keep new specs on role, label and test-id locators with web-first assertions.',
|
|
153
|
+
'Review any items under "Needs a human check" by hand.',
|
|
154
|
+
];
|
|
155
|
+
function nextSteps(groups, extra = []) {
|
|
156
|
+
const steps = [];
|
|
157
|
+
const fixes = groups.map((g) => copyFor(g.rule).fix).filter((f, i, a) => a.indexOf(f) === i);
|
|
158
|
+
const maxFromFindings = 3 - Math.min(extra.length, 2);
|
|
159
|
+
for (const fix of fixes.slice(0, maxFromFindings))
|
|
160
|
+
steps.push(fix);
|
|
161
|
+
for (const e of extra) {
|
|
162
|
+
if (steps.length < 3)
|
|
163
|
+
steps.push(e);
|
|
164
|
+
}
|
|
165
|
+
for (const s of GENERIC_STEPS) {
|
|
166
|
+
if (steps.length >= 3)
|
|
167
|
+
break;
|
|
168
|
+
steps.push(s);
|
|
169
|
+
}
|
|
170
|
+
return steps;
|
|
171
|
+
}
|
|
172
|
+
function renderBars(result) {
|
|
173
|
+
const weights = PROFILE_WEIGHTS[result.profile] ?? PROFILE_WEIGHTS.standard;
|
|
174
|
+
const bars = DIMENSIONS.map((d) => {
|
|
175
|
+
const v = result.dimensions[d.key] ?? 0;
|
|
176
|
+
const w = weights[d.key];
|
|
177
|
+
return `<div class="dim" title="${escapeHtml(d.meaning)}"><div class="dimhead"><span>${d.label}${w ? ` <span class="muted">${w}% of score</span>` : ''}</span><strong>${v}</strong></div>
|
|
178
|
+
<div class="bar" role="img" aria-label="${d.label} ${v} out of 100"><span style="width:${Math.max(0, Math.min(100, v))}%"></span></div>
|
|
179
|
+
<p class="muted sr-only">${escapeHtml(d.meaning)}</p></div>`;
|
|
180
|
+
}).join('\n');
|
|
181
|
+
return `<h2>Score by area</h2>\n<div class="dims">\n${bars}\n</div>`;
|
|
182
|
+
}
|
|
183
|
+
export function formatReport(result, opts) {
|
|
184
|
+
const { summary } = result;
|
|
185
|
+
const state = notScoredState(result);
|
|
186
|
+
const clean = summary.cleanTests;
|
|
187
|
+
const all = rankGroups(result.findings);
|
|
188
|
+
const top = summaryTop(all, result.dimensions);
|
|
189
|
+
const detail = [...new Set([...top, ...all.slice(0, DETAIL_CARDS)])];
|
|
190
|
+
const humanItems = result.findings.filter(isNeedsCheck);
|
|
191
|
+
const humanHtml = renderNeedsCheck(result.findings);
|
|
192
|
+
let headline;
|
|
193
|
+
let explain;
|
|
194
|
+
let scoreBox;
|
|
195
|
+
if (state) {
|
|
196
|
+
const ns = notScoredHeadline(result);
|
|
197
|
+
headline = ns.headline;
|
|
198
|
+
explain = ns.line;
|
|
199
|
+
scoreBox = `<div class="score notscored" role="img" aria-label="Not scored"><span class="big">--</span><span class="grade">Not scored</span></div>`;
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
headline = `${clean} of ${summary.tests} ${summary.tests === 1 ? 'test has' : 'tests have'} no issues.`;
|
|
203
|
+
explain = `A weighted average of four areas below. ${gradeBandText(result.grade, result.pass)} ${barText(result.threshold, opts.customThreshold === true, result.pass)}`;
|
|
204
|
+
scoreBox = `<div class="score g-${result.grade}" aria-hidden="true"><span class="big">${result.score}</span><span class="of">/100</span><span class="grade">${result.grade}</span></div>`;
|
|
205
|
+
}
|
|
206
|
+
const srScore = state ? '' : `<span class="sr-only">Score ${result.score} out of 100, grade ${result.grade}. </span>`;
|
|
207
|
+
const humanCount = summary.unverifiedTests;
|
|
208
|
+
const humanLine = humanCount > 0 && !state
|
|
209
|
+
? `\n<p class="check">${humanItems.length ? '<a href="#human-check">' : ''}${plural(humanCount, 'test')} ${humanCount === 1 ? 'needs' : 'need'} a human check and ${humanCount === 1 ? 'is' : 'are'} left out of the score.${humanItems.length ? '</a>' : ''}</p>`
|
|
210
|
+
: '';
|
|
211
|
+
const crashLine = result.ruleCrashes?.length ? `\n<p class="check">${escapeHtml(INCOMPLETE_NOTE)}</p>` : '';
|
|
212
|
+
const meta = state
|
|
213
|
+
? plural(summary.files, 'file')
|
|
214
|
+
: `${plural(summary.tests, 'test')} in ${plural(summary.files, 'file')}`;
|
|
215
|
+
const header = `<header>
|
|
216
|
+
<p class="eyebrow">Playwright suite score</p>
|
|
217
|
+
<p class="suite">${escapeHtml(opts.suiteName)}</p>
|
|
218
|
+
<div class="hd">
|
|
219
|
+
${scoreBox}
|
|
220
|
+
<div>
|
|
221
|
+
<h1>${srScore}${escapeHtml(headline)}</h1>
|
|
222
|
+
<p class="explain">${escapeHtml(explain)}</p>${humanLine}${crashLine}
|
|
223
|
+
<p class="meta">${meta} · scored ${escapeHtml(longDate(opts.date))}</p>
|
|
224
|
+
</div>
|
|
225
|
+
</div>
|
|
226
|
+
</header>`;
|
|
227
|
+
const footer = `<footer>Scored by QA Guardian playwright-score ${escapeHtml(opts.version)} (rules ${escapeHtml(result.scoreVersion)}) on ${escapeHtml(opts.date)}. Static analysis of the test code: the same files always give the same score. Nothing was run against your product.</footer>`;
|
|
228
|
+
const head = `<!DOCTYPE html>
|
|
229
|
+
<html lang="en">
|
|
230
|
+
<head>
|
|
231
|
+
<meta charset="utf-8">
|
|
232
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
233
|
+
<meta name="color-scheme" content="light dark">
|
|
234
|
+
<title>Playwright suite score: ${escapeHtml(opts.suiteName)}</title>
|
|
235
|
+
<style>${REPORT_CSS}</style>
|
|
236
|
+
</head>
|
|
237
|
+
<body>
|
|
238
|
+
<main>
|
|
239
|
+
`;
|
|
240
|
+
if (state === 'no-tests') {
|
|
241
|
+
return `${head}<section class="summary">\n${header}\n</section>\n${footer}\n</main>\n</body>\n</html>\n`;
|
|
242
|
+
}
|
|
243
|
+
const clearSuite = all.length === 0 && humanItems.length === 0 && !state;
|
|
244
|
+
const topHtml = all.length
|
|
245
|
+
? `<h2>${top.length > 1 ? `Top ${top.length} to fix` : 'Fix first'}</h2>\n<p class="muted legend">${escapeHtml(LEGEND)}</p>\n${renderTop3(top)}`
|
|
246
|
+
: '<h2>Issues</h2>\n<p>No issues found. Keep it that way by adding the score to CI with --threshold.</p>';
|
|
247
|
+
const extraSteps = [];
|
|
248
|
+
if (humanItems.length) {
|
|
249
|
+
extraSteps.push('Have someone confirm that the tests under "Needs a human check" really assert, since the score cannot tell.');
|
|
250
|
+
}
|
|
251
|
+
if ((result.dimensions.locators ?? 100) < 75 && !top.some((g) => g.dimension === 'locators')) {
|
|
252
|
+
extraSteps.push('Move CSS and XPath selectors to role, label or test-id locators, starting with the busiest page objects.');
|
|
253
|
+
}
|
|
254
|
+
const steps = nextSteps(top, extraSteps)
|
|
255
|
+
.map((s) => `<li>${escapeHtml(s)}</li>`)
|
|
256
|
+
.join('');
|
|
257
|
+
const nextHtml = clearSuite ? '' : `\n<h2>What to do next</h2>\n<ol>${steps}</ol>`;
|
|
258
|
+
const moreTypes = all.length - detail.length;
|
|
259
|
+
const detailsHtml = detail.length || humanHtml
|
|
260
|
+
? `<section class="details">
|
|
261
|
+
<h2>Details</h2>
|
|
262
|
+
${flakeFilesLine(result.findings)}
|
|
263
|
+
${renderDetailCards(detail, opts.readSource)}
|
|
264
|
+
${moreTypes > 0 ? `<p class="muted">${plural(moreTypes, 'more issue type')} not shown. Use --format json for every occurrence.</p>` : ''}
|
|
265
|
+
${humanHtml}
|
|
266
|
+
</section>
|
|
267
|
+
`
|
|
268
|
+
: '';
|
|
269
|
+
return `${head}<section class="summary">
|
|
270
|
+
${header}
|
|
271
|
+
${state ? '' : renderBars(result)}
|
|
272
|
+
${topHtml}${nextHtml}
|
|
273
|
+
</section>
|
|
274
|
+
${detailsHtml}${footer}
|
|
275
|
+
</main>
|
|
276
|
+
</body>
|
|
277
|
+
</html>
|
|
278
|
+
`;
|
|
279
|
+
}
|
|
@@ -1,2 +1,7 @@
|
|
|
1
1
|
import type { ScoreResult } from '../types.js';
|
|
2
|
-
export
|
|
2
|
+
export interface TextOptions {
|
|
3
|
+
suiteName?: string;
|
|
4
|
+
version?: string;
|
|
5
|
+
customThreshold?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export declare function formatText(result: ScoreResult, opts?: TextOptions): string;
|
package/dist/formatters/text.js
CHANGED
|
@@ -1,32 +1,77 @@
|
|
|
1
|
-
|
|
1
|
+
import { CONFIDENCE_LABEL, compareFindings, isNeedsCheck } from '../confidence.js';
|
|
2
|
+
import { copyFor } from '../rule-copy.js';
|
|
3
|
+
import { DIMENSIONS, notScoredHeadline, notScoredState, plural, rankGroups, INCOMPLETE_NOTE, } from '../report-model.js';
|
|
4
|
+
/** Groups shown, and locations shown per group; the rest collapse into "+N more". */
|
|
5
|
+
const MAX_GROUPS = 10;
|
|
6
|
+
const MAX_LOCATIONS = 3;
|
|
7
|
+
const MAX_PATH = 100;
|
|
8
|
+
/** Relative path, shortened in the middle to MAX_PATH. An absolute path never gets through. */
|
|
9
|
+
function shortPath(file, line) {
|
|
10
|
+
let p = file.replace(/\\/g, '/');
|
|
11
|
+
if (p.startsWith('/') || /^[A-Za-z]:\//.test(p))
|
|
12
|
+
p = p.split('/').filter(Boolean).slice(-3).join('/');
|
|
13
|
+
const suffix = line ? `:${line}` : '';
|
|
14
|
+
const budget = MAX_PATH - suffix.length;
|
|
15
|
+
if (p.length > budget) {
|
|
16
|
+
const tail = p.split('/').pop() ?? p;
|
|
17
|
+
const keep = Math.max(10, budget - tail.length - 5);
|
|
18
|
+
p = tail.length >= budget ? '...' + tail.slice(-(budget - 3)) : `${p.slice(0, keep)}/.../${tail}`;
|
|
19
|
+
}
|
|
20
|
+
return p + suffix;
|
|
21
|
+
}
|
|
22
|
+
export function formatText(result, opts = {}) {
|
|
23
|
+
const { summary } = result;
|
|
24
|
+
const state = notScoredState(result);
|
|
2
25
|
const lines = [];
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
lines.push(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
26
|
+
const counts = state
|
|
27
|
+
? plural(summary.files, 'file')
|
|
28
|
+
: `${plural(summary.tests, 'test')} in ${plural(summary.files, 'file')}`;
|
|
29
|
+
lines.push(opts.suiteName ? `${opts.suiteName} · ${counts}` : counts);
|
|
30
|
+
if (state) {
|
|
31
|
+
lines.push(`Not scored: ${notScoredHeadline(result).headline.replace(/, so this suite could not be scored\.$/, '')}.`);
|
|
32
|
+
const first = result.findings.find((f) => f.rule === 'playwright-score/parse-error');
|
|
33
|
+
if (first)
|
|
34
|
+
lines.push(` ${shortPath(first.file, first.line)}`);
|
|
35
|
+
lines.push(notScoredHeadline(result).line);
|
|
11
36
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
`
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
37
|
+
else {
|
|
38
|
+
const clean = summary.cleanTests;
|
|
39
|
+
lines.push(`Score ${result.score}/100 (${result.grade}) · ${result.pass ? 'PASS' : 'FAIL'} (${opts.customThreshold ? 'your bar' : 'bar'} ${result.threshold}) · ${clean} ${clean === 1 ? 'test has' : 'tests have'} no issues`);
|
|
40
|
+
lines.push(DIMENSIONS.filter((d) => result.dimensions[d.key] !== undefined)
|
|
41
|
+
.map((d) => `${d.label} ${result.dimensions[d.key]}`)
|
|
42
|
+
.join(' · '));
|
|
43
|
+
}
|
|
44
|
+
// 'no-tests' hard-fail findings carry the absolute working directory as their file.
|
|
45
|
+
const groups = state === 'no-tests' ? [] : rankGroups(result.findings);
|
|
46
|
+
if (groups.length) {
|
|
20
47
|
lines.push('');
|
|
21
|
-
lines.push('
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
lines.push(`
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
48
|
+
lines.push('Top issues (worst first)');
|
|
49
|
+
groups.slice(0, MAX_GROUPS).forEach((g, i) => {
|
|
50
|
+
const where = `${g.findings.length} in ${plural(g.testsAffected, 'test')}`;
|
|
51
|
+
lines.push(` ${i + 1}. ${copyFor(g.rule).title} ${where} ${CONFIDENCE_LABEL[g.confidence]}`);
|
|
52
|
+
for (const f of g.findings.slice(0, MAX_LOCATIONS))
|
|
53
|
+
lines.push(` ${shortPath(f.file, f.line)}`);
|
|
54
|
+
if (g.findings.length > MAX_LOCATIONS)
|
|
55
|
+
lines.push(` + ${g.findings.length - MAX_LOCATIONS} more`);
|
|
56
|
+
});
|
|
57
|
+
if (groups.length > MAX_GROUPS) {
|
|
58
|
+
lines.push(` + ${plural(groups.length - MAX_GROUPS, 'more issue type')}. Full list: playwright-score <path> --format report --out report.html`);
|
|
28
59
|
}
|
|
29
60
|
}
|
|
61
|
+
const human = result.findings.filter(isNeedsCheck).sort(compareFindings);
|
|
62
|
+
if (human.length && !state) {
|
|
63
|
+
const files = [...new Set(human.map((f) => f.file))];
|
|
64
|
+
lines.push('');
|
|
65
|
+
lines.push(`Needs a human check (not in the score): ${plural(human.length, 'test')} in ${plural(files.length, 'file')} ${human.length === 1 ? 'relies' : 'rely'} on helpers we could not read.`);
|
|
66
|
+
for (const f of files.slice(0, 5))
|
|
67
|
+
lines.push(` ${shortPath(f)}`);
|
|
68
|
+
if (files.length > 5)
|
|
69
|
+
lines.push(` + ${files.length - 5} more`);
|
|
70
|
+
}
|
|
71
|
+
if (result.ruleCrashes?.length) {
|
|
72
|
+
lines.push('');
|
|
73
|
+
lines.push(INCOMPLETE_NOTE);
|
|
74
|
+
}
|
|
30
75
|
if (result.skippedFiles?.length) {
|
|
31
76
|
lines.push('');
|
|
32
77
|
lines.push(`Skipped (looks like a non-Playwright test — Jest/Vitest/RTL/Cypress — ${result.skippedFiles.length}):`);
|
|
@@ -44,9 +89,9 @@ export function formatText(result) {
|
|
|
44
89
|
lines.push(` ${w}`);
|
|
45
90
|
}
|
|
46
91
|
}
|
|
92
|
+
lines.push('');
|
|
93
|
+
if (!state)
|
|
94
|
+
lines.push(`Score ${result.score}/100 (${result.grade}) · ${result.pass ? 'PASS' : 'FAIL'}`);
|
|
95
|
+
lines.push(`${opts.version ? `QA Guardian playwright-score ${opts.version} (rules ${result.scoreVersion})` : `QA Guardian playwright-score (rules ${result.scoreVersion})`}`);
|
|
47
96
|
return lines.join('\n');
|
|
48
97
|
}
|
|
49
|
-
function pathBasename(p) {
|
|
50
|
-
const parts = p.split(/[/\\]/);
|
|
51
|
-
return parts[parts.length - 1] ?? p;
|
|
52
|
-
}
|
package/dist/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import { detectExtendedTestAliases } from './test-aliases.js';
|
|
|
10
10
|
import { findPlaywrightConfig, findRepoBoundary, importsPlaywrightTestTransitively, isAncestorOrSame, parsePlaywrightConfig, resolveConfigScopedRoot, safeRealpath, testDirEscapesRepo, } from './playwright-config.js';
|
|
11
11
|
import { DEFAULT_THRESHOLDS } from './profiles.js';
|
|
12
12
|
import { computeScore } from './score-engine.js';
|
|
13
|
+
import { resetRuleCrashes, summarizeRuleCrashes } from './rules/base-plugin.js';
|
|
13
14
|
export { computeScore, MODEL, locatorsScore, gradeFromScore, ratioDimensionScore, } from './score-engine.js';
|
|
14
15
|
export { countSloc } from './sloc.js';
|
|
15
16
|
export { countLocators, analyzeSource, attributeFindingScopes, getTestSpans, } from './metrics.js';
|
|
@@ -427,6 +428,12 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles, config
|
|
|
427
428
|
* Score Playwright spec files. Deterministic and AI-free.
|
|
428
429
|
*/
|
|
429
430
|
export async function scorePaths(options) {
|
|
431
|
+
resetRuleCrashes();
|
|
432
|
+
const result = await scorePathsInner(options);
|
|
433
|
+
const crashes = summarizeRuleCrashes();
|
|
434
|
+
return crashes.length > 0 ? { ...result, ruleCrashes: crashes } : result;
|
|
435
|
+
}
|
|
436
|
+
async function scorePathsInner(options) {
|
|
430
437
|
const cwd = options.cwd ?? process.cwd();
|
|
431
438
|
const profile = options.profile ?? 'standard';
|
|
432
439
|
const threshold = options.threshold ?? DEFAULT_THRESHOLDS[profile] ?? 80;
|
|
@@ -563,12 +570,15 @@ export async function scorePaths(options) {
|
|
|
563
570
|
// (a handful of exp-decay points on an otherwise large, clean batch)
|
|
564
571
|
// would let a broken file hide inside a passing score. Hard-fail
|
|
565
572
|
// instead, same as the no-matching-files case.
|
|
566
|
-
return
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
573
|
+
return {
|
|
574
|
+
...hardFail(profile, threshold, findings, {
|
|
575
|
+
files: files.length,
|
|
576
|
+
sloc: Math.max(totalSloc, 1),
|
|
577
|
+
nativeLocators: native,
|
|
578
|
+
rawLocators: raw,
|
|
579
|
+
}, skippedFiles),
|
|
580
|
+
baseDir: filesBase,
|
|
581
|
+
};
|
|
572
582
|
}
|
|
573
583
|
const result = computeScore({
|
|
574
584
|
profile,
|
|
@@ -583,6 +593,7 @@ export async function scorePaths(options) {
|
|
|
583
593
|
});
|
|
584
594
|
return {
|
|
585
595
|
...result,
|
|
596
|
+
baseDir: filesBase,
|
|
586
597
|
...(skippedFiles.length > 0 ? { skippedFiles } : {}),
|
|
587
598
|
...(configWarnings.length > 0 ? { configWarnings } : {}),
|
|
588
599
|
};
|
package/dist/profiles.js
CHANGED
|
@@ -113,6 +113,18 @@ export const ESLINT_RULE_MAP = {
|
|
|
113
113
|
'pwscore/no-coordinate-click': { dimension: 'playwrightHygiene', severityOverride: 'warning' },
|
|
114
114
|
'pwscore/no-trivial-assertion': { dimension: 'assertions' },
|
|
115
115
|
'pwscore/no-soft-assertion-only-test': { dimension: 'assertions', severityOverride: 'warning' },
|
|
116
|
+
// QAG-287 slice 2b. no-brittle-selector is report-only: the locators ratio already counts every raw call.
|
|
117
|
+
'pwscore/no-unawaited-action': { dimension: 'playwrightHygiene' },
|
|
118
|
+
'pwscore/no-brittle-selector': { dimension: 'locators', severityOverride: 'warning', reportOnly: true },
|
|
119
|
+
'pwscore/no-shared-mutable-state': { dimension: 'structure', severityOverride: 'warning' },
|
|
120
|
+
'pwscore/serial-mode-dependency': { dimension: 'structure', severityOverride: 'info', reportOnly: true },
|
|
121
|
+
'pwscore/no-excessive-timeout': { dimension: 'playwrightHygiene', severityOverride: 'warning' },
|
|
122
|
+
// QAG-287 slice 2a: tests that would miss a real bug.
|
|
123
|
+
'pwscore/no-fixme-test': { dimension: 'structure' },
|
|
124
|
+
'pwscore/no-swallowed-assertion': { dimension: 'assertions', severityOverride: 'warning' },
|
|
125
|
+
'pwscore/no-weak-assertion': { dimension: 'assertions' },
|
|
126
|
+
'pwscore/no-unawaited-expect-value': { dimension: 'assertions' },
|
|
127
|
+
'pwscore/no-self-fulfilling-fallback': { dimension: 'assertions', severityOverride: 'info', reportOnly: true },
|
|
116
128
|
};
|
|
117
129
|
export function mapRule(ruleId) {
|
|
118
130
|
if (ESLINT_RULE_MAP[ruleId])
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { DimensionName, Finding, ScoreResult } from './types.js';
|
|
2
|
+
import { type Confidence } from './confidence.js';
|
|
3
|
+
/** Shared by the HTML report and the terminal output. */
|
|
4
|
+
export declare const INCOMPLETE_NOTE = "Some checks could not run on this suite; results may be incomplete.";
|
|
5
|
+
export declare const PARSE_ERROR_RULE = "playwright-score/parse-error";
|
|
6
|
+
export declare const DIMENSIONS: ReadonlyArray<{
|
|
7
|
+
key: DimensionName;
|
|
8
|
+
label: string;
|
|
9
|
+
meaning: string;
|
|
10
|
+
}>;
|
|
11
|
+
export interface RuleGroup {
|
|
12
|
+
rule: string;
|
|
13
|
+
/** Worst severity among the group's findings. */
|
|
14
|
+
severity: Finding['severity'];
|
|
15
|
+
confidence: Confidence;
|
|
16
|
+
/** Tests affected: distinct tests for test-scoped findings, plus one per file for hook/module-scoped ones. */
|
|
17
|
+
testsAffected: number;
|
|
18
|
+
findings: Finding[];
|
|
19
|
+
/** The occurrence shown as the example (first by severity, file, line). */
|
|
20
|
+
example: Finding;
|
|
21
|
+
dimension: DimensionName;
|
|
22
|
+
}
|
|
23
|
+
export declare function groupByRule(findings: readonly Finding[]): RuleGroup[];
|
|
24
|
+
/** Severity, then confidence, then tests affected (desc), then rule id. */
|
|
25
|
+
export declare function compareGroups(a: RuleGroup, b: RuleGroup): number;
|
|
26
|
+
/** Every rule group, worst first. Never includes needs-check findings. A file that does not parse always leads. */
|
|
27
|
+
export declare function rankGroups(findings: readonly Finding[]): RuleGroup[];
|
|
28
|
+
/** Up to `max` rule groups. Never includes needs-check findings. */
|
|
29
|
+
export declare function rankTopFindings(findings: readonly Finding[], max?: number): RuleGroup[];
|
|
30
|
+
/**
|
|
31
|
+
* The summary's top three. When the weakest area scores under 75 and none of
|
|
32
|
+
* the three belongs to it, the third slot goes to that area's best-ranked group.
|
|
33
|
+
*/
|
|
34
|
+
export declare function summaryTop(all: readonly RuleGroup[], dimensions: ScoreResult['dimensions'], n?: number): RuleGroup[];
|
|
35
|
+
/** The file with the most occurrences in the group, with the line of its first one. */
|
|
36
|
+
export declare function mostAffectedFile(g: RuleGroup): {
|
|
37
|
+
file: string;
|
|
38
|
+
line?: number;
|
|
39
|
+
count: number;
|
|
40
|
+
};
|
|
41
|
+
export declare function plural(n: number, one: string, many?: string): string;
|
|
42
|
+
export type NotScored = 'parse-error' | 'no-tests' | undefined;
|
|
43
|
+
export declare function parseErrorFileCount(result: ScoreResult): number;
|
|
44
|
+
export declare function notScoredState(result: ScoreResult): NotScored;
|
|
45
|
+
/** "B means 80-89; A is 90 and up." Built from the engine's own grade bands. */
|
|
46
|
+
export declare function gradeBandText(grade: string, pass: boolean): string;
|
|
47
|
+
export declare function barText(threshold: number, custom: boolean, pass: boolean): string;
|
|
48
|
+
export declare function notScoredHeadline(result: ScoreResult): {
|
|
49
|
+
headline: string;
|
|
50
|
+
line: string;
|
|
51
|
+
};
|
|
52
|
+
export declare function cleanCheckCalls(m: string): string | undefined;
|