@qaguardian/playwright-score 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +86 -0
  2. package/METHODOLOGY.md +41 -0
  3. package/README.md +1 -1
  4. package/dist/cli.js +61 -5
  5. package/dist/confidence.d.ts +30 -0
  6. package/dist/confidence.js +67 -0
  7. package/dist/eslint-runner.js +18 -0
  8. package/dist/formatters/json.js +4 -1
  9. package/dist/formatters/markdown.d.ts +2 -0
  10. package/dist/formatters/markdown.js +15 -4
  11. package/dist/formatters/report-css.d.ts +2 -0
  12. package/dist/formatters/report-css.js +68 -0
  13. package/dist/formatters/report.d.ts +20 -0
  14. package/dist/formatters/report.js +279 -0
  15. package/dist/formatters/text.d.ts +6 -1
  16. package/dist/formatters/text.js +73 -28
  17. package/dist/index.js +17 -6
  18. package/dist/profiles.js +12 -0
  19. package/dist/report-model.d.ts +52 -0
  20. package/dist/report-model.js +162 -0
  21. package/dist/report-source.d.ts +11 -0
  22. package/dist/report-source.js +96 -0
  23. package/dist/rule-copy.d.ts +14 -0
  24. package/dist/rule-copy.js +215 -0
  25. package/dist/rules/ast-util.d.ts +60 -0
  26. package/dist/rules/ast-util.js +140 -0
  27. package/dist/rules/base-plugin.d.ts +19 -5
  28. package/dist/rules/base-plugin.js +75 -7
  29. package/dist/rules/no-brittle-selector.d.ts +3 -0
  30. package/dist/rules/no-brittle-selector.js +150 -0
  31. package/dist/rules/no-excessive-timeout.d.ts +2 -0
  32. package/dist/rules/no-excessive-timeout.js +87 -0
  33. package/dist/rules/no-fixme-test.d.ts +14 -0
  34. package/dist/rules/no-fixme-test.js +53 -0
  35. package/dist/rules/no-self-fulfilling-fallback.d.ts +29 -0
  36. package/dist/rules/no-self-fulfilling-fallback.js +221 -0
  37. package/dist/rules/no-shared-mutable-state.d.ts +4 -0
  38. package/dist/rules/no-shared-mutable-state.js +186 -0
  39. package/dist/rules/no-swallowed-assertion.d.ts +13 -0
  40. package/dist/rules/no-swallowed-assertion.js +113 -0
  41. package/dist/rules/no-unawaited-action.d.ts +2 -0
  42. package/dist/rules/no-unawaited-action.js +77 -0
  43. package/dist/rules/no-unawaited-expect-value.d.ts +20 -0
  44. package/dist/rules/no-unawaited-expect-value.js +46 -0
  45. package/dist/rules/no-weak-assertion.d.ts +19 -0
  46. package/dist/rules/no-weak-assertion.js +82 -0
  47. package/dist/rules/rule-util.d.ts +56 -0
  48. package/dist/rules/rule-util.js +253 -0
  49. package/dist/score-engine.d.ts +5 -0
  50. package/dist/score-engine.js +9 -9
  51. package/dist/suite-name.d.ts +11 -0
  52. package/dist/suite-name.js +36 -0
  53. package/dist/test-aliases.js +144 -3
  54. package/dist/types.d.ts +14 -0
  55. package/package.json +1 -1
@@ -0,0 +1,279 @@
1
+ import { PROFILE_WEIGHTS } from '../profiles.js';
2
+ import { CONFIDENCE_LABEL, compareFindings, isNeedsCheck } from '../confidence.js';
3
+ import { copyFor } from '../rule-copy.js';
4
+ import { REPORT_CSS } from './report-css.js';
5
+ import { DIMENSIONS, barText, cleanCheckCalls, gradeBandText, mostAffectedFile, notScoredHeadline, notScoredState, plural, rankGroups, summaryTop, INCOMPLETE_NOTE, } from '../report-model.js';
6
+ export { rankTopFindings, compareGroups } from '../report-model.js';
7
+ /** Rules that make a test flaky rather than merely untidy. */
8
+ export const FLAKE_RULES = [
9
+ 'playwright/no-wait-for-timeout',
10
+ 'pwscore/no-timer-sleep',
11
+ 'playwright/no-networkidle',
12
+ 'playwright/no-force-option',
13
+ 'playwright/no-wait-for-selector',
14
+ 'playwright/missing-playwright-await',
15
+ ];
16
+ const DETAIL_CARDS = 5;
17
+ const LEGEND = 'Definite: the code does this for certain. Likely: usually a problem, but there are legitimate uses. None of these are confirmed bugs in your product; they are patterns that make bugs easier to miss or tests less reliable.';
18
+ export function escapeHtml(s) {
19
+ return s
20
+ .replace(/&/g, '&')
21
+ .replace(/</g, '&lt;')
22
+ .replace(/>/g, '&gt;')
23
+ .replace(/"/g, '&quot;')
24
+ .replace(/'/g, '&#39;');
25
+ }
26
+ const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
27
+ function longDate(iso) {
28
+ const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(iso);
29
+ const mon = m ? MONTHS[Number(m[2]) - 1] : undefined;
30
+ return m && mon ? `${Number(m[3])} ${mon} ${m[1]}` : iso;
31
+ }
32
+ function excerpt(source, line) {
33
+ const lines = source.split(/\r?\n/);
34
+ if (line < 1 || line > lines.length)
35
+ return [];
36
+ const from = Math.max(1, line - 3);
37
+ const to = Math.min(lines.length, line + 3);
38
+ const out = [];
39
+ for (let n = from; n <= to; n++) {
40
+ let text = (lines[n - 1] ?? '').replace(/\t/g, ' ');
41
+ if (text.length > 140)
42
+ text = text.slice(0, 137) + '...';
43
+ out.push({ n, text, hit: n === line });
44
+ }
45
+ return out;
46
+ }
47
+ function renderExcerpt(f, readSource) {
48
+ if (!f.line || !readSource)
49
+ return '';
50
+ const src = readSource(f.file);
51
+ if (src === undefined)
52
+ return '';
53
+ const rows = excerpt(src, f.line);
54
+ if (!rows.length)
55
+ return '';
56
+ const body = rows
57
+ .map((r) => r.hit
58
+ ? `<span class="ln hit"><i>&#9656;${r.n}</i><span class="sr-only">flagged line: </span>${escapeHtml(r.text) || ' '}</span>`
59
+ : `<span class="ln"><i>${r.n}</i>${escapeHtml(r.text) || ' '}</span>`)
60
+ .join('');
61
+ return `<pre class="code" aria-label="Code excerpt">${body}</pre>`;
62
+ }
63
+ function loc(f) {
64
+ return f.line ? `${f.file}:${f.line}` : f.file;
65
+ }
66
+ function baseName(p) {
67
+ return p.split(/[/\\]/).pop() || p;
68
+ }
69
+ function tag(g) {
70
+ return `<span class="tag">${CONFIDENCE_LABEL[g.confidence]}</span>`;
71
+ }
72
+ function renderTop3(groups) {
73
+ const items = groups
74
+ .map((g, i) => {
75
+ const c = copyFor(g.rule);
76
+ const most = mostAffectedFile(g);
77
+ const places = `${plural(g.findings.length, 'place')} in ${plural(g.testsAffected, 'test')}`;
78
+ const where = `<code>${escapeHtml(baseName(most.file))}</code>${most.line ? ` (line ${most.line})` : ''}`;
79
+ const sameFile = most.count === g.findings.length;
80
+ const count = g.findings.length === 1
81
+ ? `${places}: ${where}.`
82
+ : `${places}. ${sameFile ? 'All in' : 'Most in'} ${where}.`;
83
+ return `<li class="top"><span class="rank" aria-hidden="true">${i + 1}</span>
84
+ <div><h3>${escapeHtml(c.title)} ${tag(g)}</h3>
85
+ <p class="count">${count}</p>
86
+ <p class="why">${escapeHtml(c.why)}</p></div></li>`;
87
+ })
88
+ .join('\n');
89
+ return `<ol class="top3" role="list">\n${items}\n</ol>`;
90
+ }
91
+ function renderDetailCards(groups, readSource) {
92
+ return groups
93
+ .map((g, i) => {
94
+ const c = copyFor(g.rule);
95
+ const others = g.findings.length - 1;
96
+ return `<article class="finding sev-${g.severity}">
97
+ <h3><span class="rank" aria-hidden="true">${i + 1}</span>${escapeHtml(c.title)} ${tag(g)}</h3>
98
+ <p class="where"><code>${escapeHtml(loc(g.example))}</code></p>
99
+ ${renderExcerpt(g.example, readSource)}
100
+ <p class="why">${escapeHtml(c.why)}</p>
101
+ <p class="muted">${others > 0 ? `${plural(g.findings.length, 'occurrence')} across ${plural(g.testsAffected, 'test')}.` : 'One occurrence.'}</p>
102
+ </article>`;
103
+ })
104
+ .join('\n');
105
+ }
106
+ function flakeFilesLine(findings) {
107
+ const files = new Map();
108
+ for (const f of findings) {
109
+ if (FLAKE_RULES.includes(f.rule) && !isNeedsCheck(f))
110
+ files.set(f.file, (files.get(f.file) ?? 0) + 1);
111
+ }
112
+ if (!files.size)
113
+ return '';
114
+ const worst = [...files.entries()]
115
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
116
+ .slice(0, 3)
117
+ .map(([file, n]) => `<code>${escapeHtml(file)}</code> (${n})`)
118
+ .join(', ');
119
+ return `<p class="muted">Most flake-prone files: ${worst}.</p>`;
120
+ }
121
+ function renderNeedsCheck(findings) {
122
+ const items = findings.filter(isNeedsCheck).sort(compareFindings);
123
+ if (!items.length)
124
+ return '';
125
+ const byFile = new Map();
126
+ for (const f of items) {
127
+ const list = byFile.get(f.file);
128
+ if (list)
129
+ list.push(f);
130
+ else
131
+ byFile.set(f.file, [f]);
132
+ }
133
+ const files = [...byFile.entries()];
134
+ const shown = files.slice(0, 6);
135
+ const rows = shown
136
+ .map(([file, list]) => {
137
+ const lines = list.filter((f) => f.line).map((f) => f.line).join(', ');
138
+ const calls = cleanCheckCalls(list[0].message);
139
+ const what = calls
140
+ ? `Calls ${calls} from a helper we could not read.`
141
+ : 'Its checks could not be seen.';
142
+ return `<li><code>${escapeHtml(file)}</code> <span class="muted">${plural(list.length, 'test')}${lines ? `, line${list.length === 1 ? '' : 's'} ${escapeHtml(lines)}` : ''}</span><br><span class="muted">${escapeHtml(what)}</span></li>`;
143
+ })
144
+ .join('');
145
+ const hidden = items.length - shown.reduce((n, [, l]) => n + l.length, 0);
146
+ const more = hidden > 0 ? `<p class="muted">and ${plural(hidden, 'more test')} in ${plural(files.length - shown.length, 'other file')}.</p>` : '';
147
+ return `<h3 id="human-check">Needs a human check</h3>
148
+ <p class="note">${plural(items.length, 'test')} ${items.length === 1 ? 'relies' : 'rely'} on helpers we could not read (page objects, fixtures or imported functions). The checks probably live there, but the score cannot see them, so ${items.length === 1 ? 'this test is' : 'these tests are'} left out of the score. Ask someone to confirm each helper really checks the page.</p><ul class="plain">${rows}</ul>${more}`;
149
+ }
150
+ const GENERIC_STEPS = [
151
+ 'Re-run the score in CI with --threshold so the number cannot slip back unnoticed.',
152
+ 'Keep new specs on role, label and test-id locators with web-first assertions.',
153
+ 'Review any items under "Needs a human check" by hand.',
154
+ ];
155
+ function nextSteps(groups, extra = []) {
156
+ const steps = [];
157
+ const fixes = groups.map((g) => copyFor(g.rule).fix).filter((f, i, a) => a.indexOf(f) === i);
158
+ const maxFromFindings = 3 - Math.min(extra.length, 2);
159
+ for (const fix of fixes.slice(0, maxFromFindings))
160
+ steps.push(fix);
161
+ for (const e of extra) {
162
+ if (steps.length < 3)
163
+ steps.push(e);
164
+ }
165
+ for (const s of GENERIC_STEPS) {
166
+ if (steps.length >= 3)
167
+ break;
168
+ steps.push(s);
169
+ }
170
+ return steps;
171
+ }
172
+ function renderBars(result) {
173
+ const weights = PROFILE_WEIGHTS[result.profile] ?? PROFILE_WEIGHTS.standard;
174
+ const bars = DIMENSIONS.map((d) => {
175
+ const v = result.dimensions[d.key] ?? 0;
176
+ const w = weights[d.key];
177
+ return `<div class="dim" title="${escapeHtml(d.meaning)}"><div class="dimhead"><span>${d.label}${w ? ` <span class="muted">${w}% of score</span>` : ''}</span><strong>${v}</strong></div>
178
+ <div class="bar" role="img" aria-label="${d.label} ${v} out of 100"><span style="width:${Math.max(0, Math.min(100, v))}%"></span></div>
179
+ <p class="muted sr-only">${escapeHtml(d.meaning)}</p></div>`;
180
+ }).join('\n');
181
+ return `<h2>Score by area</h2>\n<div class="dims">\n${bars}\n</div>`;
182
+ }
183
+ export function formatReport(result, opts) {
184
+ const { summary } = result;
185
+ const state = notScoredState(result);
186
+ const clean = summary.cleanTests;
187
+ const all = rankGroups(result.findings);
188
+ const top = summaryTop(all, result.dimensions);
189
+ const detail = [...new Set([...top, ...all.slice(0, DETAIL_CARDS)])];
190
+ const humanItems = result.findings.filter(isNeedsCheck);
191
+ const humanHtml = renderNeedsCheck(result.findings);
192
+ let headline;
193
+ let explain;
194
+ let scoreBox;
195
+ if (state) {
196
+ const ns = notScoredHeadline(result);
197
+ headline = ns.headline;
198
+ explain = ns.line;
199
+ scoreBox = `<div class="score notscored" role="img" aria-label="Not scored"><span class="big">--</span><span class="grade">Not scored</span></div>`;
200
+ }
201
+ else {
202
+ headline = `${clean} of ${summary.tests} ${summary.tests === 1 ? 'test has' : 'tests have'} no issues.`;
203
+ explain = `A weighted average of four areas below. ${gradeBandText(result.grade, result.pass)} ${barText(result.threshold, opts.customThreshold === true, result.pass)}`;
204
+ scoreBox = `<div class="score g-${result.grade}" aria-hidden="true"><span class="big">${result.score}</span><span class="of">/100</span><span class="grade">${result.grade}</span></div>`;
205
+ }
206
+ const srScore = state ? '' : `<span class="sr-only">Score ${result.score} out of 100, grade ${result.grade}. </span>`;
207
+ const humanCount = summary.unverifiedTests;
208
+ const humanLine = humanCount > 0 && !state
209
+ ? `\n<p class="check">${humanItems.length ? '<a href="#human-check">' : ''}${plural(humanCount, 'test')} ${humanCount === 1 ? 'needs' : 'need'} a human check and ${humanCount === 1 ? 'is' : 'are'} left out of the score.${humanItems.length ? '</a>' : ''}</p>`
210
+ : '';
211
+ const crashLine = result.ruleCrashes?.length ? `\n<p class="check">${escapeHtml(INCOMPLETE_NOTE)}</p>` : '';
212
+ const meta = state
213
+ ? plural(summary.files, 'file')
214
+ : `${plural(summary.tests, 'test')} in ${plural(summary.files, 'file')}`;
215
+ const header = `<header>
216
+ <p class="eyebrow">Playwright suite score</p>
217
+ <p class="suite">${escapeHtml(opts.suiteName)}</p>
218
+ <div class="hd">
219
+ ${scoreBox}
220
+ <div>
221
+ <h1>${srScore}${escapeHtml(headline)}</h1>
222
+ <p class="explain">${escapeHtml(explain)}</p>${humanLine}${crashLine}
223
+ <p class="meta">${meta} · scored ${escapeHtml(longDate(opts.date))}</p>
224
+ </div>
225
+ </div>
226
+ </header>`;
227
+ const footer = `<footer>Scored by QA Guardian playwright-score ${escapeHtml(opts.version)} (rules ${escapeHtml(result.scoreVersion)}) on ${escapeHtml(opts.date)}. Static analysis of the test code: the same files always give the same score. Nothing was run against your product.</footer>`;
228
+ const head = `<!DOCTYPE html>
229
+ <html lang="en">
230
+ <head>
231
+ <meta charset="utf-8">
232
+ <meta name="viewport" content="width=device-width, initial-scale=1">
233
+ <meta name="color-scheme" content="light dark">
234
+ <title>Playwright suite score: ${escapeHtml(opts.suiteName)}</title>
235
+ <style>${REPORT_CSS}</style>
236
+ </head>
237
+ <body>
238
+ <main>
239
+ `;
240
+ if (state === 'no-tests') {
241
+ return `${head}<section class="summary">\n${header}\n</section>\n${footer}\n</main>\n</body>\n</html>\n`;
242
+ }
243
+ const clearSuite = all.length === 0 && humanItems.length === 0 && !state;
244
+ const topHtml = all.length
245
+ ? `<h2>${top.length > 1 ? `Top ${top.length} to fix` : 'Fix first'}</h2>\n<p class="muted legend">${escapeHtml(LEGEND)}</p>\n${renderTop3(top)}`
246
+ : '<h2>Issues</h2>\n<p>No issues found. Keep it that way by adding the score to CI with --threshold.</p>';
247
+ const extraSteps = [];
248
+ if (humanItems.length) {
249
+ extraSteps.push('Have someone confirm that the tests under "Needs a human check" really assert, since the score cannot tell.');
250
+ }
251
+ if ((result.dimensions.locators ?? 100) < 75 && !top.some((g) => g.dimension === 'locators')) {
252
+ extraSteps.push('Move CSS and XPath selectors to role, label or test-id locators, starting with the busiest page objects.');
253
+ }
254
+ const steps = nextSteps(top, extraSteps)
255
+ .map((s) => `<li>${escapeHtml(s)}</li>`)
256
+ .join('');
257
+ const nextHtml = clearSuite ? '' : `\n<h2>What to do next</h2>\n<ol>${steps}</ol>`;
258
+ const moreTypes = all.length - detail.length;
259
+ const detailsHtml = detail.length || humanHtml
260
+ ? `<section class="details">
261
+ <h2>Details</h2>
262
+ ${flakeFilesLine(result.findings)}
263
+ ${renderDetailCards(detail, opts.readSource)}
264
+ ${moreTypes > 0 ? `<p class="muted">${plural(moreTypes, 'more issue type')} not shown. Use --format json for every occurrence.</p>` : ''}
265
+ ${humanHtml}
266
+ </section>
267
+ `
268
+ : '';
269
+ return `${head}<section class="summary">
270
+ ${header}
271
+ ${state ? '' : renderBars(result)}
272
+ ${topHtml}${nextHtml}
273
+ </section>
274
+ ${detailsHtml}${footer}
275
+ </main>
276
+ </body>
277
+ </html>
278
+ `;
279
+ }
@@ -1,2 +1,7 @@
1
1
  import type { ScoreResult } from '../types.js';
2
- export declare function formatText(result: ScoreResult): string;
2
+ export interface TextOptions {
3
+ suiteName?: string;
4
+ version?: string;
5
+ customThreshold?: boolean;
6
+ }
7
+ export declare function formatText(result: ScoreResult, opts?: TextOptions): string;
@@ -1,32 +1,77 @@
1
- export function formatText(result) {
1
+ import { CONFIDENCE_LABEL, compareFindings, isNeedsCheck } from '../confidence.js';
2
+ import { copyFor } from '../rule-copy.js';
3
+ import { DIMENSIONS, notScoredHeadline, notScoredState, plural, rankGroups, INCOMPLETE_NOTE, } from '../report-model.js';
4
+ /** Groups shown, and locations shown per group; the rest collapse into "+N more". */
5
+ const MAX_GROUPS = 10;
6
+ const MAX_LOCATIONS = 3;
7
+ const MAX_PATH = 100;
8
+ /** Relative path, shortened in the middle to MAX_PATH. An absolute path never gets through. */
9
+ function shortPath(file, line) {
10
+ let p = file.replace(/\\/g, '/');
11
+ if (p.startsWith('/') || /^[A-Za-z]:\//.test(p))
12
+ p = p.split('/').filter(Boolean).slice(-3).join('/');
13
+ const suffix = line ? `:${line}` : '';
14
+ const budget = MAX_PATH - suffix.length;
15
+ if (p.length > budget) {
16
+ const tail = p.split('/').pop() ?? p;
17
+ const keep = Math.max(10, budget - tail.length - 5);
18
+ p = tail.length >= budget ? '...' + tail.slice(-(budget - 3)) : `${p.slice(0, keep)}/.../${tail}`;
19
+ }
20
+ return p + suffix;
21
+ }
22
+ export function formatText(result, opts = {}) {
23
+ const { summary } = result;
24
+ const state = notScoredState(result);
2
25
  const lines = [];
3
- lines.push(`Playwright Spec Score ${result.scoreVersion} · profile=${result.profile}`);
4
- lines.push(`Score: ${result.score}/100 (${result.grade}) · ${result.pass ? 'PASS' : 'FAIL'} (threshold ${result.threshold})`);
5
- lines.push('');
6
- lines.push('Dimensions:');
7
- for (const [k, v] of Object.entries(result.dimensions)) {
8
- if (v === undefined)
9
- continue;
10
- lines.push(` ${k}: ${v}`);
26
+ const counts = state
27
+ ? plural(summary.files, 'file')
28
+ : `${plural(summary.tests, 'test')} in ${plural(summary.files, 'file')}`;
29
+ lines.push(opts.suiteName ? `${opts.suiteName} · ${counts}` : counts);
30
+ if (state) {
31
+ lines.push(`Not scored: ${notScoredHeadline(result).headline.replace(/, so this suite could not be scored\.$/, '')}.`);
32
+ const first = result.findings.find((f) => f.rule === 'playwright-score/parse-error');
33
+ if (first)
34
+ lines.push(` ${shortPath(first.file, first.line)}`);
35
+ lines.push(notScoredHeadline(result).line);
11
36
  }
12
- lines.push('');
13
- lines.push(`Summary: files=${result.summary.files} tests=${result.summary.tests} sloc=${result.summary.sloc} ` +
14
- `findings=${result.summary.findings} (errors=${result.summary.errors} warnings=${result.summary.warnings}) ` +
15
- `locators native/raw=${result.summary.nativeLocators}/${result.summary.rawLocators} ` +
16
- `tests asserting=${result.summary.tests - result.summary.unassertedTests - result.summary.unverifiedTests}/${result.summary.tests}` +
17
- `${result.summary.unverifiedTests ? ` (unverified=${result.summary.unverifiedTests})` : ''} ` +
18
- `clean tests=${result.summary.cleanTests}/${result.summary.tests}`);
19
- if (result.findings.length) {
37
+ else {
38
+ const clean = summary.cleanTests;
39
+ lines.push(`Score ${result.score}/100 (${result.grade}) · ${result.pass ? 'PASS' : 'FAIL'} (${opts.customThreshold ? 'your bar' : 'bar'} ${result.threshold}) · ${clean} ${clean === 1 ? 'test has' : 'tests have'} no issues`);
40
+ lines.push(DIMENSIONS.filter((d) => result.dimensions[d.key] !== undefined)
41
+ .map((d) => `${d.label} ${result.dimensions[d.key]}`)
42
+ .join(' · '));
43
+ }
44
+ // 'no-tests' hard-fail findings carry the absolute working directory as their file.
45
+ const groups = state === 'no-tests' ? [] : rankGroups(result.findings);
46
+ if (groups.length) {
20
47
  lines.push('');
21
- lines.push('Findings:');
22
- for (const f of result.findings.slice(0, 50)) {
23
- const loc = f.line ? `:${f.line}` : '';
24
- lines.push(` [${f.severity}] ${f.rule} ${pathBasename(f.file)}${loc} — ${f.message}`);
25
- }
26
- if (result.findings.length > 50) {
27
- lines.push(` … and ${result.findings.length - 50} more`);
48
+ lines.push('Top issues (worst first)');
49
+ groups.slice(0, MAX_GROUPS).forEach((g, i) => {
50
+ const where = `${g.findings.length} in ${plural(g.testsAffected, 'test')}`;
51
+ lines.push(` ${i + 1}. ${copyFor(g.rule).title} ${where} ${CONFIDENCE_LABEL[g.confidence]}`);
52
+ for (const f of g.findings.slice(0, MAX_LOCATIONS))
53
+ lines.push(` ${shortPath(f.file, f.line)}`);
54
+ if (g.findings.length > MAX_LOCATIONS)
55
+ lines.push(` + ${g.findings.length - MAX_LOCATIONS} more`);
56
+ });
57
+ if (groups.length > MAX_GROUPS) {
58
+ lines.push(` + ${plural(groups.length - MAX_GROUPS, 'more issue type')}. Full list: playwright-score <path> --format report --out report.html`);
28
59
  }
29
60
  }
61
+ const human = result.findings.filter(isNeedsCheck).sort(compareFindings);
62
+ if (human.length && !state) {
63
+ const files = [...new Set(human.map((f) => f.file))];
64
+ lines.push('');
65
+ lines.push(`Needs a human check (not in the score): ${plural(human.length, 'test')} in ${plural(files.length, 'file')} ${human.length === 1 ? 'relies' : 'rely'} on helpers we could not read.`);
66
+ for (const f of files.slice(0, 5))
67
+ lines.push(` ${shortPath(f)}`);
68
+ if (files.length > 5)
69
+ lines.push(` + ${files.length - 5} more`);
70
+ }
71
+ if (result.ruleCrashes?.length) {
72
+ lines.push('');
73
+ lines.push(INCOMPLETE_NOTE);
74
+ }
30
75
  if (result.skippedFiles?.length) {
31
76
  lines.push('');
32
77
  lines.push(`Skipped (looks like a non-Playwright test — Jest/Vitest/RTL/Cypress — ${result.skippedFiles.length}):`);
@@ -44,9 +89,9 @@ export function formatText(result) {
44
89
  lines.push(` ${w}`);
45
90
  }
46
91
  }
92
+ lines.push('');
93
+ if (!state)
94
+ lines.push(`Score ${result.score}/100 (${result.grade}) · ${result.pass ? 'PASS' : 'FAIL'}`);
95
+ lines.push(`${opts.version ? `QA Guardian playwright-score ${opts.version} (rules ${result.scoreVersion})` : `QA Guardian playwright-score (rules ${result.scoreVersion})`}`);
47
96
  return lines.join('\n');
48
97
  }
49
- function pathBasename(p) {
50
- const parts = p.split(/[/\\]/);
51
- return parts[parts.length - 1] ?? p;
52
- }
package/dist/index.js CHANGED
@@ -10,6 +10,7 @@ import { detectExtendedTestAliases } from './test-aliases.js';
10
10
  import { findPlaywrightConfig, findRepoBoundary, importsPlaywrightTestTransitively, isAncestorOrSame, parsePlaywrightConfig, resolveConfigScopedRoot, safeRealpath, testDirEscapesRepo, } from './playwright-config.js';
11
11
  import { DEFAULT_THRESHOLDS } from './profiles.js';
12
12
  import { computeScore } from './score-engine.js';
13
+ import { resetRuleCrashes, summarizeRuleCrashes } from './rules/base-plugin.js';
13
14
  export { computeScore, MODEL, locatorsScore, gradeFromScore, ratioDimensionScore, } from './score-engine.js';
14
15
  export { countSloc } from './sloc.js';
15
16
  export { countLocators, analyzeSource, attributeFindingScopes, getTestSpans, } from './metrics.js';
@@ -427,6 +428,12 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles, config
427
428
  * Score Playwright spec files. Deterministic and AI-free.
428
429
  */
429
430
  export async function scorePaths(options) {
431
+ resetRuleCrashes();
432
+ const result = await scorePathsInner(options);
433
+ const crashes = summarizeRuleCrashes();
434
+ return crashes.length > 0 ? { ...result, ruleCrashes: crashes } : result;
435
+ }
436
+ async function scorePathsInner(options) {
430
437
  const cwd = options.cwd ?? process.cwd();
431
438
  const profile = options.profile ?? 'standard';
432
439
  const threshold = options.threshold ?? DEFAULT_THRESHOLDS[profile] ?? 80;
@@ -563,12 +570,15 @@ export async function scorePaths(options) {
563
570
  // (a handful of exp-decay points on an otherwise large, clean batch)
564
571
  // would let a broken file hide inside a passing score. Hard-fail
565
572
  // instead, same as the no-matching-files case.
566
- return hardFail(profile, threshold, findings, {
567
- files: files.length,
568
- sloc: Math.max(totalSloc, 1),
569
- nativeLocators: native,
570
- rawLocators: raw,
571
- }, skippedFiles);
573
+ return {
574
+ ...hardFail(profile, threshold, findings, {
575
+ files: files.length,
576
+ sloc: Math.max(totalSloc, 1),
577
+ nativeLocators: native,
578
+ rawLocators: raw,
579
+ }, skippedFiles),
580
+ baseDir: filesBase,
581
+ };
572
582
  }
573
583
  const result = computeScore({
574
584
  profile,
@@ -583,6 +593,7 @@ export async function scorePaths(options) {
583
593
  });
584
594
  return {
585
595
  ...result,
596
+ baseDir: filesBase,
586
597
  ...(skippedFiles.length > 0 ? { skippedFiles } : {}),
587
598
  ...(configWarnings.length > 0 ? { configWarnings } : {}),
588
599
  };
package/dist/profiles.js CHANGED
@@ -113,6 +113,18 @@ export const ESLINT_RULE_MAP = {
113
113
  'pwscore/no-coordinate-click': { dimension: 'playwrightHygiene', severityOverride: 'warning' },
114
114
  'pwscore/no-trivial-assertion': { dimension: 'assertions' },
115
115
  'pwscore/no-soft-assertion-only-test': { dimension: 'assertions', severityOverride: 'warning' },
116
+ // QAG-287 slice 2b. no-brittle-selector is report-only: the locators ratio already counts every raw call.
117
+ 'pwscore/no-unawaited-action': { dimension: 'playwrightHygiene' },
118
+ 'pwscore/no-brittle-selector': { dimension: 'locators', severityOverride: 'warning', reportOnly: true },
119
+ 'pwscore/no-shared-mutable-state': { dimension: 'structure', severityOverride: 'warning' },
120
+ 'pwscore/serial-mode-dependency': { dimension: 'structure', severityOverride: 'info', reportOnly: true },
121
+ 'pwscore/no-excessive-timeout': { dimension: 'playwrightHygiene', severityOverride: 'warning' },
122
+ // QAG-287 slice 2a: tests that would miss a real bug.
123
+ 'pwscore/no-fixme-test': { dimension: 'structure' },
124
+ 'pwscore/no-swallowed-assertion': { dimension: 'assertions', severityOverride: 'warning' },
125
+ 'pwscore/no-weak-assertion': { dimension: 'assertions' },
126
+ 'pwscore/no-unawaited-expect-value': { dimension: 'assertions' },
127
+ 'pwscore/no-self-fulfilling-fallback': { dimension: 'assertions', severityOverride: 'info', reportOnly: true },
116
128
  };
117
129
  export function mapRule(ruleId) {
118
130
  if (ESLINT_RULE_MAP[ruleId])
@@ -0,0 +1,52 @@
1
+ import type { DimensionName, Finding, ScoreResult } from './types.js';
2
+ import { type Confidence } from './confidence.js';
3
+ /** Shared by the HTML report and the terminal output. */
4
+ export declare const INCOMPLETE_NOTE = "Some checks could not run on this suite; results may be incomplete.";
5
+ export declare const PARSE_ERROR_RULE = "playwright-score/parse-error";
6
+ export declare const DIMENSIONS: ReadonlyArray<{
7
+ key: DimensionName;
8
+ label: string;
9
+ meaning: string;
10
+ }>;
11
+ export interface RuleGroup {
12
+ rule: string;
13
+ /** Worst severity among the group's findings. */
14
+ severity: Finding['severity'];
15
+ confidence: Confidence;
16
+ /** Tests affected: distinct tests for test-scoped findings, plus one per file for hook/module-scoped ones. */
17
+ testsAffected: number;
18
+ findings: Finding[];
19
+ /** The occurrence shown as the example (first by severity, file, line). */
20
+ example: Finding;
21
+ dimension: DimensionName;
22
+ }
23
+ export declare function groupByRule(findings: readonly Finding[]): RuleGroup[];
24
+ /** Severity, then confidence, then tests affected (desc), then rule id. */
25
+ export declare function compareGroups(a: RuleGroup, b: RuleGroup): number;
26
+ /** Every rule group, worst first. Never includes needs-check findings. A file that does not parse always leads. */
27
+ export declare function rankGroups(findings: readonly Finding[]): RuleGroup[];
28
+ /** Up to `max` rule groups. Never includes needs-check findings. */
29
+ export declare function rankTopFindings(findings: readonly Finding[], max?: number): RuleGroup[];
30
+ /**
31
+ * The summary's top three. When the weakest area scores under 75 and none of
32
+ * the three belongs to it, the third slot goes to that area's best-ranked group.
33
+ */
34
+ export declare function summaryTop(all: readonly RuleGroup[], dimensions: ScoreResult['dimensions'], n?: number): RuleGroup[];
35
+ /** The file with the most occurrences in the group, with the line of its first one. */
36
+ export declare function mostAffectedFile(g: RuleGroup): {
37
+ file: string;
38
+ line?: number;
39
+ count: number;
40
+ };
41
+ export declare function plural(n: number, one: string, many?: string): string;
42
+ export type NotScored = 'parse-error' | 'no-tests' | undefined;
43
+ export declare function parseErrorFileCount(result: ScoreResult): number;
44
+ export declare function notScoredState(result: ScoreResult): NotScored;
45
+ /** "B means 80-89; A is 90 and up." Built from the engine's own grade bands. */
46
+ export declare function gradeBandText(grade: string, pass: boolean): string;
47
+ export declare function barText(threshold: number, custom: boolean, pass: boolean): string;
48
+ export declare function notScoredHeadline(result: ScoreResult): {
49
+ headline: string;
50
+ line: string;
51
+ };
52
+ export declare function cleanCheckCalls(m: string): string | undefined;