@qaguardian/playwright-score 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/METHODOLOGY.md +41 -0
- package/README.md +1 -1
- package/dist/cli.js +61 -5
- package/dist/confidence.d.ts +30 -0
- package/dist/confidence.js +67 -0
- package/dist/eslint-runner.js +18 -0
- package/dist/formatters/json.js +4 -1
- package/dist/formatters/markdown.d.ts +2 -0
- package/dist/formatters/markdown.js +15 -4
- package/dist/formatters/report-css.d.ts +2 -0
- package/dist/formatters/report-css.js +68 -0
- package/dist/formatters/report.d.ts +20 -0
- package/dist/formatters/report.js +279 -0
- package/dist/formatters/text.d.ts +6 -1
- package/dist/formatters/text.js +73 -28
- package/dist/index.js +17 -6
- package/dist/profiles.js +12 -0
- package/dist/report-model.d.ts +52 -0
- package/dist/report-model.js +162 -0
- package/dist/report-source.d.ts +11 -0
- package/dist/report-source.js +96 -0
- package/dist/rule-copy.d.ts +14 -0
- package/dist/rule-copy.js +215 -0
- package/dist/rules/ast-util.d.ts +60 -0
- package/dist/rules/ast-util.js +140 -0
- package/dist/rules/base-plugin.d.ts +19 -5
- package/dist/rules/base-plugin.js +75 -7
- package/dist/rules/no-brittle-selector.d.ts +3 -0
- package/dist/rules/no-brittle-selector.js +150 -0
- package/dist/rules/no-excessive-timeout.d.ts +2 -0
- package/dist/rules/no-excessive-timeout.js +87 -0
- package/dist/rules/no-fixme-test.d.ts +14 -0
- package/dist/rules/no-fixme-test.js +53 -0
- package/dist/rules/no-self-fulfilling-fallback.d.ts +29 -0
- package/dist/rules/no-self-fulfilling-fallback.js +221 -0
- package/dist/rules/no-shared-mutable-state.d.ts +4 -0
- package/dist/rules/no-shared-mutable-state.js +186 -0
- package/dist/rules/no-swallowed-assertion.d.ts +13 -0
- package/dist/rules/no-swallowed-assertion.js +113 -0
- package/dist/rules/no-unawaited-action.d.ts +2 -0
- package/dist/rules/no-unawaited-action.js +77 -0
- package/dist/rules/no-unawaited-expect-value.d.ts +20 -0
- package/dist/rules/no-unawaited-expect-value.js +46 -0
- package/dist/rules/no-weak-assertion.d.ts +19 -0
- package/dist/rules/no-weak-assertion.js +82 -0
- package/dist/rules/rule-util.d.ts +56 -0
- package/dist/rules/rule-util.js +253 -0
- package/dist/score-engine.d.ts +5 -0
- package/dist/score-engine.js +9 -9
- package/dist/suite-name.d.ts +11 -0
- package/dist/suite-name.js +36 -0
- package/dist/test-aliases.js +144 -3
- package/dist/types.d.ts +14 -0
- package/package.json +1 -1
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { compareFindings, confidenceOf, confidenceRank, isNeedsCheck, severityRank, } from './confidence.js';
|
|
2
|
+
import { GRADE_BANDS } from './score-engine.js';
|
|
3
|
+
/** Shared by the HTML report and the terminal output. */
|
|
4
|
+
export const INCOMPLETE_NOTE = 'Some checks could not run on this suite; results may be incomplete.';
|
|
5
|
+
export const PARSE_ERROR_RULE = 'playwright-score/parse-error';
|
|
6
|
+
export const DIMENSIONS = [
|
|
7
|
+
{
|
|
8
|
+
key: 'playwrightHygiene',
|
|
9
|
+
label: 'Reliability',
|
|
10
|
+
meaning: 'Waits, forced clicks and missing awaits: the habits that make tests fail at random.',
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
key: 'assertions',
|
|
14
|
+
label: 'Checks',
|
|
15
|
+
meaning: 'Whether each test actually verifies something, with checks that retry.',
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
key: 'locators',
|
|
19
|
+
label: 'Resilience to UI changes',
|
|
20
|
+
meaning: 'How many elements are found the way a user sees them (role, label, test id) rather than by CSS or XPath.',
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
key: 'structure',
|
|
24
|
+
label: 'Maintainability',
|
|
25
|
+
meaning: 'Switched-off tests, nesting and file size.',
|
|
26
|
+
},
|
|
27
|
+
];
|
|
28
|
+
export function groupByRule(findings) {
|
|
29
|
+
const byRule = new Map();
|
|
30
|
+
for (const f of findings) {
|
|
31
|
+
const list = byRule.get(f.rule);
|
|
32
|
+
if (list)
|
|
33
|
+
list.push(f);
|
|
34
|
+
else
|
|
35
|
+
byRule.set(f.rule, [f]);
|
|
36
|
+
}
|
|
37
|
+
const groups = [];
|
|
38
|
+
for (const [rule, list] of byRule) {
|
|
39
|
+
const sorted = [...list].sort(compareFindings);
|
|
40
|
+
const tests = new Set();
|
|
41
|
+
for (const f of sorted) {
|
|
42
|
+
tests.add(f.scope === 'test' && f.testKey ? f.testKey : `file:${f.file}`);
|
|
43
|
+
}
|
|
44
|
+
groups.push({
|
|
45
|
+
rule,
|
|
46
|
+
severity: sorted[0].severity,
|
|
47
|
+
confidence: confidenceOf(sorted[0]),
|
|
48
|
+
testsAffected: tests.size,
|
|
49
|
+
findings: sorted,
|
|
50
|
+
example: sorted[0],
|
|
51
|
+
dimension: sorted[0].dimension,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
return groups;
|
|
55
|
+
}
|
|
56
|
+
/** Severity, then confidence, then tests affected (desc), then rule id. */
|
|
57
|
+
export function compareGroups(a, b) {
|
|
58
|
+
return (severityRank(a.severity) - severityRank(b.severity) ||
|
|
59
|
+
confidenceRank(a.confidence) - confidenceRank(b.confidence) ||
|
|
60
|
+
b.testsAffected - a.testsAffected ||
|
|
61
|
+
a.rule.localeCompare(b.rule));
|
|
62
|
+
}
|
|
63
|
+
/** Every rule group, worst first. Never includes needs-check findings. A file that does not parse always leads. */
|
|
64
|
+
export function rankGroups(findings) {
|
|
65
|
+
const parseFirst = (g) => (g.rule === PARSE_ERROR_RULE ? 0 : 1);
|
|
66
|
+
return groupByRule(findings.filter((f) => !isNeedsCheck(f))).sort((a, b) => parseFirst(a) - parseFirst(b) || compareGroups(a, b));
|
|
67
|
+
}
|
|
68
|
+
/** Up to `max` rule groups. Never includes needs-check findings. */
|
|
69
|
+
export function rankTopFindings(findings, max = 5) {
|
|
70
|
+
return rankGroups(findings).slice(0, max);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The summary's top three. When the weakest area scores under 75 and none of
|
|
74
|
+
* the three belongs to it, the third slot goes to that area's best-ranked group.
|
|
75
|
+
*/
|
|
76
|
+
export function summaryTop(all, dimensions, n = 3) {
|
|
77
|
+
const top = all.slice(0, n);
|
|
78
|
+
let weakest;
|
|
79
|
+
let low = 75;
|
|
80
|
+
for (const d of DIMENSIONS) {
|
|
81
|
+
const v = dimensions[d.key];
|
|
82
|
+
if (v !== undefined && v < low) {
|
|
83
|
+
low = v;
|
|
84
|
+
weakest = d.key;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (!weakest || top.length < n || top.some((g) => g.dimension === weakest))
|
|
88
|
+
return top;
|
|
89
|
+
const swap = all.slice(n).find((g) => g.dimension === weakest);
|
|
90
|
+
if (swap)
|
|
91
|
+
top[n - 1] = swap;
|
|
92
|
+
return top;
|
|
93
|
+
}
|
|
94
|
+
/** The file with the most occurrences in the group, with the line of its first one. */
|
|
95
|
+
export function mostAffectedFile(g) {
|
|
96
|
+
const counts = new Map();
|
|
97
|
+
for (const f of g.findings) {
|
|
98
|
+
const c = counts.get(f.file);
|
|
99
|
+
if (c)
|
|
100
|
+
c.n++;
|
|
101
|
+
else
|
|
102
|
+
counts.set(f.file, { n: 1, line: f.line });
|
|
103
|
+
}
|
|
104
|
+
let best;
|
|
105
|
+
for (const [file, c] of [...counts.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
106
|
+
if (!best || c.n > best.count)
|
|
107
|
+
best = { file, line: c.line, count: c.n };
|
|
108
|
+
}
|
|
109
|
+
return best;
|
|
110
|
+
}
|
|
111
|
+
export function plural(n, one, many = one + 's') {
|
|
112
|
+
return `${n} ${n === 1 ? one : many}`;
|
|
113
|
+
}
|
|
114
|
+
export function parseErrorFileCount(result) {
|
|
115
|
+
return new Set(result.findings.filter((f) => f.rule === PARSE_ERROR_RULE).map((f) => f.file)).size;
|
|
116
|
+
}
|
|
117
|
+
export function notScoredState(result) {
|
|
118
|
+
if (parseErrorFileCount(result) > 0)
|
|
119
|
+
return 'parse-error';
|
|
120
|
+
if (result.summary.tests === 0)
|
|
121
|
+
return 'no-tests';
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
/** "B means 80-89; A is 90 and up." Built from the engine's own grade bands. */
|
|
125
|
+
export function gradeBandText(grade, pass) {
|
|
126
|
+
const i = GRADE_BANDS.findIndex((b) => b.grade === grade);
|
|
127
|
+
if (i < 0)
|
|
128
|
+
return '';
|
|
129
|
+
const band = GRADE_BANDS[i];
|
|
130
|
+
const range = i === 0
|
|
131
|
+
? `${band.min}-100`
|
|
132
|
+
: band.min === 0
|
|
133
|
+
? `below ${GRADE_BANDS[i - 1].min}`
|
|
134
|
+
: `${band.min}-${GRADE_BANDS[i - 1].min - 1}`;
|
|
135
|
+
const first = `${grade} means ${range}`;
|
|
136
|
+
if (i === 0)
|
|
137
|
+
return `${first}.`;
|
|
138
|
+
const up = GRADE_BANDS[i - 1];
|
|
139
|
+
return `${first}; ${up.grade} ${pass ? `is ${up.min} and up` : `starts at ${up.min}`}.`;
|
|
140
|
+
}
|
|
141
|
+
export function barText(threshold, custom, pass) {
|
|
142
|
+
return `${pass ? 'Passes' : 'Below'} ${custom ? 'your' : 'the default'} bar of ${threshold}.`;
|
|
143
|
+
}
|
|
144
|
+
export function notScoredHeadline(result) {
|
|
145
|
+
if (notScoredState(result) === 'parse-error') {
|
|
146
|
+
const n = parseErrorFileCount(result);
|
|
147
|
+
return {
|
|
148
|
+
headline: n === 1
|
|
149
|
+
? '1 file has a syntax error, so this suite could not be scored.'
|
|
150
|
+
: `${n} files have syntax errors, so this suite could not be scored.`,
|
|
151
|
+
line: 'Fix the file below and run the score again. The other findings are shown as a preview.',
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
headline: 'No Playwright tests were found.',
|
|
156
|
+
line: 'Check the path you scored. Files that look like Jest, Vitest or Cypress tests are skipped.',
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
export function cleanCheckCalls(m) {
|
|
160
|
+
const calls = /it calls (.+?) on an object defined elsewhere/.exec(m);
|
|
161
|
+
return calls?.[1];
|
|
162
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { SourceReader } from './formatters/report.js';
|
|
2
|
+
/**
|
|
3
|
+
* Finding.file is relative to the scored files' common ancestor, recorded in
|
|
4
|
+
* the result as `baseDir`: when given, the file is read from exactly there.
|
|
5
|
+
* Without it (older results) fall back to resolving against each input path
|
|
6
|
+
* (file inputs: their directory; directory inputs: a bounded search below
|
|
7
|
+
* them), which can pick the wrong file when names repeat. Either way a
|
|
8
|
+
* finding path that is absolute or has `..` segments is refused, and the
|
|
9
|
+
* resolved file must stay inside `baseDir`.
|
|
10
|
+
*/
|
|
11
|
+
export declare function makeSourceReader(inputs: readonly string[], cwd: string, baseDir?: string): SourceReader;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
const MAX_DEPTH = 8;
|
|
4
|
+
const MAX_BYTES = 2 * 1024 * 1024;
|
|
5
|
+
function readIfFile(p) {
|
|
6
|
+
try {
|
|
7
|
+
const st = fs.statSync(p);
|
|
8
|
+
if (!st.isFile() || st.size > MAX_BYTES)
|
|
9
|
+
return undefined;
|
|
10
|
+
return fs.readFileSync(p, 'utf8');
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Walks `dir` looking for a path that ends with `rel`; skips node_modules and dot dirs. */
|
|
17
|
+
function findBySuffix(dir, rel, depth) {
|
|
18
|
+
const direct = path.join(dir, rel);
|
|
19
|
+
if (readIfFile(direct) !== undefined)
|
|
20
|
+
return direct;
|
|
21
|
+
if (depth >= MAX_DEPTH)
|
|
22
|
+
return undefined;
|
|
23
|
+
let entries;
|
|
24
|
+
try {
|
|
25
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
31
|
+
for (const e of entries) {
|
|
32
|
+
if (!e.isDirectory() || e.isSymbolicLink())
|
|
33
|
+
continue;
|
|
34
|
+
if (e.name === 'node_modules' || e.name.startsWith('.'))
|
|
35
|
+
continue;
|
|
36
|
+
const hit = findBySuffix(path.join(dir, e.name), rel, depth + 1);
|
|
37
|
+
if (hit)
|
|
38
|
+
return hit;
|
|
39
|
+
}
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Finding.file is relative to the scored files' common ancestor, recorded in
|
|
44
|
+
* the result as `baseDir`: when given, the file is read from exactly there.
|
|
45
|
+
* Without it (older results) fall back to resolving against each input path
|
|
46
|
+
* (file inputs: their directory; directory inputs: a bounded search below
|
|
47
|
+
* them), which can pick the wrong file when names repeat. Either way a
|
|
48
|
+
* finding path that is absolute or has `..` segments is refused, and the
|
|
49
|
+
* resolved file must stay inside `baseDir`.
|
|
50
|
+
*/
|
|
51
|
+
export function makeSourceReader(inputs, cwd, baseDir) {
|
|
52
|
+
const bases = inputs
|
|
53
|
+
.filter((p) => !/[*?{]/.test(p))
|
|
54
|
+
.map((p) => path.resolve(cwd, p))
|
|
55
|
+
.map((abs) => {
|
|
56
|
+
try {
|
|
57
|
+
return fs.statSync(abs).isDirectory() ? abs : path.dirname(abs);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
.filter((b) => b !== undefined);
|
|
64
|
+
const cache = new Map();
|
|
65
|
+
return (file) => {
|
|
66
|
+
if (cache.has(file))
|
|
67
|
+
return cache.get(file);
|
|
68
|
+
let text;
|
|
69
|
+
const segments = file.split(/[/\\]/);
|
|
70
|
+
if (!path.isAbsolute(file) && !segments.includes('..')) {
|
|
71
|
+
if (baseDir) {
|
|
72
|
+
const root = path.resolve(baseDir);
|
|
73
|
+
const abs = path.resolve(root, file);
|
|
74
|
+
try {
|
|
75
|
+
const real = fs.realpathSync(abs);
|
|
76
|
+
if (real.startsWith(fs.realpathSync(root) + path.sep))
|
|
77
|
+
text = readIfFile(real);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
text = undefined;
|
|
81
|
+
}
|
|
82
|
+
cache.set(file, text);
|
|
83
|
+
return text;
|
|
84
|
+
}
|
|
85
|
+
for (const base of bases) {
|
|
86
|
+
const hit = findBySuffix(base, file, 0);
|
|
87
|
+
if (hit) {
|
|
88
|
+
text = readIfFile(hit);
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
cache.set(file, text);
|
|
94
|
+
return text;
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plain-English copy for every rule in profiles.ts ESLINT_RULE_MAP: a short
|
|
3
|
+
* title, one sentence on why it matters, and one imperative fix used by the
|
|
4
|
+
* report's "What to do next". tests/report.test.ts fails if a rule in the
|
|
5
|
+
* map has no entry here.
|
|
6
|
+
*/
|
|
7
|
+
export interface RuleCopy {
|
|
8
|
+
title: string;
|
|
9
|
+
why: string;
|
|
10
|
+
fix: string;
|
|
11
|
+
}
|
|
12
|
+
export declare const RULE_COPY: Record<string, RuleCopy>;
|
|
13
|
+
export declare const FALLBACK_COPY: RuleCopy;
|
|
14
|
+
export declare function copyFor(rule: string): RuleCopy;
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
export const RULE_COPY = {
|
|
2
|
+
'playwright/no-wait-for-timeout': {
|
|
3
|
+
title: 'Fixed pause before continuing',
|
|
4
|
+
why: 'Fixed delays make runs slower and can cause random red builds, so engineers re-run pipelines and learn to ignore failures.',
|
|
5
|
+
fix: 'Replace waitForTimeout() with a web-first assertion such as expect(locator).toBeVisible().',
|
|
6
|
+
},
|
|
7
|
+
'playwright/no-networkidle': {
|
|
8
|
+
title: 'Waiting for the network to go quiet',
|
|
9
|
+
why: 'Pages that keep polling or streaming never go quiet, so tests that wait for it can time out at random and hold up releases without a real defect.',
|
|
10
|
+
fix: 'Wait for the specific element or response the test needs instead of networkidle.',
|
|
11
|
+
},
|
|
12
|
+
'playwright/no-force-option': {
|
|
13
|
+
title: 'Click forced past a blocking element',
|
|
14
|
+
why: 'Forcing a click bypasses what a real user is blocked by, so a broken or covered button can ship with a green build.',
|
|
15
|
+
fix: 'Remove { force: true } and fix what is covering the element, or wait for it to be actionable.',
|
|
16
|
+
},
|
|
17
|
+
'playwright/no-wait-for-selector': {
|
|
18
|
+
title: 'Legacy selector wait',
|
|
19
|
+
why: 'Manual waits add delay and random failures that locators already handle, so CI is slower and less trusted.',
|
|
20
|
+
fix: 'Use a locator with an expect(...).toBeVisible() assertion in place of waitForSelector.',
|
|
21
|
+
},
|
|
22
|
+
'playwright/no-wait-for-navigation': {
|
|
23
|
+
title: 'Deprecated navigation wait',
|
|
24
|
+
why: 'This wait can miss the navigation it is waiting for, which causes random failures that block releases.',
|
|
25
|
+
fix: 'Assert on the new URL or page content with expect(page).toHaveURL() instead.',
|
|
26
|
+
},
|
|
27
|
+
'playwright/missing-playwright-await': {
|
|
28
|
+
title: 'Playwright call that is never awaited',
|
|
29
|
+
why: 'The test can finish before the step runs, so it passes while the feature is broken or fails at random.',
|
|
30
|
+
fix: 'Add await to every Playwright action and expect() call that returns a promise.',
|
|
31
|
+
},
|
|
32
|
+
'playwright/no-element-handle': {
|
|
33
|
+
title: 'Element handle instead of a locator',
|
|
34
|
+
why: 'Stale handles can fail after the page re-renders, which can show up as random red builds nobody can reproduce.',
|
|
35
|
+
fix: 'Replace $() and $$() handles with page.locator() or role-based locators.',
|
|
36
|
+
},
|
|
37
|
+
'playwright/no-eval': {
|
|
38
|
+
title: 'Page code evaluated inside the test',
|
|
39
|
+
why: 'It checks a snapshot that may be out of date, so a test can pass or fail for reasons unrelated to the product.',
|
|
40
|
+
fix: 'Use locator assertions such as toHaveText() or toHaveValue() in place of $eval.',
|
|
41
|
+
},
|
|
42
|
+
'playwright/no-conditional-in-test': {
|
|
43
|
+
title: 'Test logic that branches',
|
|
44
|
+
why: 'Different runs can check different things, so a green build does not reliably mean the feature works.',
|
|
45
|
+
fix: 'Split the branches into separate tests or use test.skip(condition, reason).',
|
|
46
|
+
},
|
|
47
|
+
'playwright/no-conditional-expect': {
|
|
48
|
+
title: 'Assertion that only sometimes runs',
|
|
49
|
+
why: 'The check can be skipped silently, so a broken feature can still go out under a green build.',
|
|
50
|
+
fix: 'Move the expect out of the condition so it always runs.',
|
|
51
|
+
},
|
|
52
|
+
'playwright/no-page-pause': {
|
|
53
|
+
title: 'Debugger pause left in a test',
|
|
54
|
+
why: 'It halts the run until someone intervenes, so the CI job hangs and holds up the release pipeline.',
|
|
55
|
+
fix: 'Delete page.pause() calls before committing.',
|
|
56
|
+
},
|
|
57
|
+
'playwright/no-useless-await': {
|
|
58
|
+
title: 'await on something that is not a promise',
|
|
59
|
+
why: 'It is harmless at runtime but signals that the author was unsure what is asynchronous.',
|
|
60
|
+
fix: 'Remove await from synchronous calls such as locator() and getByRole().',
|
|
61
|
+
},
|
|
62
|
+
'playwright/expect-expect': {
|
|
63
|
+
title: 'Test with no assertion',
|
|
64
|
+
why: 'A test with no check passes even when the feature is broken, so a green build can hide a real outage.',
|
|
65
|
+
fix: 'Add at least one expect() that checks the outcome the test is named for.',
|
|
66
|
+
},
|
|
67
|
+
'playwright/prefer-web-first-assertions': {
|
|
68
|
+
title: 'Assertion that does not retry',
|
|
69
|
+
why: 'A one-shot check can fail when the page is a moment slow, so the build goes red without a real defect.',
|
|
70
|
+
fix: 'Use await expect(locator).toBeVisible() or toHaveText() in place of expect(await locator.isVisible()).',
|
|
71
|
+
},
|
|
72
|
+
'playwright/valid-expect': {
|
|
73
|
+
title: 'Malformed expect() call',
|
|
74
|
+
why: 'An incomplete check verifies nothing, so a broken feature can ship while the test reports success.',
|
|
75
|
+
fix: 'Complete each expect() with a matcher and await it.',
|
|
76
|
+
},
|
|
77
|
+
'playwright/no-standalone-expect': {
|
|
78
|
+
title: 'Assertion outside any test',
|
|
79
|
+
why: 'It runs at load time and fails the whole run with a confusing error, which costs engineers time to diagnose.',
|
|
80
|
+
fix: 'Move the expect inside a test() or a hook.',
|
|
81
|
+
},
|
|
82
|
+
'playwright/no-raw-locators': {
|
|
83
|
+
title: 'CSS or XPath locator',
|
|
84
|
+
why: 'Selectors tied to CSS or XPath can break on a redesign, so a UI change can mean unplanned test maintenance.',
|
|
85
|
+
fix: 'Use getByRole, getByLabel or getByTestId so locators follow what the user sees.',
|
|
86
|
+
},
|
|
87
|
+
'playwright/prefer-native-locators': {
|
|
88
|
+
title: 'Generic locator where a built-in one exists',
|
|
89
|
+
why: 'Selectors tied to markup can break on redesigns, so UI changes can turn into test-maintenance work.',
|
|
90
|
+
fix: 'Switch locator("text=...") style calls to getByText, getByRole or getByLabel.',
|
|
91
|
+
},
|
|
92
|
+
'playwright/prefer-locator': {
|
|
93
|
+
title: 'Page method used where a locator is better',
|
|
94
|
+
why: 'Page-level helpers do not retry, so tests can fail on slow pages and need hand-fixing.',
|
|
95
|
+
fix: 'Create a locator and call the action on it.',
|
|
96
|
+
},
|
|
97
|
+
'playwright/no-focused-test': {
|
|
98
|
+
title: 'Focused test left in (test.only)',
|
|
99
|
+
why: 'test.only silently skips every other test in the run, so the build can look green while most of the suite never ran.',
|
|
100
|
+
fix: 'Remove .only and set forbidOnly: !!process.env.CI in the Playwright config.',
|
|
101
|
+
},
|
|
102
|
+
'pwscore/no-skipped-test-declaration': {
|
|
103
|
+
title: 'Test that is switched off',
|
|
104
|
+
why: 'A skipped test protects nothing, and skipped tests tend to stay skipped, so coverage quietly shrinks.',
|
|
105
|
+
fix: 'Fix and re-enable each skipped test, or delete it.',
|
|
106
|
+
},
|
|
107
|
+
'playwright/max-nested-describe': {
|
|
108
|
+
title: 'Deeply nested describe blocks',
|
|
109
|
+
why: 'Deep nesting makes test names and shared setup hard to follow.',
|
|
110
|
+
fix: 'Flatten describe blocks to at most two levels.',
|
|
111
|
+
},
|
|
112
|
+
'playwright/require-top-level-describe': {
|
|
113
|
+
title: 'Tests outside a describe block',
|
|
114
|
+
why: 'Grouping tests by feature keeps reports and failures easy to read.',
|
|
115
|
+
fix: 'Wrap related tests in a top-level test.describe() named for the feature.',
|
|
116
|
+
},
|
|
117
|
+
'metrics/oversized-file': {
|
|
118
|
+
title: 'Very large spec file',
|
|
119
|
+
why: 'Huge files are slow to review and shard, and hide shared state, so failures are harder to trace and fix.',
|
|
120
|
+
fix: 'Split the file by feature so each spec stays readable.',
|
|
121
|
+
},
|
|
122
|
+
'pwscore/no-timer-sleep': {
|
|
123
|
+
title: 'Sleep written as a timer',
|
|
124
|
+
why: 'A timer is a fixed delay in disguise: it slows runs and can cause random red builds.',
|
|
125
|
+
fix: 'Replace the timer with an assertion on the state you are waiting for.',
|
|
126
|
+
},
|
|
127
|
+
'pwscore/no-coordinate-click': {
|
|
128
|
+
title: 'Click at screen coordinates',
|
|
129
|
+
why: 'Pixel positions can break at another screen size or layout, so the test can fail when nothing is wrong with the product.',
|
|
130
|
+
fix: 'Click a locator instead of page.mouse.click(x, y), except on canvas surfaces.',
|
|
131
|
+
},
|
|
132
|
+
'pwscore/no-trivial-assertion': {
|
|
133
|
+
title: 'Assertion that cannot fail',
|
|
134
|
+
why: 'It inflates the number of checks without checking the product, so coverage looks better than it is.',
|
|
135
|
+
fix: 'Assert on real page state, such as a heading, URL or row count.',
|
|
136
|
+
},
|
|
137
|
+
'pwscore/no-soft-assertion-only-test': {
|
|
138
|
+
title: 'Test with only soft assertions',
|
|
139
|
+
why: 'Failures are recorded but may not stop or fail the run as intended, so a regression can slip through.',
|
|
140
|
+
fix: 'Keep soft assertions for extras and end the test with a hard expect().',
|
|
141
|
+
},
|
|
142
|
+
'pwscore/eslint-disable-comment': {
|
|
143
|
+
title: 'Lint rule switched off with a comment',
|
|
144
|
+
why: 'A disable comment hides a problem from reviewers and from this score, so known issues stay unfixed.',
|
|
145
|
+
fix: 'Fix the code the comment is hiding, or justify the exception in review.',
|
|
146
|
+
},
|
|
147
|
+
'pwscore/no-unawaited-action': {
|
|
148
|
+
title: 'Click or navigation that is not awaited',
|
|
149
|
+
why: 'The test moves on before the action finishes, so it can fail at random or pass without the step having happened, and real regressions hide behind the noise.',
|
|
150
|
+
fix: 'Add await in front of the action.',
|
|
151
|
+
},
|
|
152
|
+
'pwscore/no-brittle-selector': {
|
|
153
|
+
title: 'Selector tied to page layout',
|
|
154
|
+
why: 'A cosmetic redesign can break the test even though the product still works, which costs engineering time on false alarms.',
|
|
155
|
+
fix: 'Use a role, label or test id the page owner keeps stable.',
|
|
156
|
+
},
|
|
157
|
+
'pwscore/no-shared-mutable-state': {
|
|
158
|
+
title: 'Tests that depend on each other through shared state',
|
|
159
|
+
why: 'One failure or a changed run order can make later tests fail for the wrong reason, so a failure report points at the wrong feature.',
|
|
160
|
+
fix: 'Create the data each test needs inside that test or in a fixture.',
|
|
161
|
+
},
|
|
162
|
+
'pwscore/serial-mode-dependency': {
|
|
163
|
+
title: 'Tests that must run in order',
|
|
164
|
+
why: 'One failing step skips every test after it, so a single bug can hide the state of the rest of the flow.',
|
|
165
|
+
fix: 'Make each test independent, or keep serial mode only where the flow truly needs it.',
|
|
166
|
+
},
|
|
167
|
+
'pwscore/no-excessive-timeout': {
|
|
168
|
+
title: 'Very long timeout',
|
|
169
|
+
why: 'A long timeout hides slow or hung pages and makes a broken release take many minutes to be reported.',
|
|
170
|
+
fix: 'Remove the long timeout and fix the slow step, or wait for a specific condition.',
|
|
171
|
+
},
|
|
172
|
+
'pwscore/no-fixme-test': {
|
|
173
|
+
title: 'Test switched off with fixme',
|
|
174
|
+
why: 'The feature it covers has no automated check, so a regression there can ship unnoticed.',
|
|
175
|
+
fix: 'Fix and re-enable the test, or delete it and track the gap in the backlog.',
|
|
176
|
+
},
|
|
177
|
+
'pwscore/no-swallowed-assertion': {
|
|
178
|
+
title: 'Assertion whose failure is ignored',
|
|
179
|
+
why: 'A failing check is caught and dropped, so the test passes while the feature may be broken. Deliberate soft checks are the usual exception.',
|
|
180
|
+
fix: 'Let the failure fail the test, or use expect.soft if the check is optional.',
|
|
181
|
+
},
|
|
182
|
+
'pwscore/no-weak-assertion': {
|
|
183
|
+
title: 'Assertion that can never fail',
|
|
184
|
+
why: 'The check passes whatever the page shows, so the test gives false confidence that the feature works.',
|
|
185
|
+
fix: 'Assert on a concrete value, such as an exact count, text or URL.',
|
|
186
|
+
},
|
|
187
|
+
'pwscore/no-unawaited-expect-value': {
|
|
188
|
+
title: 'Missing await inside an assertion',
|
|
189
|
+
why: 'The assertion compares a pending promise instead of the page value, so it can pass or fail without checking the feature.',
|
|
190
|
+
fix: 'Add await, or use a web-first assertion such as toHaveText.',
|
|
191
|
+
},
|
|
192
|
+
'pwscore/no-self-fulfilling-fallback': {
|
|
193
|
+
title: 'Test fills in the value it then checks',
|
|
194
|
+
why: 'If the page is missing the value, the test types it in and then confirms it, so the check passes on a broken feature.',
|
|
195
|
+
fix: 'Remove the fallback and assert on what the page shows.',
|
|
196
|
+
},
|
|
197
|
+
'playwright-score/assertion-unverified': {
|
|
198
|
+
title: 'Test whose checks we could not see',
|
|
199
|
+
why: 'The checks probably live in a helper the score cannot read, so a person should confirm the test really verifies the page.',
|
|
200
|
+
fix: 'Confirm the helper really asserts, or add an expect() to the test itself.',
|
|
201
|
+
},
|
|
202
|
+
'playwright-score/parse-error': {
|
|
203
|
+
title: 'File that could not be parsed',
|
|
204
|
+
why: 'The file cannot run, so none of its tests protect the product until the syntax error is fixed.',
|
|
205
|
+
fix: 'Fix the syntax error and re-run the score.',
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
export const FALLBACK_COPY = {
|
|
209
|
+
title: 'Other quality issue',
|
|
210
|
+
why: 'This pattern often makes tests less reliable.',
|
|
211
|
+
fix: 'Review the flagged lines and simplify them.',
|
|
212
|
+
};
|
|
213
|
+
export function copyFor(rule) {
|
|
214
|
+
return RULE_COPY[rule] ?? FALLBACK_COPY;
|
|
215
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small AST helpers shared by the QAG-287 "would miss a real bug" rules
|
|
3
|
+
* (no-swallowed-assertion, no-weak-assertion, no-unawaited-expect-value,
|
|
4
|
+
* no-self-fulfilling-fallback, no-fixme-test). Plain-object ESTree nodes;
|
|
5
|
+
* `parent` is attached by ESLint during traversal.
|
|
6
|
+
*/
|
|
7
|
+
import type { Rule } from 'eslint';
|
|
8
|
+
export type N = {
|
|
9
|
+
type: string;
|
|
10
|
+
parent?: N;
|
|
11
|
+
computed?: boolean;
|
|
12
|
+
optional?: boolean;
|
|
13
|
+
callee?: N;
|
|
14
|
+
object?: N;
|
|
15
|
+
property?: N;
|
|
16
|
+
name?: string;
|
|
17
|
+
arguments?: N[];
|
|
18
|
+
params?: N[];
|
|
19
|
+
body?: N | N[] | null;
|
|
20
|
+
operator?: string;
|
|
21
|
+
argument?: N | null;
|
|
22
|
+
value?: unknown;
|
|
23
|
+
expression?: N;
|
|
24
|
+
left?: N;
|
|
25
|
+
right?: N;
|
|
26
|
+
test?: N;
|
|
27
|
+
consequent?: N | N[];
|
|
28
|
+
alternate?: N | null;
|
|
29
|
+
block?: N;
|
|
30
|
+
handler?: (N & {
|
|
31
|
+
body?: N;
|
|
32
|
+
}) | null;
|
|
33
|
+
finalizer?: N | null;
|
|
34
|
+
id?: N;
|
|
35
|
+
init?: N | null;
|
|
36
|
+
declarations?: N[];
|
|
37
|
+
range?: [number, number];
|
|
38
|
+
};
|
|
39
|
+
export declare const FN_TYPES: Set<string>;
|
|
40
|
+
/** Walk a subtree. `visit` returns false to skip that node's children. Never descends into nested functions unless `intoFunctions`. */
|
|
41
|
+
export declare function walk(node: unknown, visit: (n: N) => boolean | void, intoFunctions?: boolean): void;
|
|
42
|
+
export declare function unwrap(node: N | null | undefined): N | undefined;
|
|
43
|
+
export declare function memberName(callee: N | undefined): string | undefined;
|
|
44
|
+
/** `expect(...)`, `expect.soft(...)`, `expect.poll(...)`. */
|
|
45
|
+
export declare function isExpectCall(node: N | undefined): boolean;
|
|
46
|
+
export interface MatcherChain {
|
|
47
|
+
expectCall: N;
|
|
48
|
+
matcher: string;
|
|
49
|
+
modifiers: string[];
|
|
50
|
+
call: N;
|
|
51
|
+
}
|
|
52
|
+
/** If `call` is `expect(x).[not.]matcher(...)`, describe it. */
|
|
53
|
+
export declare function parseMatcherCall(call: N): MatcherChain | undefined;
|
|
54
|
+
/** Scope-resolved `const x = <init>` initialiser for an identifier, if any. */
|
|
55
|
+
export declare function resolveConstInit(context: Rule.RuleContext, id: N): N | undefined;
|
|
56
|
+
/** Syntactic Locator: `page.locator(..)`, `x.getBy*(..)`, those chained with first/last/nth/filter/..., or a const initialised from one. */
|
|
57
|
+
export declare function isLocatorExpr(context: Rule.RuleContext, node: N | undefined, depth?: number): boolean;
|
|
58
|
+
/** Receiver is plausibly a Playwright Page/Locator/Frame (syntactic). */
|
|
59
|
+
export declare function isPlaywrightReceiver(context: Rule.RuleContext, node: N | undefined, depth?: number): boolean;
|
|
60
|
+
export declare function nearestFunction(node: N): N | undefined;
|