@qaguardian/playwright-score 2.1.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +159 -0
- package/METHODOLOGY.md +42 -1
- package/README.md +1 -1
- package/dist/cli.js +61 -5
- package/dist/confidence.d.ts +30 -0
- package/dist/confidence.js +67 -0
- package/dist/eslint-runner.js +18 -0
- package/dist/finding-triage.d.ts +13 -0
- package/dist/finding-triage.js +470 -0
- package/dist/formatters/json.js +4 -1
- package/dist/formatters/markdown.d.ts +2 -0
- package/dist/formatters/markdown.js +17 -5
- package/dist/formatters/report-css.d.ts +2 -0
- package/dist/formatters/report-css.js +68 -0
- package/dist/formatters/report.d.ts +20 -0
- package/dist/formatters/report.js +279 -0
- package/dist/formatters/text.d.ts +6 -1
- package/dist/formatters/text.js +73 -27
- package/dist/index.js +21 -8
- package/dist/metrics.js +33 -2
- package/dist/playwright-api-names.d.ts +10 -0
- package/dist/playwright-api-names.js +35 -0
- package/dist/profiles.js +14 -0
- package/dist/report-model.d.ts +52 -0
- package/dist/report-model.js +162 -0
- package/dist/report-source.d.ts +11 -0
- package/dist/report-source.js +96 -0
- package/dist/rule-copy.d.ts +14 -0
- package/dist/rule-copy.js +215 -0
- package/dist/rules/ast-util.d.ts +60 -0
- package/dist/rules/ast-util.js +140 -0
- package/dist/rules/base-plugin.d.ts +19 -5
- package/dist/rules/base-plugin.js +245 -8
- package/dist/rules/no-brittle-selector.d.ts +3 -0
- package/dist/rules/no-brittle-selector.js +150 -0
- package/dist/rules/no-excessive-timeout.d.ts +2 -0
- package/dist/rules/no-excessive-timeout.js +87 -0
- package/dist/rules/no-fixme-test.d.ts +14 -0
- package/dist/rules/no-fixme-test.js +53 -0
- package/dist/rules/no-self-fulfilling-fallback.d.ts +29 -0
- package/dist/rules/no-self-fulfilling-fallback.js +221 -0
- package/dist/rules/no-shared-mutable-state.d.ts +4 -0
- package/dist/rules/no-shared-mutable-state.js +186 -0
- package/dist/rules/no-swallowed-assertion.d.ts +13 -0
- package/dist/rules/no-swallowed-assertion.js +113 -0
- package/dist/rules/no-unawaited-action.d.ts +2 -0
- package/dist/rules/no-unawaited-action.js +77 -0
- package/dist/rules/no-unawaited-expect-value.d.ts +20 -0
- package/dist/rules/no-unawaited-expect-value.js +46 -0
- package/dist/rules/no-weak-assertion.d.ts +19 -0
- package/dist/rules/no-weak-assertion.js +82 -0
- package/dist/rules/rule-util.d.ts +56 -0
- package/dist/rules/rule-util.js +253 -0
- package/dist/score-engine.d.ts +5 -0
- package/dist/score-engine.js +10 -9
- package/dist/suite-name.d.ts +11 -0
- package/dist/suite-name.js +36 -0
- package/dist/test-aliases.js +144 -3
- package/dist/types.d.ts +18 -0
- package/package.json +1 -1
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { TEST_ALIAS_SCHEMA, enclosingFunction, evalNumber, hookName, isDescribeCall, isTestDeclaration, propName, testNamesFromOptions, unwrap, } from './rule-util.js';
|
|
2
|
+
/**
|
|
3
|
+
* `pwscore/no-excessive-timeout` (QAG-287). A timeout so long that a real
|
|
4
|
+
* failure takes minutes to show and a slow app is silently tolerated.
|
|
5
|
+
* Matches (literals and simple arithmetic such as `2 * 60 * 1000`, numeric
|
|
6
|
+
* separators included):
|
|
7
|
+
* - `{ timeout: N }` with N > 60_000 passed to an action, wait or
|
|
8
|
+
* matcher call: `x.click/fill/goto/waitFor*(...)`, `expect(..).toXxx(..)`,
|
|
9
|
+
* `expect.poll(..)`
|
|
10
|
+
* - `test.setTimeout(N)` / `testInfo.setTimeout(N)` with N >= 300_000
|
|
11
|
+
* - `test.slow()` with no arguments written directly at file or describe
|
|
12
|
+
* level (a conditional `test.slow(cond, why)` or one inside a test is fine)
|
|
13
|
+
* Not matched: config-file timeouts, non-literal values, `timeout: 0`.
|
|
14
|
+
*/
|
|
15
|
+
const WAIT_METHODS = new Set([
|
|
16
|
+
'goto', 'reload', 'goBack', 'goForward', 'click', 'dblclick', 'fill', 'clear', 'press',
|
|
17
|
+
'pressSequentially', 'type', 'check', 'uncheck', 'setChecked', 'selectOption', 'hover',
|
|
18
|
+
'focus', 'tap', 'dragTo', 'dragAndDrop', 'setInputFiles', 'waitFor', 'poll', 'toPass',
|
|
19
|
+
]);
|
|
20
|
+
export const noExcessiveTimeout = {
|
|
21
|
+
meta: {
|
|
22
|
+
type: 'suggestion',
|
|
23
|
+
docs: { description: 'Disallow very long literal timeouts' },
|
|
24
|
+
messages: {
|
|
25
|
+
long: 'A {{seconds}}s timeout means a failure takes that long to report and slowness is hidden. Fix the wait condition or the app instead of stretching the limit.',
|
|
26
|
+
longTest: '`setTimeout({{seconds}}s)` gives one test an extreme budget; split the test or fix the slow step.',
|
|
27
|
+
slow: 'Unconditional `test.slow()` triples every timeout in scope, hiding slowness. Apply it to the specific slow test with a reason, or fix the slowness.',
|
|
28
|
+
},
|
|
29
|
+
schema: TEST_ALIAS_SCHEMA,
|
|
30
|
+
},
|
|
31
|
+
create(context) {
|
|
32
|
+
const testNames = testNamesFromOptions(context.options);
|
|
33
|
+
const owners = new Set([...testNames, 'testInfo']);
|
|
34
|
+
return {
|
|
35
|
+
CallExpression(node) {
|
|
36
|
+
const callee = unwrap(node.callee);
|
|
37
|
+
const isExpectRoot = callee?.type === 'Identifier' && callee.name === 'expect';
|
|
38
|
+
if (callee?.type !== 'MemberExpression' && !isExpectRoot)
|
|
39
|
+
return;
|
|
40
|
+
const name = callee.type === 'MemberExpression' ? propName(callee) : undefined;
|
|
41
|
+
// test.setTimeout(N) / testInfo.setTimeout(N)
|
|
42
|
+
if (name === 'setTimeout' && callee.object?.type === 'Identifier' && owners.has(callee.object.name)) {
|
|
43
|
+
const ms = evalNumber(node.arguments?.[0]);
|
|
44
|
+
if (ms !== undefined && ms >= 300_000) {
|
|
45
|
+
context.report({ node: node, messageId: 'longTest', data: { seconds: String(ms / 1000) } });
|
|
46
|
+
}
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
// test.slow() at file/describe level
|
|
50
|
+
if (name === 'slow' && callee.object?.type === 'Identifier' && testNames.has(callee.object.name) && node.arguments.length === 0) {
|
|
51
|
+
const stmt = node.parent;
|
|
52
|
+
if (stmt?.type !== 'ExpressionStatement')
|
|
53
|
+
return;
|
|
54
|
+
const fn = enclosingFunction(node);
|
|
55
|
+
if (!fn) {
|
|
56
|
+
context.report({ node: node, messageId: 'slow' });
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const call = fn.parent;
|
|
60
|
+
if (call?.type === 'CallExpression' && isDescribeCall(call, testNames) && !hookName(call, testNames) && !isTestDeclaration(call, testNames)) {
|
|
61
|
+
context.report({ node: node, messageId: 'slow' });
|
|
62
|
+
}
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
// { timeout: N } on an action / wait / matcher
|
|
66
|
+
if (!name || !(WAIT_METHODS.has(name) || /^waitFor[A-Z]/.test(name) || /^to[A-Z]/.test(name)))
|
|
67
|
+
return;
|
|
68
|
+
for (const arg of node.arguments ?? []) {
|
|
69
|
+
const obj = unwrap(arg);
|
|
70
|
+
if (obj?.type !== 'ObjectExpression')
|
|
71
|
+
continue;
|
|
72
|
+
for (const prop of obj.properties) {
|
|
73
|
+
if (prop.type !== 'Property' || prop.computed)
|
|
74
|
+
continue;
|
|
75
|
+
const key = prop.key?.name ?? prop.key?.value;
|
|
76
|
+
if (key !== 'timeout')
|
|
77
|
+
continue;
|
|
78
|
+
const ms = evalNumber(prop.value);
|
|
79
|
+
if (ms !== undefined && ms > 60_000) {
|
|
80
|
+
context.report({ node: prop, messageId: 'long', data: { seconds: String(ms / 1000) } });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `test.fixme(...)` is the "known broken, do not run" marker. Same split as
|
|
3
|
+
* pwscore/no-skipped-test-declaration:
|
|
4
|
+
*
|
|
5
|
+
* - FLAGGED: declaration `test.fixme('t', fn)` / `test.describe.fixme(...)`
|
|
6
|
+
* (last argument is a function, 2+ arguments), and the unconditional
|
|
7
|
+
* in-body forms `test.fixme()` (no arguments) and `test.fixme(true, ...)`.
|
|
8
|
+
* - CLEAN: conditional `test.fixme(browserName === 'webkit', 'why')`
|
|
9
|
+
* (any first argument that is not the literal `true`).
|
|
10
|
+
*
|
|
11
|
+
* `test` includes `test.extend()` aliases (testAliasNames option).
|
|
12
|
+
*/
|
|
13
|
+
import type { Rule } from 'eslint';
|
|
14
|
+
export declare const noFixmeTest: Rule.RuleModule;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { FN_TYPES, memberName } from './ast-util.js';
|
|
2
|
+
function isFixmeCallee(callee, names) {
|
|
3
|
+
if (memberName(callee) !== 'fixme')
|
|
4
|
+
return false;
|
|
5
|
+
const obj = callee?.object;
|
|
6
|
+
if (obj?.type === 'Identifier' && obj.name && names.has(obj.name))
|
|
7
|
+
return true;
|
|
8
|
+
return (obj?.type === 'MemberExpression' &&
|
|
9
|
+
!obj.computed &&
|
|
10
|
+
obj.property?.name === 'describe' &&
|
|
11
|
+
obj.object?.type === 'Identifier' &&
|
|
12
|
+
!!obj.object.name &&
|
|
13
|
+
names.has(obj.object.name));
|
|
14
|
+
}
|
|
15
|
+
export const noFixmeTest = {
|
|
16
|
+
meta: {
|
|
17
|
+
type: 'suggestion',
|
|
18
|
+
docs: { description: 'Disallow unconditional test.fixme(): a test marked broken never runs' },
|
|
19
|
+
messages: {
|
|
20
|
+
declaration: 'Unexpected test.fixme() declaration: this test never runs. Fix it or delete it.',
|
|
21
|
+
unconditional: 'Unconditional test.fixme() inside a test: everything after it never runs. Use test.fixme(condition, reason) or fix the test.',
|
|
22
|
+
},
|
|
23
|
+
schema: [
|
|
24
|
+
{
|
|
25
|
+
additionalProperties: false,
|
|
26
|
+
properties: { testAliasNames: { items: { type: 'string' }, type: 'array' } },
|
|
27
|
+
type: 'object',
|
|
28
|
+
},
|
|
29
|
+
],
|
|
30
|
+
},
|
|
31
|
+
create(context) {
|
|
32
|
+
const aliases = context.options[0]?.testAliasNames ?? [];
|
|
33
|
+
const names = new Set(['test', ...aliases]);
|
|
34
|
+
return {
|
|
35
|
+
CallExpression(raw) {
|
|
36
|
+
const node = raw;
|
|
37
|
+
if (!isFixmeCallee(node.callee, names))
|
|
38
|
+
return;
|
|
39
|
+
const args = node.arguments ?? [];
|
|
40
|
+
const last = args[args.length - 1];
|
|
41
|
+
if (args.length >= 2 && last && FN_TYPES.has(last.type)) {
|
|
42
|
+
context.report({ node: raw, messageId: 'declaration' });
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (node.callee?.object?.type !== 'Identifier')
|
|
46
|
+
return; // describe.fixme() in-body form is not a thing
|
|
47
|
+
if (args.length === 0 || (args[0].type === 'Literal' && args[0].value === true)) {
|
|
48
|
+
context.report({ node: raw, messageId: 'unconditional' });
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
},
|
|
53
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "If the app did not do it, the test does it itself, then asserts."
|
|
3
|
+
*
|
|
4
|
+
* Inside a test callback, for a statement S in a block that is
|
|
5
|
+
* - `if (<negated state check>) { ...actions... }`, or the `else` branch of
|
|
6
|
+
* `if (<state check>) {...} else { ...actions... }`, or
|
|
7
|
+
* - `try { ...expect/state check... } catch { ...actions... }`
|
|
8
|
+
* ALL of the following must hold:
|
|
9
|
+
* (a) the fallback branch (not descending into nested functions) calls a
|
|
10
|
+
* content-supplying Playwright action: fill, type,
|
|
11
|
+
* pressSequentially, selectOption, setInputFiles (click-only fallbacks
|
|
12
|
+
* are the common "ensure the menu/toggle is open" setup idiom and are
|
|
13
|
+
* not flagged);
|
|
14
|
+
* (b) a later statement in the same block contains an expect(...);
|
|
15
|
+
* (c) the later expect's text shares a locator/selector expression with the
|
|
16
|
+
* state check (or try block), or with the action targets. "Locator
|
|
17
|
+
* expression" = the receiver text of a member call, e.g.
|
|
18
|
+
* `page.getByRole('textbox')` or the variable `input`; the bare `page`
|
|
19
|
+
* never counts.
|
|
20
|
+
*
|
|
21
|
+
* A "state check" is a call to isVisible/isHidden/isEnabled/isChecked/count/
|
|
22
|
+
* textContent/innerText/inputValue/getAttribute (or a const boolean derived
|
|
23
|
+
* from one). "Negated" means: `!check`, `check === false`, `count() === 0`,
|
|
24
|
+
* `count() < 1`, `.length === 0`, `text !== '...'`, `isHidden()`/
|
|
25
|
+
* `isDisabled()`, or `||`/`&&` containing one. Positive-branch conditional
|
|
26
|
+
* actions (dismiss a banner if visible) are never flagged.
|
|
27
|
+
*/
|
|
28
|
+
import type { Rule } from 'eslint';
|
|
29
|
+
export declare const noSelfFulfillingFallback: Rule.RuleModule;
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { FN_TYPES, isExpectCall, memberName, resolveConstInit, unwrap, walk } from './ast-util.js';
|
|
2
|
+
const CHECKS = new Set([
|
|
3
|
+
'isVisible', 'isHidden', 'isEnabled', 'isDisabled', 'isChecked', 'isEditable', 'count',
|
|
4
|
+
'textContent', 'innerText', 'inputValue', 'getAttribute', 'allTextContents', 'allInnerTexts',
|
|
5
|
+
]);
|
|
6
|
+
const NEGATIVE_STATE_CHECKS = new Set(['isHidden', 'isDisabled']);
|
|
7
|
+
// Only actions that SUPPLY content count. A click-only fallback is the
|
|
8
|
+
// "make sure the menu/toggle is open, then assert" setup idiom (checked on
|
|
9
|
+
// rill, mattermost: all click-only hits were setup, not self-fulfilment;
|
|
10
|
+
// `press('Escape')` in an else branch was setup too, so press is excluded).
|
|
11
|
+
const ACTIONS = new Set(['fill', 'type', 'pressSequentially', 'selectOption', 'setInputFiles']);
|
|
12
|
+
export const noSelfFulfillingFallback = {
|
|
13
|
+
meta: {
|
|
14
|
+
type: 'suggestion',
|
|
15
|
+
docs: { description: 'Disallow fallbacks that perform the expected state themselves before asserting it' },
|
|
16
|
+
messages: {
|
|
17
|
+
fallback: 'Self-fulfilling fallback: when the app does not produce this state, the test performs the actions itself and then asserts the result. The assertion can no longer detect the app failing to do it. Assert first and fail, or remove the fallback.',
|
|
18
|
+
},
|
|
19
|
+
schema: [
|
|
20
|
+
{
|
|
21
|
+
additionalProperties: false,
|
|
22
|
+
properties: { testAliasNames: { items: { type: 'string' }, type: 'array' } },
|
|
23
|
+
type: 'object',
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
},
|
|
27
|
+
create(context) {
|
|
28
|
+
const aliases = context.options[0]?.testAliasNames ?? [];
|
|
29
|
+
const testNames = new Set(['test', ...aliases]);
|
|
30
|
+
const text = (n) => context.sourceCode.getText(n).replace(/\s+/g, '');
|
|
31
|
+
function isTestCallback(fn) {
|
|
32
|
+
const call = fn.parent;
|
|
33
|
+
if (!call || call.type !== 'CallExpression' || !(call.arguments ?? []).includes(fn))
|
|
34
|
+
return false;
|
|
35
|
+
let c = call.callee;
|
|
36
|
+
if (c?.type === 'MemberExpression')
|
|
37
|
+
c = c.object?.type === 'MemberExpression' ? c.object.object : c.object;
|
|
38
|
+
return c?.type === 'Identifier' && !!c.name && testNames.has(c.name);
|
|
39
|
+
}
|
|
40
|
+
function inTest(n) {
|
|
41
|
+
for (let p = n.parent; p; p = p.parent)
|
|
42
|
+
if (FN_TYPES.has(p.type) && isTestCallback(p))
|
|
43
|
+
return true;
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
/** Does the expression contain a state check (resolving one level of const)? */
|
|
47
|
+
function hasCheck(e, depth = 0) {
|
|
48
|
+
let found = false;
|
|
49
|
+
walk(e, (n) => {
|
|
50
|
+
if (found)
|
|
51
|
+
return false;
|
|
52
|
+
if (n.type === 'CallExpression' && CHECKS.has(memberName(n.callee) ?? ''))
|
|
53
|
+
found = true;
|
|
54
|
+
else if (n.type === 'MemberExpression' && !n.computed && n.property?.name === 'length')
|
|
55
|
+
found = true;
|
|
56
|
+
else if (n.type === 'Identifier' && depth < 2) {
|
|
57
|
+
const init = resolveConstInit(context, n);
|
|
58
|
+
if (init && hasCheck(init, depth + 1))
|
|
59
|
+
found = true;
|
|
60
|
+
}
|
|
61
|
+
return undefined;
|
|
62
|
+
});
|
|
63
|
+
return found;
|
|
64
|
+
}
|
|
65
|
+
function isZeroish(n) {
|
|
66
|
+
return !!n && n.type === 'Literal' && (n.value === 0 || n.value === '' || n.value === false);
|
|
67
|
+
}
|
|
68
|
+
/** The test expression is true when the app did NOT produce the state. */
|
|
69
|
+
function negated(e, depth = 0) {
|
|
70
|
+
const n = unwrap(e);
|
|
71
|
+
if (!n || depth > 3)
|
|
72
|
+
return false;
|
|
73
|
+
if (n.type === 'UnaryExpression' && n.operator === '!')
|
|
74
|
+
return hasCheck(n.argument ?? undefined);
|
|
75
|
+
if (n.type === 'LogicalExpression')
|
|
76
|
+
return negated(n.left, depth + 1) || negated(n.right, depth + 1);
|
|
77
|
+
if (n.type === 'BinaryExpression') {
|
|
78
|
+
const op = n.operator;
|
|
79
|
+
const l = unwrap(n.left);
|
|
80
|
+
const r = unwrap(n.right);
|
|
81
|
+
if ((op === '===' || op === '==') && hasCheck(l) && isZeroish(r))
|
|
82
|
+
return true;
|
|
83
|
+
if ((op === '<' && hasCheck(l) && r?.type === 'Literal' && r.value === 1))
|
|
84
|
+
return true;
|
|
85
|
+
if ((op === '!==' || op === '!=') && hasCheck(l) && r?.type === 'Literal' && typeof r.value === 'string')
|
|
86
|
+
return true;
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
if (n.type === 'CallExpression')
|
|
90
|
+
return NEGATIVE_STATE_CHECKS.has(memberName(n.callee) ?? '');
|
|
91
|
+
if (n.type === 'Identifier' && depth < 2) {
|
|
92
|
+
const init = resolveConstInit(context, n);
|
|
93
|
+
// `const hidden = await x.isHidden(); if (hidden)`
|
|
94
|
+
return init ? negated(init, depth + 1) : false;
|
|
95
|
+
}
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
function positive(e) {
|
|
99
|
+
return hasCheck(e) && !negated(e);
|
|
100
|
+
}
|
|
101
|
+
/** Receiver texts of member calls and expect() subjects in a subtree. */
|
|
102
|
+
function keys(root, depth = 0) {
|
|
103
|
+
const out = new Set();
|
|
104
|
+
walk(root, (n) => {
|
|
105
|
+
if (n.type === 'CallExpression') {
|
|
106
|
+
if (isExpectCall(n)) {
|
|
107
|
+
const a = unwrap((n.arguments ?? [])[0]);
|
|
108
|
+
if (a)
|
|
109
|
+
addKey(out, a, depth);
|
|
110
|
+
}
|
|
111
|
+
else if (n.callee?.type === 'MemberExpression') {
|
|
112
|
+
addKey(out, n.callee.object, depth);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
return out;
|
|
117
|
+
}
|
|
118
|
+
function addKey(out, n, depth) {
|
|
119
|
+
const u = unwrap(n);
|
|
120
|
+
if (!u)
|
|
121
|
+
return;
|
|
122
|
+
const t = text(u);
|
|
123
|
+
if (t === 'page' || t === 'this.page' || t === 'expect' || t === 'test' || t.length < 3)
|
|
124
|
+
return;
|
|
125
|
+
out.add(t);
|
|
126
|
+
if (u.type === 'Identifier' && depth < 2) {
|
|
127
|
+
const init = resolveConstInit(context, u);
|
|
128
|
+
if (init)
|
|
129
|
+
for (const k of keys(init, depth + 1))
|
|
130
|
+
out.add(k);
|
|
131
|
+
if (init)
|
|
132
|
+
addKey(out, init, depth + 1);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function overlap(a, b) {
|
|
136
|
+
for (const x of a)
|
|
137
|
+
for (const y of b)
|
|
138
|
+
if (x === y)
|
|
139
|
+
return true;
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
function hasAction(branch) {
|
|
143
|
+
let has = false;
|
|
144
|
+
const targets = new Set();
|
|
145
|
+
walk(branch, (n) => {
|
|
146
|
+
if (n.type === 'CallExpression' && ACTIONS.has(memberName(n.callee) ?? '')) {
|
|
147
|
+
has = true;
|
|
148
|
+
addKey(targets, n.callee?.object, 0);
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
return { has, targets };
|
|
152
|
+
}
|
|
153
|
+
function expectsAfter(siblings, from) {
|
|
154
|
+
const found = [];
|
|
155
|
+
for (const s of siblings.slice(from + 1)) {
|
|
156
|
+
walk(s, (n) => {
|
|
157
|
+
if (isExpectCall(n))
|
|
158
|
+
found.push(n);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
return found;
|
|
162
|
+
}
|
|
163
|
+
function evaluate(stmt, siblings, idx) {
|
|
164
|
+
let fallback;
|
|
165
|
+
let condKeys = new Set();
|
|
166
|
+
if (stmt.type === 'IfStatement') {
|
|
167
|
+
const test = stmt.test;
|
|
168
|
+
if (negated(test))
|
|
169
|
+
fallback = stmt.consequent;
|
|
170
|
+
else if (positive(test) && stmt.alternate)
|
|
171
|
+
fallback = stmt.alternate;
|
|
172
|
+
if (!fallback)
|
|
173
|
+
return;
|
|
174
|
+
condKeys = keys(test);
|
|
175
|
+
// keys() only sees member-call receivers/expect subjects; a const-derived test needs its init too.
|
|
176
|
+
walk(test, (n) => {
|
|
177
|
+
if (n.type === 'Identifier') {
|
|
178
|
+
const init = resolveConstInit(context, n);
|
|
179
|
+
if (init)
|
|
180
|
+
for (const k of keys(init))
|
|
181
|
+
condKeys.add(k);
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
else if (stmt.type === 'TryStatement' && stmt.handler) {
|
|
186
|
+
let guarded = false;
|
|
187
|
+
walk(stmt.block, (n) => {
|
|
188
|
+
if (isExpectCall(n) || (n.type === 'CallExpression' && CHECKS.has(memberName(n.callee) ?? '')))
|
|
189
|
+
guarded = true;
|
|
190
|
+
});
|
|
191
|
+
if (!guarded)
|
|
192
|
+
return;
|
|
193
|
+
fallback = stmt.handler.body;
|
|
194
|
+
condKeys = keys(stmt.block);
|
|
195
|
+
}
|
|
196
|
+
else
|
|
197
|
+
return;
|
|
198
|
+
const act = hasAction(fallback);
|
|
199
|
+
if (!act.has)
|
|
200
|
+
return;
|
|
201
|
+
const expects = expectsAfter(siblings, idx);
|
|
202
|
+
if (expects.length === 0)
|
|
203
|
+
return;
|
|
204
|
+
const later = new Set();
|
|
205
|
+
for (const e of expects)
|
|
206
|
+
for (const k of keys(e))
|
|
207
|
+
later.add(k);
|
|
208
|
+
if (overlap(condKeys, later) || overlap(act.targets, later)) {
|
|
209
|
+
context.report({ node: stmt, messageId: 'fallback' });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function onBlock(raw) {
|
|
213
|
+
const block = raw;
|
|
214
|
+
if (!inTest(block))
|
|
215
|
+
return;
|
|
216
|
+
const body = (Array.isArray(block.body) ? block.body : []);
|
|
217
|
+
body.forEach((s, i) => evaluate(s, body, i));
|
|
218
|
+
}
|
|
219
|
+
return { BlockStatement: onBlock };
|
|
220
|
+
},
|
|
221
|
+
};
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { TEST_ALIAS_SCHEMA, enclosingFunction, hookName, isDescribeCall, isInSerialScope, isSerialConfigureCall, isSerialDescribeCall, isTestDeclaration, propName, testNamesFromOptions, unwrap, } from './rule-util.js';
|
|
2
|
+
/**
|
|
3
|
+
* `pwscore/no-shared-mutable-state` (QAG-287). A module-level or
|
|
4
|
+
* describe-level `let`/`var` (or a `const` bound to an object/array
|
|
5
|
+
* literal, `new Map()` or `new Set()`) that one `test(...)` body assigns or
|
|
6
|
+
* mutates and a DIFFERENT `test(...)` body reads without writing it
|
|
7
|
+
* itself. Run alone, in another order or on another worker, the reader sees
|
|
8
|
+
* the initial value, so the suite is order-dependent.
|
|
9
|
+
*
|
|
10
|
+
* Not matched: variables in serial mode (`test.describe.serial`, or
|
|
11
|
+
* `test.describe.configure({ mode: 'serial' })` at file level or in an
|
|
12
|
+
* enclosing describe); assignment only inside `beforeEach`/`beforeAll`/`afterEach`
|
|
13
|
+
* (setup); a variable that any `beforeEach` assigns (it is reset before each
|
|
14
|
+
* test); a reader that also writes the variable itself (its own setup);
|
|
15
|
+
* variables declared inside a test.
|
|
16
|
+
* Mutation = assignment, `++`/`--`, member assignment (`v.x = 1`), or a
|
|
17
|
+
* call of `push/pop/shift/unshift/splice/set/add/delete/clear/sort/reverse`.
|
|
18
|
+
*
|
|
19
|
+
* Companion `pwscore/serial-mode-dependency` is report-only information:
|
|
20
|
+
* `test.describe.configure({ mode: 'serial' })` and `test.describe.serial()`
|
|
21
|
+
* declare that tests depend on order; not an error on its own.
|
|
22
|
+
*/
|
|
23
|
+
const MUTATORS = new Set(['push', 'pop', 'shift', 'unshift', 'splice', 'set', 'add', 'delete', 'clear', 'sort', 'reverse']);
|
|
24
|
+
export const noSharedMutableState = {
|
|
25
|
+
meta: {
|
|
26
|
+
type: 'suggestion',
|
|
27
|
+
docs: { description: 'Disallow state written in one test and read in another' },
|
|
28
|
+
messages: {
|
|
29
|
+
shared: '`{{name}}` is written in test "{{writer}}" and read in this test. Tests that share state depend on run order and break under retries or parallel workers. Create the state inside each test (or a fixture/beforeEach).',
|
|
30
|
+
},
|
|
31
|
+
schema: TEST_ALIAS_SCHEMA,
|
|
32
|
+
},
|
|
33
|
+
create(context) {
|
|
34
|
+
const testNames = testNamesFromOptions(context.options);
|
|
35
|
+
/** The test(...) call (and its title) whose callback contains `node`, if any. */
|
|
36
|
+
const owningTest = (node) => {
|
|
37
|
+
let fn = enclosingFunction(node);
|
|
38
|
+
while (fn) {
|
|
39
|
+
const call = fn.parent;
|
|
40
|
+
if (call?.type === 'CallExpression' && call.arguments.includes(fn)) {
|
|
41
|
+
if (isTestDeclaration(call, testNames))
|
|
42
|
+
return call;
|
|
43
|
+
if (hookName(call, testNames))
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
fn = enclosingFunction(fn);
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
};
|
|
50
|
+
const owningHook = (node) => {
|
|
51
|
+
let fn = enclosingFunction(node);
|
|
52
|
+
while (fn) {
|
|
53
|
+
const call = fn.parent;
|
|
54
|
+
if (call?.type === 'CallExpression' && call.arguments.includes(fn)) {
|
|
55
|
+
const h = hookName(call, testNames);
|
|
56
|
+
if (h)
|
|
57
|
+
return h;
|
|
58
|
+
if (isTestDeclaration(call, testNames))
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
fn = enclosingFunction(fn);
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
};
|
|
65
|
+
/** Declared directly at module level or directly in a describe callback. */
|
|
66
|
+
const isSharedScope = (decl) => {
|
|
67
|
+
const fn = enclosingFunction(decl);
|
|
68
|
+
if (!fn)
|
|
69
|
+
return true;
|
|
70
|
+
const call = fn.parent;
|
|
71
|
+
return call?.type === 'CallExpression' && call.arguments.includes(fn) && isDescribeCall(call, testNames);
|
|
72
|
+
};
|
|
73
|
+
const isMutableInit = (kind, init) => {
|
|
74
|
+
if (kind !== 'const')
|
|
75
|
+
return true;
|
|
76
|
+
const i = unwrap(init);
|
|
77
|
+
return (!!i &&
|
|
78
|
+
(i.type === 'ObjectExpression' ||
|
|
79
|
+
i.type === 'ArrayExpression' ||
|
|
80
|
+
(i.type === 'NewExpression' && i.callee?.type === 'Identifier' && (i.callee.name === 'Map' || i.callee.name === 'Set'))));
|
|
81
|
+
};
|
|
82
|
+
const titleOf = (call) => {
|
|
83
|
+
const t = call.arguments?.[0];
|
|
84
|
+
return t?.type === 'Literal' && typeof t.value === 'string' ? t.value : 'another test';
|
|
85
|
+
};
|
|
86
|
+
/** Is this reference a write/mutation of the variable? */
|
|
87
|
+
const isMutation = (id, ref) => {
|
|
88
|
+
if (ref.isWrite())
|
|
89
|
+
return true;
|
|
90
|
+
let child = id;
|
|
91
|
+
let parent = id.parent;
|
|
92
|
+
while (parent?.type === 'MemberExpression' && parent.object === child) {
|
|
93
|
+
const gp = parent.parent;
|
|
94
|
+
if (gp?.type === 'AssignmentExpression' && gp.left === parent)
|
|
95
|
+
return true;
|
|
96
|
+
if (gp?.type === 'UpdateExpression')
|
|
97
|
+
return true;
|
|
98
|
+
if (gp?.type === 'CallExpression' && gp.callee === parent) {
|
|
99
|
+
const name = propName(parent);
|
|
100
|
+
if (name && MUTATORS.has(name) && parent.object === id)
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
child = parent;
|
|
104
|
+
parent = gp;
|
|
105
|
+
}
|
|
106
|
+
return false;
|
|
107
|
+
};
|
|
108
|
+
return {
|
|
109
|
+
VariableDeclaration(node) {
|
|
110
|
+
if (!isSharedScope(node))
|
|
111
|
+
return;
|
|
112
|
+
for (const decl of node.declarations) {
|
|
113
|
+
if (decl.id?.type !== 'Identifier' || !isMutableInit(node.kind, decl.init))
|
|
114
|
+
continue;
|
|
115
|
+
const [variable] = context.sourceCode.getDeclaredVariables(decl);
|
|
116
|
+
if (!variable)
|
|
117
|
+
continue;
|
|
118
|
+
// Serial mode is Playwright's documented way to share state in order.
|
|
119
|
+
if (isInSerialScope(decl, testNames))
|
|
120
|
+
continue;
|
|
121
|
+
const writes = new Map(); // test call -> write node
|
|
122
|
+
const reads = new Map(); // test call -> first read identifier
|
|
123
|
+
let hookReset = false;
|
|
124
|
+
for (const ref of variable.references) {
|
|
125
|
+
const id = ref.identifier;
|
|
126
|
+
if (id === decl.id)
|
|
127
|
+
continue;
|
|
128
|
+
const test = owningTest(id);
|
|
129
|
+
const mutation = isMutation(id, ref);
|
|
130
|
+
if (!test) {
|
|
131
|
+
if (mutation && owningHook(id) === 'beforeEach')
|
|
132
|
+
hookReset = true;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (mutation) {
|
|
136
|
+
if (!writes.has(test))
|
|
137
|
+
writes.set(test, id);
|
|
138
|
+
}
|
|
139
|
+
else if (ref.isRead() && !reads.has(test)) {
|
|
140
|
+
reads.set(test, id);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (hookReset || writes.size === 0)
|
|
144
|
+
continue;
|
|
145
|
+
for (const [test, id] of reads) {
|
|
146
|
+
if (writes.has(test))
|
|
147
|
+
continue;
|
|
148
|
+
// Serial mode orders the tests that share state: skip a writer/reader
|
|
149
|
+
// pair only when BOTH sit in a serial scope.
|
|
150
|
+
const readerSerial = isInSerialScope(test, testNames);
|
|
151
|
+
const writer = [...writes.keys()].find((w) => !(readerSerial && isInSerialScope(w, testNames)));
|
|
152
|
+
if (!writer)
|
|
153
|
+
continue;
|
|
154
|
+
context.report({
|
|
155
|
+
node: id,
|
|
156
|
+
messageId: 'shared',
|
|
157
|
+
data: { name: decl.id.name, writer: titleOf(writer) },
|
|
158
|
+
});
|
|
159
|
+
break; // one finding per variable
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
/** Report-only companion: serial mode declares order dependence. */
|
|
167
|
+
export const serialModeDependency = {
|
|
168
|
+
meta: {
|
|
169
|
+
type: 'suggestion',
|
|
170
|
+
docs: { description: 'Report serial-mode describe blocks (tests depend on order)' },
|
|
171
|
+
messages: {
|
|
172
|
+
serial: 'Serial mode: the tests in this scope depend on running in order, so one failure skips the rest and retries rerun the whole group. Information only.',
|
|
173
|
+
},
|
|
174
|
+
schema: TEST_ALIAS_SCHEMA,
|
|
175
|
+
},
|
|
176
|
+
create(context) {
|
|
177
|
+
const testNames = testNamesFromOptions(context.options);
|
|
178
|
+
return {
|
|
179
|
+
CallExpression(node) {
|
|
180
|
+
if (isSerialDescribeCall(node, testNames) || isSerialConfigureCall(node, testNames)) {
|
|
181
|
+
context.report({ node: node, messageId: 'serial' });
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
};
|
|
185
|
+
},
|
|
186
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An assertion whose failure is caught and discarded cannot fail the test.
|
|
3
|
+
*
|
|
4
|
+
* Matches:
|
|
5
|
+
* 1. `try { ... expect(x).matcher() ... } catch { ... }` where the catch
|
|
6
|
+
* body does not throw, call expect / test.fail / a fail-ish helper,
|
|
7
|
+
* return a value, or assign/mutate outer state (any of which may carry
|
|
8
|
+
* the failure to a later assertion). Skipped inside loops (retry idiom).
|
|
9
|
+
* 2. `await expect(x).matcher().catch(<fn that does not throw>)` as a
|
|
10
|
+
* statement (result discarded).
|
|
11
|
+
*/
|
|
12
|
+
import type { Rule } from 'eslint';
|
|
13
|
+
export declare const noSwallowedAssertion: Rule.RuleModule;
|