@qaguardian/playwright-score 0.1.14 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METHODOLOGY.md +23 -27
- package/README.md +55 -29
- package/dist/cli.js +11 -5
- package/dist/eslint-runner.js +27 -23
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -4
- package/dist/profiles.js +0 -31
- package/dist/rules/base-plugin.d.ts +2 -2
- package/dist/score-engine.js +7 -10
- package/dist/types.d.ts +2 -3
- package/package.json +2 -3
- package/dist/rules/guardian-plugin.d.ts +0 -29
- package/dist/rules/guardian-plugin.js +0 -386
package/METHODOLOGY.md
CHANGED
|
@@ -11,14 +11,11 @@ Any change to formulas, weights, SLOC definition, caps, or constants requires a
|
|
|
11
11
|
|
|
12
12
|
---
|
|
13
13
|
|
|
14
|
-
##
|
|
14
|
+
## Profile
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
| `guardian` | `standard` + QA Guardian house conventions (optional; clearly labeled) |
|
|
20
|
-
|
|
21
|
-
House rules are **not** claimed to be official Playwright doctrine.
|
|
16
|
+
`standard` (the only profile): community Playwright best practices
|
|
17
|
+
(`eslint-plugin-playwright`) plus suite-level metrics (locator ratio,
|
|
18
|
+
assertion-delegation tracing, Page Object Model import resolution).
|
|
22
19
|
|
|
23
20
|
---
|
|
24
21
|
|
|
@@ -34,8 +31,6 @@ House rules are **not** claimed to be official Playwright doctrine.
|
|
|
34
31
|
|
|
35
32
|
## Dimension weights
|
|
36
33
|
|
|
37
|
-
### `standard`
|
|
38
|
-
|
|
39
34
|
| Dimension | Weight |
|
|
40
35
|
|---|---|
|
|
41
36
|
| playwrightHygiene | 40 |
|
|
@@ -43,16 +38,6 @@ House rules are **not** claimed to be official Playwright doctrine.
|
|
|
43
38
|
| locators | 20 |
|
|
44
39
|
| structure | 15 |
|
|
45
40
|
|
|
46
|
-
### `guardian`
|
|
47
|
-
|
|
48
|
-
| Dimension | Weight |
|
|
49
|
-
|---|---|
|
|
50
|
-
| playwrightHygiene | 30 |
|
|
51
|
-
| assertions | 15 |
|
|
52
|
-
| locators | 15 |
|
|
53
|
-
| structure | 15 |
|
|
54
|
-
| guardianConventions | 25 |
|
|
55
|
-
|
|
56
41
|
---
|
|
57
42
|
|
|
58
43
|
## Locators dimension (ratio)
|
|
@@ -88,7 +73,7 @@ Used for penalty density. JSDoc / decorative comments do not inflate slots.
|
|
|
88
73
|
|
|
89
74
|
## Penalty dimensions
|
|
90
75
|
|
|
91
|
-
Applies to: `playwrightHygiene`, `assertions`, `structure
|
|
76
|
+
Applies to: `playwrightHygiene`, `assertions`, `structure`.
|
|
92
77
|
|
|
93
78
|
### Constants (frozen sqs-v1)
|
|
94
79
|
|
|
@@ -159,11 +144,13 @@ These examples feel fair: tiny single error stays high; multi-error drops materi
|
|
|
159
144
|
|
|
160
145
|
## Default thresholds
|
|
161
146
|
|
|
162
|
-
| Context |
|
|
163
|
-
|
|
164
|
-
| CLI default |
|
|
165
|
-
|
|
166
|
-
|
|
147
|
+
| Context | Threshold |
|
|
148
|
+
|---|---|
|
|
149
|
+
| CLI default | 80 |
|
|
150
|
+
|
|
151
|
+
QA Guardian's own internal codegen pipeline (`playwright_runner`) layers a
|
|
152
|
+
private house-rules gate on top of `standard` — see the README's
|
|
153
|
+
"QA Guardian integration" section. That layer isn't part of this package.
|
|
167
154
|
|
|
168
155
|
---
|
|
169
156
|
|
|
@@ -174,5 +161,14 @@ See package rule docs / source `profiles.ts`. High level:
|
|
|
174
161
|
- **playwrightHygiene:** waits, force, networkidle, missing await, handles, conditionals
|
|
175
162
|
- **assertions:** expect-expect, empty tests, web-first preference
|
|
176
163
|
- **locators:** ratio metric only (for score); raw-locator ESLint in findings
|
|
177
|
-
- **structure:** focused/skipped, oversized file, describe shape
|
|
178
|
-
|
|
164
|
+
- **structure:** focused/skipped, oversized file, describe shape
|
|
165
|
+
|
|
166
|
+
### Active rule set (frozen)
|
|
167
|
+
|
|
168
|
+
The exact ESLint rules that produce sqs-v1 findings are enabled explicitly
|
|
169
|
+
in `eslint-runner.ts` with fixed severities — the score never inherits
|
|
170
|
+
eslint-plugin-playwright's `recommended` set at install time, so identical
|
|
171
|
+
code scores identically regardless of which plugin version npm resolves.
|
|
172
|
+
(Versions ≤0.2.0 intended to inherit `recommended` but a config-shape bug
|
|
173
|
+
dropped it entirely; 0.3.0 froze the intended set explicitly. See
|
|
174
|
+
CHANGELOG.)
|
package/README.md
CHANGED
|
@@ -9,14 +9,51 @@
|
|
|
9
9
|
**Website:** [qaguardian.com/open-source/playwright-score](https://qaguardian.com/open-source/playwright-score)
|
|
10
10
|
**Built by:** [QA Guardian](https://qaguardian.com) — managed Playwright E2E (AI drafts, engineers verify, you own the code)
|
|
11
11
|
|
|
12
|
-
Lint + score Playwright tests
|
|
13
|
-
|
|
14
|
-
-
|
|
15
|
-
|
|
12
|
+
Lint + score Playwright tests against community best practices via
|
|
13
|
+
[`eslint-plugin-playwright`](https://github.com/playwright-community/eslint-plugin-playwright)
|
|
14
|
+
plus suite-level metrics (locator ratio, assertion-delegation tracing,
|
|
15
|
+
Page Object Model import resolution, ...).
|
|
16
16
|
|
|
17
17
|
The score never calls an LLM. AI may *generate* or *repair* code using findings; **rules grade code**.
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
## Real-world results
|
|
20
|
+
|
|
21
|
+
17 public Playwright suites, scored with the published package against each
|
|
22
|
+
project's actual source — chosen to include both well-known,
|
|
23
|
+
heavily-engineered platforms and smaller, less mature projects. Not curated
|
|
24
|
+
to look good: 3 of 17 fail the default threshold, for real, verifiable
|
|
25
|
+
reasons (see [VALIDATION.md](./VALIDATION.md)).
|
|
26
|
+
|
|
27
|
+
**17 suites · 1,228 files · 6,037 tests · 14/17 passed (80% threshold)**
|
|
28
|
+
|
|
29
|
+
| Repo (source scanned) | Score | Grade | Result |
|
|
30
|
+
|---|---:|:-:|:-:|
|
|
31
|
+
| [Playwright (own TodoMVC example)](https://github.com/microsoft/playwright/tree/main/examples/todomvc) | 98/100 | A | PASS |
|
|
32
|
+
| [Supabase](https://github.com/supabase/supabase/tree/master/e2e) | 97/100 | A | PASS |
|
|
33
|
+
| [Documenso](https://github.com/documenso/documenso/tree/main/packages/app-tests) | 96/100 | A | PASS |
|
|
34
|
+
| [Storybook](https://github.com/storybookjs/storybook/tree/next/code/e2e-internal) | 96/100 | A | PASS |
|
|
35
|
+
| [freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp/tree/main/e2e) | 96/100 | A | PASS |
|
|
36
|
+
| [n8n](https://github.com/n8n-io/n8n/tree/master/packages/testing/playwright) | 95/100 | A | PASS |
|
|
37
|
+
| [dub](https://github.com/dubinc/dub/tree/main/apps/web/playwright) | 95/100 | A | PASS |
|
|
38
|
+
| [novu](https://github.com/novuhq/novu/tree/next/apps/dashboard/tests) | 93/100 | A | PASS |
|
|
39
|
+
| [Grafana](https://github.com/grafana/grafana/tree/main/e2e-playwright) | 91/100 | A | PASS |
|
|
40
|
+
| [PostHog](https://github.com/PostHog/posthog/tree/master/playwright) | 90/100 | A | PASS |
|
|
41
|
+
| [sencho](https://github.com/Studio-Saelix/sencho/tree/main/e2e) | 90/100 | A | PASS |
|
|
42
|
+
| [Mattermost](https://github.com/mattermost/mattermost/tree/master/e2e-tests/playwright) | 89/100 | B | PASS |
|
|
43
|
+
| [Immich](https://github.com/immich-app/immich/tree/main/e2e/src/specs) | 89/100 | B | PASS |
|
|
44
|
+
| [cal.com](https://github.com/calcom/cal.com/tree/main/apps/web/playwright) | 84/100 | B | PASS |
|
|
45
|
+
| [livecodes](https://github.com/live-codes/livecodes/tree/develop/e2e) | 78/100 | C | FAIL |
|
|
46
|
+
| [openplayerjs](https://github.com/openplayerjs/openplayerjs/tree/master/e2e) | 77/100 | C | FAIL |
|
|
47
|
+
| [TheCyberHub](https://github.com/th3cyb3rhub/TheCyberHub/tree/dev/e2e) | 68/100 | D | FAIL |
|
|
48
|
+
|
|
49
|
+
Repo names link to the exact source scanned. Full methodology, findings
|
|
50
|
+
breakdown, and `scripts/validate-corpus.sh` to reproduce every number here
|
|
51
|
+
yourself live in [VALIDATION.md](./VALIDATION.md). A self-contained visual
|
|
52
|
+
report of an earlier five-suite audit lives at
|
|
53
|
+
[`docs/scorecard.html`](./docs/scorecard.html) — host it wherever
|
|
54
|
+
(GitHub Pages, qaguardian.com, ...).
|
|
55
|
+
|
|
56
|
+
See [METHODOLOGY.md](./METHODOLOGY.md) for the frozen formula, or the full product write-up on the [landing page](https://qaguardian.com/open-source/playwright-score).
|
|
20
57
|
|
|
21
58
|
## Install
|
|
22
59
|
|
|
@@ -33,12 +70,12 @@ Package: [@qaguardian/playwright-score](https://www.npmjs.com/package/@qaguardia
|
|
|
33
70
|
npx -p @qaguardian/playwright-score playwright-score ./tests --profile standard --threshold 80
|
|
34
71
|
|
|
35
72
|
# After local install
|
|
36
|
-
npx playwright-score ./flow.spec.ts --
|
|
73
|
+
npx playwright-score ./flow.spec.ts --format json --out report.json
|
|
37
74
|
```
|
|
38
75
|
|
|
39
76
|
| Flag | Description |
|
|
40
77
|
|---|---|
|
|
41
|
-
| `--profile standard
|
|
78
|
+
| `--profile standard` | Scoring profile (default, and only, profile) |
|
|
42
79
|
| `--threshold <n>` | Pass bar 0–100 |
|
|
43
80
|
| `--format text\|json\|markdown\|sarif` | Output format |
|
|
44
81
|
| `--out <file>` | Write report to file |
|
|
@@ -69,16 +106,15 @@ keep a sticky PR comment with the full findings up to date:
|
|
|
69
106
|
- uses: qa-guardian/playwright-score@v1
|
|
70
107
|
with:
|
|
71
108
|
paths: tests e2e
|
|
72
|
-
|
|
73
|
-
# threshold: 80 # defaults to the profile's default
|
|
109
|
+
# threshold: 80 # defaults to 80
|
|
74
110
|
mode: gate # gate: fail CI below threshold · warn: report only
|
|
75
111
|
```
|
|
76
112
|
|
|
77
113
|
| Input | Description | Default |
|
|
78
114
|
|---|---|---|
|
|
79
115
|
| `paths` | Space-separated paths/globs to score | `tests` |
|
|
80
|
-
| `profile` | `standard`
|
|
81
|
-
| `threshold` | Pass bar 0–100 |
|
|
116
|
+
| `profile` | `standard` (default, and only, profile) | `standard` |
|
|
117
|
+
| `threshold` | Pass bar 0–100 | `80` |
|
|
82
118
|
| `mode` | `gate` (fail CI below threshold) \| `warn` (report only) | `gate` |
|
|
83
119
|
| `comment` | Post/update a sticky PR comment | `true` |
|
|
84
120
|
| `version` | `@qaguardian/playwright-score` version to run | `latest` |
|
|
@@ -97,13 +133,17 @@ section above — same score, same exit codes:
|
|
|
97
133
|
|
|
98
134
|
## QA Guardian integration
|
|
99
135
|
|
|
100
|
-
|
|
136
|
+
QA Guardian's own codegen pipeline (`playwright_runner`) dogfoods this
|
|
137
|
+
package for the `standard` profile, layering its own private house-rules
|
|
138
|
+
gate on top internally — that layer isn't part of this package (it's
|
|
139
|
+
product-specific, e.g. "timeouts must be exactly 2000 or 20000ms", not
|
|
140
|
+
Playwright best practice) and isn't published here. `playwright_runner`
|
|
141
|
+
sets:
|
|
101
142
|
|
|
102
143
|
| Env | Values | Default |
|
|
103
144
|
|---|---|---|
|
|
104
145
|
| `SPEC_SCORE_MODE` | `off` \| `warn` \| `gate` | `warn` |
|
|
105
|
-
| `
|
|
106
|
-
| `SPEC_SCORE_THRESHOLD` | `0`–`100` | `75` (guardian) / `80` (standard) |
|
|
146
|
+
| `SPEC_SCORE_THRESHOLD` | `0`–`100` | `80` |
|
|
107
147
|
|
|
108
148
|
- **`warn`**: log score; never fail the run; findings still inject into heal/generate repair prompts
|
|
109
149
|
- **`gate`**: fail the run when score < threshold
|
|
@@ -114,21 +154,7 @@ When used from `playwright_runner`, set:
|
|
|
114
154
|
# Local from monorepo
|
|
115
155
|
cd playwright-score && npm run build
|
|
116
156
|
cd ../playwright_runner && npm install
|
|
117
|
-
SPEC_SCORE_MODE=warn
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
## Profiles
|
|
121
|
-
|
|
122
|
-
| Profile | Use when |
|
|
123
|
-
|---|---|
|
|
124
|
-
| `standard` | Any Playwright repo — best practices + locator mix + empty-expect detection |
|
|
125
|
-
| `guardian` | QA Guardian (or similar) codegen quality gate |
|
|
126
|
-
|
|
127
|
-
Disable a guardian secrets false positive:
|
|
128
|
-
|
|
129
|
-
```ts
|
|
130
|
-
// spec-score-disable-next-line guardian/no-hardcoded-secrets
|
|
131
|
-
const password = 'intentional-fixture-value-xyz';
|
|
157
|
+
SPEC_SCORE_MODE=warn node ...
|
|
132
158
|
```
|
|
133
159
|
|
|
134
160
|
## Development
|
package/dist/cli.js
CHANGED
|
@@ -18,8 +18,8 @@ Usage:
|
|
|
18
18
|
npx playwright-score <paths...> [options]
|
|
19
19
|
|
|
20
20
|
Options:
|
|
21
|
-
--profile <standard
|
|
22
|
-
--threshold <n> Pass threshold 0-100 (default: 80
|
|
21
|
+
--profile <standard> Scoring profile (default, and only: standard)
|
|
22
|
+
--threshold <n> Pass threshold 0-100 (default: 80)
|
|
23
23
|
--format <text|json|markdown|sarif> Output format (default: text)
|
|
24
24
|
--out <file> Write report to file
|
|
25
25
|
--version Print version
|
|
@@ -70,12 +70,18 @@ async function main() {
|
|
|
70
70
|
for (let i = 0; i < args.length; i++) {
|
|
71
71
|
const a = args[i];
|
|
72
72
|
if (a === '--profile') {
|
|
73
|
-
|
|
74
|
-
if (
|
|
75
|
-
console.error(
|
|
73
|
+
const p = args[++i];
|
|
74
|
+
if (p === 'guardian') {
|
|
75
|
+
console.error('Profile "guardian" is no longer public — it was QA Guardian\'s internal house-rules layer, never official Playwright doctrine, and doesn\'t belong in a general-purpose tool. See CHANGELOG.md. Use --profile standard.');
|
|
76
76
|
process.exitCode = 2;
|
|
77
77
|
return;
|
|
78
78
|
}
|
|
79
|
+
if (p !== 'standard') {
|
|
80
|
+
console.error(`Invalid profile: ${p}`);
|
|
81
|
+
process.exitCode = 2;
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
profile = p;
|
|
79
85
|
}
|
|
80
86
|
else if (a === '--threshold') {
|
|
81
87
|
threshold = Number(args[++i]);
|
package/dist/eslint-runner.js
CHANGED
|
@@ -2,12 +2,9 @@ import path from 'node:path';
|
|
|
2
2
|
import { ESLint } from 'eslint';
|
|
3
3
|
import { commonAncestorDir } from './fs-util.js';
|
|
4
4
|
import playwright from 'eslint-plugin-playwright';
|
|
5
|
-
import { guardianPlugin, guardianRuleConfigs, } from './rules/guardian-plugin.js';
|
|
6
5
|
import { basePlugin } from './rules/base-plugin.js';
|
|
7
6
|
import { mapRule } from './profiles.js';
|
|
8
|
-
function buildConfig(
|
|
9
|
-
const playwrightRecommended = playwright.configs?.['flat/recommended'] ??
|
|
10
|
-
playwright.configs?.recommended;
|
|
7
|
+
function buildConfig(assertFunctionNames) {
|
|
11
8
|
const base = [
|
|
12
9
|
{
|
|
13
10
|
files: ['**/*.{ts,tsx,js,jsx,mjs,cjs}'],
|
|
@@ -19,7 +16,6 @@ function buildConfig(profile, assertFunctionNames) {
|
|
|
19
16
|
plugins: {
|
|
20
17
|
playwright,
|
|
21
18
|
pwscore: basePlugin,
|
|
22
|
-
...(profile === 'guardian' ? { guardian: guardianPlugin } : {}),
|
|
23
19
|
},
|
|
24
20
|
rules: {
|
|
25
21
|
// Ensure key hygiene rules even if recommended set differs by
|
|
@@ -91,26 +87,34 @@ function buildConfig(profile, assertFunctionNames) {
|
|
|
91
87
|
'playwright/prefer-native-locators': 'warn',
|
|
92
88
|
// Matches eslint-plugin-playwright's own recommended severity.
|
|
93
89
|
'playwright/prefer-web-first-assertions': 'error',
|
|
94
|
-
|
|
90
|
+
// The rules below complete the sqs-v1 rule set from
|
|
91
|
+
// profiles.ts's ESLINT_RULE_MAP. Earlier versions (<=0.2.0)
|
|
92
|
+
// expected them to arrive via eslint-plugin-playwright's
|
|
93
|
+
// flat/recommended config, but the spread only handled an array
|
|
94
|
+
// shape while the plugin exports a single config object — so the
|
|
95
|
+
// entire recommended set was silently dropped and these rules
|
|
96
|
+
// never ran (METHODOLOGY.md's hygiene dimension lists
|
|
97
|
+
// "conditionals" and "handles", yet neither could ever fire).
|
|
98
|
+
// They are enabled explicitly and frozen here on purpose:
|
|
99
|
+
// eslint-plugin-playwright is a ^range dependency, and inheriting
|
|
100
|
+
// whatever its recommended set says at install time would make
|
|
101
|
+
// identical code score differently across installs. Severities
|
|
102
|
+
// mirror upstream flat/recommended as of eslint-plugin-playwright
|
|
103
|
+
// 2.x. require-top-level-describe stays disabled deliberately —
|
|
104
|
+
// it is stylistic, not in upstream recommended, and would
|
|
105
|
+
// penalize idiomatic describe-less suites.
|
|
106
|
+
'playwright/no-conditional-in-test': 'warn',
|
|
107
|
+
'playwright/no-conditional-expect': 'warn',
|
|
108
|
+
'playwright/no-element-handle': 'warn',
|
|
109
|
+
'playwright/no-eval': 'warn',
|
|
110
|
+
'playwright/no-page-pause': 'warn',
|
|
111
|
+
'playwright/no-useless-await': 'warn',
|
|
112
|
+
'playwright/valid-expect': 'error',
|
|
113
|
+
'playwright/no-standalone-expect': 'error',
|
|
114
|
+
'playwright/max-nested-describe': 'warn',
|
|
95
115
|
},
|
|
96
116
|
},
|
|
97
117
|
];
|
|
98
|
-
// Prefer spreading official recommended flat config when available
|
|
99
|
-
if (Array.isArray(playwrightRecommended)) {
|
|
100
|
-
const configs = playwrightRecommended;
|
|
101
|
-
if (profile === 'guardian') {
|
|
102
|
-
return [
|
|
103
|
-
...configs,
|
|
104
|
-
{
|
|
105
|
-
plugins: { guardian: guardianPlugin },
|
|
106
|
-
rules: { ...guardianRuleConfigs },
|
|
107
|
-
},
|
|
108
|
-
// our severity tweaks
|
|
109
|
-
base[0],
|
|
110
|
-
];
|
|
111
|
-
}
|
|
112
|
-
return [...configs, base[0]];
|
|
113
|
-
}
|
|
114
118
|
return base;
|
|
115
119
|
}
|
|
116
120
|
function toSeverity(eslintSeverity, override) {
|
|
@@ -138,7 +142,7 @@ export async function runEslint(options) {
|
|
|
138
142
|
const cwd = options.files.length > 0
|
|
139
143
|
? commonAncestorDir(options.files)
|
|
140
144
|
: (options.cwd ?? process.cwd());
|
|
141
|
-
const overrideConfig = buildConfig(options.
|
|
145
|
+
const overrideConfig = buildConfig(options.assertFunctionNames ?? []);
|
|
142
146
|
// Try to attach typescript-eslint parser for .ts files
|
|
143
147
|
try {
|
|
144
148
|
const tsParser = await import('@typescript-eslint/parser');
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ScoreOptions, ScoreResult } from './types.js';
|
|
2
2
|
export type { ScoreResult, ScoreOptions, Finding, ProfileName } from './types.js';
|
|
3
|
-
export { computeScore, SQS_V1, locatorsScore, gradeFromScore } from './score-engine.js';
|
|
3
|
+
export { computeScore, SQS_V1, locatorsScore, gradeFromScore, penaltyDimensionScore, applyPerRuleCap, } from './score-engine.js';
|
|
4
4
|
export { countSloc } from './sloc.js';
|
|
5
5
|
export { countLocators, analyzeSource } from './metrics.js';
|
|
6
6
|
export { formatJson } from './formatters/json.js';
|
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { collectLocallyImportedFiles } from './import-graph.js';
|
|
|
7
7
|
import { analyzeSource, countLocators, findLocalAssertionHelperNames, looksLikeNonPlaywrightTest, } from './metrics.js';
|
|
8
8
|
import { DEFAULT_THRESHOLDS } from './profiles.js';
|
|
9
9
|
import { computeScore } from './score-engine.js';
|
|
10
|
-
export { computeScore, SQS_V1, locatorsScore, gradeFromScore } from './score-engine.js';
|
|
10
|
+
export { computeScore, SQS_V1, locatorsScore, gradeFromScore, penaltyDimensionScore, applyPerRuleCap, } from './score-engine.js';
|
|
11
11
|
export { countSloc } from './sloc.js';
|
|
12
12
|
export { countLocators, analyzeSource } from './metrics.js';
|
|
13
13
|
export { formatJson } from './formatters/json.js';
|
|
@@ -132,7 +132,6 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
|
132
132
|
assertions: 0,
|
|
133
133
|
locators: 0,
|
|
134
134
|
structure: 0,
|
|
135
|
-
...(profile === 'guardian' ? { guardianConventions: 0 } : {}),
|
|
136
135
|
},
|
|
137
136
|
findings,
|
|
138
137
|
...(skippedFiles && skippedFiles.length > 0 ? { skippedFiles } : {}),
|
|
@@ -242,7 +241,6 @@ export async function scorePaths(options) {
|
|
|
242
241
|
totalTests += m.tests;
|
|
243
242
|
native += m.locators.native;
|
|
244
243
|
raw += m.locators.raw;
|
|
245
|
-
// metrics/no-empty-test duplicates guardian/require-expect under guardian — keep both mild
|
|
246
244
|
metricFindings.push(...m.findings);
|
|
247
245
|
}
|
|
248
246
|
for (const [file, source] of importedFiles) {
|
|
@@ -250,7 +248,6 @@ export async function scorePaths(options) {
|
|
|
250
248
|
native += counts.native;
|
|
251
249
|
raw += counts.raw;
|
|
252
250
|
}
|
|
253
|
-
// Dedupe empty-test style: if guardian require-expect already fired, still ok
|
|
254
251
|
const findings = [...eslintFindings, ...metricFindings];
|
|
255
252
|
const parseErrors = findings.filter((f) => f.rule === 'playwright-score/parse-error');
|
|
256
253
|
if (parseErrors.length > 0) {
|
package/dist/profiles.js
CHANGED
|
@@ -5,17 +5,9 @@ export const PROFILE_WEIGHTS = {
|
|
|
5
5
|
locators: 20,
|
|
6
6
|
structure: 15,
|
|
7
7
|
},
|
|
8
|
-
guardian: {
|
|
9
|
-
playwrightHygiene: 30,
|
|
10
|
-
assertions: 15,
|
|
11
|
-
locators: 15,
|
|
12
|
-
structure: 15,
|
|
13
|
-
guardianConventions: 25,
|
|
14
|
-
},
|
|
15
8
|
};
|
|
16
9
|
export const DEFAULT_THRESHOLDS = {
|
|
17
10
|
standard: 80,
|
|
18
|
-
guardian: 75,
|
|
19
11
|
};
|
|
20
12
|
/**
|
|
21
13
|
* ESLint rule → dimension map (plugin rule ids without prefix handled in runner).
|
|
@@ -80,26 +72,6 @@ export const ESLINT_RULE_MAP = {
|
|
|
80
72
|
dimension: 'structure',
|
|
81
73
|
severityOverride: 'warning',
|
|
82
74
|
},
|
|
83
|
-
// Guardian custom rules
|
|
84
|
-
'guardian/no-wait-for-load-state': { dimension: 'guardianConventions' },
|
|
85
|
-
'guardian/no-date-now-id': { dimension: 'guardianConventions' },
|
|
86
|
-
'guardian/require-expect': { dimension: 'guardianConventions' },
|
|
87
|
-
'guardian/no-hardcoded-secrets': {
|
|
88
|
-
dimension: 'guardianConventions',
|
|
89
|
-
severityOverride: 'warning',
|
|
90
|
-
},
|
|
91
|
-
'guardian/require-test-step': {
|
|
92
|
-
dimension: 'guardianConventions',
|
|
93
|
-
severityOverride: 'warning',
|
|
94
|
-
},
|
|
95
|
-
'guardian/one-describe-one-test': {
|
|
96
|
-
dimension: 'guardianConventions',
|
|
97
|
-
severityOverride: 'warning',
|
|
98
|
-
},
|
|
99
|
-
'guardian/no-generic-long-timeout': {
|
|
100
|
-
dimension: 'guardianConventions',
|
|
101
|
-
severityOverride: 'warning',
|
|
102
|
-
},
|
|
103
75
|
// Metrics injected as synthetic findings
|
|
104
76
|
'metrics/oversized-file': {
|
|
105
77
|
dimension: 'structure',
|
|
@@ -109,9 +81,6 @@ export const ESLINT_RULE_MAP = {
|
|
|
109
81
|
export function mapRule(ruleId) {
|
|
110
82
|
if (ESLINT_RULE_MAP[ruleId])
|
|
111
83
|
return ESLINT_RULE_MAP[ruleId];
|
|
112
|
-
if (ruleId.startsWith('guardian/')) {
|
|
113
|
-
return { dimension: 'guardianConventions', severityOverride: 'warning' };
|
|
114
|
-
}
|
|
115
84
|
if (ruleId.startsWith('playwright/')) {
|
|
116
85
|
return { dimension: 'playwrightHygiene', severityOverride: 'warning' };
|
|
117
86
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* Corrections to upstream eslint-plugin-playwright behavior, not house
|
|
3
|
+
* style — always active.
|
|
4
4
|
*/
|
|
5
5
|
import type { Rule } from 'eslint';
|
|
6
6
|
export declare const basePlugin: {
|
package/dist/score-engine.js
CHANGED
|
@@ -10,12 +10,7 @@ export const SQS_V1 = {
|
|
|
10
10
|
MIN_SLOTS: 4,
|
|
11
11
|
K: 0.4,
|
|
12
12
|
};
|
|
13
|
-
const PENALTY_DIMENSIONS = [
|
|
14
|
-
'playwrightHygiene',
|
|
15
|
-
'assertions',
|
|
16
|
-
'structure',
|
|
17
|
-
'guardianConventions',
|
|
18
|
-
];
|
|
13
|
+
const PENALTY_DIMENSIONS = ['playwrightHygiene', 'assertions', 'structure'];
|
|
19
14
|
export function gradeFromScore(score) {
|
|
20
15
|
if (score >= 90)
|
|
21
16
|
return 'A';
|
|
@@ -69,16 +64,18 @@ export function penaltyDimensionScore(findings, dimension, sloc) {
|
|
|
69
64
|
return Math.min(100, Math.max(0, score));
|
|
70
65
|
}
|
|
71
66
|
export function computeScore(input) {
|
|
72
|
-
|
|
67
|
+
// Falls back to `standard`'s weights for any unrecognized profile string
|
|
68
|
+
// rather than throwing — a defensive guard against exactly the failure
|
|
69
|
+
// mode a profile removal creates: a caller pinned to an old contract
|
|
70
|
+
// (e.g. `profile: 'guardian'`, removed in 0.2.0) degrades to a working
|
|
71
|
+
// score instead of a hard crash.
|
|
72
|
+
const weights = PROFILE_WEIGHTS[input.profile] ?? PROFILE_WEIGHTS.standard;
|
|
73
73
|
const dims = {
|
|
74
74
|
playwrightHygiene: penaltyDimensionScore(input.findings, 'playwrightHygiene', input.sloc),
|
|
75
75
|
assertions: penaltyDimensionScore(input.findings, 'assertions', input.sloc),
|
|
76
76
|
locators: locatorsScore(input.nativeLocators, input.rawLocators),
|
|
77
77
|
structure: penaltyDimensionScore(input.findings, 'structure', input.sloc),
|
|
78
78
|
};
|
|
79
|
-
if (input.profile === 'guardian') {
|
|
80
|
-
dims.guardianConventions = penaltyDimensionScore(input.findings, 'guardianConventions', input.sloc);
|
|
81
|
-
}
|
|
82
79
|
let score = 0;
|
|
83
80
|
for (const [dim, weight] of Object.entries(weights)) {
|
|
84
81
|
const key = dim;
|
package/dist/types.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export type ScoreVersion = 'sqs-v1';
|
|
2
|
-
export type ProfileName = 'standard'
|
|
2
|
+
export type ProfileName = 'standard';
|
|
3
3
|
export type Grade = 'A' | 'B' | 'C' | 'D' | 'F';
|
|
4
4
|
export type Severity = 'error' | 'warning' | 'info';
|
|
5
|
-
export type DimensionName = 'playwrightHygiene' | 'assertions' | 'locators' | 'structure'
|
|
5
|
+
export type DimensionName = 'playwrightHygiene' | 'assertions' | 'locators' | 'structure';
|
|
6
6
|
export interface Finding {
|
|
7
7
|
rule: string;
|
|
8
8
|
severity: Severity;
|
|
@@ -19,7 +19,6 @@ export interface ScoreDimensions {
|
|
|
19
19
|
assertions: number;
|
|
20
20
|
locators: number;
|
|
21
21
|
structure: number;
|
|
22
|
-
guardianConventions?: number;
|
|
23
22
|
}
|
|
24
23
|
export interface ScoreSummary {
|
|
25
24
|
files: number;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qaguardian/playwright-score",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Deterministic, AI-free Playwright spec quality score (sqs-v1).
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Deterministic, AI-free Playwright spec quality score (sqs-v1).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
@@ -26,7 +26,6 @@
|
|
|
26
26
|
"build:tests": "tsc -p tsconfig.tests.json",
|
|
27
27
|
"test": "npm run build && npm run build:tests && node --test dist-tests/tests/*.js",
|
|
28
28
|
"score": "npm run build && node bin/playwright-score.js",
|
|
29
|
-
"dogfood": "npm run build && bash scripts/dogfood.sh",
|
|
30
29
|
"prepublishOnly": "npm run build"
|
|
31
30
|
},
|
|
32
31
|
"keywords": [
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Guardian house-style ESLint rules for profile=guardian.
|
|
3
|
-
* Embedded in playwright-score — not claimed as official Playwright doctrine.
|
|
4
|
-
*/
|
|
5
|
-
import type { Rule } from 'eslint';
|
|
6
|
-
export declare const guardianPlugin: {
|
|
7
|
-
meta: {
|
|
8
|
-
name: string;
|
|
9
|
-
version: string;
|
|
10
|
-
};
|
|
11
|
-
rules: {
|
|
12
|
-
'no-wait-for-load-state': Rule.RuleModule;
|
|
13
|
-
'no-date-now-id': Rule.RuleModule;
|
|
14
|
-
'no-hardcoded-secrets': Rule.RuleModule;
|
|
15
|
-
'no-generic-long-timeout': Rule.RuleModule;
|
|
16
|
-
'one-describe-one-test': Rule.RuleModule;
|
|
17
|
-
'require-test-step': Rule.RuleModule;
|
|
18
|
-
'require-expect': Rule.RuleModule;
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
export declare const guardianRuleConfigs: {
|
|
22
|
-
readonly 'guardian/no-wait-for-load-state': "error";
|
|
23
|
-
readonly 'guardian/no-date-now-id': "error";
|
|
24
|
-
readonly 'guardian/no-hardcoded-secrets': "warn";
|
|
25
|
-
readonly 'guardian/no-generic-long-timeout': "warn";
|
|
26
|
-
readonly 'guardian/one-describe-one-test': "warn";
|
|
27
|
-
readonly 'guardian/require-test-step': "warn";
|
|
28
|
-
readonly 'guardian/require-expect': "error";
|
|
29
|
-
};
|
|
@@ -1,386 +0,0 @@
|
|
|
1
|
-
const DUMMY_SECRETS = new Set([
|
|
2
|
-
'password',
|
|
3
|
-
'password123',
|
|
4
|
-
'test',
|
|
5
|
-
'test123',
|
|
6
|
-
'secret',
|
|
7
|
-
'changeme',
|
|
8
|
-
'admin',
|
|
9
|
-
'admin123',
|
|
10
|
-
'pass',
|
|
11
|
-
'pwd',
|
|
12
|
-
'123456',
|
|
13
|
-
'qwerty',
|
|
14
|
-
'',
|
|
15
|
-
].map((s) => s.toLowerCase()));
|
|
16
|
-
// Playwright's actual modifier API is test.fixme() — not test.fix().
|
|
17
|
-
const TEST_MODIFIERS = new Set(['only', 'skip', 'fixme', 'slow']);
|
|
18
|
-
const DESCRIBE_MODIFIERS = new Set(['serial', 'parallel', 'only', 'fixme']);
|
|
19
|
-
/** Matches a bare test(...) call. */
|
|
20
|
-
function isBareTestCall(node) {
|
|
21
|
-
return node.callee?.type === 'Identifier' && node.callee.name === 'test';
|
|
22
|
-
}
|
|
23
|
-
/** Matches test.describe(...) and test.describe.serial/parallel/only/fixme(...). */
|
|
24
|
-
function isDescribeCall(node) {
|
|
25
|
-
const callee = node.callee;
|
|
26
|
-
if (!callee || callee.type !== 'MemberExpression' || callee.computed)
|
|
27
|
-
return false;
|
|
28
|
-
const obj = callee.object;
|
|
29
|
-
const prop = callee.property;
|
|
30
|
-
if (!obj || !prop || prop.type !== 'Identifier')
|
|
31
|
-
return false;
|
|
32
|
-
// test.describe(...)
|
|
33
|
-
if (obj.type === 'Identifier' && obj.name === 'test' && prop.name === 'describe') {
|
|
34
|
-
return true;
|
|
35
|
-
}
|
|
36
|
-
// test.describe.serial/parallel/only/fixme(...)
|
|
37
|
-
if (obj.type === 'MemberExpression' &&
|
|
38
|
-
!obj.computed &&
|
|
39
|
-
obj.object?.type === 'Identifier' &&
|
|
40
|
-
obj.object.name === 'test' &&
|
|
41
|
-
obj.property?.type === 'Identifier' &&
|
|
42
|
-
obj.property.name === 'describe' &&
|
|
43
|
-
DESCRIBE_MODIFIERS.has(prop.name ?? '')) {
|
|
44
|
-
return true;
|
|
45
|
-
}
|
|
46
|
-
return false;
|
|
47
|
-
}
|
|
48
|
-
/** Matches specifically test.describe.serial(...). */
|
|
49
|
-
function isSerialDescribeCall(node) {
|
|
50
|
-
const callee = node.callee;
|
|
51
|
-
if (!callee || callee.type !== 'MemberExpression' || callee.computed)
|
|
52
|
-
return false;
|
|
53
|
-
const obj = callee.object;
|
|
54
|
-
const prop = callee.property;
|
|
55
|
-
return (obj?.type === 'MemberExpression' &&
|
|
56
|
-
!obj.computed &&
|
|
57
|
-
obj.object?.type === 'Identifier' &&
|
|
58
|
-
obj.object.name === 'test' &&
|
|
59
|
-
obj.property?.type === 'Identifier' &&
|
|
60
|
-
obj.property.name === 'describe' &&
|
|
61
|
-
prop?.type === 'Identifier' &&
|
|
62
|
-
prop.name === 'serial');
|
|
63
|
-
}
|
|
64
|
-
/** Matches test.only/skip/fixme/slow(...) (a modifier form of a real test declaration). */
|
|
65
|
-
function isTestModifierCall(node) {
|
|
66
|
-
const callee = node.callee;
|
|
67
|
-
if (!callee || callee.type !== 'MemberExpression' || callee.computed)
|
|
68
|
-
return false;
|
|
69
|
-
const obj = callee.object;
|
|
70
|
-
const prop = callee.property;
|
|
71
|
-
return (obj?.type === 'Identifier' &&
|
|
72
|
-
obj.name === 'test' &&
|
|
73
|
-
prop?.type === 'Identifier' &&
|
|
74
|
-
TEST_MODIFIERS.has(prop.name ?? ''));
|
|
75
|
-
}
|
|
76
|
-
/** Matches test.step(...). */
|
|
77
|
-
function isTestStepCall(node) {
|
|
78
|
-
const callee = node.callee;
|
|
79
|
-
return (callee?.type === 'MemberExpression' &&
|
|
80
|
-
!callee.computed &&
|
|
81
|
-
callee.object?.type === 'Identifier' &&
|
|
82
|
-
callee.object.name === 'test' &&
|
|
83
|
-
callee.property?.type === 'Identifier' &&
|
|
84
|
-
callee.property.name === 'step');
|
|
85
|
-
}
|
|
86
|
-
function hasDisableNextLine(sourceCode, line, ruleId) {
|
|
87
|
-
const lines = sourceCode.getText().split(/\r?\n/);
|
|
88
|
-
if (line < 2)
|
|
89
|
-
return false;
|
|
90
|
-
const prev = lines[line - 2] ?? '';
|
|
91
|
-
return (prev.includes('spec-score-disable-next-line') &&
|
|
92
|
-
(prev.includes(ruleId) || prev.includes('guardian/')));
|
|
93
|
-
}
|
|
94
|
-
const noWaitForLoadState = {
|
|
95
|
-
meta: {
|
|
96
|
-
type: 'problem',
|
|
97
|
-
docs: { description: 'Disallow page.waitForLoadState()' },
|
|
98
|
-
messages: {
|
|
99
|
-
banned: 'waitForLoadState is banned under guardian profile. Use web-first assertions.',
|
|
100
|
-
},
|
|
101
|
-
schema: [],
|
|
102
|
-
},
|
|
103
|
-
create(context) {
|
|
104
|
-
return {
|
|
105
|
-
CallExpression(node) {
|
|
106
|
-
const callee = node.callee;
|
|
107
|
-
if (callee.type === 'MemberExpression' &&
|
|
108
|
-
!callee.computed &&
|
|
109
|
-
callee.property.type === 'Identifier' &&
|
|
110
|
-
callee.property.name === 'waitForLoadState') {
|
|
111
|
-
context.report({ node, messageId: 'banned' });
|
|
112
|
-
}
|
|
113
|
-
},
|
|
114
|
-
};
|
|
115
|
-
},
|
|
116
|
-
};
|
|
117
|
-
const noDateNowId = {
|
|
118
|
-
meta: {
|
|
119
|
-
type: 'problem',
|
|
120
|
-
docs: { description: 'Disallow Date.now() for unique ids' },
|
|
121
|
-
messages: {
|
|
122
|
-
banned: 'Date.now() is banned for unique ids. Use crypto.randomUUID().split("-")[0].',
|
|
123
|
-
},
|
|
124
|
-
schema: [],
|
|
125
|
-
},
|
|
126
|
-
create(context) {
|
|
127
|
-
return {
|
|
128
|
-
CallExpression(node) {
|
|
129
|
-
const callee = node.callee;
|
|
130
|
-
if (callee.type === 'MemberExpression' &&
|
|
131
|
-
!callee.computed &&
|
|
132
|
-
callee.object.type === 'Identifier' &&
|
|
133
|
-
callee.object.name === 'Date' &&
|
|
134
|
-
callee.property.type === 'Identifier' &&
|
|
135
|
-
callee.property.name === 'now') {
|
|
136
|
-
context.report({ node, messageId: 'banned' });
|
|
137
|
-
}
|
|
138
|
-
},
|
|
139
|
-
};
|
|
140
|
-
},
|
|
141
|
-
};
|
|
142
|
-
// Whole words only (after splitting camelCase/snake_case/kebab-case), not a
|
|
143
|
-
// bare substring match — a substring match on "auth" flags "authorName",
|
|
144
|
-
// "isAuthenticated", every "Author"/"authenticate"/"authorize" field in a
|
|
145
|
-
// typical app under test, which is a lot of unrelated real-world test data.
|
|
146
|
-
const SENSITIVE_WORDS = new Set([
|
|
147
|
-
'password',
|
|
148
|
-
'passwd',
|
|
149
|
-
'secret',
|
|
150
|
-
'secrets',
|
|
151
|
-
'token',
|
|
152
|
-
'tokens',
|
|
153
|
-
'auth',
|
|
154
|
-
'authorization',
|
|
155
|
-
'credential',
|
|
156
|
-
'credentials',
|
|
157
|
-
]);
|
|
158
|
-
function splitIdentifierWords(identifier) {
|
|
159
|
-
return identifier
|
|
160
|
-
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
161
|
-
.replace(/[_-]+/g, ' ')
|
|
162
|
-
.split(/\s+/)
|
|
163
|
-
.filter(Boolean)
|
|
164
|
-
.map((w) => w.toLowerCase());
|
|
165
|
-
}
|
|
166
|
-
function looksLikeSensitiveKey(rawKey) {
|
|
167
|
-
const words = splitIdentifierWords(rawKey);
|
|
168
|
-
if (words.some((w) => SENSITIVE_WORDS.has(w)))
|
|
169
|
-
return true;
|
|
170
|
-
// apiKey / api_key / API-KEY — "key" alone is too generic (primaryKey,
|
|
171
|
-
// sortKey, ...) so only flag it directly adjacent to "api".
|
|
172
|
-
for (let i = 0; i < words.length - 1; i++) {
|
|
173
|
-
if (words[i] === 'api' && words[i + 1] === 'key')
|
|
174
|
-
return true;
|
|
175
|
-
}
|
|
176
|
-
return false;
|
|
177
|
-
}
|
|
178
|
-
const noHardcodedSecrets = {
|
|
179
|
-
meta: {
|
|
180
|
-
type: 'suggestion',
|
|
181
|
-
docs: { description: 'Flag likely hardcoded secrets (with allowlist)' },
|
|
182
|
-
messages: {
|
|
183
|
-
secret: 'Possible hardcoded secret value. Use env/fixtures or allowlist dummy data.',
|
|
184
|
-
},
|
|
185
|
-
schema: [],
|
|
186
|
-
},
|
|
187
|
-
create(context) {
|
|
188
|
-
const sourceCode = context.sourceCode ?? context.getSourceCode();
|
|
189
|
-
function check(node, rawKey) {
|
|
190
|
-
if (node.type !== 'Literal' || typeof node.value !== 'string')
|
|
191
|
-
return;
|
|
192
|
-
const value = String(node.value);
|
|
193
|
-
if (DUMMY_SECRETS.has(value.toLowerCase()))
|
|
194
|
-
return;
|
|
195
|
-
if (value.length < 6)
|
|
196
|
-
return;
|
|
197
|
-
if (!looksLikeSensitiveKey(rawKey ?? ''))
|
|
198
|
-
return;
|
|
199
|
-
const line = node.loc?.start.line ?? 1;
|
|
200
|
-
if (hasDisableNextLine(sourceCode, line, 'guardian/no-hardcoded-secrets'))
|
|
201
|
-
return;
|
|
202
|
-
context.report({ node: node, messageId: 'secret' });
|
|
203
|
-
}
|
|
204
|
-
return {
|
|
205
|
-
Property(node) {
|
|
206
|
-
if (node.key.type === 'Identifier') {
|
|
207
|
-
check(node.value, node.key.name);
|
|
208
|
-
}
|
|
209
|
-
else if (node.key.type === 'Literal') {
|
|
210
|
-
check(node.value, String(node.key.value));
|
|
211
|
-
}
|
|
212
|
-
},
|
|
213
|
-
VariableDeclarator(node) {
|
|
214
|
-
if (node.id.type === 'Identifier' && node.init) {
|
|
215
|
-
check(node.init, node.id.name);
|
|
216
|
-
}
|
|
217
|
-
},
|
|
218
|
-
};
|
|
219
|
-
},
|
|
220
|
-
};
|
|
221
|
-
const noGenericLongTimeout = {
|
|
222
|
-
meta: {
|
|
223
|
-
type: 'suggestion',
|
|
224
|
-
docs: { description: 'Only allow timeout 2000 or 20000' },
|
|
225
|
-
messages: {
|
|
226
|
-
timeout: 'Avoid generic timeouts; allowed values under guardian: 2000 (audit) or 20000 (backend).',
|
|
227
|
-
},
|
|
228
|
-
schema: [],
|
|
229
|
-
},
|
|
230
|
-
create(context) {
|
|
231
|
-
const allowed = new Set([2000, 20000]);
|
|
232
|
-
return {
|
|
233
|
-
Property(node) {
|
|
234
|
-
if (node.key.type !== 'Identifier' ||
|
|
235
|
-
node.key.name !== 'timeout' ||
|
|
236
|
-
node.value.type !== 'Literal' ||
|
|
237
|
-
typeof node.value.value !== 'number' ||
|
|
238
|
-
allowed.has(node.value.value)) {
|
|
239
|
-
return;
|
|
240
|
-
}
|
|
241
|
-
// Only flag timeout options passed inline to a call, e.g.
|
|
242
|
-
// locator.click({ timeout }), expect(x).toBeVisible({ timeout }),
|
|
243
|
-
// test('x', { timeout }, fn) — not unrelated business data that
|
|
244
|
-
// happens to have a field named "timeout" (session config, a
|
|
245
|
-
// fixture payload, a mocked API response body, ...).
|
|
246
|
-
const objectExpr = node.parent;
|
|
247
|
-
const call = objectExpr?.parent;
|
|
248
|
-
if (objectExpr?.type !== 'ObjectExpression' ||
|
|
249
|
-
call?.type !== 'CallExpression' ||
|
|
250
|
-
!call.arguments?.includes(objectExpr)) {
|
|
251
|
-
return;
|
|
252
|
-
}
|
|
253
|
-
context.report({ node, messageId: 'timeout' });
|
|
254
|
-
},
|
|
255
|
-
};
|
|
256
|
-
},
|
|
257
|
-
};
|
|
258
|
-
const oneDescribeOneTest = {
|
|
259
|
-
meta: {
|
|
260
|
-
type: 'suggestion',
|
|
261
|
-
docs: { description: 'Prefer one test.describe and one test per file' },
|
|
262
|
-
messages: {
|
|
263
|
-
describes: 'Guardian profile prefers exactly one test.describe() per file (found {{count}}).',
|
|
264
|
-
tests: 'Guardian profile prefers exactly one test() per file (found {{count}}).',
|
|
265
|
-
},
|
|
266
|
-
schema: [],
|
|
267
|
-
},
|
|
268
|
-
create(context) {
|
|
269
|
-
let describes = 0;
|
|
270
|
-
let tests = 0;
|
|
271
|
-
let usesSerialDescribe = false;
|
|
272
|
-
return {
|
|
273
|
-
CallExpression(node) {
|
|
274
|
-
if (isDescribeCall(node)) {
|
|
275
|
-
describes++;
|
|
276
|
-
if (isSerialDescribeCall(node))
|
|
277
|
-
usesSerialDescribe = true;
|
|
278
|
-
}
|
|
279
|
-
if (isBareTestCall(node) || isTestModifierCall(node))
|
|
280
|
-
tests++;
|
|
281
|
-
},
|
|
282
|
-
'Program:exit'() {
|
|
283
|
-
if (describes > 1) {
|
|
284
|
-
context.report({
|
|
285
|
-
loc: { line: 1, column: 0 },
|
|
286
|
-
messageId: 'describes',
|
|
287
|
-
data: { count: String(describes) },
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
// test.describe.serial(...) is Playwright's documented mechanism
|
|
291
|
-
// for one user journey split into multiple ordered, dependent
|
|
292
|
-
// test() calls — the same "one scenario per file" intent as this
|
|
293
|
-
// rule, expressed a different way. Don't nag a pattern the
|
|
294
|
-
// framework itself recommends for exactly this case.
|
|
295
|
-
if (tests > 1 && !usesSerialDescribe) {
|
|
296
|
-
context.report({
|
|
297
|
-
loc: { line: 1, column: 0 },
|
|
298
|
-
messageId: 'tests',
|
|
299
|
-
data: { count: String(tests) },
|
|
300
|
-
});
|
|
301
|
-
}
|
|
302
|
-
},
|
|
303
|
-
};
|
|
304
|
-
},
|
|
305
|
-
};
|
|
306
|
-
const requireTestStep = {
|
|
307
|
-
meta: {
|
|
308
|
-
type: 'suggestion',
|
|
309
|
-
docs: { description: 'Warn when file uses test() but no test.step()' },
|
|
310
|
-
messages: {
|
|
311
|
-
missing: 'Consider wrapping actions in test.step() for clearer structure.',
|
|
312
|
-
},
|
|
313
|
-
schema: [],
|
|
314
|
-
},
|
|
315
|
-
create(context) {
|
|
316
|
-
let hasTest = false;
|
|
317
|
-
let hasStep = false;
|
|
318
|
-
return {
|
|
319
|
-
CallExpression(node) {
|
|
320
|
-
if (isBareTestCall(node) || isTestModifierCall(node))
|
|
321
|
-
hasTest = true;
|
|
322
|
-
if (isTestStepCall(node))
|
|
323
|
-
hasStep = true;
|
|
324
|
-
},
|
|
325
|
-
'Program:exit'() {
|
|
326
|
-
if (hasTest && !hasStep) {
|
|
327
|
-
context.report({ loc: { line: 1, column: 0 }, messageId: 'missing' });
|
|
328
|
-
}
|
|
329
|
-
},
|
|
330
|
-
};
|
|
331
|
-
},
|
|
332
|
-
};
|
|
333
|
-
const requireExpect = {
|
|
334
|
-
meta: {
|
|
335
|
-
type: 'problem',
|
|
336
|
-
docs: { description: 'Require expect() in files with test()' },
|
|
337
|
-
messages: {
|
|
338
|
-
missing: 'Guardian profile requires at least one expect() in the spec.',
|
|
339
|
-
},
|
|
340
|
-
schema: [],
|
|
341
|
-
},
|
|
342
|
-
create(context) {
|
|
343
|
-
let hasTest = false;
|
|
344
|
-
let hasExpect = false;
|
|
345
|
-
return {
|
|
346
|
-
CallExpression(node) {
|
|
347
|
-
if (isBareTestCall(node) || isTestModifierCall(node))
|
|
348
|
-
hasTest = true;
|
|
349
|
-
if (node.callee?.type === 'Identifier' && node.callee.name === 'expect') {
|
|
350
|
-
hasExpect = true;
|
|
351
|
-
}
|
|
352
|
-
if (node.callee?.type === 'MemberExpression' &&
|
|
353
|
-
node.callee.object?.type === 'Identifier' &&
|
|
354
|
-
node.callee.object.name === 'expect') {
|
|
355
|
-
hasExpect = true;
|
|
356
|
-
}
|
|
357
|
-
},
|
|
358
|
-
'Program:exit'() {
|
|
359
|
-
if (hasTest && !hasExpect) {
|
|
360
|
-
context.report({ loc: { line: 1, column: 0 }, messageId: 'missing' });
|
|
361
|
-
}
|
|
362
|
-
},
|
|
363
|
-
};
|
|
364
|
-
},
|
|
365
|
-
};
|
|
366
|
-
export const guardianPlugin = {
|
|
367
|
-
meta: { name: 'guardian', version: '0.1.0' },
|
|
368
|
-
rules: {
|
|
369
|
-
'no-wait-for-load-state': noWaitForLoadState,
|
|
370
|
-
'no-date-now-id': noDateNowId,
|
|
371
|
-
'no-hardcoded-secrets': noHardcodedSecrets,
|
|
372
|
-
'no-generic-long-timeout': noGenericLongTimeout,
|
|
373
|
-
'one-describe-one-test': oneDescribeOneTest,
|
|
374
|
-
'require-test-step': requireTestStep,
|
|
375
|
-
'require-expect': requireExpect,
|
|
376
|
-
},
|
|
377
|
-
};
|
|
378
|
-
export const guardianRuleConfigs = {
|
|
379
|
-
'guardian/no-wait-for-load-state': 'error',
|
|
380
|
-
'guardian/no-date-now-id': 'error',
|
|
381
|
-
'guardian/no-hardcoded-secrets': 'warn',
|
|
382
|
-
'guardian/no-generic-long-timeout': 'warn',
|
|
383
|
-
'guardian/one-describe-one-test': 'warn',
|
|
384
|
-
'guardian/require-test-step': 'warn',
|
|
385
|
-
'guardian/require-expect': 'error',
|
|
386
|
-
};
|