@qaguardian/playwright-score 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METHODOLOGY.md +92 -164
- package/README.md +20 -19
- package/dist/cli.js +2 -2
- package/dist/eslint-runner.js +2 -2
- package/dist/formatters/text.js +2 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +20 -6
- package/dist/metrics.d.ts +37 -0
- package/dist/metrics.js +108 -0
- package/dist/score-engine.d.ts +42 -45
- package/dist/score-engine.js +116 -85
- package/dist/types.d.ts +13 -4
- package/package.json +2 -2
package/METHODOLOGY.md
CHANGED
|
@@ -1,217 +1,145 @@
|
|
|
1
|
-
# Playwright Spec Score
|
|
1
|
+
# Playwright Spec Score Methodology
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**Scoring model:** v3 (2026-08-27)
|
|
4
4
|
**Package:** `playwright-score`
|
|
5
5
|
**Landing page:** https://qaguardian.com/open-source/playwright-score
|
|
6
6
|
**Maintainer:** [QA Guardian](https://qaguardian.com)
|
|
7
7
|
**Deterministic:** same inputs → same score
|
|
8
8
|
**AI-free:** no LLM calls in the scoring path
|
|
9
9
|
|
|
10
|
-
Any change
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
finding-density penalty (see the Assertions dimension section). All other
|
|
15
|
-
formulas, weights, and constants are unchanged from sqs-v1.
|
|
10
|
+
Any change that alters what a given suite scores is a new model version
|
|
11
|
+
(v4, …), documented in the CHANGELOG. Model history: v1/v2 (published as
|
|
12
|
+
"sqs-v1"/"sqs-v2" through package 0.4.0) scored finding *density per line
|
|
13
|
+
of code*; v3 replaced that entirely — see "Why v3" at the bottom.
|
|
16
14
|
|
|
17
15
|
---
|
|
18
16
|
|
|
19
|
-
##
|
|
17
|
+
## The model in one sentence
|
|
20
18
|
|
|
21
|
-
|
|
22
|
-
(`eslint-plugin-playwright`) plus suite-level metrics (locator ratio,
|
|
23
|
-
assertion-delegation tracing, Page Object Model import resolution).
|
|
19
|
+
**Your score is the weighted share of your tests that are clean.**
|
|
24
20
|
|
|
25
21
|
---
|
|
26
22
|
|
|
27
|
-
##
|
|
23
|
+
## How it works
|
|
28
24
|
|
|
29
|
-
|
|
30
|
-
- **grade:** A (90–100), B (80–89), C (70–79), D (60–69), F (0–59)
|
|
31
|
-
- **pass:** `score >= threshold`
|
|
32
|
-
- **dimensions:** per-dimension 0–100
|
|
33
|
-
- **findings:** full list (uncapped) for humans/agents
|
|
25
|
+
### 1. Findings are attributed to tests
|
|
34
26
|
|
|
35
|
-
|
|
27
|
+
Every file is parsed and each test declaration's source span is recorded
|
|
28
|
+
(`test(...)`, `test.only/skip/fixme/slow(...)` — any form carrying a
|
|
29
|
+
function body). Each lint/metric finding is then attributed to:
|
|
36
30
|
|
|
37
|
-
|
|
31
|
+
- **the test it sits in**, when its line falls inside a test span;
|
|
32
|
+
- **every test in the file**, when it sits in a `beforeEach`/`afterEach`/
|
|
33
|
+
`beforeAll`/`afterAll` hook — setup problems affect every test that runs
|
|
34
|
+
through them;
|
|
35
|
+
- **the file, once** (module scope), for anything else — imports,
|
|
36
|
+
`describe` bodies, file-level metrics like oversized-file.
|
|
38
37
|
|
|
39
|
-
|
|
40
|
-
|---|---|
|
|
41
|
-
| playwrightHygiene | 40 |
|
|
42
|
-
| assertions | 25 |
|
|
43
|
-
| locators | 20 |
|
|
44
|
-
| structure | 15 |
|
|
38
|
+
### 2. Tests accumulate demerits
|
|
45
39
|
|
|
46
|
-
|
|
40
|
+
Within each scored dimension, a test's demerit is the sum of its findings'
|
|
41
|
+
units, capped at 1:
|
|
47
42
|
|
|
48
|
-
|
|
43
|
+
| Severity | Demerit |
|
|
44
|
+
|---|---|
|
|
45
|
+
| error | 1.0 |
|
|
46
|
+
| warning | 0.4 |
|
|
47
|
+
| info / report-only | 0 |
|
|
49
48
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
dimension, the primary signal is a ratio:
|
|
49
|
+
A test is at worst *fully flawed* (demerit 1) — never more. That cap is
|
|
50
|
+
both the anti-nuke mechanism (one loop-generated repeated finding cannot
|
|
51
|
+
zero a suite) and the anti-gaming mechanism (there is no per-line density
|
|
52
|
+
left to dilute by padding files with clean code). Module-scope findings
|
|
53
|
+
are capped at 1 demerit per file per dimension.
|
|
56
54
|
|
|
57
|
-
|
|
58
|
-
unassertedTests = count(playwright/expect-expect findings) // UNCAPPED census
|
|
59
|
-
coverage = tests === 0 ? 1 : clamp01((tests - unassertedTests) / tests)
|
|
55
|
+
### 3. Dimensions are ratios
|
|
60
56
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
no-standalone-expect, prefer-web-first-assertions
|
|
64
|
-
|
|
65
|
-
assertionsScore = clamp(0, 100, round(100 * coverage * e^(-K * auxLoad)))
|
|
57
|
+
```
|
|
58
|
+
dimensionScore = round(100 × (1 − totalDemerit / tests)) // clamped 0–100
|
|
66
59
|
```
|
|
67
60
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
- `expect-expect` findings are excluded from the density term — they ARE
|
|
71
|
-
the coverage term; counting them twice would double-penalize.
|
|
72
|
-
- The census is deliberately uncapped: the per-(file, rule) cap exists to
|
|
73
|
-
stop one repeated smell from nuking a density score, but hiding 2 of 5
|
|
74
|
-
assertion-free tests would misstate coverage.
|
|
75
|
-
- All of expect-expect's delegation machinery (same-file helper discovery,
|
|
76
|
-
imported-helper tracing, `assertFunctionPatterns`) applies before the
|
|
77
|
-
census is taken, so delegated assertions do not count as unasserted.
|
|
78
|
-
- `tests === 0` (nothing matched but helpers) keeps coverage neutral at 1.
|
|
79
|
-
|
|
80
|
-
Worked example (the motivating case): 3 tests, 2 without assertions, no
|
|
81
|
-
aux findings → coverage = 1/3 → **assertions = 33** (sqs-v1 said 82).
|
|
82
|
-
|
|
83
|
-
---
|
|
84
|
-
|
|
85
|
-
## Locators dimension (ratio)
|
|
61
|
+
With zero tests the denominator is 1, so a findings-bearing zero-test
|
|
62
|
+
input still cannot score clean.
|
|
86
63
|
|
|
87
|
-
|
|
64
|
+
The **locators** dimension is independent of demerits — it is the usage
|
|
65
|
+
ratio it always was:
|
|
88
66
|
|
|
89
67
|
```
|
|
90
|
-
native =
|
|
91
|
-
raw =
|
|
92
|
-
total = native + raw
|
|
68
|
+
native = getByRole|getByLabel|getByTestId|getByText|getByPlaceholder|getByAltText|getByTitle calls
|
|
69
|
+
raw = .locator(...) calls + legacy string-selector actions (page.click('#x'), …)
|
|
93
70
|
|
|
94
|
-
locatorsScore = total === 0 ? 100 : round(100
|
|
71
|
+
locatorsScore = total === 0 ? 100 : round(100 × native / total)
|
|
95
72
|
```
|
|
96
73
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
74
|
+
(`no-raw-locators` / `prefer-native-locators` / `prefer-locator` findings
|
|
75
|
+
appear in `findings[]` for repair guidance but are report-only — the ratio
|
|
76
|
+
owns this dimension; counting them as demerits too would double-penalize.)
|
|
100
77
|
|
|
101
|
-
|
|
78
|
+
### 4. Weighted sum
|
|
102
79
|
|
|
103
|
-
|
|
80
|
+
| Dimension | Weight | Demerit sources |
|
|
81
|
+
|---|---:|---|
|
|
82
|
+
| playwrightHygiene | 40 | hard waits, networkidle, force, missing await, element handles, eval, conditionals, page.pause, useless await |
|
|
83
|
+
| assertions | 25 | a test with no recognized assertion is fully flawed (`expect-expect`, after all delegation resolution); `valid-expect`, `no-standalone-expect`, `prefer-web-first-assertions` add partial demerits |
|
|
84
|
+
| locators | 20 | usage ratio (above) |
|
|
85
|
+
| structure | 15 | focused tests, always-skipped declarations, nested-describe depth, oversized files (module scope) |
|
|
104
86
|
|
|
105
87
|
```
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
sloc = max(sloc, 1)
|
|
88
|
+
score = round(Σ (weight_d / 100) × dimensionScore_d)
|
|
89
|
+
pass = score >= threshold // CLI default 80
|
|
90
|
+
grade = A ≥90 · B ≥80 · C ≥70 · D ≥60 · F <60
|
|
110
91
|
```
|
|
111
92
|
|
|
112
|
-
Used for penalty density. JSDoc / decorative comments do not inflate slots.
|
|
113
|
-
|
|
114
93
|
---
|
|
115
94
|
|
|
116
|
-
##
|
|
117
|
-
|
|
118
|
-
Applies to: `playwrightHygiene`, `structure`, and the *aux* term of
|
|
119
|
-
`assertions` (see above; the assertions primary term is a coverage ratio).
|
|
95
|
+
## Worked example
|
|
120
96
|
|
|
121
|
-
|
|
97
|
+
3 tests. Test 1 has two hard-wait errors and no assertion. Test 2 is an
|
|
98
|
+
always-skipped declaration with a conditional and no assertion. Test 3 is
|
|
99
|
+
clean. Locators: 4 native, 4 raw.
|
|
122
100
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
| `MAX_FINDINGS_PER_RULE_PER_FILE` | 3 (for penalty math only) |
|
|
129
|
-
| `SLOT_DIVISOR` | 25 |
|
|
130
|
-
| `MIN_SLOTS` | 4 |
|
|
131
|
-
| `K` | 0.4 |
|
|
101
|
+
- **hygiene**: test 1 → min(1, 2×1.0) = 1; test 2 → 0.4 → total 1.4/3 → **53**
|
|
102
|
+
- **assertions**: tests 1, 2 unasserted → 2/3 → **33**
|
|
103
|
+
- **locators**: 4/8 → **50**
|
|
104
|
+
- **structure**: skip warning 0.4/3 → **87**
|
|
105
|
+
- **score** = 0.40·53 + 0.25·33 + 0.20·50 + 0.15·87 = **53 → F**
|
|
132
106
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
1. **Cap** findings per `(file, ruleId)` to 3 for penalty math (report all in `findings[]`).
|
|
136
|
-
2. **Units:** `rawUnits = E * 1.0 + W * 0.4`
|
|
137
|
-
3. **Slots:** `slots = max(sloc / 25, 4)`
|
|
138
|
-
4. **Load:** `load = rawUnits / slots`
|
|
139
|
-
5. **Score:** `dimensionScore = clamp(0, 100, round(100 * exp(-0.4 * load)))`
|
|
140
|
-
|
|
141
|
-
### Final score
|
|
142
|
-
|
|
143
|
-
```
|
|
144
|
-
score = round(Σ (weight_d / 100) * dimensionScore_d)
|
|
145
|
-
pass = score >= threshold
|
|
146
|
-
```
|
|
107
|
+
The same suite scored 78/C under the v1 density model. "Two of your three
|
|
108
|
+
tests are broken" should not read as a near-B.
|
|
147
109
|
|
|
148
110
|
---
|
|
149
111
|
|
|
150
|
-
##
|
|
112
|
+
## SLOC
|
|
151
113
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
- All penalty dims = 100
|
|
157
|
-
- Locators = 100
|
|
158
|
-
- **standard score ≈ 100**
|
|
159
|
-
|
|
160
|
-
### 2. Tiny file, 1 hygiene error (~20 SLOC)
|
|
161
|
-
|
|
162
|
-
- slots = max(20/25, 4) = 4
|
|
163
|
-
- rawUnits = 1.0 → load = 0.25 → hygiene = round(100 * e^(-0.1)) ≈ **90**
|
|
164
|
-
- Other dims 100
|
|
165
|
-
- standard ≈ 0.4*90 + 0.25*100 + 0.2*100 + 0.15*100 = **96**
|
|
166
|
-
|
|
167
|
-
### 3. Medium file, 5 distinct hygiene errors (~100 SLOC)
|
|
168
|
-
|
|
169
|
-
- slots = max(100/25, 4) = 4
|
|
170
|
-
- rawUnits = 5 → load = 1.25 → hygiene = round(100 * e^(-0.5)) ≈ **61**
|
|
171
|
-
- standard ≈ 0.4*61 + 60 = **84** if other dims perfect
|
|
172
|
-
|
|
173
|
-
### 4. Large file, 20× same warning (capped to 3)
|
|
174
|
-
|
|
175
|
-
- 20 warnings same rule → cap 3 → rawUnits = 3 * 0.4 = 1.2
|
|
176
|
-
- sloc = 500 → slots = 20 → load = 0.06 → score ≈ **98** on that dim
|
|
177
|
-
- Cap prevents auto-zero
|
|
178
|
-
|
|
179
|
-
### 5. Locator ratio 2 native / 10 total
|
|
180
|
-
|
|
181
|
-
- locatorsScore = round(100 * 2/10) = **20**
|
|
182
|
-
- Independent of caps
|
|
183
|
-
|
|
184
|
-
These examples feel fair: tiny single error stays high; multi-error drops materialy; repeated same-rule does not nuke large files; locator mix is a pure ratio.
|
|
114
|
+
`summary.sloc` (non-blank, non-comment lines) is reported for context and
|
|
115
|
+
feeds the oversized-file check (>400 SLOC per file → structure warning,
|
|
116
|
+
module scope). It plays no other role in scoring.
|
|
185
117
|
|
|
186
118
|
---
|
|
187
119
|
|
|
188
|
-
##
|
|
120
|
+
## Active rule set (frozen)
|
|
189
121
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
QA Guardian's own internal codegen pipeline (`playwright_runner`) layers a
|
|
195
|
-
private house-rules gate on top of `standard` — see the README's
|
|
196
|
-
"QA Guardian integration" section. That layer isn't part of this package.
|
|
122
|
+
The exact ESLint rules that produce findings are enabled explicitly in
|
|
123
|
+
`eslint-runner.ts` with fixed severities — the score never inherits
|
|
124
|
+
eslint-plugin-playwright's `recommended` set at install time, so identical
|
|
125
|
+
code scores identically regardless of which plugin version npm resolves.
|
|
197
126
|
|
|
198
127
|
---
|
|
199
128
|
|
|
200
|
-
##
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
CHANGELOG.)
|
|
129
|
+
## Why v3
|
|
130
|
+
|
|
131
|
+
v1/v2 scored findings per 25 lines of code through an exponential decay
|
|
132
|
+
with a minimum-slots floor. Three structural problems, each verified with
|
|
133
|
+
real inputs:
|
|
134
|
+
|
|
135
|
+
1. **Density is not how anyone judges a test suite.** "2 of 3 tests have
|
|
136
|
+
hard waits" is what a reviewer says; "0.5 penalty units per slot" is
|
|
137
|
+
not. v2 already fixed this for assertions (coverage ratio); v3 applies
|
|
138
|
+
the same per-test framing everywhere.
|
|
139
|
+
2. **Small suites were structurally under-penalized** — the minimum-slots
|
|
140
|
+
floor meant a 3-test suite with 2 errors, 2 assertion-free tests, and a
|
|
141
|
+
skipped test scored 78/C.
|
|
142
|
+
3. **Density is gameable**: adding clean lines of code (or clean files)
|
|
143
|
+
diluted `findings/SLOC` and raised the score without fixing anything.
|
|
144
|
+
A per-test ratio has no such lever — padding with fake tests costs
|
|
145
|
+
assertions coverage instead.
|
package/README.md
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
[](https://www.npmjs.com/package/@qaguardian/playwright-score)
|
|
5
5
|
[](./LICENSE)
|
|
6
6
|
|
|
7
|
-
**Deterministic, AI-free quality score for Playwright specs**
|
|
7
|
+
**Deterministic, AI-free quality score for Playwright specs** — the
|
|
8
|
+
weighted share of your tests that are clean.
|
|
8
9
|
|
|
9
10
|
**Website:** [qaguardian.com/open-source/playwright-score](https://qaguardian.com/open-source/playwright-score)
|
|
10
11
|
**Built by:** [QA Guardian](https://qaguardian.com) — managed Playwright E2E (AI drafts, engineers verify, you own the code)
|
|
@@ -21,30 +22,30 @@ The score never calls an LLM. AI may *generate* or *repair* code using findings;
|
|
|
21
22
|
17 public Playwright suites, scored with the published package against each
|
|
22
23
|
project's actual source — chosen to include both well-known,
|
|
23
24
|
heavily-engineered platforms and smaller, less mature projects. Not curated
|
|
24
|
-
to look good:
|
|
25
|
+
to look good: 5 of 17 fail the default threshold, for real, verifiable
|
|
25
26
|
reasons (see [VALIDATION.md](./VALIDATION.md)).
|
|
26
27
|
|
|
27
|
-
**17 suites · 1,
|
|
28
|
+
**17 suites · 1,230 files · 5,654 tests · 12/17 passed (80% threshold)**
|
|
28
29
|
|
|
29
30
|
| Repo (source scanned) | Score | Grade | Result |
|
|
30
31
|
|---|---:|:-:|:-:|
|
|
31
32
|
| [Playwright (own TodoMVC example)](https://github.com/microsoft/playwright/tree/main/examples/todomvc) | 98/100 | A | PASS |
|
|
32
|
-
| [
|
|
33
|
-
| [Documenso](https://github.com/documenso/documenso/tree/main/packages/app-tests) | 96/100 | A | PASS |
|
|
34
|
-
| [Storybook](https://github.com/storybookjs/storybook/tree/next/code/e2e-internal) | 96/100 | A | PASS |
|
|
35
|
-
| [freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp/tree/main/e2e) | 96/100 | A | PASS |
|
|
33
|
+
| [Storybook](https://github.com/storybookjs/storybook/tree/next/code/e2e-internal) | 95/100 | A | PASS |
|
|
36
34
|
| [n8n](https://github.com/n8n-io/n8n/tree/master/packages/testing/playwright) | 95/100 | A | PASS |
|
|
37
|
-
| [
|
|
38
|
-
| [
|
|
39
|
-
| [
|
|
40
|
-
| [
|
|
41
|
-
| [
|
|
42
|
-
| [
|
|
43
|
-
| [
|
|
44
|
-
| [
|
|
45
|
-
| [
|
|
46
|
-
| [
|
|
47
|
-
| [
|
|
35
|
+
| [freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp/tree/main/e2e) | 94/100 | A | PASS |
|
|
36
|
+
| [dub](https://github.com/dubinc/dub/tree/main/apps/web/playwright) | 94/100 | A | PASS |
|
|
37
|
+
| [Documenso](https://github.com/documenso/documenso/tree/main/packages/app-tests) | 93/100 | A | PASS |
|
|
38
|
+
| [Supabase](https://github.com/supabase/supabase/tree/master/e2e) | 91/100 | A | PASS |
|
|
39
|
+
| [PostHog](https://github.com/PostHog/posthog/tree/master/playwright) | 88/100 | B | PASS |
|
|
40
|
+
| [Grafana](https://github.com/grafana/grafana/tree/main/e2e-playwright) | 87/100 | B | PASS |
|
|
41
|
+
| [sencho](https://github.com/Studio-Saelix/sencho/tree/main/e2e) | 85/100 | B | PASS |
|
|
42
|
+
| [Mattermost](https://github.com/mattermost/mattermost/tree/master/e2e-tests/playwright) | 83/100 | B | PASS |
|
|
43
|
+
| [novu](https://github.com/novuhq/novu/tree/next/apps/dashboard/tests) | 82/100 | B | PASS |
|
|
44
|
+
| [Immich](https://github.com/immich-app/immich/tree/main/e2e/src/specs) | 78/100 | C | FAIL |
|
|
45
|
+
| [cal.com](https://github.com/calcom/cal.com/tree/main/apps/web/playwright) | 73/100 | C | FAIL |
|
|
46
|
+
| [openplayerjs](https://github.com/openplayerjs/openplayerjs/tree/master/e2e) | 73/100 | C | FAIL |
|
|
47
|
+
| [livecodes](https://github.com/live-codes/livecodes/tree/develop/e2e) | 70/100 | C | FAIL |
|
|
48
|
+
| [TheCyberHub](https://github.com/th3cyb3rhub/TheCyberHub/tree/dev/e2e) | 56/100 | F | FAIL |
|
|
48
49
|
|
|
49
50
|
Repo names link to the exact source scanned. Full methodology, findings
|
|
50
51
|
breakdown, and `scripts/validate-corpus.sh` to reproduce every number here
|
|
@@ -53,7 +54,7 @@ report of an earlier five-suite audit lives at
|
|
|
53
54
|
[`docs/scorecard.html`](./docs/scorecard.html) — host it wherever
|
|
54
55
|
(GitHub Pages, qaguardian.com, ...).
|
|
55
56
|
|
|
56
|
-
See [METHODOLOGY.md](./METHODOLOGY.md) for the
|
|
57
|
+
See [METHODOLOGY.md](./METHODOLOGY.md) for the full scoring model, or the product write-up on the [landing page](https://qaguardian.com/open-source/playwright-score).
|
|
57
58
|
|
|
58
59
|
## Install
|
|
59
60
|
|
package/dist/cli.js
CHANGED
|
@@ -10,7 +10,7 @@ import { formatText } from './formatters/text.js';
|
|
|
10
10
|
import { formatMarkdown } from './formatters/markdown.js';
|
|
11
11
|
import { formatSarif } from './formatters/sarif.js';
|
|
12
12
|
function printHelp() {
|
|
13
|
-
console.log(`@qaguardian/playwright-score — deterministic Playwright spec quality score
|
|
13
|
+
console.log(`@qaguardian/playwright-score — deterministic Playwright spec quality score
|
|
14
14
|
|
|
15
15
|
Usage:
|
|
16
16
|
npx -p @qaguardian/playwright-score playwright-score <paths...> [options]
|
|
@@ -31,7 +31,7 @@ Exit codes:
|
|
|
31
31
|
2 tool error
|
|
32
32
|
|
|
33
33
|
Website: https://qaguardian.com/open-source/playwright-score
|
|
34
|
-
Methodology: METHODOLOGY.md (
|
|
34
|
+
Methodology: METHODOLOGY.md (scoring model v3)
|
|
35
35
|
`);
|
|
36
36
|
}
|
|
37
37
|
// Deliberately never process.exit() right after a console.log/console.error
|
package/dist/eslint-runner.js
CHANGED
|
@@ -23,7 +23,7 @@ function buildConfig(assertFunctionNames) {
|
|
|
23
23
|
// no-wait-for-selector, expect-expect) are intentionally stricter
|
|
24
24
|
// than eslint-plugin-playwright's own "recommended" severity —
|
|
25
25
|
// these are patterns Playwright's own docs call out as anti-
|
|
26
|
-
// patterns (see playwright.dev/docs/best-practices), so
|
|
26
|
+
// patterns (see playwright.dev/docs/best-practices), so this scorer
|
|
27
27
|
// treats them as errors rather than warnings. no-networkidle and
|
|
28
28
|
// no-raw-locators aren't part of upstream "recommended" at all but
|
|
29
29
|
// are enabled here for the same reason.
|
|
@@ -87,7 +87,7 @@ function buildConfig(assertFunctionNames) {
|
|
|
87
87
|
'playwright/prefer-native-locators': 'warn',
|
|
88
88
|
// Matches eslint-plugin-playwright's own recommended severity.
|
|
89
89
|
'playwright/prefer-web-first-assertions': 'error',
|
|
90
|
-
// The rules below complete the
|
|
90
|
+
// The rules below complete the intended rule set from
|
|
91
91
|
// profiles.ts's ESLINT_RULE_MAP. Earlier versions (<=0.2.0)
|
|
92
92
|
// expected them to arrive via eslint-plugin-playwright's
|
|
93
93
|
// flat/recommended config, but the spread only handled an array
|
package/dist/formatters/text.js
CHANGED
|
@@ -13,7 +13,8 @@ export function formatText(result) {
|
|
|
13
13
|
lines.push(`Summary: files=${result.summary.files} tests=${result.summary.tests} sloc=${result.summary.sloc} ` +
|
|
14
14
|
`findings=${result.summary.findings} (errors=${result.summary.errors} warnings=${result.summary.warnings}) ` +
|
|
15
15
|
`locators native/raw=${result.summary.nativeLocators}/${result.summary.rawLocators} ` +
|
|
16
|
-
`tests asserting=${result.summary.tests - result.summary.unassertedTests}/${result.summary.tests}`
|
|
16
|
+
`tests asserting=${result.summary.tests - result.summary.unassertedTests}/${result.summary.tests} ` +
|
|
17
|
+
`clean tests=${result.summary.cleanTests}/${result.summary.tests}`);
|
|
17
18
|
if (result.findings.length) {
|
|
18
19
|
lines.push('');
|
|
19
20
|
lines.push('Findings:');
|
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { ScoreOptions, ScoreResult } from './types.js';
|
|
2
2
|
export type { ScoreResult, ScoreOptions, Finding, ProfileName } from './types.js';
|
|
3
|
-
export { computeScore,
|
|
3
|
+
export { computeScore, MODEL, locatorsScore, gradeFromScore, ratioDimensionScore, } from './score-engine.js';
|
|
4
4
|
export { countSloc } from './sloc.js';
|
|
5
|
-
export { countLocators, analyzeSource } from './metrics.js';
|
|
5
|
+
export { countLocators, analyzeSource, attributeFindingScopes, getTestSpans, } from './metrics.js';
|
|
6
6
|
export { formatJson } from './formatters/json.js';
|
|
7
7
|
export { formatText } from './formatters/text.js';
|
|
8
8
|
export { formatMarkdown } from './formatters/markdown.js';
|
|
9
9
|
export { formatSarif } from './formatters/sarif.js';
|
|
10
10
|
/**
|
|
11
|
-
* Score Playwright spec files. Deterministic and AI-free
|
|
11
|
+
* Score Playwright spec files. Deterministic and AI-free.
|
|
12
12
|
*/
|
|
13
13
|
export declare function scorePaths(options: ScoreOptions): Promise<ScoreResult>;
|
package/dist/index.js
CHANGED
|
@@ -4,12 +4,12 @@ import { globSync } from 'glob';
|
|
|
4
4
|
import { runEslint } from './eslint-runner.js';
|
|
5
5
|
import { commonAncestorDir } from './fs-util.js';
|
|
6
6
|
import { collectLocallyImportedFiles } from './import-graph.js';
|
|
7
|
-
import { analyzeSource, countLocators, findLocalAssertionHelperNames, looksLikeNonPlaywrightTest, } from './metrics.js';
|
|
7
|
+
import { analyzeSource, attributeFindingScopes, countLocators, findLocalAssertionHelperNames, getTestSpans, looksLikeNonPlaywrightTest, } from './metrics.js';
|
|
8
8
|
import { DEFAULT_THRESHOLDS } from './profiles.js';
|
|
9
9
|
import { computeScore } from './score-engine.js';
|
|
10
|
-
export { computeScore,
|
|
10
|
+
export { computeScore, MODEL, locatorsScore, gradeFromScore, ratioDimensionScore, } from './score-engine.js';
|
|
11
11
|
export { countSloc } from './sloc.js';
|
|
12
|
-
export { countLocators, analyzeSource } from './metrics.js';
|
|
12
|
+
export { countLocators, analyzeSource, attributeFindingScopes, getTestSpans, } from './metrics.js';
|
|
13
13
|
export { formatJson } from './formatters/json.js';
|
|
14
14
|
export { formatText } from './formatters/text.js';
|
|
15
15
|
export { formatMarkdown } from './formatters/markdown.js';
|
|
@@ -110,7 +110,7 @@ function expandPaths(inputs, cwd) {
|
|
|
110
110
|
}
|
|
111
111
|
function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
112
112
|
return {
|
|
113
|
-
scoreVersion: '
|
|
113
|
+
scoreVersion: 'v3',
|
|
114
114
|
profile,
|
|
115
115
|
score: 0,
|
|
116
116
|
grade: 'F',
|
|
@@ -126,6 +126,7 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
|
126
126
|
nativeLocators: 0,
|
|
127
127
|
rawLocators: 0,
|
|
128
128
|
unassertedTests: 0,
|
|
129
|
+
cleanTests: 0,
|
|
129
130
|
...extra,
|
|
130
131
|
},
|
|
131
132
|
dimensions: {
|
|
@@ -139,7 +140,7 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
|
139
140
|
};
|
|
140
141
|
}
|
|
141
142
|
/**
|
|
142
|
-
* Score Playwright spec files. Deterministic and AI-free
|
|
143
|
+
* Score Playwright spec files. Deterministic and AI-free.
|
|
143
144
|
*/
|
|
144
145
|
export async function scorePaths(options) {
|
|
145
146
|
const cwd = options.cwd ?? process.cwd();
|
|
@@ -234,15 +235,24 @@ export async function scorePaths(options) {
|
|
|
234
235
|
let native = 0;
|
|
235
236
|
let raw = 0;
|
|
236
237
|
const metricFindings = [];
|
|
238
|
+
const fileTests = {};
|
|
239
|
+
const fileSpans = new Map();
|
|
237
240
|
for (const file of files) {
|
|
238
241
|
const source = sources.get(file) ?? '';
|
|
239
242
|
const relFile = path.relative(filesBase, file) || path.basename(file);
|
|
240
243
|
const m = analyzeSource(source, relFile);
|
|
241
244
|
totalSloc += m.sloc;
|
|
242
|
-
totalTests += m.tests;
|
|
243
245
|
native += m.locators.native;
|
|
244
246
|
raw += m.locators.raw;
|
|
245
247
|
metricFindings.push(...m.findings);
|
|
248
|
+
const spans = getTestSpans(source, relFile);
|
|
249
|
+
fileSpans.set(relFile, spans);
|
|
250
|
+
// AST spans are the authoritative test count (they are also the
|
|
251
|
+
// attribution targets); the regex count is the fallback for a file
|
|
252
|
+
// the parser can't read — which hard-fails below anyway.
|
|
253
|
+
const testCount = spans.tests.length > 0 ? spans.tests.length : m.tests;
|
|
254
|
+
fileTests[relFile] = testCount;
|
|
255
|
+
totalTests += testCount;
|
|
246
256
|
}
|
|
247
257
|
for (const [file, source] of importedFiles) {
|
|
248
258
|
const counts = countLocators(source, file);
|
|
@@ -250,6 +260,9 @@ export async function scorePaths(options) {
|
|
|
250
260
|
raw += counts.raw;
|
|
251
261
|
}
|
|
252
262
|
const findings = [...eslintFindings, ...metricFindings];
|
|
263
|
+
// Attribute each finding to the test it sits in (model v3) — see
|
|
264
|
+
// attributeFindingScopes in metrics.ts.
|
|
265
|
+
attributeFindingScopes(findings, fileSpans);
|
|
253
266
|
const parseErrors = findings.filter((f) => f.rule === 'playwright-score/parse-error');
|
|
254
267
|
if (parseErrors.length > 0) {
|
|
255
268
|
// A file that isn't valid JS/TS can't be executed, let alone scored on
|
|
@@ -271,6 +284,7 @@ export async function scorePaths(options) {
|
|
|
271
284
|
sloc: Math.max(totalSloc, 1),
|
|
272
285
|
files: files.length,
|
|
273
286
|
tests: totalTests,
|
|
287
|
+
fileTests,
|
|
274
288
|
nativeLocators: native,
|
|
275
289
|
rawLocators: raw,
|
|
276
290
|
});
|
package/dist/metrics.d.ts
CHANGED
|
@@ -38,6 +38,43 @@ export declare function countLocators(source: string, file?: string): LocatorCou
|
|
|
38
38
|
* function against each imported dependency file too).
|
|
39
39
|
*/
|
|
40
40
|
export declare function findLocalAssertionHelperNames(source: string, file?: string): string[];
|
|
41
|
+
export interface SourceSpan {
|
|
42
|
+
startLine: number;
|
|
43
|
+
endLine: number;
|
|
44
|
+
}
|
|
45
|
+
export interface TestSpans {
|
|
46
|
+
/** Outermost test declaration spans (test/test.only/test.skip/test.fixme/
|
|
47
|
+
* test.slow calls that carry a function body). Order is source order and
|
|
48
|
+
* is the stable per-file test index used for finding attribution. */
|
|
49
|
+
tests: SourceSpan[];
|
|
50
|
+
/** Hook spans (test.beforeEach/afterEach/beforeAll/afterAll and their
|
|
51
|
+
* bare-identifier forms) — findings inside these affect every test in
|
|
52
|
+
* the file, not one. */
|
|
53
|
+
hooks: SourceSpan[];
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* AST spans of test declarations and hooks, for attributing findings to
|
|
57
|
+
* the test they actually sit in. Only calls carrying a function argument
|
|
58
|
+
* count — that is what makes `test.skip('t', async () => {...})` (an
|
|
59
|
+
* always-skipped declaration) a span while the documented runtime form
|
|
60
|
+
* `test.skip(condition, reason)` inside a test body is not, and likewise
|
|
61
|
+
* for `test.fixme()` used as a statement. Spans nested inside another test
|
|
62
|
+
* span (rare, but e.g. a helper declaring a test inside a test would be
|
|
63
|
+
* malformed anyway) are dropped so each line maps to at most one test.
|
|
64
|
+
*/
|
|
65
|
+
export declare function getTestSpans(source: string, file?: string): TestSpans;
|
|
66
|
+
/**
|
|
67
|
+
* Model-v3 finding attribution (see score-engine.ts): a finding inside a
|
|
68
|
+
* test span belongs to that test (`scope: 'test'` + a stable
|
|
69
|
+
* `file#index` testKey); inside a hook span it affects every test in the
|
|
70
|
+
* file (`scope: 'hook'`); anything else — imports, describe bodies,
|
|
71
|
+
* file-level metrics — is `scope: 'module'`, counted once per file.
|
|
72
|
+
* Exported so layers composing extra rules on top (e.g. a private
|
|
73
|
+
* house-rules dimension) attribute their own findings with the same
|
|
74
|
+
* logic the package itself uses, rather than reimplementing it.
|
|
75
|
+
* Mutates the findings in place and returns them.
|
|
76
|
+
*/
|
|
77
|
+
export declare function attributeFindingScopes(findings: Finding[], spansByFile: Map<string, TestSpans>): Finding[];
|
|
41
78
|
/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
|
|
42
79
|
export declare function countTests(source: string): number;
|
|
43
80
|
export declare function findOversizedFile(source: string, file: string, maxSloc?: number): Finding[];
|
package/dist/metrics.js
CHANGED
|
@@ -35,12 +35,18 @@ const CYPRESS_API_RE = /\bcy\.(?:visit|get|contains|click|type|should|intercept|
|
|
|
35
35
|
export function looksLikeNonPlaywrightTest(source) {
|
|
36
36
|
return NON_PLAYWRIGHT_IMPORT_RE.test(source) || RTL_API_RE.test(source) || CYPRESS_API_RE.test(source);
|
|
37
37
|
}
|
|
38
|
+
// The full getBy* family Playwright ships. getByAltText/getByTitle are
|
|
39
|
+
// just as native as the rest — leaving them out meant an image-heavy suite
|
|
40
|
+
// (alt/title locators plus one .locator() fallback) scored 0 on the
|
|
41
|
+
// locators dimension for perfectly idiomatic code.
|
|
38
42
|
const NATIVE_METHODS = new Set([
|
|
39
43
|
'getByRole',
|
|
40
44
|
'getByLabel',
|
|
41
45
|
'getByTestId',
|
|
42
46
|
'getByText',
|
|
43
47
|
'getByPlaceholder',
|
|
48
|
+
'getByAltText',
|
|
49
|
+
'getByTitle',
|
|
44
50
|
]);
|
|
45
51
|
/**
|
|
46
52
|
* Generic ESTree walk (no visitor-keys dependency). `parent` is excluded so
|
|
@@ -313,6 +319,108 @@ export function findLocalAssertionHelperNames(source, file) {
|
|
|
313
319
|
}
|
|
314
320
|
return [...names];
|
|
315
321
|
}
|
|
322
|
+
const TEST_MODIFIERS = new Set(['only', 'skip', 'fixme', 'slow']);
|
|
323
|
+
const HOOK_NAMES = new Set(['beforeEach', 'afterEach', 'beforeAll', 'afterAll']);
|
|
324
|
+
function hasFunctionArg(node) {
|
|
325
|
+
const args = node.arguments;
|
|
326
|
+
return (args ?? []).some((a) => a?.type === 'ArrowFunctionExpression' || a?.type === 'FunctionExpression');
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* AST spans of test declarations and hooks, for attributing findings to
|
|
330
|
+
* the test they actually sit in. Only calls carrying a function argument
|
|
331
|
+
* count — that is what makes `test.skip('t', async () => {...})` (an
|
|
332
|
+
* always-skipped declaration) a span while the documented runtime form
|
|
333
|
+
* `test.skip(condition, reason)` inside a test body is not, and likewise
|
|
334
|
+
* for `test.fixme()` used as a statement. Spans nested inside another test
|
|
335
|
+
* span (rare, but e.g. a helper declaring a test inside a test would be
|
|
336
|
+
* malformed anyway) are dropped so each line maps to at most one test.
|
|
337
|
+
*/
|
|
338
|
+
export function getTestSpans(source, file) {
|
|
339
|
+
const tests = [];
|
|
340
|
+
const hooks = [];
|
|
341
|
+
const ast = parseSource(source, file);
|
|
342
|
+
if (!ast)
|
|
343
|
+
return { tests, hooks };
|
|
344
|
+
walk(ast, (node) => {
|
|
345
|
+
if (node.type !== 'CallExpression')
|
|
346
|
+
return;
|
|
347
|
+
const callee = node.callee;
|
|
348
|
+
const loc = node.loc;
|
|
349
|
+
if (!loc?.start?.line || !loc?.end?.line)
|
|
350
|
+
return;
|
|
351
|
+
const span = { startLine: loc.start.line, endLine: loc.end.line };
|
|
352
|
+
let isTest = false;
|
|
353
|
+
let isHook = false;
|
|
354
|
+
if (callee?.type === 'Identifier') {
|
|
355
|
+
const name = callee.name;
|
|
356
|
+
if (name === 'test' || name === 'it')
|
|
357
|
+
isTest = true;
|
|
358
|
+
else if (name && HOOK_NAMES.has(name))
|
|
359
|
+
isHook = true;
|
|
360
|
+
}
|
|
361
|
+
else if (callee?.type === 'MemberExpression' && !callee.computed) {
|
|
362
|
+
const obj = callee.object;
|
|
363
|
+
const prop = callee.property;
|
|
364
|
+
if (obj?.type === 'Identifier' && (obj.name === 'test' || obj.name === 'it') && prop?.type === 'Identifier') {
|
|
365
|
+
if (TEST_MODIFIERS.has(prop.name ?? ''))
|
|
366
|
+
isTest = true;
|
|
367
|
+
else if (HOOK_NAMES.has(prop.name ?? ''))
|
|
368
|
+
isHook = true;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
if (!isTest && !isHook)
|
|
372
|
+
return;
|
|
373
|
+
if (!hasFunctionArg(node))
|
|
374
|
+
return;
|
|
375
|
+
if (isTest)
|
|
376
|
+
tests.push(span);
|
|
377
|
+
else
|
|
378
|
+
hooks.push(span);
|
|
379
|
+
});
|
|
380
|
+
// Keep outermost test spans only, in source order.
|
|
381
|
+
tests.sort((a, b) => a.startLine - b.startLine || b.endLine - a.endLine);
|
|
382
|
+
const outer = [];
|
|
383
|
+
for (const s of tests) {
|
|
384
|
+
const last = outer[outer.length - 1];
|
|
385
|
+
if (last && s.startLine >= last.startLine && s.endLine <= last.endLine)
|
|
386
|
+
continue;
|
|
387
|
+
outer.push(s);
|
|
388
|
+
}
|
|
389
|
+
return { tests: outer, hooks };
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Model-v3 finding attribution (see score-engine.ts): a finding inside a
|
|
393
|
+
* test span belongs to that test (`scope: 'test'` + a stable
|
|
394
|
+
* `file#index` testKey); inside a hook span it affects every test in the
|
|
395
|
+
* file (`scope: 'hook'`); anything else — imports, describe bodies,
|
|
396
|
+
* file-level metrics — is `scope: 'module'`, counted once per file.
|
|
397
|
+
* Exported so layers composing extra rules on top (e.g. a private
|
|
398
|
+
* house-rules dimension) attribute their own findings with the same
|
|
399
|
+
* logic the package itself uses, rather than reimplementing it.
|
|
400
|
+
* Mutates the findings in place and returns them.
|
|
401
|
+
*/
|
|
402
|
+
export function attributeFindingScopes(findings, spansByFile) {
|
|
403
|
+
for (const f of findings) {
|
|
404
|
+
const spans = spansByFile.get(f.file);
|
|
405
|
+
if (!spans || f.line === undefined) {
|
|
406
|
+
f.scope = 'module';
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const line = f.line;
|
|
410
|
+
const idx = spans.tests.findIndex((t) => line >= t.startLine && line <= t.endLine);
|
|
411
|
+
if (idx >= 0) {
|
|
412
|
+
f.scope = 'test';
|
|
413
|
+
f.testKey = `${f.file}#${idx}`;
|
|
414
|
+
}
|
|
415
|
+
else if (spans.hooks.some((h) => line >= h.startLine && line <= h.endLine)) {
|
|
416
|
+
f.scope = 'hook';
|
|
417
|
+
}
|
|
418
|
+
else {
|
|
419
|
+
f.scope = 'module';
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
return findings;
|
|
423
|
+
}
|
|
316
424
|
const TEST_NON_DECL_RE = /\btest\.(describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info|expect)\b/;
|
|
317
425
|
/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
|
|
318
426
|
export function countTests(source) {
|
package/dist/score-engine.d.ts
CHANGED
|
@@ -1,55 +1,51 @@
|
|
|
1
1
|
import type { DimensionName, Finding, Grade, ProfileName, ScoreResult } from './types.js';
|
|
2
|
-
/**
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
*
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
2
|
+
/**
|
|
3
|
+
* Scoring model v3 — "the weighted share of your tests that are clean."
|
|
4
|
+
*
|
|
5
|
+
* Every finding is attributed to the test it actually sits in (AST test
|
|
6
|
+
* spans, resolved in index.ts). Each test accumulates demerit units for a
|
|
7
|
+
* dimension — an error is a full demerit (1.0), a warning 0.4 — capped at
|
|
8
|
+
* 1 per test: a test is at worst fully flawed, never more. Findings in a
|
|
9
|
+
* before/after hook demerit every test in that file (setup problems affect
|
|
10
|
+
* every test that runs through them); module-level findings count once per
|
|
11
|
+
* file. A dimension's score is simply:
|
|
12
|
+
*
|
|
13
|
+
* score = 100 × (1 − totalDemerit / tests)
|
|
14
|
+
*
|
|
15
|
+
* The locators dimension stays the native/(native+raw) usage ratio.
|
|
16
|
+
* The final score is the weighted sum of dimensions.
|
|
17
|
+
*
|
|
18
|
+
* There is deliberately no per-SLOC density, no slot divisor, no minimum
|
|
19
|
+
* slots, no exponential decay, and no per-rule finding caps (the per-test
|
|
20
|
+
* cap of 1 is the anti-nuke mechanism, and it is also the anti-gaming
|
|
21
|
+
* mechanism: padding a suite with clean lines no longer dilutes anything,
|
|
22
|
+
* and padding it with fake clean tests costs assertions coverage). Earlier
|
|
23
|
+
* models (≤0.4.0, published as "sqs-v1"/"sqs-v2") used density math and
|
|
24
|
+
* scored e.g. a 3-test suite with two hard-wait errors, two assertion-free
|
|
25
|
+
* tests, and a skipped test at 78/C; model v3 scores it 53/F, which is
|
|
26
|
+
* what it is.
|
|
27
|
+
*/
|
|
28
|
+
export declare const MODEL: {
|
|
29
|
+
readonly scoreVersion: "v3";
|
|
30
|
+
readonly ERROR_DEMERIT: 1;
|
|
31
|
+
readonly WARNING_DEMERIT: 0.4;
|
|
32
|
+
readonly INFO_DEMERIT: 0;
|
|
33
|
+
/** A single test's demerit ceiling per dimension. */
|
|
34
|
+
readonly MAX_DEMERIT_PER_TEST: 1;
|
|
35
|
+
/** Module-level findings' demerit ceiling per (file, dimension). */
|
|
36
|
+
readonly MAX_DEMERIT_PER_FILE_MODULE: 1;
|
|
24
37
|
};
|
|
25
38
|
declare const PENALTY_DIMENSIONS: DimensionName[];
|
|
26
39
|
export declare function gradeFromScore(score: number): Grade;
|
|
27
40
|
export declare function locatorsScore(native: number, raw: number): number;
|
|
41
|
+
/** Tests per file, keyed by the same relative file path findings carry. */
|
|
42
|
+
export type FileTestCounts = Record<string, number>;
|
|
28
43
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* sqs-v1 fed "test has no assertions" through the same density math as
|
|
33
|
-
* hygiene smells, which misread coverage badly on small suites: 2 of 3
|
|
34
|
-
* tests asserting nothing scored 82/100 on this dimension. Whether a test
|
|
35
|
-
* asserts anything is a fraction of tests, not a findings-per-SLOC rate —
|
|
36
|
-
* so, like the locators dimension, the primary signal is now a ratio:
|
|
37
|
-
*
|
|
38
|
-
* coverage = (tests - unassertedTests) / tests // uncapped census
|
|
39
|
-
* aux = density load of the OTHER assertion rules
|
|
40
|
-
* (valid-expect, no-standalone-expect,
|
|
41
|
-
* prefer-web-first-assertions), capped as usual
|
|
42
|
-
* score = round(100 * coverage * e^(-K * aux))
|
|
43
|
-
*
|
|
44
|
-
* expect-expect findings are excluded from the density term — they ARE the
|
|
45
|
-
* coverage term, and counting them twice would double-penalize.
|
|
46
|
-
* tests === 0 (nothing but helpers matched) keeps coverage at 1 and lets
|
|
47
|
-
* the aux findings, if any, carry the dimension.
|
|
44
|
+
* 100 × (1 − demerit share). With zero tests there is nothing to take a
|
|
45
|
+
* share of — module/hook demerits then act on a denominator of 1 so a
|
|
46
|
+
* findings-bearing zero-test input still can't score clean.
|
|
48
47
|
*/
|
|
49
|
-
export declare function
|
|
50
|
-
/** Cap findings per (file, rule) for penalty math only */
|
|
51
|
-
export declare function applyPerRuleCap(findings: Finding[]): Finding[];
|
|
52
|
-
export declare function penaltyDimensionScore(findings: Finding[], dimension: DimensionName, sloc: number): number;
|
|
48
|
+
export declare function ratioDimensionScore(findings: Finding[], dimension: DimensionName, fileTests: FileTestCounts, tests: number): number;
|
|
53
49
|
export declare function computeScore(input: {
|
|
54
50
|
profile: ProfileName;
|
|
55
51
|
threshold: number;
|
|
@@ -57,6 +53,7 @@ export declare function computeScore(input: {
|
|
|
57
53
|
sloc: number;
|
|
58
54
|
files: number;
|
|
59
55
|
tests: number;
|
|
56
|
+
fileTests: FileTestCounts;
|
|
60
57
|
nativeLocators: number;
|
|
61
58
|
rawLocators: number;
|
|
62
59
|
}): ScoreResult;
|
package/dist/score-engine.js
CHANGED
|
@@ -1,27 +1,40 @@
|
|
|
1
1
|
import { PROFILE_WEIGHTS } from './profiles.js';
|
|
2
|
-
/** sqs-v2 frozen constants — change requires sqs-v3 */
|
|
3
|
-
export const SQS_V2 = {
|
|
4
|
-
scoreVersion: 'sqs-v2',
|
|
5
|
-
ERROR_UNIT: 1.0,
|
|
6
|
-
WARNING_UNIT: 0.4,
|
|
7
|
-
INFO_UNIT: 0.0,
|
|
8
|
-
MAX_FINDINGS_PER_RULE_PER_FILE: 3,
|
|
9
|
-
SLOT_DIVISOR: 25,
|
|
10
|
-
MIN_SLOTS: 4,
|
|
11
|
-
K: 0.4,
|
|
12
|
-
};
|
|
13
|
-
/** @deprecated sqs-v1 alias kept for one release so pinned callers get a
|
|
14
|
-
* type-level nudge instead of a hard break; constants are identical. */
|
|
15
|
-
export const SQS_V1 = SQS_V2;
|
|
16
2
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
3
|
+
* Scoring model v3 — "the weighted share of your tests that are clean."
|
|
4
|
+
*
|
|
5
|
+
* Every finding is attributed to the test it actually sits in (AST test
|
|
6
|
+
* spans, resolved in index.ts). Each test accumulates demerit units for a
|
|
7
|
+
* dimension — an error is a full demerit (1.0), a warning 0.4 — capped at
|
|
8
|
+
* 1 per test: a test is at worst fully flawed, never more. Findings in a
|
|
9
|
+
* before/after hook demerit every test in that file (setup problems affect
|
|
10
|
+
* every test that runs through them); module-level findings count once per
|
|
11
|
+
* file. A dimension's score is simply:
|
|
12
|
+
*
|
|
13
|
+
* score = 100 × (1 − totalDemerit / tests)
|
|
14
|
+
*
|
|
15
|
+
* The locators dimension stays the native/(native+raw) usage ratio.
|
|
16
|
+
* The final score is the weighted sum of dimensions.
|
|
17
|
+
*
|
|
18
|
+
* There is deliberately no per-SLOC density, no slot divisor, no minimum
|
|
19
|
+
* slots, no exponential decay, and no per-rule finding caps (the per-test
|
|
20
|
+
* cap of 1 is the anti-nuke mechanism, and it is also the anti-gaming
|
|
21
|
+
* mechanism: padding a suite with clean lines no longer dilutes anything,
|
|
22
|
+
* and padding it with fake clean tests costs assertions coverage). Earlier
|
|
23
|
+
* models (≤0.4.0, published as "sqs-v1"/"sqs-v2") used density math and
|
|
24
|
+
* scored e.g. a 3-test suite with two hard-wait errors, two assertion-free
|
|
25
|
+
* tests, and a skipped test at 78/C; model v3 scores it 53/F, which is
|
|
26
|
+
* what it is.
|
|
23
27
|
*/
|
|
24
|
-
const
|
|
28
|
+
export const MODEL = {
|
|
29
|
+
scoreVersion: 'v3',
|
|
30
|
+
ERROR_DEMERIT: 1.0,
|
|
31
|
+
WARNING_DEMERIT: 0.4,
|
|
32
|
+
INFO_DEMERIT: 0.0,
|
|
33
|
+
/** A single test's demerit ceiling per dimension. */
|
|
34
|
+
MAX_DEMERIT_PER_TEST: 1.0,
|
|
35
|
+
/** Module-level findings' demerit ceiling per (file, dimension). */
|
|
36
|
+
MAX_DEMERIT_PER_FILE_MODULE: 1.0,
|
|
37
|
+
};
|
|
25
38
|
const PENALTY_DIMENSIONS = ['playwrightHygiene', 'assertions', 'structure'];
|
|
26
39
|
export function gradeFromScore(score) {
|
|
27
40
|
if (score >= 90)
|
|
@@ -40,73 +53,80 @@ export function locatorsScore(native, raw) {
|
|
|
40
53
|
return 100;
|
|
41
54
|
return Math.round((100 * native) / total);
|
|
42
55
|
}
|
|
43
|
-
|
|
44
|
-
* sqs-v2 assertions dimension: a per-test coverage ratio, decayed by the
|
|
45
|
-
* density of the remaining assertion-quality findings.
|
|
46
|
-
*
|
|
47
|
-
* sqs-v1 fed "test has no assertions" through the same density math as
|
|
48
|
-
* hygiene smells, which misread coverage badly on small suites: 2 of 3
|
|
49
|
-
* tests asserting nothing scored 82/100 on this dimension. Whether a test
|
|
50
|
-
* asserts anything is a fraction of tests, not a findings-per-SLOC rate —
|
|
51
|
-
* so, like the locators dimension, the primary signal is now a ratio:
|
|
52
|
-
*
|
|
53
|
-
* coverage = (tests - unassertedTests) / tests // uncapped census
|
|
54
|
-
* aux = density load of the OTHER assertion rules
|
|
55
|
-
* (valid-expect, no-standalone-expect,
|
|
56
|
-
* prefer-web-first-assertions), capped as usual
|
|
57
|
-
* score = round(100 * coverage * e^(-K * aux))
|
|
58
|
-
*
|
|
59
|
-
* expect-expect findings are excluded from the density term — they ARE the
|
|
60
|
-
* coverage term, and counting them twice would double-penalize.
|
|
61
|
-
* tests === 0 (nothing but helpers matched) keeps coverage at 1 and lets
|
|
62
|
-
* the aux findings, if any, carry the dimension.
|
|
63
|
-
*/
|
|
64
|
-
export function assertionsScore(findings, tests, sloc) {
|
|
65
|
-
const unasserted = findings.filter((f) => f.rule === UNASSERTED_TEST_RULE && !f.reportOnly).length;
|
|
66
|
-
const coverage = tests <= 0 ? 1 : Math.min(1, Math.max(0, (tests - unasserted) / tests));
|
|
67
|
-
const aux = findings.filter((f) => f.dimension === 'assertions' && f.rule !== UNASSERTED_TEST_RULE);
|
|
68
|
-
const capped = applyPerRuleCap(aux);
|
|
69
|
-
let rawUnits = 0;
|
|
70
|
-
for (const f of capped)
|
|
71
|
-
rawUnits += severityUnit(f.severity);
|
|
72
|
-
const slots = Math.max(sloc / SQS_V2.SLOT_DIVISOR, SQS_V2.MIN_SLOTS);
|
|
73
|
-
const load = rawUnits / slots;
|
|
74
|
-
const score = Math.round(100 * coverage * Math.exp(-SQS_V2.K * load));
|
|
75
|
-
return Math.min(100, Math.max(0, score));
|
|
76
|
-
}
|
|
77
|
-
function severityUnit(severity) {
|
|
56
|
+
function severityDemerit(severity) {
|
|
78
57
|
if (severity === 'error')
|
|
79
|
-
return
|
|
58
|
+
return MODEL.ERROR_DEMERIT;
|
|
80
59
|
if (severity === 'warning')
|
|
81
|
-
return
|
|
82
|
-
return
|
|
60
|
+
return MODEL.WARNING_DEMERIT;
|
|
61
|
+
return MODEL.INFO_DEMERIT;
|
|
83
62
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
const
|
|
87
|
-
const
|
|
63
|
+
function demeritsForDimension(findings, dimension, fileTests) {
|
|
64
|
+
const perTest = new Map();
|
|
65
|
+
const perFileHook = new Map();
|
|
66
|
+
const perFileModule = new Map();
|
|
88
67
|
for (const f of findings) {
|
|
89
|
-
if (f.reportOnly)
|
|
68
|
+
if (f.dimension !== dimension || f.reportOnly)
|
|
90
69
|
continue;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
if (n >= SQS_V2.MAX_FINDINGS_PER_RULE_PER_FILE)
|
|
70
|
+
const d = severityDemerit(f.severity);
|
|
71
|
+
if (d === 0)
|
|
94
72
|
continue;
|
|
95
|
-
|
|
96
|
-
|
|
73
|
+
if (f.scope === 'test' && f.testKey) {
|
|
74
|
+
perTest.set(f.testKey, (perTest.get(f.testKey) ?? 0) + d);
|
|
75
|
+
}
|
|
76
|
+
else if (f.scope === 'hook') {
|
|
77
|
+
perFileHook.set(f.file, (perFileHook.get(f.file) ?? 0) + d);
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
perFileModule.set(f.file, (perFileModule.get(f.file) ?? 0) + d);
|
|
81
|
+
}
|
|
97
82
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
83
|
+
let total = 0;
|
|
84
|
+
const flawedTestKeys = new Set();
|
|
85
|
+
// Per-test demerits: each test's own findings plus its file's hook
|
|
86
|
+
// findings, capped at 1 — a test is at worst fully flawed.
|
|
87
|
+
for (const [file, count] of Object.entries(fileTests)) {
|
|
88
|
+
const hookUnits = perFileHook.get(file) ?? 0;
|
|
89
|
+
for (let i = 0; i < count; i++) {
|
|
90
|
+
const key = `${file}#${i}`;
|
|
91
|
+
const units = (perTest.get(key) ?? 0) + hookUnits;
|
|
92
|
+
if (units <= 0)
|
|
93
|
+
continue;
|
|
94
|
+
total += Math.min(MODEL.MAX_DEMERIT_PER_TEST, units);
|
|
95
|
+
flawedTestKeys.add(key);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
// A test-scoped finding whose file has no counted tests (should not
|
|
99
|
+
// happen, but attribution and counting are separate passes) still
|
|
100
|
+
// counts, capped the same way.
|
|
101
|
+
for (const [key, units] of perTest) {
|
|
102
|
+
const file = key.slice(0, key.lastIndexOf('#'));
|
|
103
|
+
if (fileTests[file] !== undefined)
|
|
104
|
+
continue;
|
|
105
|
+
total += Math.min(MODEL.MAX_DEMERIT_PER_TEST, units);
|
|
106
|
+
flawedTestKeys.add(key);
|
|
107
|
+
}
|
|
108
|
+
// Hook findings in a file with zero tests (setup-only file swept in):
|
|
109
|
+
// count once per file, like module scope.
|
|
110
|
+
for (const [file, units] of perFileHook) {
|
|
111
|
+
if ((fileTests[file] ?? 0) > 0)
|
|
112
|
+
continue;
|
|
113
|
+
total += Math.min(MODEL.MAX_DEMERIT_PER_FILE_MODULE, units);
|
|
106
114
|
}
|
|
107
|
-
|
|
108
|
-
const
|
|
109
|
-
|
|
115
|
+
// Module-level findings count once per file.
|
|
116
|
+
for (const units of perFileModule.values()) {
|
|
117
|
+
total += Math.min(MODEL.MAX_DEMERIT_PER_FILE_MODULE, units);
|
|
118
|
+
}
|
|
119
|
+
return { total, flawedTestKeys };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* 100 × (1 − demerit share). With zero tests there is nothing to take a
|
|
123
|
+
* share of — module/hook demerits then act on a denominator of 1 so a
|
|
124
|
+
* findings-bearing zero-test input still can't score clean.
|
|
125
|
+
*/
|
|
126
|
+
export function ratioDimensionScore(findings, dimension, fileTests, tests) {
|
|
127
|
+
const { total } = demeritsForDimension(findings, dimension, fileTests);
|
|
128
|
+
const denominator = Math.max(tests, 1);
|
|
129
|
+
const score = Math.round(100 * (1 - total / denominator));
|
|
110
130
|
return Math.min(100, Math.max(0, score));
|
|
111
131
|
}
|
|
112
132
|
export function computeScore(input) {
|
|
@@ -117,10 +137,10 @@ export function computeScore(input) {
|
|
|
117
137
|
// score instead of a hard crash.
|
|
118
138
|
const weights = PROFILE_WEIGHTS[input.profile] ?? PROFILE_WEIGHTS.standard;
|
|
119
139
|
const dims = {
|
|
120
|
-
playwrightHygiene:
|
|
121
|
-
assertions:
|
|
140
|
+
playwrightHygiene: ratioDimensionScore(input.findings, 'playwrightHygiene', input.fileTests, input.tests),
|
|
141
|
+
assertions: ratioDimensionScore(input.findings, 'assertions', input.fileTests, input.tests),
|
|
122
142
|
locators: locatorsScore(input.nativeLocators, input.rawLocators),
|
|
123
|
-
structure:
|
|
143
|
+
structure: ratioDimensionScore(input.findings, 'structure', input.fileTests, input.tests),
|
|
124
144
|
};
|
|
125
145
|
let score = 0;
|
|
126
146
|
for (const [dim, weight] of Object.entries(weights)) {
|
|
@@ -133,8 +153,18 @@ export function computeScore(input) {
|
|
|
133
153
|
score = Math.round(score);
|
|
134
154
|
const errors = input.findings.filter((f) => f.severity === 'error').length;
|
|
135
155
|
const warnings = input.findings.filter((f) => f.severity === 'warning').length;
|
|
156
|
+
// Clean tests: no demerit in any penalty dimension. (Locator usage is a
|
|
157
|
+
// suite-level ratio, not a per-test property.)
|
|
158
|
+
const flawed = new Set();
|
|
159
|
+
for (const dim of PENALTY_DIMENSIONS) {
|
|
160
|
+
for (const key of demeritsForDimension(input.findings, dim, input.fileTests)
|
|
161
|
+
.flawedTestKeys) {
|
|
162
|
+
flawed.add(key);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
const cleanTests = Math.max(0, input.tests - flawed.size);
|
|
136
166
|
return {
|
|
137
|
-
scoreVersion:
|
|
167
|
+
scoreVersion: MODEL.scoreVersion,
|
|
138
168
|
profile: input.profile,
|
|
139
169
|
score,
|
|
140
170
|
grade: gradeFromScore(score),
|
|
@@ -149,7 +179,8 @@ export function computeScore(input) {
|
|
|
149
179
|
warnings,
|
|
150
180
|
nativeLocators: input.nativeLocators,
|
|
151
181
|
rawLocators: input.rawLocators,
|
|
152
|
-
unassertedTests: input.findings.filter((f) => f.rule ===
|
|
182
|
+
unassertedTests: input.findings.filter((f) => f.rule === 'playwright/expect-expect' && !f.reportOnly).length,
|
|
183
|
+
cleanTests,
|
|
153
184
|
},
|
|
154
185
|
dimensions: dims,
|
|
155
186
|
findings: input.findings,
|
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type ScoreVersion = '
|
|
1
|
+
export type ScoreVersion = 'v3';
|
|
2
2
|
export type ProfileName = 'standard';
|
|
3
3
|
export type Grade = 'A' | 'B' | 'C' | 'D' | 'F';
|
|
4
4
|
export type Severity = 'error' | 'warning' | 'info';
|
|
@@ -13,6 +13,13 @@ export interface Finding {
|
|
|
13
13
|
dimension: DimensionName;
|
|
14
14
|
/** When true, finding is listed but does not contribute penalty units */
|
|
15
15
|
reportOnly?: boolean;
|
|
16
|
+
/** Where the finding sits: inside a specific test ('test', with testKey),
|
|
17
|
+
* inside a before/after hook ('hook' — affects every test in the file),
|
|
18
|
+
* or at module level ('module' — counted once per file). Set during
|
|
19
|
+
* attribution in index.ts; findings without it are treated as 'module'. */
|
|
20
|
+
scope?: 'test' | 'hook' | 'module';
|
|
21
|
+
/** `${file}#${testIndex}` when scope === 'test'. */
|
|
22
|
+
testKey?: string;
|
|
16
23
|
}
|
|
17
24
|
export interface ScoreDimensions {
|
|
18
25
|
playwrightHygiene: number;
|
|
@@ -29,10 +36,12 @@ export interface ScoreSummary {
|
|
|
29
36
|
warnings: number;
|
|
30
37
|
nativeLocators: number;
|
|
31
38
|
rawLocators: number;
|
|
32
|
-
/**
|
|
33
|
-
*
|
|
34
|
-
* playwright/expect-expect finding count). */
|
|
39
|
+
/** Tests with no recognized assertion (uncapped playwright/expect-expect
|
|
40
|
+
* census, after delegation resolution). */
|
|
35
41
|
unassertedTests: number;
|
|
42
|
+
/** Tests with no demerit in any dimension — the headline the score is
|
|
43
|
+
* built from: the weighted share of tests that are clean. */
|
|
44
|
+
cleanTests: number;
|
|
36
45
|
}
|
|
37
46
|
export interface ScoreResult {
|
|
38
47
|
scoreVersion: ScoreVersion;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qaguardian/playwright-score",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Deterministic, AI-free Playwright spec quality score
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Deterministic, AI-free Playwright spec quality score — the weighted share of your tests that are clean.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|