@qaguardian/playwright-score 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +131 -3
- package/README.md +50 -63
- package/dist/eslint-runner.d.ts +2 -0
- package/dist/eslint-runner.js +22 -4
- package/dist/formatters/markdown.js +8 -0
- package/dist/formatters/text.js +7 -0
- package/dist/import-graph.d.ts +1 -0
- package/dist/import-graph.js +1 -1
- package/dist/index.js +123 -10
- package/dist/metrics.d.ts +17 -4
- package/dist/metrics.js +27 -8
- package/dist/playwright-config.d.ts +114 -0
- package/dist/playwright-config.js +520 -0
- package/dist/rules/base-plugin.js +46 -12
- package/dist/test-aliases.d.ts +18 -0
- package/dist/test-aliases.js +218 -0
- package/dist/types.d.ts +8 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,136 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented here. Any change that
|
|
4
|
-
alters what a given suite scores
|
|
5
|
-
|
|
4
|
+
alters what a given suite scores *by changing a rule's weight or
|
|
5
|
+
severity* is a new scoring-model version (see
|
|
6
|
+
[METHODOLOGY.md](./METHODOLOGY.md)), not a quiet patch. A release that
|
|
7
|
+
leaves every rule's weight and severity untouched but changes which files
|
|
8
|
+
or which test names get recognized in the first place — discovery and
|
|
9
|
+
parsing accuracy, not grading policy — can still move a suite's score
|
|
10
|
+
without itself being a new model version; 2.1.0 below is exactly that
|
|
11
|
+
case, and says so explicitly.
|
|
12
|
+
|
|
13
|
+
## 2.1.0 — 2026-09-24
|
|
14
|
+
|
|
15
|
+
Scoring model stays v4 — nothing here changes a rule's weight or
|
|
16
|
+
severity, only whether a suite is *recognized* correctly in the first
|
|
17
|
+
place (see the note at the top of this file reconciling that with "any
|
|
18
|
+
score-changing release is a new model version"). Every fix below made at
|
|
19
|
+
least one real corpus repo's score more accurate — not uniformly higher,
|
|
20
|
+
and not less strict or more lenient by policy; see "Behaviour change"
|
|
21
|
+
below and VALIDATION.md's "Bugs found this way, and fixed" for the full
|
|
22
|
+
story (corpus repo, exact before/after) behind each one.
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
- **Custom `test.extend()` fixture names.** A suite that always declares
|
|
26
|
+
its tests via a custom-named fixture (`const loggedTest =
|
|
27
|
+
test.extend(...)`, imported into every spec file — sometimes through a
|
|
28
|
+
re-export barrel) had every `expect()` inside a completely normal test
|
|
29
|
+
flagged by upstream `eslint-plugin-playwright`'s `no-standalone-expect`
|
|
30
|
+
and sibling rules (which key off the literal identifier `test`, and
|
|
31
|
+
can only dereference a same-file extend chain), and undercounted by
|
|
32
|
+
this package's own test-span/`countTests` logic too.
|
|
33
|
+
`src/test-aliases.ts` resolves these aliases — same-file and
|
|
34
|
+
cross-file, through imports and re-export barrels — and feeds both
|
|
35
|
+
`eslint-plugin-playwright`'s `globalAliases.test` setting and this
|
|
36
|
+
package's own AST rules. Scores rise for affected suites:
|
|
37
|
+
`argos-ci/argos` 57/F → 95/A, `apache/superset` 70/C → 77/C.
|
|
38
|
+
- **Discovery now honours the target's own `playwright.config.*`
|
|
39
|
+
`testDir`/`testMatch`/`testIgnore`.** Spec discovery previously matched
|
|
40
|
+
by filename suffix only, so a directory holding a different test
|
|
41
|
+
runner's files under a similar naming convention got swept in
|
|
42
|
+
alongside the real suite. `src/playwright-config.ts` statically parses
|
|
43
|
+
(never executes — a scored repo's config is untrusted input) the
|
|
44
|
+
nearest `playwright.config.{ts,js,mjs,cjs}` at or above the scanned
|
|
45
|
+
directory and scopes discovery to what the target itself declares.
|
|
46
|
+
When no config is found at all, this falls back to the previous
|
|
47
|
+
filename-based behavior silently, exactly as before this release; when
|
|
48
|
+
a config *is* found but can't be read statically (a dynamic value or
|
|
49
|
+
an unsupported shape, e.g. a `RegExp` `testMatch`), it falls back the
|
|
50
|
+
same way but now also adds an entry to the new, additive, optional
|
|
51
|
+
`ScoreResult.configWarnings` field so the caller can see why. Includes
|
|
52
|
+
three fixes found making this work against real configs: matching
|
|
53
|
+
Playwright's own `createFileMatcher` auto-`**/`-prefix behavior for a
|
|
54
|
+
`testMatch`/`testIgnore` pattern with no leading `**/`; globbing from
|
|
55
|
+
the config's `testDir` rather than the caller's own (possibly narrower
|
|
56
|
+
or differently-rooted) scanned directory; and failing *open* (keeping a
|
|
57
|
+
file) rather than treating an unresolvable relative import as proof a
|
|
58
|
+
file isn't a real spec, since a sparse/partial checkout can legitimately
|
|
59
|
+
be missing a fixtures module the real project has. This moves a
|
|
60
|
+
suite's score in *either* direction depending on whether the old
|
|
61
|
+
filename-only discovery had been over- or under-matching relative to
|
|
62
|
+
what the config itself declares — see "Behaviour change" below for a
|
|
63
|
+
concrete case where it lowers a score.
|
|
64
|
+
- **Discovery no longer misses a plain CommonJS suite.** A follow-on to
|
|
65
|
+
the config-scoped discovery fix above: the safety net that confirms a
|
|
66
|
+
config-matched file transitively imports `@playwright/test` only
|
|
67
|
+
walked ES `import`/`export ... from` syntax, so a suite written
|
|
68
|
+
entirely in CommonJS (`require()`, no `import` anywhere) read as
|
|
69
|
+
*definitive negative evidence* of not being a Playwright spec — the
|
|
70
|
+
opposite of the truth — and hard-failed to 0 files the moment
|
|
71
|
+
config-scoped discovery activated for it. Fixed by also recognizing a
|
|
72
|
+
top-level `require('x')` call the same way as an ES import.
|
|
73
|
+
- **Config discovery and a config's `testDir` can no longer escape the
|
|
74
|
+
scanned repo.** Hardening fixes for a public tool that reads an
|
|
75
|
+
untrusted repo's own config. `findPlaywrightConfig` stops walking
|
|
76
|
+
upward at the first ancestor directory containing `.git` (a directory
|
|
77
|
+
for a normal clone, a file for a worktree/submodule), so a coincidental
|
|
78
|
+
`playwright.config.*` sitting above the actual repo root — a shared CI
|
|
79
|
+
runner, a monorepo checkout nested inside another one — is never picked
|
|
80
|
+
up (this guarantee needs a `.git` somewhere in the ancestor chain).
|
|
81
|
+
The bug this closes: when no `.git` existed anywhere in the ancestor
|
|
82
|
+
chain, the boundary used to clamp a config's `testDir` fell back to the
|
|
83
|
+
filesystem root (or 20 levels up), which is effectively no clamp at
|
|
84
|
+
all, so a `testDir: '/'` could make the glob walk the whole filesystem
|
|
85
|
+
(in the worst case, a hang). For the `testDir` clamp the fallback is
|
|
86
|
+
now the directory actually passed to the scorer, or the config file's
|
|
87
|
+
own directory when that's the wider of the two, never the filesystem
|
|
88
|
+
root. The config search itself (a bounded series of existence checks)
|
|
89
|
+
keeps its wider fallback so monorepo configs above the scanned
|
|
90
|
+
directory are still found.
|
|
91
|
+
Symlinks got the same treatment: a `testDir` is now checked *after
|
|
92
|
+
resolving symlinks* on both sides of the ancestor check, not just
|
|
93
|
+
lexically, so a `testDir` that's a symlink to somewhere outside the
|
|
94
|
+
repo can't launder an escape the plain path comparison would have
|
|
95
|
+
missed (the glob's own `follow` option, already `false` by default for
|
|
96
|
+
`**` patterns, is also now pinned explicitly). An escaping `testDir` —
|
|
97
|
+
absolute (`'/'`), climbing out (`'../../..'`), or via a symlink — gets a
|
|
98
|
+
`configWarnings` entry and falls back to filename-based discovery
|
|
99
|
+
rather than globbing the clamped repo root with testMatch/testIgnore
|
|
100
|
+
patterns written for a different directory. To be clear about the
|
|
101
|
+
actual exposure this closes: a scored *result* was never able to
|
|
102
|
+
include a file whose path lies outside the scan root either way (a
|
|
103
|
+
symlinked file inside the repo is a separate, tracked follow-up) — index.ts always
|
|
104
|
+
filters expanded matches back down to the caller's own requested
|
|
105
|
+
directory — the risk was the walk/glob itself reading (or hanging on)
|
|
106
|
+
parts of the filesystem well outside the repo, not a leaked finding in
|
|
107
|
+
the output. Neither case is exercised by any corpus repo (a legitimate
|
|
108
|
+
config never does this); found and fixed during this release's own
|
|
109
|
+
review, not from a corpus regression.
|
|
110
|
+
|
|
111
|
+
### Behaviour change
|
|
112
|
+
- **Config-scoped discovery can lower a suite's score, not just raise
|
|
113
|
+
it.** Scoping to what a target's own `playwright.config.*` actually
|
|
114
|
+
declares is strictly more accurate than the old filename-suffix match,
|
|
115
|
+
but "more accurate" isn't the same as "friendlier": `mattermost/
|
|
116
|
+
mattermost` moves **84/B → 79/C (a pass→fail flip)** because its
|
|
117
|
+
top-level config's `testDir` doesn't cover the real, correctly-written
|
|
118
|
+
`upgrade-specs/` suite it runs only through per-project `testDir`
|
|
119
|
+
overrides — `projects[].testDir` isn't read yet (see VALIDATION.md's
|
|
120
|
+
"Known limitations"). A future config parser that reads
|
|
121
|
+
`projects[].testDir` would recover this repo's full suite and its
|
|
122
|
+
higher score.
|
|
123
|
+
|
|
124
|
+
### Corpus
|
|
125
|
+
Grown from 50 (2.0.0) to 100 public suites: 34 added in an interim,
|
|
126
|
+
unpublished pass (84 total — see VALIDATION.md for the full accounting of
|
|
127
|
+
that pass), then 16 more here for brand
|
|
128
|
+
recognition (Nextcloud, Ghost, Pinterest, Microsoft, The Guardian,
|
|
129
|
+
Shopify ×2, Adobe, Automattic, BBC, Google, Datadog, Twilio) plus three
|
|
130
|
+
named competitors (Checkly, Currents, LambdaTest), scored under the
|
|
131
|
+
exact same rules as everyone else. Every one of the 100 entries in
|
|
132
|
+
VALIDATION.md now carries the exact commit SHA scored and the date it
|
|
133
|
+
was tested, not just an aggregate "last run" date.
|
|
6
134
|
|
|
7
135
|
## 2.0.0 — 2026-09-23
|
|
8
136
|
|
|
@@ -37,7 +165,7 @@ own official examples must still score >= 90/A (see
|
|
|
37
165
|
`tests/official-examples.test.ts`, vendored from microsoft/playwright),
|
|
38
166
|
and a hand-written "gold" fixture must still reach exactly 100.
|
|
39
167
|
|
|
40
|
-
### Added — four rules narrowing the
|
|
168
|
+
### Added — four rules narrowing the gameability gap (scored under `standard`)
|
|
41
169
|
Narrowing, not closing: each rule matches specific known patterns (see the
|
|
42
170
|
respellings each one closed in this same release), not the general class
|
|
43
171
|
of "code that fakes stability or coverage" — a determined author can
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/qa-guardian/playwright-score/actions/workflows/ci.yml)
|
|
4
4
|
[](https://www.npmjs.com/package/@qaguardian/playwright-score)
|
|
5
|
-
[](
|
|
5
|
+
[](https://github.com/qa-guardian/playwright-score/blob/main/LICENSE)
|
|
6
6
|
|
|
7
7
|
**Deterministic, AI-free quality score for Playwright specs** — the
|
|
8
8
|
weighted share of your tests that are clean.
|
|
@@ -19,76 +19,63 @@ The score never calls an LLM. AI may *generate* or *repair* code using findings;
|
|
|
19
19
|
|
|
20
20
|
## Real-world results
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
100 public Playwright suites, scored with the published package against each
|
|
23
23
|
project's actual source — chosen to include both well-known,
|
|
24
|
-
heavily-engineered platforms and smaller, less mature projects
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
heavily-engineered platforms and smaller, less mature projects, plus three
|
|
25
|
+
named testing-infra competitors scored under the exact same rules as
|
|
26
|
+
everyone else. Not curated to look good: 35 of 100 fail the default
|
|
27
|
+
threshold, for real, verifiable reasons (see
|
|
28
|
+
[VALIDATION.md](https://github.com/qa-guardian/playwright-score/blob/main/VALIDATION.md),
|
|
29
|
+
which also has the full before/after against the last published version
|
|
30
|
+
and every one of the 100 entries, not just the 15 shown below).
|
|
28
31
|
|
|
29
|
-
**
|
|
32
|
+
**100 suites · 7,281 files · 39,059 tests · 65/100 pass (80% threshold) ·
|
|
33
|
+
tested on 2026-09-24**
|
|
34
|
+
|
|
35
|
+
Top scorers (everyone at 97+; the next tier down is a 6-way tie at 96 — see
|
|
36
|
+
VALIDATION.md for the full 100):
|
|
37
|
+
|
|
38
|
+
| Repo (source scanned) | Score | Grade | Result |
|
|
39
|
+
|---|---:|:-:|:-:|
|
|
40
|
+
| [pinterest/gestalt](https://github.com/pinterest/gestalt/tree/22874a7522d1/playwright/accessibility) | 100/100 | A | PASS |
|
|
41
|
+
| [TryGhost/Ghost](https://github.com/TryGhost/Ghost/tree/9d7248b56d53/e2e/tests) | 99/100 | A | PASS |
|
|
42
|
+
| [hashicorp/vault](https://github.com/hashicorp/vault/tree/f03da019aba5/ui/e2e) | 98/100 | A | PASS |
|
|
43
|
+
| [microsoft/playwright (TodoMVC example)](https://github.com/microsoft/playwright/tree/7563c5a67eab/examples/todomvc) | 98/100 | A | PASS |
|
|
44
|
+
| [plausible/analytics](https://github.com/plausible/analytics/tree/8f3970104aa7/e2e) | 98/100 | A | PASS |
|
|
45
|
+
| [segmentio/analytics-next](https://github.com/segmentio/analytics-next/tree/2d6d55301804/packages/browser-integration-tests/src) | 98/100 | A | PASS |
|
|
46
|
+
| [cloudflare/templates](https://github.com/cloudflare/templates/tree/a0bb6ef9a990/playwright-tests) | 97/100 | A | PASS |
|
|
47
|
+
| [glpi-project/glpi](https://github.com/glpi-project/glpi/tree/9e8a9570a90b/tests/e2e/specs) | 97/100 | A | PASS |
|
|
48
|
+
| [shopify/hydrogen](https://github.com/shopify/hydrogen/tree/3ef7ec23096a/e2e/specs) | 97/100 | A | PASS |
|
|
49
|
+
|
|
50
|
+
Bottom 5 scorers:
|
|
30
51
|
|
|
31
52
|
| Repo (source scanned) | Score | Grade | Result |
|
|
32
53
|
|---|---:|:-:|:-:|
|
|
33
|
-
| [
|
|
34
|
-
| [
|
|
35
|
-
| [
|
|
36
|
-
| [
|
|
37
|
-
| [
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
| [Element (Matrix client)](https://github.com/element-hq/element-web/tree/3153cc6ae123/apps/web/playwright) | 89/100 | B | PASS |
|
|
54
|
-
| [Clerk](https://github.com/clerk/javascript/tree/81a641348101/integration/tests) | 87/100 | B | PASS |
|
|
55
|
-
| [Grafana](https://github.com/grafana/grafana/tree/1fd1d6398053/e2e-playwright) | 87/100 | B | PASS |
|
|
56
|
-
| [PostHog](https://github.com/PostHog/posthog/tree/854a4262ec8b/playwright) | 87/100 | B | PASS |
|
|
57
|
-
| [Formbricks](https://github.com/formbricks/formbricks/tree/55ade3bc2a5a/apps/web/playwright) | 85/100 | B | PASS |
|
|
58
|
-
| [Bruno](https://github.com/usebruno/bruno/tree/f1b433f7b06f/tests) | 84/100 | B | PASS |
|
|
59
|
-
| [Mattermost](https://github.com/mattermost/mattermost/tree/f0b63f8ed11e/e2e-tests/playwright) | 84/100 | B | PASS |
|
|
60
|
-
| [Saleor Dashboard](https://github.com/saleor/saleor-dashboard/tree/3508127f50dc/playwright) | 84/100 | B | PASS |
|
|
61
|
-
| [WordPress.com Calypso](https://github.com/Automattic/wp-calypso/tree/af6245bcd266/test/e2e) | 84/100 | B | PASS |
|
|
62
|
-
| [sencho](https://github.com/Studio-Saelix/sencho/tree/489e1a9a9a90/e2e) | 84/100 | B | PASS |
|
|
63
|
-
| [Handsontable](https://github.com/handsontable/handsontable/tree/40c2738b2e87/tests/e2e) | 83/100 | B | PASS |
|
|
64
|
-
| [Rocket.Chat](https://github.com/RocketChat/Rocket.Chat/tree/828fa06f72d5/apps/meteor/tests/e2e) | 83/100 | B | PASS |
|
|
65
|
-
| [novu](https://github.com/novuhq/novu/tree/6fc9d02ea001/apps/dashboard/tests) | 82/100 | B | PASS |
|
|
66
|
-
| [Appsmith](https://github.com/appsmithorg/appsmith/tree/a72a95b73a99/app/client/playwright/tests) | 80/100 | B | PASS |
|
|
67
|
-
| [Payload CMS](https://github.com/payloadcms/payload/tree/5f9bcac2f07b/test/admin/e2e) | 79/100 | C | FAIL |
|
|
68
|
-
| [Immich](https://github.com/immich-app/immich/tree/f48d4b332127/e2e/src/specs) | 78/100 | C | FAIL |
|
|
69
|
-
| [VS Code (component fixtures)](https://github.com/microsoft/vscode/tree/7b24b6303075/test/componentFixtures/playwright) | 77/100 | C | FAIL |
|
|
70
|
-
| [Astro](https://github.com/withastro/astro/tree/40896acb7449/packages/astro/e2e) | 75/100 | C | FAIL |
|
|
71
|
-
| [AFFiNE](https://github.com/toeverything/AFFiNE/tree/d897bb3d8409/tests/blocksuite/e2e) | 72/100 | C | FAIL |
|
|
72
|
-
| [Builder.io SDKs](https://github.com/BuilderIO/builder/tree/51cd0521bbf1/packages/sdks-tests/src/e2e-tests) | 72/100 | C | FAIL |
|
|
73
|
-
| [openplayerjs](https://github.com/openplayerjs/openplayerjs/tree/ee1673106237/e2e) | 72/100 | C | FAIL |
|
|
74
|
-
| [tldraw](https://github.com/tldraw/tldraw/tree/a6cfe996e1df/apps/examples/e2e) | 72/100 | C | FAIL |
|
|
75
|
-
| [cal.com](https://github.com/calcom/cal.com/tree/54343aa685ae/apps/web/playwright) | 71/100 | C | FAIL |
|
|
76
|
-
| [Qwik](https://github.com/QwikDev/qwik/tree/02b5c6b18fe7/e2e) | 70/100 | C | FAIL |
|
|
77
|
-
| [Wekan](https://github.com/wekan/wekan/tree/84414711b7ae/tests/playwright/specs) | 70/100 | C | FAIL |
|
|
78
|
-
| [livecodes](https://github.com/live-codes/livecodes/tree/747054828256/e2e) | 70/100 | C | FAIL |
|
|
79
|
-
| [OpenReplay](https://github.com/openreplay/openreplay/tree/3fce37d89113/frontend/tests/playwright) | 66/100 | D | FAIL |
|
|
80
|
-
| [Flagsmith](https://github.com/Flagsmith/flagsmith/tree/66dc2e9ed00b/frontend/e2e/tests) | 62/100 | D | FAIL |
|
|
81
|
-
| [Activepieces](https://github.com/activepieces/activepieces/tree/ab8cd5c9069e/packages/tests-e2e) | 58/100 | F | FAIL |
|
|
82
|
-
| [TheCyberHub](https://github.com/th3cyb3rhub/TheCyberHub/tree/1cf418654f16/e2e) | 56/100 | F | FAIL |
|
|
83
|
-
|
|
84
|
-
Repo names link to the exact source scanned. Full methodology, findings
|
|
85
|
-
breakdown, and `scripts/validate-corpus.sh` to reproduce every number here
|
|
86
|
-
yourself live in [VALIDATION.md](./VALIDATION.md). A self-contained visual
|
|
54
|
+
| [Flagsmith/flagsmith](https://github.com/Flagsmith/flagsmith/tree/ea5ad826e85b/frontend/e2e/tests) | 62/100 | D | FAIL |
|
|
55
|
+
| [activepieces/activepieces](https://github.com/activepieces/activepieces/tree/9e493a87f471/packages/tests-e2e) | 58/100 | F | FAIL |
|
|
56
|
+
| [th3cyb3rhub/TheCyberHub](https://github.com/th3cyb3rhub/TheCyberHub/tree/1cf418654f16/e2e) | 56/100 | F | FAIL |
|
|
57
|
+
| [DataDog/documentation](https://github.com/DataDog/documentation/tree/6cd10f40cf46/hugo/e2e) | 50/100 | F | FAIL |
|
|
58
|
+
| [getsentry/spotlight](https://github.com/getsentry/spotlight/tree/3dd5eb5b315f/packages/spotlight/tests) | 33/100 | F | FAIL |
|
|
59
|
+
|
|
60
|
+
Repo names link to the exact commit scanned. Selected by a mix of
|
|
61
|
+
mechanical GitHub search (real `@playwright/test` usage, hand-verified)
|
|
62
|
+
and, for the most recent 16, hand-picked brand recognition — three of the
|
|
63
|
+
100 are named competitors' own example suites (Checkly, Currents,
|
|
64
|
+
LambdaTest), scored with no separate bar. Full methodology, the complete
|
|
65
|
+
100-repo table, findings breakdown, and
|
|
66
|
+
`scripts/validate-corpus.sh` to reproduce every number here yourself live
|
|
67
|
+
in
|
|
68
|
+
[VALIDATION.md](https://github.com/qa-guardian/playwright-score/blob/main/VALIDATION.md).
|
|
69
|
+
A machine-readable
|
|
70
|
+
[`validation.json`](https://github.com/qa-guardian/playwright-score/blob/main/validation.json)
|
|
71
|
+
carries the same per-repo data for tooling. See
|
|
72
|
+
[CHANGELOG.md](https://github.com/qa-guardian/playwright-score/blob/main/CHANGELOG.md)
|
|
73
|
+
for what changed release to release. A self-contained visual
|
|
87
74
|
report of an earlier five-suite audit lives at
|
|
88
|
-
[`docs/scorecard.html`](
|
|
75
|
+
[`docs/scorecard.html`](https://github.com/qa-guardian/playwright-score/blob/main/docs/scorecard.html) — host it wherever
|
|
89
76
|
(GitHub Pages, qaguardian.com, ...).
|
|
90
77
|
|
|
91
|
-
See [METHODOLOGY.md](
|
|
78
|
+
See [METHODOLOGY.md](https://github.com/qa-guardian/playwright-score/blob/main/METHODOLOGY.md) for the full scoring model, or the product write-up on the [landing page](https://qaguardian.com/open-source/playwright-score).
|
|
92
79
|
|
|
93
80
|
## Install
|
|
94
81
|
|
package/dist/eslint-runner.d.ts
CHANGED
|
@@ -9,4 +9,6 @@ export declare function runEslint(options: {
|
|
|
9
9
|
cwd?: string;
|
|
10
10
|
/** See findLocalAssertionHelperNames in metrics.ts. */
|
|
11
11
|
assertFunctionNames?: string[];
|
|
12
|
+
/** See detectExtendedTestAliases in test-aliases.ts. */
|
|
13
|
+
testAliasNames?: string[];
|
|
12
14
|
}): Promise<Finding[]>;
|
package/dist/eslint-runner.js
CHANGED
|
@@ -4,10 +4,28 @@ import { commonAncestorDir } from './fs-util.js';
|
|
|
4
4
|
import playwright from 'eslint-plugin-playwright';
|
|
5
5
|
import { basePlugin } from './rules/base-plugin.js';
|
|
6
6
|
import { mapRule } from './profiles.js';
|
|
7
|
-
function buildConfig(assertFunctionNames) {
|
|
7
|
+
function buildConfig(assertFunctionNames, testAliasNames) {
|
|
8
8
|
const base = [
|
|
9
9
|
{
|
|
10
10
|
files: ['**/*.{ts,tsx,js,jsx,mjs,cjs}'],
|
|
11
|
+
// Identifiers bound to a `test.extend(...)`-derived fixture under a
|
|
12
|
+
// custom name (test-aliases.ts) — upstream eslint-plugin-playwright
|
|
13
|
+
// already dereferences a *same-file* extend chain internally, but
|
|
14
|
+
// can't follow one across files (its own `dereference` is scoped to
|
|
15
|
+
// the current file); this setting is what makes a cross-file alias
|
|
16
|
+
// (`import { loggedTest } from './fixtures'`) resolve to `test` for
|
|
17
|
+
// every rule that keys off the literal identifier — no-standalone-
|
|
18
|
+
// expect, no-conditional-expect, no-conditional-in-test, valid-expect,
|
|
19
|
+
// no-focused-test, and others built on the same parseFnCall/
|
|
20
|
+
// resolveToPlaywrightFn helper. Verified against two real corpus
|
|
21
|
+
// repos (argos-ci/argos's `loggedTest`, apache/superset's
|
|
22
|
+
// `testWithAssets`) where this exact cross-file shape was the actual
|
|
23
|
+
// false-positive cause.
|
|
24
|
+
settings: {
|
|
25
|
+
playwright: {
|
|
26
|
+
globalAliases: { test: testAliasNames },
|
|
27
|
+
},
|
|
28
|
+
},
|
|
11
29
|
// A pre-2.0.0 review found `/* eslint-disable */` on line 1 of a
|
|
12
30
|
// 10-test file (waitForTimeout + expect(true).toBe(true) in every
|
|
13
31
|
// test) took it from 35/F to 99/A — every rule this package enables
|
|
@@ -83,7 +101,7 @@ function buildConfig(assertFunctionNames) {
|
|
|
83
101
|
// with the same severity for the actual anti-pattern (an
|
|
84
102
|
// always-skipped test declaration) only.
|
|
85
103
|
'playwright/no-skipped-test': 'off',
|
|
86
|
-
'pwscore/no-skipped-test-declaration': 'warn',
|
|
104
|
+
'pwscore/no-skipped-test-declaration': ['warn', { testAliasNames }],
|
|
87
105
|
'playwright/no-raw-locators': 'warn',
|
|
88
106
|
// Both were mapped in profiles.ts (report-only, so they don't
|
|
89
107
|
// double-penalize the ratio-based locators dimension) but never
|
|
@@ -135,7 +153,7 @@ function buildConfig(assertFunctionNames) {
|
|
|
135
153
|
'pwscore/no-timer-sleep': 'error',
|
|
136
154
|
'pwscore/no-coordinate-click': 'warn',
|
|
137
155
|
'pwscore/no-trivial-assertion': 'error',
|
|
138
|
-
'pwscore/no-soft-assertion-only-test': 'warn',
|
|
156
|
+
'pwscore/no-soft-assertion-only-test': ['warn', { testAliasNames }],
|
|
139
157
|
},
|
|
140
158
|
},
|
|
141
159
|
];
|
|
@@ -166,7 +184,7 @@ export async function runEslint(options) {
|
|
|
166
184
|
const cwd = options.files.length > 0
|
|
167
185
|
? commonAncestorDir(options.files)
|
|
168
186
|
: (options.cwd ?? process.cwd());
|
|
169
|
-
const overrideConfig = buildConfig(options.assertFunctionNames ?? []);
|
|
187
|
+
const overrideConfig = buildConfig(options.assertFunctionNames ?? [], options.testAliasNames ?? []);
|
|
170
188
|
// Try to attach typescript-eslint parser for .ts files
|
|
171
189
|
try {
|
|
172
190
|
const tsParser = await import('@typescript-eslint/parser');
|
|
@@ -32,5 +32,13 @@ export function formatMarkdown(result) {
|
|
|
32
32
|
lines.push('');
|
|
33
33
|
lines.push('</details>');
|
|
34
34
|
}
|
|
35
|
+
if (result.configWarnings?.length) {
|
|
36
|
+
lines.push('');
|
|
37
|
+
lines.push('## Config warnings');
|
|
38
|
+
lines.push('');
|
|
39
|
+
for (const w of result.configWarnings) {
|
|
40
|
+
lines.push(`- ${w}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
35
43
|
return lines.join('\n');
|
|
36
44
|
}
|
package/dist/formatters/text.js
CHANGED
|
@@ -36,6 +36,13 @@ export function formatText(result) {
|
|
|
36
36
|
lines.push(` … and ${result.skippedFiles.length - 20} more`);
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
+
if (result.configWarnings?.length) {
|
|
40
|
+
lines.push('');
|
|
41
|
+
lines.push('Config warnings:');
|
|
42
|
+
for (const w of result.configWarnings) {
|
|
43
|
+
lines.push(` ${w}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
39
46
|
return lines.join('\n');
|
|
40
47
|
}
|
|
41
48
|
function pathBasename(p) {
|
package/dist/import-graph.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare function resolveRelativeImport(fromFile: string, specifier: string, boundaryDir: string): string | undefined;
|
|
1
2
|
/**
|
|
2
3
|
* A Page Object Model is common in real Playwright suites: spec files
|
|
3
4
|
* interact with the page entirely through methods on an imported class
|
package/dist/import-graph.js
CHANGED
|
@@ -35,7 +35,7 @@ function extractRelativeImportSpecifiers(source, file) {
|
|
|
35
35
|
}
|
|
36
36
|
return specifiers;
|
|
37
37
|
}
|
|
38
|
-
function resolveRelativeImport(fromFile, specifier, boundaryDir) {
|
|
38
|
+
export function resolveRelativeImport(fromFile, specifier, boundaryDir) {
|
|
39
39
|
const base = path.resolve(path.dirname(fromFile), specifier);
|
|
40
40
|
const candidates = [];
|
|
41
41
|
for (const ext of RESOLVE_EXTENSIONS)
|
package/dist/index.js
CHANGED
|
@@ -5,6 +5,8 @@ import { runEslint } from './eslint-runner.js';
|
|
|
5
5
|
import { commonAncestorDir } from './fs-util.js';
|
|
6
6
|
import { collectLocallyImportedFiles } from './import-graph.js';
|
|
7
7
|
import { analyzeSource, attributeFindingScopes, countLocators, findLocalAssertionHelperNames, getTestSpans, looksLikeNonPlaywrightTest, } from './metrics.js';
|
|
8
|
+
import { detectExtendedTestAliases } from './test-aliases.js';
|
|
9
|
+
import { findPlaywrightConfig, findRepoBoundary, importsPlaywrightTestTransitively, parsePlaywrightConfig, resolveConfigScopedRoot, testDirEscapesRepo, } from './playwright-config.js';
|
|
8
10
|
import { DEFAULT_THRESHOLDS } from './profiles.js';
|
|
9
11
|
import { computeScore } from './score-engine.js';
|
|
10
12
|
export { computeScore, MODEL, locatorsScore, gradeFromScore, ratioDimensionScore, } from './score-engine.js';
|
|
@@ -67,6 +69,7 @@ function expandPaths(inputs, cwd) {
|
|
|
67
69
|
const explicit = new Set();
|
|
68
70
|
const expanded = new Set();
|
|
69
71
|
const scanRootDirs = new Set();
|
|
72
|
+
const configWarnings = [];
|
|
70
73
|
for (const input of inputs) {
|
|
71
74
|
const abs = path.isAbsolute(input) ? input : path.resolve(cwd, input);
|
|
72
75
|
if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
|
|
@@ -81,14 +84,109 @@ function expandPaths(inputs, cwd) {
|
|
|
81
84
|
// needs this wider boundary, not the narrower common ancestor of
|
|
82
85
|
// only the matched spec files.
|
|
83
86
|
scanRootDirs.add(abs);
|
|
84
|
-
|
|
87
|
+
// A playwright.config.* at or above this directory, when it can be
|
|
88
|
+
// read statically, is authoritative about what this target itself
|
|
89
|
+
// considers a spec — see playwright-config.ts. Falls back to the
|
|
90
|
+
// filename-suffix SPEC_GLOBS (today's behavior) whenever a config
|
|
91
|
+
// isn't found, doesn't overlap this scan, or can't be parsed
|
|
92
|
+
// statically (a dynamic/unsupported shape) — the last case surfaces
|
|
93
|
+
// a warning rather than silently guessing.
|
|
94
|
+
let globCwd = abs;
|
|
95
|
+
let specGlobs = SPEC_GLOBS;
|
|
96
|
+
let ignoreGlobs = DEFAULT_IGNORE_GLOBS;
|
|
97
|
+
let configApplied = false;
|
|
98
|
+
// Boundary for the transitive-import safety net below — widened to
|
|
99
|
+
// the config's own directory (not `abs`), for the same reason
|
|
100
|
+
// import-graph.ts already widens its own POM boundary past the
|
|
101
|
+
// matched spec files' common ancestor: a fixtures module a spec
|
|
102
|
+
// imports test/expect from commonly lives in a directory *sibling*
|
|
103
|
+
// to testDir (`playwright/fixtures` next to `playwright/tests`), not
|
|
104
|
+
// inside it. Verified against a real config (appsmithorg/appsmith)
|
|
105
|
+
// where every real spec imports `test`/`expect` from `../../fixtures`
|
|
106
|
+
// — bounding resolution to `abs` (testDir itself) made every file
|
|
107
|
+
// fail the check and silently zeroed the whole suite, the same
|
|
108
|
+
// false-negative failure mode this safety net exists to prevent.
|
|
109
|
+
let importSafetyNetBoundary = abs;
|
|
110
|
+
const configFile = findPlaywrightConfig(abs);
|
|
111
|
+
if (configFile) {
|
|
112
|
+
const parsedConfig = parsePlaywrightConfig(configFile);
|
|
113
|
+
if (!parsedConfig) {
|
|
114
|
+
configWarnings.push(`Found ${path.relative(cwd, configFile) || configFile} but could not statically parse testDir/testMatch/testIgnore (a dynamic value or unsupported shape) — falling back to filename-based spec discovery for ${path.relative(cwd, abs) || abs}.`);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
// Fallback boundary for findRepoBoundary when no .git turns up:
|
|
118
|
+
// the config file's own directory when it sits above `abs`
|
|
119
|
+
// (found by walking up from abs, so it's always an ancestor-or-
|
|
120
|
+
// same of it), else `abs` itself. Never the filesystem root or
|
|
121
|
+
// 20 levels up — see findRepoBoundary's doc comment.
|
|
122
|
+
const repoRoot = findRepoBoundary(abs, path.dirname(configFile));
|
|
123
|
+
if (testDirEscapesRepo(parsedConfig.testDirAbs, repoRoot)) {
|
|
124
|
+
// The config's testDir resolves outside the repo boundary
|
|
125
|
+
// (directly, or via a symlink — see testDirEscapesRepo). Its
|
|
126
|
+
// testMatch/testIgnore were written for that escaped testDir,
|
|
127
|
+
// so applying them to the clamped repoRoot instead would be a
|
|
128
|
+
// silent mismatch; fall back to filename-based discovery.
|
|
129
|
+
configWarnings.push(`Found ${path.relative(cwd, configFile) || configFile} but its testDir (${parsedConfig.testDirAbs}) resolves outside the repository (${repoRoot}) — falling back to filename-based spec discovery for ${path.relative(cwd, abs) || abs}.`);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
const scopedRoot = resolveConfigScopedRoot(abs, parsedConfig.testDirAbs, repoRoot);
|
|
133
|
+
if (scopedRoot) {
|
|
134
|
+
globCwd = scopedRoot;
|
|
135
|
+
if (parsedConfig.testMatch)
|
|
136
|
+
specGlobs = parsedConfig.testMatch;
|
|
137
|
+
if (parsedConfig.testIgnore) {
|
|
138
|
+
ignoreGlobs = [...DEFAULT_IGNORE_GLOBS, ...parsedConfig.testIgnore];
|
|
139
|
+
}
|
|
140
|
+
configApplied = true;
|
|
141
|
+
importSafetyNetBoundary = path.dirname(configFile);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
for (const g of specGlobs) {
|
|
85
147
|
for (const f of globSync(g, {
|
|
86
|
-
cwd:
|
|
148
|
+
cwd: globCwd,
|
|
87
149
|
absolute: true,
|
|
88
150
|
nodir: true,
|
|
89
|
-
ignore:
|
|
151
|
+
ignore: ignoreGlobs,
|
|
152
|
+
// glob's own default for `follow` is already false (don't
|
|
153
|
+
// descend into a symlinked directory while expanding `**`), but
|
|
154
|
+
// this is untrusted-repo territory (see testDirEscapesRepo
|
|
155
|
+
// above), so pin it explicitly rather than relying on a default
|
|
156
|
+
// that could change upstream.
|
|
157
|
+
follow: false,
|
|
90
158
|
})) {
|
|
91
|
-
|
|
159
|
+
const resolved = path.resolve(f);
|
|
160
|
+
// resolveConfigScopedRoot always globs from testDir (Playwright's
|
|
161
|
+
// own testMatch/testIgnore semantics — see its doc comment), which
|
|
162
|
+
// can be wider than the directory the caller actually pointed
|
|
163
|
+
// `scorePaths` at (a narrower `abs`, e.g. one subproject's specs
|
|
164
|
+
// inside a monorepo testDir that also covers others). Filter back
|
|
165
|
+
// down to what the caller asked for so scoping to testDir for
|
|
166
|
+
// correct path matching never silently widens the scored set.
|
|
167
|
+
if (configApplied && globCwd !== abs) {
|
|
168
|
+
const rel = path.relative(abs, resolved);
|
|
169
|
+
if (rel.startsWith('..') || path.isAbsolute(rel))
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
// Extra safety net, only once config-based discovery is actually
|
|
173
|
+
// driving this scan: testMatch can still be broader than intended
|
|
174
|
+
// (e.g. a generic '**/*.ts'); a file that imports neither
|
|
175
|
+
// '@playwright/test' directly nor (transitively, through a local
|
|
176
|
+
// module) re-exports it isn't a Playwright spec regardless of
|
|
177
|
+
// what the config's glob matched.
|
|
178
|
+
if (configApplied) {
|
|
179
|
+
let source;
|
|
180
|
+
try {
|
|
181
|
+
source = fs.readFileSync(resolved, 'utf8');
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (!importsPlaywrightTestTransitively(resolved, source, importSafetyNetBoundary))
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
expanded.add(resolved);
|
|
92
190
|
}
|
|
93
191
|
}
|
|
94
192
|
continue;
|
|
@@ -111,9 +209,10 @@ function expandPaths(inputs, cwd) {
|
|
|
111
209
|
explicit: [...explicit].sort(),
|
|
112
210
|
expanded: [...expanded].sort(),
|
|
113
211
|
scanRootDirs: [...scanRootDirs].sort(),
|
|
212
|
+
configWarnings,
|
|
114
213
|
};
|
|
115
214
|
}
|
|
116
|
-
function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
215
|
+
function hardFail(profile, threshold, findings, extra = {}, skippedFiles, configWarnings) {
|
|
117
216
|
return {
|
|
118
217
|
scoreVersion: 'v4',
|
|
119
218
|
profile,
|
|
@@ -142,6 +241,7 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
|
|
|
142
241
|
},
|
|
143
242
|
findings,
|
|
144
243
|
...(skippedFiles && skippedFiles.length > 0 ? { skippedFiles } : {}),
|
|
244
|
+
...(configWarnings && configWarnings.length > 0 ? { configWarnings } : {}),
|
|
145
245
|
};
|
|
146
246
|
}
|
|
147
247
|
/**
|
|
@@ -151,7 +251,7 @@ export async function scorePaths(options) {
|
|
|
151
251
|
const cwd = options.cwd ?? process.cwd();
|
|
152
252
|
const profile = options.profile ?? 'standard';
|
|
153
253
|
const threshold = options.threshold ?? DEFAULT_THRESHOLDS[profile] ?? 80;
|
|
154
|
-
const { explicit, expanded, scanRootDirs } = expandPaths(options.paths, cwd);
|
|
254
|
+
const { explicit, expanded, scanRootDirs, configWarnings } = expandPaths(options.paths, cwd);
|
|
155
255
|
if (explicit.length === 0 && expanded.length === 0) {
|
|
156
256
|
// Hard-fail: empty match must never look like a healthy suite (was ~99 PASS).
|
|
157
257
|
return hardFail(profile, threshold, [
|
|
@@ -162,7 +262,7 @@ export async function scorePaths(options) {
|
|
|
162
262
|
file: cwd,
|
|
163
263
|
dimension: 'structure',
|
|
164
264
|
},
|
|
165
|
-
]);
|
|
265
|
+
], {}, undefined, configWarnings);
|
|
166
266
|
}
|
|
167
267
|
// Directory/glob matches are excluded only on positive evidence of a
|
|
168
268
|
// non-Playwright test framework — see looksLikeNonPlaywrightTest. Paths
|
|
@@ -229,11 +329,20 @@ export async function scorePaths(options) {
|
|
|
229
329
|
assertFunctionNames.add(name);
|
|
230
330
|
}
|
|
231
331
|
}
|
|
332
|
+
// Identifiers bound to a `test.extend(...)`-derived fixture under a
|
|
333
|
+
// custom name (same-file, or imported from a local fixtures module —
|
|
334
|
+
// see test-aliases.ts), so a suite that always declares its tests via
|
|
335
|
+
// `loggedTest(...)` instead of `test(...)` is recognized correctly by
|
|
336
|
+
// both eslint-plugin-playwright's rules (fed via runEslint's settings
|
|
337
|
+
// below) and this package's own AST checks (getTestSpans/countTests,
|
|
338
|
+
// base-plugin.ts).
|
|
339
|
+
const testAliasNames = detectExtendedTestAliases(sources, importedFiles, importBoundary);
|
|
232
340
|
const eslintFindings = await runEslint({
|
|
233
341
|
files,
|
|
234
342
|
profile,
|
|
235
343
|
cwd,
|
|
236
344
|
assertFunctionNames: [...assertFunctionNames],
|
|
345
|
+
testAliasNames,
|
|
237
346
|
});
|
|
238
347
|
let totalSloc = 0;
|
|
239
348
|
let totalTests = 0;
|
|
@@ -245,12 +354,12 @@ export async function scorePaths(options) {
|
|
|
245
354
|
for (const file of files) {
|
|
246
355
|
const source = sources.get(file) ?? '';
|
|
247
356
|
const relFile = path.relative(filesBase, file) || path.basename(file);
|
|
248
|
-
const m = analyzeSource(source, relFile);
|
|
357
|
+
const m = analyzeSource(source, relFile, testAliasNames);
|
|
249
358
|
totalSloc += m.sloc;
|
|
250
359
|
native += m.locators.native;
|
|
251
360
|
raw += m.locators.raw;
|
|
252
361
|
metricFindings.push(...m.findings);
|
|
253
|
-
const spans = getTestSpans(source, relFile);
|
|
362
|
+
const spans = getTestSpans(source, relFile, testAliasNames);
|
|
254
363
|
fileSpans.set(relFile, spans);
|
|
255
364
|
// AST spans are the authoritative test count (they are also the
|
|
256
365
|
// attribution targets); the regex count is the fallback for a file
|
|
@@ -293,5 +402,9 @@ export async function scorePaths(options) {
|
|
|
293
402
|
nativeLocators: native,
|
|
294
403
|
rawLocators: raw,
|
|
295
404
|
});
|
|
296
|
-
return
|
|
405
|
+
return {
|
|
406
|
+
...result,
|
|
407
|
+
...(skippedFiles.length > 0 ? { skippedFiles } : {}),
|
|
408
|
+
...(configWarnings.length > 0 ? { configWarnings } : {}),
|
|
409
|
+
};
|
|
297
410
|
}
|