@qaguardian/playwright-score 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +817 -0
- package/METHODOLOGY.md +53 -6
- package/README.md +52 -31
- package/dist/cli.js +8 -2
- package/dist/eslint-runner.d.ts +2 -0
- package/dist/eslint-runner.js +57 -3
- package/dist/formatters/markdown.js +8 -0
- package/dist/formatters/text.js +7 -0
- package/dist/import-graph.d.ts +1 -0
- package/dist/import-graph.js +1 -1
- package/dist/index.js +130 -12
- package/dist/metrics.d.ts +18 -4
- package/dist/metrics.js +59 -9
- package/dist/playwright-config.d.ts +114 -0
- package/dist/playwright-config.js +520 -0
- package/dist/profiles.d.ts +15 -0
- package/dist/profiles.js +34 -0
- package/dist/rules/base-plugin.d.ts +4 -0
- package/dist/rules/base-plugin.js +698 -7
- package/dist/score-engine.d.ts +9 -1
- package/dist/score-engine.js +9 -1
- package/dist/test-aliases.d.ts +18 -0
- package/dist/test-aliases.js +218 -0
- package/dist/types.d.ts +21 -1
- package/package.json +3 -2
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { parseForESLint } from '@typescript-eslint/parser';
|
|
4
|
+
import { resolveRelativeImport } from './import-graph.js';
|
|
5
|
+
// Spec discovery (index.ts's SPEC_GLOBS) matches by filename suffix only —
|
|
6
|
+
// it never reads the scanned repo's own playwright.config.*, so it can't
|
|
7
|
+
// tell a real Playwright suite's *.e2e.ts apart from an unrelated *.spec.ts
|
|
8
|
+
// file sitting in the same directory under a *different* test runner (Ionic
|
|
9
|
+
// Framework's core/: 419 real *.e2e.ts Playwright specs alongside 60 Jest
|
|
10
|
+
// *.spec.ts unit tests, both matched by the default globs). This module
|
|
11
|
+
// reads testDir/testMatch/testIgnore *statically* — parsed as source, never
|
|
12
|
+
// executed (a scored repo's config is untrusted input) — so index.ts can
|
|
13
|
+
// scope discovery to what the target itself considers its test files.
|
|
14
|
+
const CONFIG_FILENAMES = [
|
|
15
|
+
'playwright.config.ts',
|
|
16
|
+
'playwright.config.js',
|
|
17
|
+
'playwright.config.mjs',
|
|
18
|
+
'playwright.config.cjs',
|
|
19
|
+
];
|
|
20
|
+
/** How far up from the scan root to look for a config — generous relative
|
|
21
|
+
* to any real monorepo nesting depth, still a hard bound. */
|
|
22
|
+
const MAX_ANCESTOR_LOOKUP = 20;
|
|
23
|
+
/**
|
|
24
|
+
* Walks upward from `startDir` to find this scan's repo boundary — the
|
|
25
|
+
* first ancestor directory that contains a `.git` entry (a directory for a
|
|
26
|
+
* normal clone, a file for a worktree/submodule — either counts as "this is
|
|
27
|
+
* a repo root").
|
|
28
|
+
*
|
|
29
|
+
* When no `.git` turns up within MAX_ANCESTOR_LOOKUP hops or before
|
|
30
|
+
* reaching the filesystem root, the *caller* decides what's safe to fall
|
|
31
|
+
* back to via `fallbackBoundary`:
|
|
32
|
+
*
|
|
33
|
+
* - `findPlaywrightConfig`'s own internal use (below) omits it, keeping
|
|
34
|
+
* this function's legacy behavior of returning whatever directory the
|
|
35
|
+
* climb stopped at. That's fine there — it only ever bounds a loop of
|
|
36
|
+
* cheap `fs.statSync` existence checks for a handful of fixed filenames,
|
|
37
|
+
* never a glob or a file read, so a wide bound doesn't let an untrusted
|
|
38
|
+
* repo make this tool touch anything outside itself.
|
|
39
|
+
* - Every caller that uses the return value as an actual security
|
|
40
|
+
* boundary — most importantly `resolveConfigScopedRoot`'s `repoRoot`,
|
|
41
|
+
* which clamps a config's `testDir` before it becomes a glob root (see
|
|
42
|
+
* its own doc comment) — MUST pass an explicit `fallbackBoundary`, and
|
|
43
|
+
* it must never be the filesystem root or some arbitrary ancestor 20
|
|
44
|
+
* levels up (that would make the "repo boundary" nearly unbounded and
|
|
45
|
+
* defeat the escape checks it exists to support). index.ts passes the
|
|
46
|
+
* scanned directory, or, when a config file was found above it, that
|
|
47
|
+
* config file's own directory — whichever of the two is the ancestor
|
|
48
|
+
* (the config file is always found by walking up from the scan root, so
|
|
49
|
+
* its directory is always at-or-above it).
|
|
50
|
+
*/
|
|
51
|
+
export function findRepoBoundary(startDir, fallbackBoundary) {
|
|
52
|
+
let dir = startDir;
|
|
53
|
+
let last = startDir;
|
|
54
|
+
for (let i = 0; i < MAX_ANCESTOR_LOOKUP; i++) {
|
|
55
|
+
last = dir;
|
|
56
|
+
if (fs.existsSync(path.join(dir, '.git')))
|
|
57
|
+
return dir;
|
|
58
|
+
const parent = path.dirname(dir);
|
|
59
|
+
if (parent === dir)
|
|
60
|
+
break; // reached the filesystem root — no .git found
|
|
61
|
+
dir = parent;
|
|
62
|
+
}
|
|
63
|
+
return fallbackBoundary ?? last;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Walks upward from `scanRootAbs` (the directory the caller pointed at, or
|
|
67
|
+
* its own directory tree) looking for a playwright.config.* file "at or
|
|
68
|
+
* above" it — the common real-world shape is a monorepo root config one or
|
|
69
|
+
* more levels above the actual spec directory being scanned. Never searches
|
|
70
|
+
* above the repo boundary (see findRepoBoundary): a config living outside
|
|
71
|
+
* the scanned repo (e.g. a coincidental playwright.config.* sitting in some
|
|
72
|
+
* unrelated ancestor directory on a shared CI runner) must never be picked
|
|
73
|
+
* up and applied to this scan.
|
|
74
|
+
*/
|
|
75
|
+
export function findPlaywrightConfig(scanRootAbs) {
|
|
76
|
+
const boundary = findRepoBoundary(scanRootAbs);
|
|
77
|
+
let dir = scanRootAbs;
|
|
78
|
+
for (let i = 0; i < MAX_ANCESTOR_LOOKUP; i++) {
|
|
79
|
+
for (const name of CONFIG_FILENAMES) {
|
|
80
|
+
const candidate = path.join(dir, name);
|
|
81
|
+
try {
|
|
82
|
+
if (fs.statSync(candidate).isFile())
|
|
83
|
+
return candidate;
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
// doesn't exist — try the next filename/directory
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (dir === boundary)
|
|
90
|
+
break; // never search above the repo root
|
|
91
|
+
const parent = path.dirname(dir);
|
|
92
|
+
if (parent === dir)
|
|
93
|
+
break;
|
|
94
|
+
dir = parent;
|
|
95
|
+
}
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
function isStringLiteral(node) {
|
|
99
|
+
return !!node && node.type === 'Literal' && typeof node.value === 'string';
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* A string literal, or an array of only string literals — Playwright's own
|
|
103
|
+
* `testMatch`/`testIgnore` also accept a `RegExp`, which this deliberately
|
|
104
|
+
* does not support (a regex can embed arbitrary logic no static reader
|
|
105
|
+
* should try to evaluate): a config using one is treated as unparseable as
|
|
106
|
+
* a whole (see parsePlaywrightConfig) rather than silently ignoring just
|
|
107
|
+
* that field, so the fallback is obvious rather than a partial, confusing
|
|
108
|
+
* hybrid.
|
|
109
|
+
*/
|
|
110
|
+
/**
|
|
111
|
+
* Playwright's own `createFileMatcher` (packages/playwright/src/util.ts)
|
|
112
|
+
* prepends `**\/` to any string pattern that doesn't already start with
|
|
113
|
+
* it, before handing it to minimatch — i.e. a pattern with no leading
|
|
114
|
+
* `**\/` still matches at any depth, not just files directly under
|
|
115
|
+
* testDir. A plain glob library (this package uses `glob`) does NOT do
|
|
116
|
+
* this on its own: `*e2e.spec.ts` only matches a file sitting directly in
|
|
117
|
+
* the glob's cwd, since a single `*` never crosses a `/`. Verified against
|
|
118
|
+
* a real config (payloadcms/payload's `test/playwright.config.ts`:
|
|
119
|
+
* `testMatch: ['*e2e.spec.ts', '*perf.spec.ts']`, every real spec several
|
|
120
|
+
* directories below testDir) that silently matched zero files without
|
|
121
|
+
* this — the exact same failure mode this feature exists to fix, caused
|
|
122
|
+
* by the feature itself. Applied to testIgnore too, since Playwright
|
|
123
|
+
* matches both through the same `createFileMatcher`.
|
|
124
|
+
*/
|
|
125
|
+
function toRecursiveGlob(pattern) {
|
|
126
|
+
return pattern.startsWith('**/') ? pattern : `**/${pattern}`;
|
|
127
|
+
}
|
|
128
|
+
function extractStringOrStringArray(node) {
|
|
129
|
+
if (!node)
|
|
130
|
+
return undefined;
|
|
131
|
+
if (isStringLiteral(node))
|
|
132
|
+
return [toRecursiveGlob(node.value)];
|
|
133
|
+
if (node.type === 'ArrayExpression') {
|
|
134
|
+
const elements = node.elements ?? [];
|
|
135
|
+
const values = [];
|
|
136
|
+
for (const el of elements) {
|
|
137
|
+
const candidate = el ?? undefined;
|
|
138
|
+
if (!isStringLiteral(candidate))
|
|
139
|
+
return undefined;
|
|
140
|
+
values.push(toRecursiveGlob(candidate.value));
|
|
141
|
+
}
|
|
142
|
+
return values;
|
|
143
|
+
}
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
/** Finds the plain object literal passed to defineConfig(...), or used
|
|
147
|
+
* directly as `export default {...}` / `module.exports = {...}`. Returns
|
|
148
|
+
* undefined for anything else (a spread, a conditional, a function call
|
|
149
|
+
* result other than defineConfig, a variable reference, ...) — deliberately
|
|
150
|
+
* conservative: a config shape we can't confidently read statically must
|
|
151
|
+
* fall back, never guess. */
|
|
152
|
+
function findConfigObjectLiteral(body) {
|
|
153
|
+
const unwrapToObjectLiteral = (node) => {
|
|
154
|
+
if (!node)
|
|
155
|
+
return undefined;
|
|
156
|
+
if (node.type === 'ObjectExpression')
|
|
157
|
+
return node;
|
|
158
|
+
if (node.type === 'CallExpression') {
|
|
159
|
+
// defineConfig({...}) (any callee shape — named import, namespace
|
|
160
|
+
// access, etc.) — only the single object-literal argument shape is
|
|
161
|
+
// supported; anything else (spread args, no args, a non-object arg)
|
|
162
|
+
// is unparseable.
|
|
163
|
+
const args = node.arguments ?? [];
|
|
164
|
+
if (args.length === 1 && args[0]?.type === 'ObjectExpression')
|
|
165
|
+
return args[0];
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
return undefined;
|
|
169
|
+
};
|
|
170
|
+
for (const node of body) {
|
|
171
|
+
if (node.type === 'ExportDefaultDeclaration') {
|
|
172
|
+
const decl = node.declaration;
|
|
173
|
+
const obj = unwrapToObjectLiteral(decl);
|
|
174
|
+
if (obj)
|
|
175
|
+
return obj;
|
|
176
|
+
}
|
|
177
|
+
if (node.type === 'ExpressionStatement') {
|
|
178
|
+
const expr = node.expression;
|
|
179
|
+
if (expr?.type === 'AssignmentExpression' &&
|
|
180
|
+
expr.left?.type === 'MemberExpression') {
|
|
181
|
+
const left = expr.left;
|
|
182
|
+
const obj = left.object;
|
|
183
|
+
const prop = left.property;
|
|
184
|
+
// module.exports = {...} / module.exports = defineConfig({...})
|
|
185
|
+
if (obj?.type === 'Identifier' && obj.name === 'module' && prop?.type === 'Identifier' && prop.name === 'exports') {
|
|
186
|
+
const right = expr.right;
|
|
187
|
+
const resolved = unwrapToObjectLiteral(right);
|
|
188
|
+
if (resolved)
|
|
189
|
+
return resolved;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Parses `testDir`/`testMatch`/`testIgnore` out of a playwright.config.*
|
|
198
|
+
* file's source, statically (AST-based, never executed — a scored repo's
|
|
199
|
+
* config is untrusted input, same reasoning as never running its tests).
|
|
200
|
+
* Returns undefined when the shape can't be read with confidence (a
|
|
201
|
+
* dynamic value from `process.env`, a spread, an unsupported top-level
|
|
202
|
+
* export shape, ...) — callers fall back to today's filename-suffix
|
|
203
|
+
* discovery for the whole scan, with a warning, rather than acting on a
|
|
204
|
+
* partial or guessed reading.
|
|
205
|
+
*/
|
|
206
|
+
export function parsePlaywrightConfig(configFile) {
|
|
207
|
+
let source;
|
|
208
|
+
try {
|
|
209
|
+
source = fs.readFileSync(configFile, 'utf8');
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return undefined;
|
|
213
|
+
}
|
|
214
|
+
let ast;
|
|
215
|
+
try {
|
|
216
|
+
ast = parseForESLint(source, {
|
|
217
|
+
ecmaVersion: 2022,
|
|
218
|
+
sourceType: 'module',
|
|
219
|
+
}).ast;
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
const configObject = findConfigObjectLiteral(ast.body ?? []);
|
|
225
|
+
if (!configObject)
|
|
226
|
+
return undefined;
|
|
227
|
+
const properties = configObject.properties ?? [];
|
|
228
|
+
let testDir;
|
|
229
|
+
let testMatch;
|
|
230
|
+
let testIgnore;
|
|
231
|
+
let sawTestMatchKey = false;
|
|
232
|
+
let sawTestIgnoreKey = false;
|
|
233
|
+
for (const prop of properties) {
|
|
234
|
+
// A spread (`...shared`) or computed key makes the object's real shape
|
|
235
|
+
// impossible to read statically — bail on the whole config rather than
|
|
236
|
+
// silently ignoring part of it.
|
|
237
|
+
if (prop.type === 'SpreadElement')
|
|
238
|
+
return undefined;
|
|
239
|
+
if (prop.type !== 'Property')
|
|
240
|
+
continue;
|
|
241
|
+
const computed = prop.computed;
|
|
242
|
+
const key = prop.key;
|
|
243
|
+
if (computed)
|
|
244
|
+
return undefined;
|
|
245
|
+
const keyName = key?.type === 'Identifier'
|
|
246
|
+
? key.name
|
|
247
|
+
: isStringLiteral(key)
|
|
248
|
+
? key.value
|
|
249
|
+
: undefined;
|
|
250
|
+
if (!keyName)
|
|
251
|
+
continue;
|
|
252
|
+
const value = prop.value;
|
|
253
|
+
if (keyName === 'testDir') {
|
|
254
|
+
if (!isStringLiteral(value))
|
|
255
|
+
return undefined;
|
|
256
|
+
testDir = value.value;
|
|
257
|
+
}
|
|
258
|
+
else if (keyName === 'testMatch') {
|
|
259
|
+
sawTestMatchKey = true;
|
|
260
|
+
testMatch = extractStringOrStringArray(value);
|
|
261
|
+
if (testMatch === undefined)
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
else if (keyName === 'testIgnore') {
|
|
265
|
+
sawTestIgnoreKey = true;
|
|
266
|
+
testIgnore = extractStringOrStringArray(value);
|
|
267
|
+
if (testIgnore === undefined)
|
|
268
|
+
return undefined;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
const configDir = path.dirname(configFile);
|
|
272
|
+
const testDirAbs = testDir ? path.resolve(configDir, testDir) : configDir;
|
|
273
|
+
return {
|
|
274
|
+
testDirAbs,
|
|
275
|
+
testMatch: sawTestMatchKey ? testMatch : undefined,
|
|
276
|
+
testIgnore: sawTestIgnoreKey ? testIgnore : undefined,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
function isAncestorOrSame(ancestor, descendant) {
|
|
280
|
+
const rel = path.relative(ancestor, descendant);
|
|
281
|
+
return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
|
|
282
|
+
}
|
|
283
|
+
/** Resolves symlinks; falls back to the input path unchanged when it
|
|
284
|
+
* doesn't exist on disk (matches the existing "testDir doesn't exist"
|
|
285
|
+
* behavior elsewhere in this file rather than throwing). */
|
|
286
|
+
function safeRealpath(p) {
|
|
287
|
+
try {
|
|
288
|
+
return fs.realpathSync(p);
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return p;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* True when `testDirAbs`, resolved through any symlinks, is NOT inside
|
|
296
|
+
* (or equal to) `repoRoot`, also resolved through symlinks. A scored
|
|
297
|
+
* repo's config is untrusted input, and a lexical ancestor check alone
|
|
298
|
+
* isn't enough — a `testDir` that lexically sits inside the repo can still
|
|
299
|
+
* be a symlink whose real target is elsewhere on disk (e.g. `tests/` ->
|
|
300
|
+
* `/tmp/somewhere-else`), which would otherwise let a crafted repo make
|
|
301
|
+
* this public tool glob outside its own boundary. Both sides are
|
|
302
|
+
* realpath'd (never just testDirAbs) because `repoRoot` itself could in
|
|
303
|
+
* principle be reached through a symlinked path.
|
|
304
|
+
*/
|
|
305
|
+
export function testDirEscapesRepo(testDirAbs, repoRoot) {
|
|
306
|
+
const realRepoRoot = safeRealpath(repoRoot);
|
|
307
|
+
const realTestDirAbs = safeRealpath(testDirAbs);
|
|
308
|
+
return !isAncestorOrSame(realRepoRoot, realTestDirAbs);
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Chooses which directory to actually glob from, given the directory the
|
|
312
|
+
* caller pointed `scorePaths` at (`scanRootAbs`) and the config's resolved
|
|
313
|
+
* `testDir`. Always testDirAbs, never the caller's narrower `scanRootAbs`
|
|
314
|
+
* — Playwright itself unconditionally resolves testMatch/testIgnore
|
|
315
|
+
* relative to testDir, and a testMatch entry is not always a bare
|
|
316
|
+
* basename pattern; it can embed real path segments anchored at testDir
|
|
317
|
+
* (verified against a real config, PostHog/posthog's `playwright/
|
|
318
|
+
* playwright.config.ts`: `testDir: '..'` — the repo root — with
|
|
319
|
+
* `testMatch: ['playwright/e2e/**\/*.spec.ts', 'products/*\/frontend/e2e/
|
|
320
|
+
* **\/*.spec.ts']`; globbing from the caller's narrower `scanRootAbs`
|
|
321
|
+
* instead of testDirAbs made the `playwright/e2e/` segment never match
|
|
322
|
+
* anything, silently zeroing a real, correctly-discovered config). The
|
|
323
|
+
* caller (index.ts) is responsible for filtering the resulting absolute
|
|
324
|
+
* paths back down to its own requested subtree when `scanRootAbs` is
|
|
325
|
+
* narrower than `testDirAbs`, so scoping to testDir here never widens
|
|
326
|
+
* what actually gets scored beyond what the caller asked for.
|
|
327
|
+
*
|
|
328
|
+
* Returns undefined when testDir doesn't exist on disk, or doesn't
|
|
329
|
+
* overlap the requested scan root at all in either direction — the
|
|
330
|
+
* config doesn't describe this scan, so the caller should ignore it
|
|
331
|
+
* entirely rather than force a mismatch.
|
|
332
|
+
*
|
|
333
|
+
* `repoRoot` (see findRepoBoundary) clamps `testDirAbs` before any of the
|
|
334
|
+
* above, via `testDirEscapesRepo` (realpath'd on both sides, so a
|
|
335
|
+
* symlinked `testDir` can't launder an escape either): a scored repo's own
|
|
336
|
+
* config is untrusted input, and a `testDir` that resolves outside the
|
|
337
|
+
* repo (an absolute `'/'`, a relative `'../../..'` that climbs past the
|
|
338
|
+
* repo root, a symlink pointing elsewhere, ...) must never turn into a
|
|
339
|
+
* glob root — that would let a crafted config make this public tool read
|
|
340
|
+
* or scan arbitrary parts of the filesystem it's running on. When
|
|
341
|
+
* testDirAbs falls outside repoRoot, repoRoot itself is used as the
|
|
342
|
+
* effective testDir instead of rejecting the config outright, so a
|
|
343
|
+
* merely-overshooting `testDir` still gets *some* config-scoped result
|
|
344
|
+
* (bounded to the repo) rather than silently falling all the way back to
|
|
345
|
+
* unscoped filename discovery. index.ts additionally treats this
|
|
346
|
+
* escaped-and-clamped case as "the config doesn't cleanly describe this
|
|
347
|
+
* scan" and falls back to filename-based discovery with a configWarning,
|
|
348
|
+
* rather than applying the config's testMatch/testIgnore (written for the
|
|
349
|
+
* escaped testDir) to the clamped repoRoot instead.
|
|
350
|
+
*/
|
|
351
|
+
export function resolveConfigScopedRoot(scanRootAbs, testDirAbs, repoRoot) {
|
|
352
|
+
const clampedTestDirAbs = testDirEscapesRepo(testDirAbs, repoRoot) ? repoRoot : testDirAbs;
|
|
353
|
+
if (!isAncestorOrSame(clampedTestDirAbs, scanRootAbs) && !isAncestorOrSame(scanRootAbs, clampedTestDirAbs)) {
|
|
354
|
+
return undefined;
|
|
355
|
+
}
|
|
356
|
+
try {
|
|
357
|
+
if (fs.statSync(clampedTestDirAbs).isDirectory())
|
|
358
|
+
return clampedTestDirAbs;
|
|
359
|
+
}
|
|
360
|
+
catch {
|
|
361
|
+
// testDir doesn't exist on disk — nothing to scope to.
|
|
362
|
+
}
|
|
363
|
+
return undefined;
|
|
364
|
+
}
|
|
365
|
+
const RESOLVE_DEPTH = 4;
|
|
366
|
+
const RESOLVE_FILE_CAP = 100;
|
|
367
|
+
/**
|
|
368
|
+
* True unless `file` has *definitive negative evidence* of not being a
|
|
369
|
+
* Playwright spec: no direct `@playwright/test` import, no relative
|
|
370
|
+
* import at all, or every relative import resolves (on disk) to a file
|
|
371
|
+
* that itself, transitively, doesn't import `@playwright/test` either.
|
|
372
|
+
* Used only as an extra safety net once config-based discovery is already
|
|
373
|
+
* active (see index.ts) — testMatch/testIgnore can still be broader than
|
|
374
|
+
* intended (e.g. a generic `**\/*.ts`), and this catches a plain non-test
|
|
375
|
+
* source file that pattern would otherwise sweep in.
|
|
376
|
+
*
|
|
377
|
+
* Deliberately fails OPEN (keeps the file) whenever a relative import
|
|
378
|
+
* can't be resolved on disk, rather than treating that as negative
|
|
379
|
+
* evidence — same "denylist, not allowlist" philosophy metrics.ts's
|
|
380
|
+
* `looksLikeNonPlaywrightTest` already documents ("a missed rare/unusual
|
|
381
|
+
* file just scores harmlessly, where a falsely-skipped real spec silently
|
|
382
|
+
* stops being scored at all"). This matters in practice, not just in
|
|
383
|
+
* theory: verified against a real config (appsmithorg/appsmith) whose
|
|
384
|
+
* every spec imports `test`/`expect` from a sibling `../../fixtures`
|
|
385
|
+
* module — under `scripts/validate-corpus.sh`'s sparse checkout (only the
|
|
386
|
+
* declared subpath is fetched), that fixtures file genuinely doesn't
|
|
387
|
+
* exist on disk, so resolution legitimately fails even though the import
|
|
388
|
+
* is completely real; a full (non-sparse) checkout, which is what a real
|
|
389
|
+
* customer's CI actually has, would resolve it fine. Treating an
|
|
390
|
+
* unresolvable import as "not a test" turned a correctly-configured real
|
|
391
|
+
* suite into a hard 0-files fail.
|
|
392
|
+
*/
|
|
393
|
+
/**
|
|
394
|
+
* Pulls the string-literal specifier out of a CommonJS `require('x')` call
|
|
395
|
+
* expression (any position — a bare `require(...)`, not just a call whose
|
|
396
|
+
* result is assigned to something).
|
|
397
|
+
*/
|
|
398
|
+
function requireCallSpecifier(node) {
|
|
399
|
+
if (!node || node.type !== 'CallExpression')
|
|
400
|
+
return undefined;
|
|
401
|
+
const callee = node.callee;
|
|
402
|
+
if (callee?.type !== 'Identifier' || callee.name !== 'require')
|
|
403
|
+
return undefined;
|
|
404
|
+
const args = node.arguments ?? [];
|
|
405
|
+
const arg = args[0];
|
|
406
|
+
if (arg?.type === 'Literal' && typeof arg.value === 'string') {
|
|
407
|
+
return arg.value;
|
|
408
|
+
}
|
|
409
|
+
return undefined;
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Top-level-only `require(...)` specifiers in a CommonJS file — mirrors the
|
|
413
|
+
* ES `import`/`export ... from` sources this function already walks at the
|
|
414
|
+
* same (non-nested) depth. Covers the two shapes real fixtures/spec files
|
|
415
|
+
* actually use: `const { test } = require('./x')` and a bare
|
|
416
|
+
* `require('./x')` statement. Verified against a real repo
|
|
417
|
+
* (wekan/wekan's `tests/playwright/`, plain CommonJS `.js` specs/fixtures,
|
|
418
|
+
* no `import` anywhere) that this function silently zeroed to "not a
|
|
419
|
+
* Playwright spec" for every file before this fix — `hasAnyRelativeImport`
|
|
420
|
+
* never became true (no ES import syntax present at all), so the "no
|
|
421
|
+
* @playwright/test found + nothing to resolve" fallback read as definitive
|
|
422
|
+
* negative evidence instead of the true reason: this loop just wasn't
|
|
423
|
+
* looking at CommonJS's own import syntax.
|
|
424
|
+
*/
|
|
425
|
+
function requireSpecifiersInBody(body) {
|
|
426
|
+
const specifiers = [];
|
|
427
|
+
for (const node of body) {
|
|
428
|
+
if (node.type === 'ExpressionStatement') {
|
|
429
|
+
const spec = requireCallSpecifier(node.expression);
|
|
430
|
+
if (spec)
|
|
431
|
+
specifiers.push(spec);
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
if (node.type === 'VariableDeclaration') {
|
|
435
|
+
for (const decl of node.declarations ?? []) {
|
|
436
|
+
const spec = requireCallSpecifier(decl.init);
|
|
437
|
+
if (spec)
|
|
438
|
+
specifiers.push(spec);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return specifiers;
|
|
443
|
+
}
|
|
444
|
+
export function importsPlaywrightTestTransitively(file, source, boundaryDir) {
|
|
445
|
+
const visited = new Set([file]);
|
|
446
|
+
const queue = [{ file, source, depth: 0 }];
|
|
447
|
+
let checked = 0;
|
|
448
|
+
let hasAnyRelativeImport = false;
|
|
449
|
+
let everyRelativeImportResolved = true;
|
|
450
|
+
while (queue.length > 0 && checked < RESOLVE_FILE_CAP) {
|
|
451
|
+
const next = queue.shift();
|
|
452
|
+
if (!next)
|
|
453
|
+
break;
|
|
454
|
+
checked++;
|
|
455
|
+
let ast;
|
|
456
|
+
try {
|
|
457
|
+
ast = parseForESLint(next.source, {
|
|
458
|
+
ecmaVersion: 2022,
|
|
459
|
+
sourceType: 'module',
|
|
460
|
+
ecmaFeatures: { jsx: /\.(tsx|jsx)$/.test(next.file) },
|
|
461
|
+
}).ast;
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
// Can't parse this hop — inconclusive, not negative evidence.
|
|
465
|
+
everyRelativeImportResolved = false;
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
const body = ast.body ?? [];
|
|
469
|
+
const requireSrcs = requireSpecifiersInBody(body);
|
|
470
|
+
for (const node of body) {
|
|
471
|
+
if (node.type !== 'ImportDeclaration' && node.type !== 'ExportNamedDeclaration' && node.type !== 'ExportAllDeclaration') {
|
|
472
|
+
continue;
|
|
473
|
+
}
|
|
474
|
+
const src = node.source?.value;
|
|
475
|
+
if (typeof src !== 'string')
|
|
476
|
+
continue;
|
|
477
|
+
requireSrcs.push(src);
|
|
478
|
+
}
|
|
479
|
+
for (const src of requireSrcs) {
|
|
480
|
+
if (src === '@playwright/test')
|
|
481
|
+
return true;
|
|
482
|
+
if (!src.startsWith('.'))
|
|
483
|
+
continue;
|
|
484
|
+
hasAnyRelativeImport = true;
|
|
485
|
+
if (next.depth >= RESOLVE_DEPTH) {
|
|
486
|
+
everyRelativeImportResolved = false;
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
const resolved = resolveRelativeImport(next.file, src, boundaryDir);
|
|
490
|
+
if (!resolved) {
|
|
491
|
+
// Doesn't exist on disk (or climbs outside boundaryDir) — could be
|
|
492
|
+
// a real fixtures module missing only because of how this scan was
|
|
493
|
+
// set up (sparse checkout, a narrower boundary than the real
|
|
494
|
+
// project root, ...). Inconclusive, not negative evidence.
|
|
495
|
+
everyRelativeImportResolved = false;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
if (visited.has(resolved))
|
|
499
|
+
continue;
|
|
500
|
+
visited.add(resolved);
|
|
501
|
+
let depSource;
|
|
502
|
+
try {
|
|
503
|
+
depSource = fs.readFileSync(resolved, 'utf8');
|
|
504
|
+
}
|
|
505
|
+
catch {
|
|
506
|
+
everyRelativeImportResolved = false;
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
queue.push({ file: resolved, source: depSource, depth: next.depth + 1 });
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
if (checked >= RESOLVE_FILE_CAP && queue.length > 0)
|
|
513
|
+
everyRelativeImportResolved = false;
|
|
514
|
+
// No @playwright/test found anywhere reachable: keep the file unless
|
|
515
|
+
// every relative import chain fully resolved and still came up empty
|
|
516
|
+
// (definitive negative evidence) — a file with no relative imports at
|
|
517
|
+
// all and no @playwright/test import is also definitive negative
|
|
518
|
+
// evidence (nothing left to resolve).
|
|
519
|
+
return hasAnyRelativeImport && !everyRelativeImportResolved;
|
|
520
|
+
}
|
package/dist/profiles.d.ts
CHANGED
|
@@ -1,4 +1,19 @@
|
|
|
1
1
|
import type { DimensionName, ProfileName } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* `strict` was planned for 1.1.0 (never published) to gate four
|
|
4
|
+
* contentious rules behind an opt-in profile, report-only under
|
|
5
|
+
* `standard`. 2.0.0 (model v4, see
|
|
6
|
+
* CHANGELOG.md) scores all of them under `standard` directly — owner
|
|
7
|
+
* direction: public scores should reflect quality, full stop, not be
|
|
8
|
+
* softened to avoid moving anyone's number. With nothing left for a
|
|
9
|
+
* second profile to gate (every rule now counts the same way regardless
|
|
10
|
+
* of which profile runs), `strict` would be byte-for-byte identical to
|
|
11
|
+
* `standard` — dead weight, not a real choice — so it's removed rather
|
|
12
|
+
* than kept as a confusing no-op alias. `--profile strict` errors with a
|
|
13
|
+
* migration message (see cli.ts); the library API falls back to
|
|
14
|
+
* `standard` weights for any unrecognized profile string, same as the
|
|
15
|
+
* `guardian` profile's removal in 1.0.0.
|
|
16
|
+
*/
|
|
2
17
|
export declare const PROFILE_WEIGHTS: Record<ProfileName, Partial<Record<DimensionName, number>>>;
|
|
3
18
|
export declare const DEFAULT_THRESHOLDS: Record<ProfileName, number>;
|
|
4
19
|
/** Map eslint ruleId → dimension + whether report-only for scoring */
|
package/dist/profiles.js
CHANGED
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `strict` was planned for 1.1.0 (never published) to gate four
|
|
3
|
+
* contentious rules behind an opt-in profile, report-only under
|
|
4
|
+
* `standard`. 2.0.0 (model v4, see
|
|
5
|
+
* CHANGELOG.md) scores all of them under `standard` directly — owner
|
|
6
|
+
* direction: public scores should reflect quality, full stop, not be
|
|
7
|
+
* softened to avoid moving anyone's number. With nothing left for a
|
|
8
|
+
* second profile to gate (every rule now counts the same way regardless
|
|
9
|
+
* of which profile runs), `strict` would be byte-for-byte identical to
|
|
10
|
+
* `standard` — dead weight, not a real choice — so it's removed rather
|
|
11
|
+
* than kept as a confusing no-op alias. `--profile strict` errors with a
|
|
12
|
+
* migration message (see cli.ts); the library API falls back to
|
|
13
|
+
* `standard` weights for any unrecognized profile string, same as the
|
|
14
|
+
* `guardian` profile's removal in 1.0.0.
|
|
15
|
+
*/
|
|
1
16
|
export const PROFILE_WEIGHTS = {
|
|
2
17
|
standard: {
|
|
3
18
|
playwrightHygiene: 40,
|
|
@@ -77,6 +92,25 @@ export const ESLINT_RULE_MAP = {
|
|
|
77
92
|
dimension: 'structure',
|
|
78
93
|
severityOverride: 'warning',
|
|
79
94
|
},
|
|
95
|
+
// v4 (2.0.0): the four QAG-196 gameability rules and the
|
|
96
|
+
// eslint-disable-comment finding are scored the same as everything
|
|
97
|
+
// else above — no more report-only/profile split. See CHANGELOG.md's
|
|
98
|
+
// 2.0.0 entry and METHODOLOGY.md's "Why v4".
|
|
99
|
+
'pwscore/no-timer-sleep': { dimension: 'playwrightHygiene' },
|
|
100
|
+
// Demoted to warning (0.4 demerit, not 1.0): page.mouse.click/move/down/
|
|
101
|
+
// up(x, y) is also the *correct* way to interact with a canvas/drawing
|
|
102
|
+
// surface (no addressable DOM element to target with a locator), not
|
|
103
|
+
// just a brittle-DOM-app anti-pattern. Verified against the 50-repo
|
|
104
|
+
// corpus: two infinite-canvas editors (tldraw, AFFiNE) accounted for
|
|
105
|
+
// 428/459 of this rule's hits pre-v4. A full per-test demerit would
|
|
106
|
+
// treat every legitimate canvas interaction as a full flaw; a partial
|
|
107
|
+
// one still costs real points (a canvas-heavy test loses up to 0.4 per
|
|
108
|
+
// dimension-capped test) without a brittle "is this a canvas app?"
|
|
109
|
+
// filename/import heuristic that would itself become a new gameability
|
|
110
|
+
// surface (name your test file "canvas-*" to suppress the rule).
|
|
111
|
+
'pwscore/no-coordinate-click': { dimension: 'playwrightHygiene', severityOverride: 'warning' },
|
|
112
|
+
'pwscore/no-trivial-assertion': { dimension: 'assertions' },
|
|
113
|
+
'pwscore/no-soft-assertion-only-test': { dimension: 'assertions', severityOverride: 'warning' },
|
|
80
114
|
};
|
|
81
115
|
export function mapRule(ruleId) {
|
|
82
116
|
if (ESLINT_RULE_MAP[ruleId])
|
|
@@ -10,5 +10,9 @@ export declare const basePlugin: {
|
|
|
10
10
|
};
|
|
11
11
|
rules: {
|
|
12
12
|
'no-skipped-test-declaration': Rule.RuleModule;
|
|
13
|
+
'no-timer-sleep': Rule.RuleModule;
|
|
14
|
+
'no-coordinate-click': Rule.RuleModule;
|
|
15
|
+
'no-trivial-assertion': Rule.RuleModule;
|
|
16
|
+
'no-soft-assertion-only-test': Rule.RuleModule;
|
|
13
17
|
};
|
|
14
18
|
};
|