@scantrix/cli 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +219 -0
- package/dist/astConfigParser.js +308 -0
- package/dist/astRuleHelpers.js +1451 -0
- package/dist/auditConfig.js +81 -0
- package/dist/ciExtractor.js +327 -0
- package/dist/cli.js +156 -0
- package/dist/configExtractor.js +261 -0
- package/dist/cypressExtractor.js +217 -0
- package/dist/diffTracker.js +310 -0
- package/dist/report.js +1904 -0
- package/dist/sarifFormatter.js +88 -0
- package/dist/scanResult.js +45 -0
- package/dist/scanner.js +3519 -0
- package/dist/scoring.js +206 -0
- package/dist/sinks/index.js +29 -0
- package/dist/sinks/jsonSink.js +28 -0
- package/dist/sinks/types.js +2 -0
- package/docs/high-res-icon.svg +26 -0
- package/docs/scantrix-logo-light.svg +64 -0
- package/docs/scantrix-logo.svg +64 -0
- package/package.json +55 -0
package/dist/scanner.js
ADDED
|
@@ -0,0 +1,3519 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.retriesLikelyEnabled = retriesLikelyEnabled;
|
|
7
|
+
exports.scanRepo = scanRepo;
|
|
8
|
+
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
9
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const fs_1 = require("fs");
|
|
12
|
+
const https_1 = __importDefault(require("https"));
|
|
13
|
+
const configExtractor_1 = require("./configExtractor");
|
|
14
|
+
const ciExtractor_1 = require("./ciExtractor");
|
|
15
|
+
const cypressExtractor_1 = require("./cypressExtractor");
|
|
16
|
+
const diffTracker_1 = require("./diffTracker");
|
|
17
|
+
const auditConfig_1 = require("./auditConfig");
|
|
18
|
+
const astRuleHelpers_1 = require("./astRuleHelpers");
|
|
19
|
+
function parseSemverFromText(text) {
|
|
20
|
+
if (!text)
|
|
21
|
+
return undefined;
|
|
22
|
+
const m = String(text).match(/(\d+)\.(\d+)\.(\d+)/);
|
|
23
|
+
if (!m)
|
|
24
|
+
return undefined;
|
|
25
|
+
const major = Number(m[1]);
|
|
26
|
+
const minor = Number(m[2]);
|
|
27
|
+
const patch = Number(m[3]);
|
|
28
|
+
if (![major, minor, patch].every((n) => Number.isFinite(n)))
|
|
29
|
+
return undefined;
|
|
30
|
+
return { major, minor, patch };
|
|
31
|
+
}
|
|
32
|
+
function compareSemver(a, b) {
|
|
33
|
+
if (a.major !== b.major)
|
|
34
|
+
return a.major - b.major;
|
|
35
|
+
if (a.minor !== b.minor)
|
|
36
|
+
return a.minor - b.minor;
|
|
37
|
+
return a.patch - b.patch;
|
|
38
|
+
}
|
|
39
|
+
function isPinnedVersionRange(range) {
|
|
40
|
+
const r = range.trim();
|
|
41
|
+
// Treat exact versions (e.g. 1.2.3) as pinned.
|
|
42
|
+
// Caret/tilde ranges generally float within the major/minor, so we keep alerts conservative.
|
|
43
|
+
return /^\d+\.\d+\.\d+/.test(r) || /^=\s*\d+\.\d+\.\d+/.test(r);
|
|
44
|
+
}
|
|
45
|
+
async function fetchJson(url, timeoutMs = 4500) {
|
|
46
|
+
return await new Promise((resolve, reject) => {
|
|
47
|
+
const req = https_1.default.get(url, {
|
|
48
|
+
headers: {
|
|
49
|
+
"accept": "application/json",
|
|
50
|
+
"user-agent": "scantrix",
|
|
51
|
+
},
|
|
52
|
+
}, (res) => {
|
|
53
|
+
const { statusCode } = res;
|
|
54
|
+
if (!statusCode || statusCode < 200 || statusCode >= 300) {
|
|
55
|
+
res.resume();
|
|
56
|
+
reject(new Error(`HTTP ${statusCode ?? "(unknown)"} for ${url}`));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const chunks = [];
|
|
60
|
+
res.on("data", (d) => chunks.push(Buffer.isBuffer(d) ? d : Buffer.from(d)));
|
|
61
|
+
res.on("end", () => {
|
|
62
|
+
try {
|
|
63
|
+
const body = Buffer.concat(chunks).toString("utf8");
|
|
64
|
+
resolve(JSON.parse(body));
|
|
65
|
+
}
|
|
66
|
+
catch (e) {
|
|
67
|
+
reject(e);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
req.on("error", reject);
|
|
72
|
+
req.setTimeout(timeoutMs, () => {
|
|
73
|
+
req.destroy(new Error(`Timeout after ${timeoutMs}ms for ${url}`));
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
const npmInfoCache = new Map();
|
|
78
|
+
async function getNpmPackageInfo(packageName) {
|
|
79
|
+
if (npmInfoCache.has(packageName))
|
|
80
|
+
return npmInfoCache.get(packageName);
|
|
81
|
+
// npm registry expects scoped packages as %40scope%2Fname
|
|
82
|
+
const encoded = encodeURIComponent(packageName);
|
|
83
|
+
const url = `https://registry.npmjs.org/${encoded}`;
|
|
84
|
+
try {
|
|
85
|
+
const json = await fetchJson(url);
|
|
86
|
+
const latest = json?.["dist-tags"]?.latest;
|
|
87
|
+
let deprecated;
|
|
88
|
+
if (typeof latest === "string" && json?.versions?.[latest]?.deprecated) {
|
|
89
|
+
deprecated = String(json.versions[latest].deprecated);
|
|
90
|
+
}
|
|
91
|
+
const out = {
|
|
92
|
+
latest: typeof latest === "string" ? latest : undefined,
|
|
93
|
+
deprecated,
|
|
94
|
+
};
|
|
95
|
+
npmInfoCache.set(packageName, out);
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
const fallback = { latest: undefined, deprecated: undefined };
|
|
100
|
+
npmInfoCache.set(packageName, fallback);
|
|
101
|
+
return fallback;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
function pickPlaywrightRelatedPackages(pkg, pwConfig) {
|
|
105
|
+
const deps = { ...(pkg?.dependencies ?? {}), ...(pkg?.devDependencies ?? {}) };
|
|
106
|
+
const candidates = new Map();
|
|
107
|
+
// Always consider these if present.
|
|
108
|
+
for (const name of ["@playwright/test", "playwright", "playwright-core"]) {
|
|
109
|
+
const r = deps[name];
|
|
110
|
+
if (typeof r === "string" && r.trim())
|
|
111
|
+
candidates.set(name, r);
|
|
112
|
+
}
|
|
113
|
+
// Add anything that looks like a Playwright plugin/reporter dependency.
|
|
114
|
+
for (const [name, range] of Object.entries(deps)) {
|
|
115
|
+
if (typeof range !== "string" || !range.trim())
|
|
116
|
+
continue;
|
|
117
|
+
if (/playwright/i.test(name) && !candidates.has(name))
|
|
118
|
+
candidates.set(name, range);
|
|
119
|
+
}
|
|
120
|
+
// Reporter packages detected in config (if the package is declared).
|
|
121
|
+
for (const rep of pwConfig?.reporters ?? []) {
|
|
122
|
+
const name = rep.name;
|
|
123
|
+
if (typeof name === "string" && name.startsWith("@") && deps[name] && !candidates.has(name)) {
|
|
124
|
+
candidates.set(name, deps[name]);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return [...candidates.entries()].map(([name, range]) => ({ name, range }));
|
|
128
|
+
}
|
|
129
|
+
async function maybeAddOutdatedDependencyFinding(params) {
|
|
130
|
+
const { pkg, pkgPath, pwConfig, findings, enabled } = params;
|
|
131
|
+
if (!enabled)
|
|
132
|
+
return;
|
|
133
|
+
if (!pkg || !pkgPath)
|
|
134
|
+
return;
|
|
135
|
+
const packages = pickPlaywrightRelatedPackages(pkg, pwConfig);
|
|
136
|
+
if (!packages.length)
|
|
137
|
+
return;
|
|
138
|
+
const outdatedEvidence = [];
|
|
139
|
+
const deprecatedEvidence = [];
|
|
140
|
+
let worstSeverity = "low";
|
|
141
|
+
for (const p of packages) {
|
|
142
|
+
const info = await getNpmPackageInfo(p.name);
|
|
143
|
+
// ── DEP-002: npm-deprecated package ──
|
|
144
|
+
if (info.deprecated) {
|
|
145
|
+
const snippet = `${p.name}: DEPRECATED — ${info.deprecated}`;
|
|
146
|
+
deprecatedEvidence.push({ file: normalize(pkgPath), line: 1, snippet });
|
|
147
|
+
}
|
|
148
|
+
// ── DEP-001: outdated version check ──
|
|
149
|
+
if (!info.latest)
|
|
150
|
+
continue;
|
|
151
|
+
const latestV = parseSemverFromText(info.latest);
|
|
152
|
+
const currentV = parseSemverFromText(p.range);
|
|
153
|
+
if (!latestV || !currentV)
|
|
154
|
+
continue;
|
|
155
|
+
// Conservative: if you use ^ or ~, only alert on major bumps.
|
|
156
|
+
const pinned = isPinnedVersionRange(p.range);
|
|
157
|
+
const majorBehind = latestV.major > currentV.major;
|
|
158
|
+
const minorBehind = latestV.major === currentV.major && latestV.minor > currentV.minor;
|
|
159
|
+
if (!majorBehind && !(pinned && minorBehind))
|
|
160
|
+
continue;
|
|
161
|
+
const sev = majorBehind ? "medium" : "low";
|
|
162
|
+
if (sev === "medium")
|
|
163
|
+
worstSeverity = "medium";
|
|
164
|
+
const snippet = `${p.name}: specified '${p.range}' vs latest '${info.latest}'`;
|
|
165
|
+
outdatedEvidence.push({ file: normalize(pkgPath), line: 1, snippet });
|
|
166
|
+
}
|
|
167
|
+
// ── Emit DEP-001 if outdated deps found ──
|
|
168
|
+
if (outdatedEvidence.length) {
|
|
169
|
+
const evidence = outdatedEvidence.slice(0, 25);
|
|
170
|
+
findings.push({
|
|
171
|
+
findingId: "DEP-001",
|
|
172
|
+
severity: worstSeverity,
|
|
173
|
+
title: "Outdated Playwright plugins/dependencies detected",
|
|
174
|
+
description: "Some Playwright-related dependencies appear behind the latest published versions. Staying current helps reduce flakiness, improves performance, and avoids known bugs.",
|
|
175
|
+
recommendation: "Review the packages listed and consider upgrading to the latest stable versions. If you intentionally pin versions, document the rationale and schedule periodic upgrade windows.",
|
|
176
|
+
evidence,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
// ── Emit DEP-002 if deprecated packages found ──
|
|
180
|
+
if (deprecatedEvidence.length) {
|
|
181
|
+
findings.push({
|
|
182
|
+
findingId: "DEP-002",
|
|
183
|
+
severity: "high",
|
|
184
|
+
title: "Deprecated npm package detected",
|
|
185
|
+
description: "One or more Playwright-related dependencies are marked as deprecated on npm. Deprecated packages may be unmaintained, contain known vulnerabilities, or have an official replacement.",
|
|
186
|
+
recommendation: "Check the npm deprecation notice for each package and migrate to the recommended replacement. Remove deprecated packages from your dependency tree as soon as possible.",
|
|
187
|
+
evidence: deprecatedEvidence.slice(0, 25),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const RULES = [
|
|
192
|
+
{
|
|
193
|
+
id: "PW-FLAKE-001",
|
|
194
|
+
title: "Hard waits detected (waitForTimeout / setTimeout)",
|
|
195
|
+
severity: "high",
|
|
196
|
+
description: "Hard waits increase flakiness and slow tests. They introduce non-deterministic delays that mask timing bugs and inflate CI run times.",
|
|
197
|
+
recommendation: "Replace hard waits with Playwright auto-waiting + explicit expect().\nhttps://playwright.dev/docs/api/class-page#page-wait-for-timeout",
|
|
198
|
+
patterns: [
|
|
199
|
+
/waitForTimeout\s*\(/g,
|
|
200
|
+
/\bsetTimeout\s*\(/g,
|
|
201
|
+
/new\s+Promise\s*\(\s*resolve\s*=>\s*setTimeout/g,
|
|
202
|
+
/new\s+Promise\s*\(\s*r\s*=>\s*setTimeout/g,
|
|
203
|
+
],
|
|
204
|
+
// Focus on test files and POMs - exclude helper utilities where delays may be legitimate
|
|
205
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/fixtures/**/*.ts"],
|
|
206
|
+
maxEvidence: 25,
|
|
207
|
+
testContentOnly: true,
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
id: "PW-FLAKE-002",
|
|
211
|
+
title: "Force-click usage detected",
|
|
212
|
+
severity: "low",
|
|
213
|
+
description: "Force clicks bypass Playwright’s actionability checks. While there are legitimate uses (e.g., clicking outside a modal to close it), widespread usage may hide real UI issues.",
|
|
214
|
+
recommendation: "Review each force-click for legitimacy. Prefer expect(locator).toBeVisible()/toBeEnabled() and stable locators. Legitimate cases include clicking overlays or testing dismiss-on-outside-click behavior.",
|
|
215
|
+
patterns: [/force\s*:\s*true/g],
|
|
216
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/fixtures/**/*.ts"],
|
|
217
|
+
maxEvidence: 25,
|
|
218
|
+
framework: "playwright",
|
|
219
|
+
testContentOnly: true,
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
id: "PW-LOC-001",
|
|
223
|
+
title: "XPath selectors detected (locator('//...'))",
|
|
224
|
+
severity: "medium",
|
|
225
|
+
description: "XPath selectors tend to be brittle and harder to maintain vs role/label/testid locators.",
|
|
226
|
+
recommendation: "Prefer getByRole/getByLabel/getByTestId and scoped locators. Introduce data-testid where needed. Focus refactoring on frequently changed test files first. \nhttps://playwright.dev/docs/locators#quick-guide",
|
|
227
|
+
patterns: [
|
|
228
|
+
/\blocator\s*\(\s*['"`]\s*\/\/[^'"`]+['"`]\s*\)/g,
|
|
229
|
+
/\bpage\.locator\s*\(\s*['"`]\s*\/\/[^'"`]+['"`]\s*\)/g,
|
|
230
|
+
/\bxpath\s*=\s*['"`][^'"`]+['"`]/g,
|
|
231
|
+
],
|
|
232
|
+
fileGlobs: ["**/*.ts", "**/*.js", "**/*.tsx", "**/*.jsx"],
|
|
233
|
+
maxEvidence: 25,
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
id: "PW-PERF-001",
|
|
237
|
+
title: "page.goto() using unreliable 'networkidle' wait strategy",
|
|
238
|
+
severity: "medium",
|
|
239
|
+
description: "page.goto() defaults to waitUntil: 'load', which is reliable and sufficient for most use cases. Using waitUntil: 'networkidle' is explicitly discouraged by Playwright — it waits for zero network activity for 500 ms, which is unreliable in modern SPAs with background requests, analytics, and websockets.",
|
|
240
|
+
recommendation: "Remove the waitUntil: 'networkidle' option from page.goto() calls and rely on the default ('load'), or use 'domcontentloaded' if you need faster navigation. Better: use page.waitForResponse() to wait for specific API calls, or expect(locator).toBeVisible() for specific UI elements.\nhttps://playwright.dev/docs/api/class-page#page-goto",
|
|
241
|
+
patterns: [
|
|
242
|
+
// Match page.goto() calls that explicitly use networkidle.
|
|
243
|
+
// [\s\S]{0,200}? caps how far ahead we look so the regex doesn't span
|
|
244
|
+
// across multiple unrelated page.goto() calls.
|
|
245
|
+
/page\.goto\s*\([\s\S]{0,200}?waitUntil\s*:\s*['"`]networkidle['"`]/g,
|
|
246
|
+
],
|
|
247
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
248
|
+
maxEvidence: 25,
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "PW-PERF-002",
|
|
252
|
+
title: "Hardcoded URLs detected in tests",
|
|
253
|
+
severity: "medium",
|
|
254
|
+
description: "Tests contain hardcoded URLs (http://, https://). This makes it difficult to run tests against different environments (dev, staging, production) and violates the DRY principle.",
|
|
255
|
+
recommendation: "Configure baseURL in playwright.config.ts and use relative paths in tests (e.g., page.goto('/login') instead of page.goto('https://example.com/login')). Use environment variables for environment-specific configuration.",
|
|
256
|
+
patterns: [
|
|
257
|
+
/page\.goto\s*\(\s*['"`]https?:\/\/[^'"`]+['"`]/g,
|
|
258
|
+
/(?:const|let|var)\s+\w+\s*=\s*['"`]https?:\/\/[^'"`]+['"`]/g,
|
|
259
|
+
],
|
|
260
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
261
|
+
maxEvidence: 25,
|
|
262
|
+
},
|
|
263
|
+
{
|
|
264
|
+
id: "PW-FLAKE-003",
|
|
265
|
+
title: "test.only(), test.skip(), or test.describe.skip() left in codebase",
|
|
266
|
+
severity: "high",
|
|
267
|
+
description: "test.only() silently runs only one test — CI pass rates become meaningless. test.skip() and test.describe.skip() permanently disable tests without visibility. All erode suite coverage.",
|
|
268
|
+
recommendation: "Remove test.only() before merging. Replace test.skip()/test.describe.skip() with test.fixme() (appears in reports) or use grep/tag-based filtering for selective execution.\nhttps://playwright.dev/docs/test-annotations#skip-a-test",
|
|
269
|
+
patterns: [
|
|
270
|
+
/\btest\.only\s*\(/g,
|
|
271
|
+
/\btest\.describe\.only\s*\(/g,
|
|
272
|
+
/\btest\.skip\s*\(\s*['"`]/g,
|
|
273
|
+
/\btest\.describe\.skip\s*\(/g,
|
|
274
|
+
],
|
|
275
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
276
|
+
maxEvidence: 25,
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
id: "PW-FLAKE-004",
|
|
280
|
+
title: "page.waitForLoadState('networkidle') usage",
|
|
281
|
+
severity: "high",
|
|
282
|
+
description: "Playwright docs explicitly discourage 'networkidle'. It waits for zero network activity for 500 ms, which is unreliable in modern SPAs with background requests, analytics, and websockets.",
|
|
283
|
+
recommendation: "Replace with specific waits: page.waitForResponse() for API calls, expect(locator).toBeVisible() for UI elements, or waitForLoadState('domcontentloaded').\nhttps://playwright.dev/docs/api/class-page#page-wait-for-load-state",
|
|
284
|
+
patterns: [
|
|
285
|
+
/waitForLoadState\s*\(\s*['"`]networkidle['"`]\s*\)/g,
|
|
286
|
+
/waitUntil\s*:\s*['"`]networkidle['"`]/g,
|
|
287
|
+
],
|
|
288
|
+
fileGlobs: ["**/*.ts", "**/*.js"],
|
|
289
|
+
maxEvidence: 25,
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
id: "PW-LOC-003",
|
|
293
|
+
title: "Index-based selectors (nth, first, last)",
|
|
294
|
+
severity: "medium",
|
|
295
|
+
description: "Index-based selectors like .nth(3), .first(), .last() are order-dependent and fragile. DOM order changes (new elements, reordering) silently break tests without clear diagnostics.",
|
|
296
|
+
recommendation: "Replace index-based selectors with semantic locators: getByRole, getByText, getByTestId, or .filter({ hasText: ... }) for specificity.\nhttps://playwright.dev/docs/locators#filtering-locators",
|
|
297
|
+
patterns: [
|
|
298
|
+
/\.nth\s*\(\s*\d+\s*\)/g,
|
|
299
|
+
/\.first\s*\(\s*\)/g,
|
|
300
|
+
/\.last\s*\(\s*\)/g,
|
|
301
|
+
],
|
|
302
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
303
|
+
maxEvidence: 25,
|
|
304
|
+
},
|
|
305
|
+
// ── Deprecated API Rules ─────────────────────────────────────────────
|
|
306
|
+
{
|
|
307
|
+
id: "PW-DEPREC-001",
|
|
308
|
+
title: "Deprecated selector engine methods ($, $$, $eval, $$eval)",
|
|
309
|
+
severity: "medium",
|
|
310
|
+
description: "page.$(), page.$$(), page.$eval(), page.$$eval() and their ElementHandle equivalents are deprecated. These methods return ElementHandle objects, which are the legacy way to interact with DOM elements. Locators are the modern, auto-waiting replacement.",
|
|
311
|
+
recommendation: "Replace all $/$$ usage with page.locator():\n" +
|
|
312
|
+
" page.$('selector') → page.locator('selector')\n" +
|
|
313
|
+
" page.$$('selector') → page.locator('selector').all()\n" +
|
|
314
|
+
" page.$eval(sel, fn) → page.locator(sel).evaluate(fn)\n" +
|
|
315
|
+
" page.$$eval(sel, fn) → page.locator(sel).evaluateAll(fn)\n" +
|
|
316
|
+
"Deprecated Page methods: https://playwright.dev/docs/api/class-page#page-query-selector\n" +
|
|
317
|
+
"Deprecated ElementHandle: https://playwright.dev/docs/next/api/class-elementhandle#deprecated",
|
|
318
|
+
patterns: [
|
|
319
|
+
/\.\$\(/g,
|
|
320
|
+
/\.\$\$\(/g,
|
|
321
|
+
/\.\$eval\s*\(/g,
|
|
322
|
+
/\.\$\$eval\s*\(/g,
|
|
323
|
+
],
|
|
324
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts"],
|
|
325
|
+
maxEvidence: 25,
|
|
326
|
+
},
|
|
327
|
+
{
|
|
328
|
+
id: "PW-DEPREC-002",
|
|
329
|
+
title: "Deprecated page.action(selector) convenience methods",
|
|
330
|
+
severity: "medium",
|
|
331
|
+
description: "Deprecated Page methods that accept a selector string as the first argument have been detected. In modern Playwright, use page.locator(selector) to create a Locator, then call the action method on it. The Locator API provides auto-waiting, actionability checks, and better error messages.",
|
|
332
|
+
recommendation: "Refactor page.action('selector') → page.locator('selector').action():\n" +
|
|
333
|
+
" page.click('sel') → page.locator('sel').click()\n" +
|
|
334
|
+
" page.fill('sel', 'val') → page.locator('sel').fill('val')\n" +
|
|
335
|
+
" page.type('sel', 'text') → page.locator('sel').pressSequentially('text')\n" +
|
|
336
|
+
" page.hover('sel') → page.locator('sel').hover()\n" +
|
|
337
|
+
" page.check('sel') → page.locator('sel').check()\n" +
|
|
338
|
+
" page.selectOption('sel') → page.locator('sel').selectOption()\n" +
|
|
339
|
+
" page.getAttribute('sel') → page.locator('sel').getAttribute()\n" +
|
|
340
|
+
" page.textContent('sel') → page.locator('sel').textContent()\n" +
|
|
341
|
+
" page.isVisible('sel') → page.locator('sel').isVisible()\n" +
|
|
342
|
+
"Full list: https://playwright.dev/docs/api/class-page#page-click\n" +
|
|
343
|
+
"Migration guide: https://playwright.dev/docs/locators",
|
|
344
|
+
patterns: [
|
|
345
|
+
/\bpage\.(click|dblclick|fill|focus|hover|check|uncheck|tap|type|press|selectOption|setChecked|setInputFiles|dispatchEvent|getAttribute|innerHTML|innerText|inputValue|textContent|isChecked|isDisabled|isEditable|isEnabled|isHidden|isVisible|screenshot)\s*\(\s*['"]/g,
|
|
346
|
+
],
|
|
347
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts"],
|
|
348
|
+
maxEvidence: 25,
|
|
349
|
+
},
|
|
350
|
+
{
|
|
351
|
+
id: "PW-DEPREC-003",
|
|
352
|
+
title: "Deprecated page.waitForNavigation() / page.waitForSelector()",
|
|
353
|
+
severity: "medium",
|
|
354
|
+
description: "page.waitForNavigation() and page.waitForSelector() are deprecated. These are legacy wait mechanisms with race condition risks (the event may fire before the wait is registered).",
|
|
355
|
+
recommendation: "Replace with modern equivalents:\n" +
|
|
356
|
+
" page.waitForNavigation() → page.waitForURL('**/expected-path')\n" +
|
|
357
|
+
" page.waitForSelector(sel) → page.locator(sel).waitFor()\n" +
|
|
358
|
+
"Note: page.waitForTimeout() is also deprecated (flagged separately by PW-FLAKE-001).\n" +
|
|
359
|
+
"waitForNavigation: https://playwright.dev/docs/api/class-page#page-wait-for-navigation\n" +
|
|
360
|
+
"waitForSelector: https://playwright.dev/docs/api/class-page#page-wait-for-selector",
|
|
361
|
+
patterns: [
|
|
362
|
+
/\bpage\.waitForNavigation\s*\(/g,
|
|
363
|
+
/\.waitForNavigation\s*\(/g,
|
|
364
|
+
/\bpage\.waitForSelector\s*\(/g,
|
|
365
|
+
/\.waitForSelector\s*\(\s*['"]/g,
|
|
366
|
+
],
|
|
367
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts"],
|
|
368
|
+
maxEvidence: 25,
|
|
369
|
+
},
|
|
370
|
+
{
|
|
371
|
+
id: "PW-DEPREC-004",
|
|
372
|
+
title: "test.describe.serial usage (isolation risk)",
|
|
373
|
+
severity: "medium",
|
|
374
|
+
description: "test.describe.serial() creates sequential test dependencies, if one test fails, subsequent tests in the group are skipped. The Playwright docs state: 'Using serial is not recommended. It is usually better to make your tests isolated.' Note: test.describe.parallel() and test.describe.serial() are not deprecated but the modern equivalent is test.describe.configure({ mode: 'parallel' | 'serial' }).",
|
|
375
|
+
recommendation: "Refactor serial test groups to be fully isolated so each test can run independently. If you need serial mode for a legitimate reason (e.g. multi-step wizard), prefer test.describe.configure({ mode: 'serial' }) which is the modern syntax.\nhttps://playwright.dev/docs/test-retries#serial-mode\nhttps://playwright.dev/docs/test-parallel",
|
|
376
|
+
patterns: [
|
|
377
|
+
/\btest\.describe\.serial\b/g,
|
|
378
|
+
],
|
|
379
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
380
|
+
maxEvidence: 25,
|
|
381
|
+
},
|
|
382
|
+
{
|
|
383
|
+
id: "PW-DEPREC-005",
|
|
384
|
+
title: "Deprecated snapshotDir config option",
|
|
385
|
+
severity: "low",
|
|
386
|
+
description: "The snapshotDir option in playwright.config.ts is deprecated. It was replaced by the more flexible snapshotPathTemplate option which provides greater control over snapshot file paths.",
|
|
387
|
+
recommendation: "Replace snapshotDir with snapshotPathTemplate:\n" +
|
|
388
|
+
" // Before (deprecated):\n" +
|
|
389
|
+
" snapshotDir: './snapshots'\n" +
|
|
390
|
+
" // After:\n" +
|
|
391
|
+
" snapshotPathTemplate: '{snapshotDir}/{testFileDir}/{testFileName}-snapshots/{arg}{ext}'\n" +
|
|
392
|
+
"https://playwright.dev/docs/api/class-testconfig#test-config-snapshot-dir",
|
|
393
|
+
patterns: [
|
|
394
|
+
/\bsnapshotDir\s*[=:]/g,
|
|
395
|
+
],
|
|
396
|
+
fileGlobs: [
|
|
397
|
+
"playwright.config.ts", "playwright.config.js",
|
|
398
|
+
"**/playwright.config.ts", "**/playwright.config.js",
|
|
399
|
+
"**/playwright*.config.ts", "**/playwright*.config.js",
|
|
400
|
+
],
|
|
401
|
+
maxEvidence: 5,
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
id: "PW-DEPREC-006",
|
|
405
|
+
title: "Deprecated BrowserContext methods (setHTTPCredentials, backgroundPages)",
|
|
406
|
+
severity: "low",
|
|
407
|
+
description: "Deprecated BrowserContext methods detected: setHTTPCredentials() was replaced by httpCredentials in browser.newContext(). backgroundPages() and the 'backgroundpage' event are deprecated (Chromium-only, use service workers instead).",
|
|
408
|
+
recommendation: "Replace deprecated BrowserContext methods:\n" +
|
|
409
|
+
" browserContext.setHTTPCredentials({...}) → browser.newContext({ httpCredentials: {...} })\n" +
|
|
410
|
+
" browserContext.backgroundPages() → (use service workers or Chrome DevTools Protocol)\n" +
|
|
411
|
+
" browserContext.on('backgroundpage', ...) → (use service workers or Chrome DevTools Protocol)\n" +
|
|
412
|
+
"setHTTPCredentials: https://playwright.dev/docs/next/api/class-browsercontext#browser-context-set-http-credentials\n" +
|
|
413
|
+
"backgroundPages: https://playwright.dev/docs/next/api/class-browsercontext#browser-context-background-pages\n" +
|
|
414
|
+
"on('backgroundpage'): https://playwright.dev/docs/next/api/class-browsercontext#browser-context-event-background-page",
|
|
415
|
+
patterns: [
|
|
416
|
+
/\.setHTTPCredentials\s*\(/g,
|
|
417
|
+
/\.backgroundPages\s*\(/g,
|
|
418
|
+
/\.on\s*\(\s*['"]backgroundpage['"]/g,
|
|
419
|
+
],
|
|
420
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts"],
|
|
421
|
+
maxEvidence: 15,
|
|
422
|
+
},
|
|
423
|
+
// ── New rules from Playwright docs audit (2026-02-14) ────────────────
|
|
424
|
+
{
|
|
425
|
+
id: "PW-FLAKE-005",
|
|
426
|
+
title: "Non-retrying assertions used (manual await + toBe instead of web-first assertion)",
|
|
427
|
+
severity: "high",
|
|
428
|
+
description: "Using non-retrying assertions like expect(await locator.isVisible()).toBe(true) can lead to flaky tests. Playwright docs explicitly warn: 'using non-retrying assertions can lead to a flaky test.' Web-first assertions (e.g., await expect(locator).toBeVisible()) automatically retry until the condition is met.",
|
|
429
|
+
recommendation: "Replace manual await + toBe patterns with web-first assertions:\n" +
|
|
430
|
+
" expect(await locator.isVisible()).toBe(true) → await expect(locator).toBeVisible()\n" +
|
|
431
|
+
" expect(await locator.textContent()).toBe(...) → await expect(locator).toHaveText(...)\n" +
|
|
432
|
+
" expect(await locator.getAttribute(...)).toBe(..) → await expect(locator).toHaveAttribute(...)\n" +
|
|
433
|
+
" expect(await page.title()).toBe(...) → await expect(page).toHaveTitle(...)\n" +
|
|
434
|
+
" expect(await page.url()).toBe(...) → await expect(page).toHaveURL(...)\n" +
|
|
435
|
+
" expect(await locator.inputValue()).toBe(...) → await expect(locator).toHaveValue(...)\n" +
|
|
436
|
+
" expect(await locator.isEnabled()).toBe(true) → await expect(locator).toBeEnabled()\n" +
|
|
437
|
+
"https://playwright.dev/docs/test-assertions#auto-retrying-assertions",
|
|
438
|
+
patterns: [
|
|
439
|
+
/expect\s*\(\s*await\s+.+?\.isVisible\s*\(\s*\)\s*\)/g,
|
|
440
|
+
/expect\s*\(\s*await\s+.+?\.isEnabled\s*\(\s*\)\s*\)/g,
|
|
441
|
+
/expect\s*\(\s*await\s+.+?\.isDisabled\s*\(\s*\)\s*\)/g,
|
|
442
|
+
/expect\s*\(\s*await\s+.+?\.isChecked\s*\(\s*\)\s*\)/g,
|
|
443
|
+
/expect\s*\(\s*await\s+.+?\.isHidden\s*\(\s*\)\s*\)/g,
|
|
444
|
+
/expect\s*\(\s*await\s+.+?\.textContent\s*\(\s*\)\s*\)/g,
|
|
445
|
+
/expect\s*\(\s*await\s+.+?\.innerText\s*\(\s*\)\s*\)/g,
|
|
446
|
+
/expect\s*\(\s*await\s+.+?\.inputValue\s*\(\s*\)\s*\)/g,
|
|
447
|
+
/expect\s*\(\s*await\s+.+?\.getAttribute\s*\(/g,
|
|
448
|
+
/expect\s*\(\s*await\s+.+?\.title\s*\(\s*\)\s*\)/g,
|
|
449
|
+
/expect\s*\(\s*await\s+.+?\.url\s*\(\s*\)\s*\)/g,
|
|
450
|
+
],
|
|
451
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
452
|
+
maxEvidence: 25,
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
id: "PW-FLAKE-006",
|
|
456
|
+
title: "test.fixme() or test.describe.fixme() markers indicate deferred broken tests",
|
|
457
|
+
severity: "low",
|
|
458
|
+
description: "test.fixme() and test.describe.fixme() mark tests as 'known to fail' so Playwright skips them. While better than test.skip() for visibility, accumulated fixme markers indicate growing technical debt in the test suite. *Note - tests marked with test.fixme() are not assessed in this audit.",
|
|
459
|
+
recommendation: "Track test.fixme()/test.describe.fixme() tests in your backlog and aim to fix or remove them periodically. If a test has been fixme'd for a long time, consider whether it's still relevant.\nhttps://playwright.dev/docs/api/class-test#test-fixme-1",
|
|
460
|
+
patterns: [
|
|
461
|
+
/\btest\.fixme\s*\(/g,
|
|
462
|
+
/\btest\.describe\.fixme\s*\(/g,
|
|
463
|
+
],
|
|
464
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
465
|
+
maxEvidence: 25,
|
|
466
|
+
},
|
|
467
|
+
{
|
|
468
|
+
id: "PW-FLAKE-009",
|
|
469
|
+
title: "Debug pause left in code (page.pause)",
|
|
470
|
+
severity: "high",
|
|
471
|
+
description: "page.pause() halts test execution and opens the Playwright Inspector. It is a debug only method that must be removed before committing. Any test containing it, will hang indefinitely in CI.",
|
|
472
|
+
recommendation: "Remove all page.pause() calls. Use Playwright's trace viewer or --debug flag for local debugging instead.\nhttps://playwright.dev/docs/debug",
|
|
473
|
+
patterns: [/\.pause\s*\(\s*\)/g],
|
|
474
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js",
|
|
475
|
+
"**/POMs/**/*.ts", "**/pages/**/*.ts", "**/fixtures/**/*.ts"],
|
|
476
|
+
maxEvidence: 25,
|
|
477
|
+
testContentOnly: true,
|
|
478
|
+
},
|
|
479
|
+
// PW-FLAKE-010 moved to AST-based detection (see combined AST loop below)
|
|
480
|
+
{
|
|
481
|
+
id: "PW-DEPREC-007",
|
|
482
|
+
title: "Deprecated page.type() / locator.type() usage",
|
|
483
|
+
severity: "medium",
|
|
484
|
+
description: "page.type(), frame.type(), locator.type(), and elementHandle.type() were deprecated in Playwright v1.39. Use locator.fill() for setting input values (much faster) or locator.pressSequentially() when character-by-character typing is needed.",
|
|
485
|
+
recommendation: "Replace .type() calls with locator.fill() for setting input values:\n" +
|
|
486
|
+
" await page.type('#input', 'text') → await page.locator('#input').fill('text')\n" +
|
|
487
|
+
" await locator.type('text') → await locator.fill('text')\n" +
|
|
488
|
+
"Use locator.pressSequentially() only if the page has special keyboard handling.\nhttps://playwright.dev/docs/input#type-characters",
|
|
489
|
+
patterns: [
|
|
490
|
+
/\bpage\.type\s*\(/g,
|
|
491
|
+
/\bframe\.type\s*\(/g,
|
|
492
|
+
/\.type\s*\(\s*['"`]/g,
|
|
493
|
+
],
|
|
494
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts"],
|
|
495
|
+
maxEvidence: 25,
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
id: "PW-DEPREC-008",
|
|
499
|
+
title: "Deprecated layout selectors (:left-of, :right-of, :above, :below, :near)",
|
|
500
|
+
severity: "medium",
|
|
501
|
+
description: "Layout pseudo-class selectors (:left-of, :right-of, :above, :below, :near) are deprecated per the Playwright docs and may be removed in a future version.",
|
|
502
|
+
recommendation: "Replace layout selectors with semantic locators (getByRole, getByLabel, getByTestId) or use locator.filter() with hasText/has options for relative positioning.\nhttps://playwright.dev/docs/other-locators#css-matching-elements-based-on-layout",
|
|
503
|
+
patterns: [
|
|
504
|
+
/:left-of\s*\(/g,
|
|
505
|
+
/:right-of\s*\(/g,
|
|
506
|
+
/:above\s*\(/g,
|
|
507
|
+
/:below\s*\(/g,
|
|
508
|
+
/:near\s*\(/g,
|
|
509
|
+
],
|
|
510
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js", "**/POMs/**/*.ts", "**/pages/**/*.ts"],
|
|
511
|
+
maxEvidence: 25,
|
|
512
|
+
},
|
|
513
|
+
// ── Cypress Rules ────────────────────────────────────────────────────
|
|
514
|
+
{
|
|
515
|
+
id: "CY-FLAKE-001",
|
|
516
|
+
framework: "cypress",
|
|
517
|
+
title: "Hard waits detected in Cypress tests (cy.wait with numeric ms)",
|
|
518
|
+
severity: "high",
|
|
519
|
+
description: "cy.wait() with a numeric argument introduces brittle hard waits identical to Playwright's waitForTimeout. These slow tests and mask real timing issues.",
|
|
520
|
+
recommendation: "Replace cy.wait(ms) with cy.intercept() + cy.wait('@alias') to wait for specific network requests, or use cy.get().should() assertions that auto-retry. Hard waits should never be used for synchronization.\nhttps://docs.cypress.io/guides/references/best-practices#Unnecessary-Waiting",
|
|
521
|
+
patterns: [
|
|
522
|
+
/\bcy\.wait\s*\(\s*\d+\s*\)/g,
|
|
523
|
+
/\bcy\.wait\s*\(\s*\d+\s*,/g,
|
|
524
|
+
],
|
|
525
|
+
fileGlobs: [
|
|
526
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
527
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
528
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
529
|
+
],
|
|
530
|
+
maxEvidence: 25,
|
|
531
|
+
},
|
|
532
|
+
{
|
|
533
|
+
id: "CY-FLAKE-002",
|
|
534
|
+
framework: "cypress",
|
|
535
|
+
title: "Fragile CSS selectors in Cypress tests (missing data-cy / data-testid)",
|
|
536
|
+
severity: "medium",
|
|
537
|
+
description: "cy.get() calls use CSS class or tag selectors that are tightly coupled to styling and DOM structure. These break when CSS or markup changes, even if the feature works correctly.",
|
|
538
|
+
recommendation: "Use dedicated test attributes: cy.get('[data-cy=\"submit-btn\"]') or cy.get('[data-testid=\"submit-btn\"]'). Add data-cy attributes to your application markup.\nhttps://docs.cypress.io/guides/references/best-practices#Selecting-Elements",
|
|
539
|
+
patterns: [
|
|
540
|
+
// cy.get('.class-name') or cy.get('tag') without data- attributes
|
|
541
|
+
/\bcy\.get\s*\(\s*['"`]\.[\w-]+['"`]\s*\)/g,
|
|
542
|
+
/\bcy\.get\s*\(\s*['"`]#[\w-]+['"`]\s*\)/g,
|
|
543
|
+
],
|
|
544
|
+
fileGlobs: [
|
|
545
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
546
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
547
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
548
|
+
],
|
|
549
|
+
maxEvidence: 25,
|
|
550
|
+
},
|
|
551
|
+
{
|
|
552
|
+
id: "CY-FLAKE-003",
|
|
553
|
+
framework: "cypress",
|
|
554
|
+
title: "Force-click usage detected in Cypress tests",
|
|
555
|
+
severity: "medium",
|
|
556
|
+
description: "Cypress commands using { force: true } bypass actionability checks (visibility, disability, cover). This hides genuine UI issues such as overlapping elements, disabled buttons, or elements rendered off screen.",
|
|
557
|
+
recommendation: "Investigate why the element isn't actionable. Fix the underlying DOM/CSS issue rather than forcing clicks. If the element is in a scrollable container, Cypress auto-scrolls force should rarely be needed.\nhttps://docs.cypress.io/guides/core-concepts/interacting-with-elements#Forcing",
|
|
558
|
+
patterns: [/force\s*:\s*true/g],
|
|
559
|
+
fileGlobs: [
|
|
560
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
561
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
562
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
563
|
+
],
|
|
564
|
+
maxEvidence: 25,
|
|
565
|
+
},
|
|
566
|
+
{
|
|
567
|
+
id: "CY-LOC-001",
|
|
568
|
+
framework: "cypress",
|
|
569
|
+
title: "XPath selectors detected in Cypress tests",
|
|
570
|
+
severity: "low",
|
|
571
|
+
description: "Cypress tests use cy.xpath() or XPath expressions. XPath selectors are brittle, hard to read, and not natively supported by Cypress (requires cypress-xpath plugin).",
|
|
572
|
+
recommendation: "Replace XPath selectors with cy.get('[data-cy=\"...\"]') or cy.contains(). Cypress's built-in commands with data-cy attributes are faster and more maintainable.\nhttps://docs.cypress.io/guides/references/best-practices#Selecting-Elements",
|
|
573
|
+
patterns: [
|
|
574
|
+
/\bcy\.xpath\s*\(/g,
|
|
575
|
+
/\bcy\.get\s*\(\s*['"`]\/\/[^'"`]+['"`]\s*\)/g,
|
|
576
|
+
],
|
|
577
|
+
fileGlobs: [
|
|
578
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
579
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
580
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
581
|
+
],
|
|
582
|
+
maxEvidence: 25,
|
|
583
|
+
},
|
|
584
|
+
// ── Selenium Rules ───────────────────────────────────────────────────
|
|
585
|
+
{
|
|
586
|
+
id: "SE-FLAKE-001",
|
|
587
|
+
framework: "selenium",
|
|
588
|
+
title: "Hard waits detected in Selenium tests (Thread.sleep / time.sleep)",
|
|
589
|
+
severity: "high",
|
|
590
|
+
description: "Thread.sleep() (Java), time.sleep() (Python), or driver.sleep() (JS) in Selenium tests introduce non-deterministic waits that slow execution and mask timing bugs.",
|
|
591
|
+
recommendation: "Replace hard sleeps with explicit waits: WebDriverWait (Python/Java), driver.wait() (JS). Wait for specific conditions (element visible, clickable, text present) rather than arbitrary durations.\nhttps://www.selenium.dev/documentation/webdriver/waits/",
|
|
592
|
+
patterns: [
|
|
593
|
+
/\bThread\.sleep\s*\(/g,
|
|
594
|
+
/\btime\.sleep\s*\(/g,
|
|
595
|
+
/\bdriver\.sleep\s*\(/g,
|
|
596
|
+
],
|
|
597
|
+
fileGlobs: ["**/*.py", "**/*.java", "**/*.ts", "**/*.js"],
|
|
598
|
+
maxEvidence: 25,
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
id: "SE-FLAKE-002",
|
|
602
|
+
framework: "selenium",
|
|
603
|
+
title: "Implicit waits detected in Selenium tests",
|
|
604
|
+
severity: "medium",
|
|
605
|
+
description: "implicitly_wait() / implicitlyWait() sets a global timeout for all element lookups. Mixing implicit and explicit waits causes unpredictable wait behavior and doubles timeout durations.",
|
|
606
|
+
recommendation: "Remove implicit waits entirely and use explicit waits (WebDriverWait / driver.wait) for each interaction. Explicit waits are more predictable and allow waiting for specific conditions.\nhttps://www.selenium.dev/documentation/webdriver/waits/#implicit-waits",
|
|
607
|
+
patterns: [
|
|
608
|
+
/\.implicitly_wait\s*\(/g,
|
|
609
|
+
/\.implicitlyWait\s*\(/g,
|
|
610
|
+
/implicitly_wait\s*\(\s*\d+/g,
|
|
611
|
+
],
|
|
612
|
+
fileGlobs: ["**/*.py", "**/*.java", "**/*.ts", "**/*.js"],
|
|
613
|
+
maxEvidence: 25,
|
|
614
|
+
},
|
|
615
|
+
{
|
|
616
|
+
id: "SE-LOC-001",
|
|
617
|
+
framework: "selenium",
|
|
618
|
+
title: "XPath selectors detected in Selenium tests",
|
|
619
|
+
severity: "low",
|
|
620
|
+
description: "Selenium tests use By.xpath() or find_element(By.XPATH, ...) selectors. XPath expressions are verbose, fragile to DOM changes, and often slower than CSS-based alternatives.",
|
|
621
|
+
recommendation: "Prefer By.css_selector / By.CSS_SELECTOR, By.id, By.name, or data-testid attributes. CSS selectors are faster, more readable, and less sensitive to DOM structure changes.\nhttps://www.selenium.dev/documentation/webdriver/elements/locators/",
|
|
622
|
+
patterns: [
|
|
623
|
+
/By\.xpath\s*\(/g,
|
|
624
|
+
/By\.XPATH\s*,/g,
|
|
625
|
+
/find_element\s*\(\s*By\.XPATH/g,
|
|
626
|
+
/find_elements?\s*\(\s*['"]xpath['"]\s*,/g,
|
|
627
|
+
],
|
|
628
|
+
fileGlobs: ["**/*.py", "**/*.java", "**/*.ts", "**/*.js"],
|
|
629
|
+
maxEvidence: 25,
|
|
630
|
+
},
|
|
631
|
+
// ── Cross-framework: Emoji in logs ─────────────────────────────────
|
|
632
|
+
{
|
|
633
|
+
id: "ARCH-011",
|
|
634
|
+
title: "Emoji characters in log messages",
|
|
635
|
+
severity: "low",
|
|
636
|
+
description: "Console output contains emoji/pictographic characters. Depending on your CI pipeline's log encoding, emoji can render as garbled characters (mojibake) or cause parsing issues in log aggregators, SIEM tools, and plain-text report archivers.",
|
|
637
|
+
recommendation: "Replace emoji with plain-text equivalents in log messages:\n" +
|
|
638
|
+
" console.log('✅ Test passed') → console.log('[PASS] Test passed')\n" +
|
|
639
|
+
" console.log('❌ Test failed') → console.log('[FAIL] Test failed')\n" +
|
|
640
|
+
" console.log('🚀 Starting...') → console.log('[START] Starting...')\n" +
|
|
641
|
+
" console.log('⚠️ Warning') → console.log('[WARN] Warning')\n" +
|
|
642
|
+
"If emoji are intentional (e.g., for local dev UX), guard them behind an environment check or use a logger that strips non-ASCII for CI output.",
|
|
643
|
+
patterns: [
|
|
644
|
+
/console\.(?:log|warn|error|info|debug)\s*\(.*\p{Extended_Pictographic}/gu,
|
|
645
|
+
],
|
|
646
|
+
fileGlobs: ["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"],
|
|
647
|
+
maxEvidence: 25,
|
|
648
|
+
},
|
|
649
|
+
];
|
|
650
|
+
function normalize(p) {
|
|
651
|
+
return p.replace(/\\/g, "/");
|
|
652
|
+
}
|
|
653
|
+
function isIgnored(filePath) {
|
|
654
|
+
const p = normalize(filePath).toLowerCase();
|
|
655
|
+
return (p.includes("/node_modules/") ||
|
|
656
|
+
p.includes("/dist/") ||
|
|
657
|
+
p.includes("/build/") ||
|
|
658
|
+
p.includes("/.git/") ||
|
|
659
|
+
p.includes("/playwright-report/") ||
|
|
660
|
+
p.includes("/test-results/") ||
|
|
661
|
+
p.includes("/coverage/") ||
|
|
662
|
+
p.includes("/.next/") ||
|
|
663
|
+
p.includes("/.cache/") ||
|
|
664
|
+
p.endsWith(".lock") ||
|
|
665
|
+
p.endsWith(".png") ||
|
|
666
|
+
p.endsWith(".jpg") ||
|
|
667
|
+
p.endsWith(".jpeg") ||
|
|
668
|
+
p.endsWith(".webp") ||
|
|
669
|
+
p.endsWith(".mp4") ||
|
|
670
|
+
p.endsWith(".zip"));
|
|
671
|
+
}
|
|
672
|
+
/**
|
|
673
|
+
* Returns true if file content contains test framework DSL or page-object patterns.
|
|
674
|
+
* Used to skip utility/infrastructure files that happen to match rule fileGlobs
|
|
675
|
+
* (e.g., resource managers living in fixtures/ directories).
|
|
676
|
+
*/
|
|
677
|
+
function hasTestContent(content) {
|
|
678
|
+
// Test framework DSL: test(), it(), describe()
|
|
679
|
+
if (/\b(?:test|it)\s*\(/.test(content))
|
|
680
|
+
return true;
|
|
681
|
+
if (/\b(?:test\.)?describe\s*\(/.test(content))
|
|
682
|
+
return true;
|
|
683
|
+
// Playwright extended API: test.extend<, test.beforeAll(, test.use(, test.step(, etc.
|
|
684
|
+
if (/\btest\.(?:extend|beforeAll|beforeEach|afterAll|afterEach|step|use)\s*[(<]/.test(content))
|
|
685
|
+
return true;
|
|
686
|
+
// Page object pattern: class with this.page (POMs should still be scanned)
|
|
687
|
+
if (/\bclass\s+\w+/.test(content) && /\bthis\.page\b/.test(content))
|
|
688
|
+
return true;
|
|
689
|
+
return false;
|
|
690
|
+
}
|
|
691
|
+
function lineNumberAtIndex(text, idx) {
|
|
692
|
+
let line = 1;
|
|
693
|
+
for (let i = 0; i < idx && i < text.length; i++) {
|
|
694
|
+
if (text.charCodeAt(i) === 10)
|
|
695
|
+
line++;
|
|
696
|
+
}
|
|
697
|
+
return line;
|
|
698
|
+
}
|
|
699
|
+
function snippetAroundLine(lines, lineIdx) {
|
|
700
|
+
const line = lines[lineIdx] ?? "";
|
|
701
|
+
return line.trim().slice(0, 220);
|
|
702
|
+
}
|
|
703
|
+
// Check if a line is likely a comment or contains contextual false-positive indicators
|
|
704
|
+
function isLikelyFalsePositive(lines, lineIdx, ruleId) {
|
|
705
|
+
const line = (lines[lineIdx] ?? "").trim();
|
|
706
|
+
// Skip commented-out code
|
|
707
|
+
if (line.startsWith("//") || line.startsWith("/*") || line.startsWith("*")) {
|
|
708
|
+
return true;
|
|
709
|
+
}
|
|
710
|
+
// PW-FLAKE-001: Retry/polling context exclusion.
|
|
711
|
+
// Suppress when the matched line ±10 is inside an actual loop structure
|
|
712
|
+
// (while/for/do) AND mentions retry/polling intent. The wide window catches
|
|
713
|
+
// real backoff loops where the setTimeout may be many lines from the while.
|
|
714
|
+
if (ruleId === "PW-FLAKE-001") {
|
|
715
|
+
const start = Math.max(0, lineIdx - 10);
|
|
716
|
+
const end = Math.min(lines.length - 1, lineIdx + 10);
|
|
717
|
+
const nearContext = lines.slice(start, end + 1).join("\n").toLowerCase();
|
|
718
|
+
const hasLoopStructure = /\b(while|for|do)\b/.test(nearContext);
|
|
719
|
+
const hasRetryIntent = nearContext.includes("retry") ||
|
|
720
|
+
nearContext.includes("attempt") ||
|
|
721
|
+
nearContext.includes("polling") ||
|
|
722
|
+
nearContext.includes("backoff") ||
|
|
723
|
+
nearContext.includes("reconnect");
|
|
724
|
+
if (hasLoopStructure && hasRetryIntent) {
|
|
725
|
+
return true;
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
// PW-FLAKE-003: Don't flag test.skip() when used as conditional skip
|
|
729
|
+
// (test.skip(condition) is OK — it's the permanent test.skip('name',...) that's bad)
|
|
730
|
+
if (ruleId === "PW-FLAKE-003") {
|
|
731
|
+
// test.skip() as a method call inside test body (conditional) is fine
|
|
732
|
+
if (/test\.skip\s*\(\s*\)/.test(line))
|
|
733
|
+
return true;
|
|
734
|
+
// test.skip(boolean_expression) inside a test body
|
|
735
|
+
if (/test\.skip\s*\(\s*(?:true|false|!|process|env)/.test(line))
|
|
736
|
+
return true;
|
|
737
|
+
}
|
|
738
|
+
// PW-DEPREC-001: Suppress $/$$ inside page.evaluate() / page.evaluateHandle()
|
|
739
|
+
// where $ may be jQuery or a native browser API, not Playwright's deprecated method.
|
|
740
|
+
if (ruleId === "PW-DEPREC-001") {
|
|
741
|
+
// Check surrounding 3 lines for evaluate context
|
|
742
|
+
const start = Math.max(0, lineIdx - 3);
|
|
743
|
+
const context = lines.slice(start, lineIdx + 1).join("\n");
|
|
744
|
+
if (/\.(evaluate|evaluateHandle|evaluateAll)\s*\(/.test(context))
|
|
745
|
+
return true;
|
|
746
|
+
// Also skip if it looks like a CSS selector string referencing $ (e.g., 'input[value$="..."]')
|
|
747
|
+
if (/\[[\w-]+\$=/.test(line))
|
|
748
|
+
return true;
|
|
749
|
+
}
|
|
750
|
+
// PW-FLAKE-010 suppression removed — now handled by AST-based detection
|
|
751
|
+
// PW-DEPREC-003: Suppress waitForSelector inside page.evaluate() or when followed by
|
|
752
|
+
// a comment indicating intentional legacy usage
|
|
753
|
+
if (ruleId === "PW-DEPREC-003") {
|
|
754
|
+
const start = Math.max(0, lineIdx - 3);
|
|
755
|
+
const context = lines.slice(start, lineIdx + 1).join("\n");
|
|
756
|
+
if (/\.(evaluate|evaluateHandle)\s*\(/.test(context))
|
|
757
|
+
return true;
|
|
758
|
+
}
|
|
759
|
+
return false;
|
|
760
|
+
}
|
|
761
|
+
async function collectEvidence(file, content, patterns, maxEvidence, ruleId) {
|
|
762
|
+
const evidence = [];
|
|
763
|
+
const lines = content.split(/\r?\n/);
|
|
764
|
+
for (const pat of patterns) {
|
|
765
|
+
pat.lastIndex = 0;
|
|
766
|
+
let m;
|
|
767
|
+
while ((m = pat.exec(content)) !== null) {
|
|
768
|
+
const line = lineNumberAtIndex(content, m.index);
|
|
769
|
+
// Skip false positives based on context
|
|
770
|
+
if (ruleId && isLikelyFalsePositive(lines, line - 1, ruleId)) {
|
|
771
|
+
if (m.index === pat.lastIndex)
|
|
772
|
+
pat.lastIndex++;
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
evidence.push({
|
|
776
|
+
file,
|
|
777
|
+
line,
|
|
778
|
+
snippet: snippetAroundLine(lines, line - 1),
|
|
779
|
+
});
|
|
780
|
+
if (evidence.length >= maxEvidence)
|
|
781
|
+
return evidence;
|
|
782
|
+
if (m.index === pat.lastIndex)
|
|
783
|
+
pat.lastIndex++;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
return evidence;
|
|
787
|
+
}
|
|
788
|
+
/** Count pattern matches in a file without building evidence (used for totalOccurrences). */
|
|
789
|
+
function countMatches(content, patterns, ruleId) {
|
|
790
|
+
let count = 0;
|
|
791
|
+
const lines = content.split(/\r?\n/);
|
|
792
|
+
for (const pat of patterns) {
|
|
793
|
+
pat.lastIndex = 0;
|
|
794
|
+
let m;
|
|
795
|
+
while ((m = pat.exec(content)) !== null) {
|
|
796
|
+
const line = lineNumberAtIndex(content, m.index);
|
|
797
|
+
if (ruleId && isLikelyFalsePositive(lines, line - 1, ruleId)) {
|
|
798
|
+
if (m.index === pat.lastIndex)
|
|
799
|
+
pat.lastIndex++;
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
count++;
|
|
803
|
+
if (m.index === pat.lastIndex)
|
|
804
|
+
pat.lastIndex++;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
return count;
|
|
808
|
+
}
|
|
809
|
+
function hasFinding(findings, id) {
|
|
810
|
+
return findings.some((f) => f.findingId === id && f.evidence.length > 0);
|
|
811
|
+
}
|
|
812
|
+
/** True when the expression depends on runtime env vars (can't determine value statically). */
|
|
813
|
+
function isEnvDriven(expr) {
|
|
814
|
+
return /process\.env\b/.test(expr) || /\bparseInt\s*\(/.test(expr);
|
|
815
|
+
}
|
|
816
|
+
function retriesLikelyEnabled(retriesExpr) {
|
|
817
|
+
if (!retriesExpr)
|
|
818
|
+
return false;
|
|
819
|
+
if (/^\s*0\s*$/.test(retriesExpr))
|
|
820
|
+
return false;
|
|
821
|
+
if (retriesExpr.includes("?") && retriesExpr.includes(":")) {
|
|
822
|
+
// For simple numeric ternaries (e.g. `process.env.CI ? 2 : 0`), only
|
|
823
|
+
// treat retries as enabled when at least one branch is greater than zero.
|
|
824
|
+
const m = retriesExpr.match(/\?\s*(\d+)\s*:\s*(\d+)\s*$/);
|
|
825
|
+
if (m)
|
|
826
|
+
return Number(m[1]) > 0 || Number(m[2]) > 0;
|
|
827
|
+
return true;
|
|
828
|
+
}
|
|
829
|
+
const nums = [...retriesExpr.matchAll(/\b(\d+)\b/g)].map((m) => Number(m[1]));
|
|
830
|
+
if (nums.some((n) => Number.isFinite(n) && n > 0))
|
|
831
|
+
return true;
|
|
832
|
+
return true;
|
|
833
|
+
}
|
|
834
|
+
function analyzeTestFile(content) {
|
|
835
|
+
// Count test() and test.only() calls
|
|
836
|
+
const testMatches = content.match(/\btest\s*\(/g) || [];
|
|
837
|
+
const testOnlyMatches = content.match(/\btest\.only\s*\(/g) || [];
|
|
838
|
+
const testCount = testMatches.length + testOnlyMatches.length;
|
|
839
|
+
// Count describe() and test.describe() blocks
|
|
840
|
+
const describeMatches = content.match(/\b(?:test\.)?describe\s*\(/g) || [];
|
|
841
|
+
const describeCount = describeMatches.length;
|
|
842
|
+
// Check for lifecycle hooks
|
|
843
|
+
const hasBeforeEach = /\btest\.beforeEach\s*\(|\bbeforeEach\s*\(/i.test(content);
|
|
844
|
+
const hasAfterEach = /\btest\.afterEach\s*\(|\bafterEach\s*\(/i.test(content);
|
|
845
|
+
const hasBeforeAll = /\btest\.beforeAll\s*\(|\bbeforeAll\s*\(/i.test(content);
|
|
846
|
+
const hasAfterAll = /\btest\.afterAll\s*\(|\bafterAll\s*\(/i.test(content);
|
|
847
|
+
// Check for Playwright fixtures usage (test.extend or importing extended test)
|
|
848
|
+
const usesFixtures = /\btest\.extend\s*</.test(content) ||
|
|
849
|
+
/import\s*\{[^}]*\btest\b[^}]*\}\s*from\s*['"][^'"]*fixtures/.test(content) ||
|
|
850
|
+
/import\s+\{[^}]*test[^}]*\}\s+from\s+['"]\.\.?\//.test(content);
|
|
851
|
+
// Check if test function signature includes { page } destructuring (indicates fixture usage)
|
|
852
|
+
const usesPageFixture = /test\s*\([^)]*,\s*async\s*\(\s*\{[^}]*\bpage\b/.test(content);
|
|
853
|
+
return {
|
|
854
|
+
testCount,
|
|
855
|
+
describeCount,
|
|
856
|
+
hasBeforeEach,
|
|
857
|
+
hasAfterEach,
|
|
858
|
+
hasBeforeAll,
|
|
859
|
+
hasAfterAll,
|
|
860
|
+
usesFixtures,
|
|
861
|
+
usesPageFixture,
|
|
862
|
+
};
|
|
863
|
+
}
|
|
864
|
+
function analyzeTestCoupling(filePath, content) {
|
|
865
|
+
const importsFromTestFiles = [];
|
|
866
|
+
// Find imports from .spec.ts or .test.ts files
|
|
867
|
+
const importMatches = content.matchAll(/import\s+.*\s+from\s+['"]([^'"]+\.(?:spec|test)(?:\.ts)?)['"]/g);
|
|
868
|
+
for (const match of importMatches) {
|
|
869
|
+
importsFromTestFiles.push(match[1]);
|
|
870
|
+
}
|
|
871
|
+
// Also check for require statements
|
|
872
|
+
const requireMatches = content.matchAll(/require\s*\(\s*['"]([^'"]+\.(?:spec|test)(?:\.ts)?)['"]\s*\)/g);
|
|
873
|
+
for (const match of requireMatches) {
|
|
874
|
+
importsFromTestFiles.push(match[1]);
|
|
875
|
+
}
|
|
876
|
+
return {
|
|
877
|
+
file: filePath,
|
|
878
|
+
importsFromTestFiles,
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
function findHardcodedData(filePath, content) {
|
|
882
|
+
const matches = [];
|
|
883
|
+
const lines = content.split(/\r?\n/);
|
|
884
|
+
const patterns = [
|
|
885
|
+
// Email addresses (but not example.com or test domains)
|
|
886
|
+
{ type: 'email', regex: /['"`][\w.+-]+@(?!example\.com|test\.com|localhost)[\w.-]+\.[a-z]{2,}['"`]/gi },
|
|
887
|
+
// Passwords in assignments
|
|
888
|
+
{ type: 'password', regex: /(?:password|passwd|pwd)\s*[=:]\s*['"`](?!process\.env)[^'"`]{4,}['"`]/gi },
|
|
889
|
+
// API keys (common patterns)
|
|
890
|
+
{ type: 'api_key', regex: /(?:api[_-]?key|apikey|secret[_-]?key|access[_-]?token)\s*[=:]\s*['"`](?!process\.env)[A-Za-z0-9_-]{16,}['"`]/gi },
|
|
891
|
+
// Hardcoded URLs with credentials
|
|
892
|
+
{ type: 'url', regex: /['"`]https?:\/\/[^:]+:[^@]+@[^'"`]+['"`]/gi },
|
|
893
|
+
// Phone numbers — only flag when variable name suggests phone/tel/mobile
|
|
894
|
+
// (plain digit strings produce too many false positives: IDs, zip codes, etc.)
|
|
895
|
+
{ type: 'phone', regex: /(?:phone|tel|mobile|fax)\s*[=:]\s*['"`]\+?\d[\d\s().-]{6,}['"`]/gi },
|
|
896
|
+
];
|
|
897
|
+
for (let i = 0; i < lines.length; i++) {
|
|
898
|
+
const line = lines[i];
|
|
899
|
+
// Skip comments
|
|
900
|
+
if (line.trim().startsWith('//') || line.trim().startsWith('*'))
|
|
901
|
+
continue;
|
|
902
|
+
for (const { type, regex } of patterns) {
|
|
903
|
+
regex.lastIndex = 0;
|
|
904
|
+
if (regex.test(line)) {
|
|
905
|
+
// Additional filtering for false positives
|
|
906
|
+
if (type === 'email' && (line.includes('@example') || line.includes('@test') || line.includes('faker')))
|
|
907
|
+
continue;
|
|
908
|
+
if (type === 'password' && line.includes('process.env'))
|
|
909
|
+
continue;
|
|
910
|
+
matches.push({
|
|
911
|
+
file: filePath,
|
|
912
|
+
line: i + 1,
|
|
913
|
+
type,
|
|
914
|
+
snippet: line.trim().slice(0, 150),
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
return matches.slice(0, 20); // Limit to 20 matches per file
|
|
920
|
+
}
|
|
921
|
+
function analyzeFixtureTyping(filePath, content) {
|
|
922
|
+
// Check for 'any' type usage
|
|
923
|
+
const anyMatches = content.match(/:\s*any\b/g) || [];
|
|
924
|
+
const anyTypeCount = anyMatches.length;
|
|
925
|
+
// Check for proper fixture typing (test.extend<{...}>)
|
|
926
|
+
const hasProperTyping = /test\.extend\s*<\s*\{/.test(content);
|
|
927
|
+
return {
|
|
928
|
+
file: filePath,
|
|
929
|
+
hasAnyTypes: anyTypeCount > 0,
|
|
930
|
+
anyTypeCount,
|
|
931
|
+
hasProperTyping,
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
function analyzeApiMocking(files, contents) {
|
|
935
|
+
const mockingFiles = [];
|
|
936
|
+
let hasRouteMocking = false;
|
|
937
|
+
let hasRequestInterception = false;
|
|
938
|
+
for (const file of files) {
|
|
939
|
+
const content = contents.get(file);
|
|
940
|
+
if (!content)
|
|
941
|
+
continue;
|
|
942
|
+
// Check for route mocking
|
|
943
|
+
if (/page\.route\s*\(/.test(content) || /context\.route\s*\(/.test(content)) {
|
|
944
|
+
hasRouteMocking = true;
|
|
945
|
+
mockingFiles.push(file);
|
|
946
|
+
}
|
|
947
|
+
// Check for request interception
|
|
948
|
+
if (/page\.on\s*\(\s*['"]request['"]/.test(content) || /page\.on\s*\(\s*['"]response['"]/.test(content)) {
|
|
949
|
+
hasRequestInterception = true;
|
|
950
|
+
if (!mockingFiles.includes(file))
|
|
951
|
+
mockingFiles.push(file);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
return {
|
|
955
|
+
hasRouteMocking,
|
|
956
|
+
hasRequestInterception,
|
|
957
|
+
mockingFiles,
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
function analyzeErrorHandling(filePath, content) {
|
|
961
|
+
// Check for generic throws (throw new Error without custom class)
|
|
962
|
+
const genericThrowMatches = content.match(/throw\s+new\s+Error\s*\(/g) || [];
|
|
963
|
+
const genericThrowCount = genericThrowMatches.length;
|
|
964
|
+
// Check for custom error classes
|
|
965
|
+
const hasCustomErrors = /class\s+\w+Error\s+extends\s+Error/.test(content) ||
|
|
966
|
+
/throw\s+new\s+(?!Error\b)\w+Error\s*\(/.test(content);
|
|
967
|
+
return {
|
|
968
|
+
file: filePath,
|
|
969
|
+
hasGenericThrows: genericThrowCount > 0,
|
|
970
|
+
genericThrowCount,
|
|
971
|
+
hasCustomErrors,
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
function findBrowserResourceLeaks(filePath, content) {
|
|
975
|
+
const results = [];
|
|
976
|
+
// Skip files that only use fixture destructuring (auto-cleanup)
|
|
977
|
+
if (/test\s*\([^)]*,\s*async\s*\(\s*\{[^}]*\b(?:page|browser|context|request)\b/.test(content) &&
|
|
978
|
+
!/chromium\.launch|firefox\.launch|webkit\.launch|browser\.newContext\(|browser\.newPage\(|context\.newPage\(|request\.newContext\(/.test(content)) {
|
|
979
|
+
return results;
|
|
980
|
+
}
|
|
981
|
+
const creationPatterns = [
|
|
982
|
+
/\b(?:chromium|firefox|webkit)\.launch\s*\(/g,
|
|
983
|
+
/\bbrowser\.newContext\s*\(/g,
|
|
984
|
+
/\bbrowser\.newPage\s*\(/g,
|
|
985
|
+
/\bcontext\.newPage\s*\(/g,
|
|
986
|
+
/\brequest\.newContext\s*\(/g,
|
|
987
|
+
];
|
|
988
|
+
const cleanupPatterns = [
|
|
989
|
+
/\bbrowser\.close\s*\(/,
|
|
990
|
+
/\bcontext\.close\s*\(/,
|
|
991
|
+
/\bpage\.close\s*\(/,
|
|
992
|
+
/\.dispose\s*\(/,
|
|
993
|
+
];
|
|
994
|
+
const hookPattern = /\b(?:afterEach|afterAll|test\.afterEach|test\.afterAll)\s*\(/;
|
|
995
|
+
const lines = content.split(/\r?\n/);
|
|
996
|
+
const foundCreations = [];
|
|
997
|
+
for (const re of creationPatterns) {
|
|
998
|
+
let m;
|
|
999
|
+
while ((m = re.exec(content)) !== null) {
|
|
1000
|
+
const lineNum = lineNumberAtIndex(content, m.index);
|
|
1001
|
+
const lineText = lines[lineNum - 1]?.trim() ?? "";
|
|
1002
|
+
if (!lineText.startsWith("//") && !lineText.startsWith("*")) {
|
|
1003
|
+
foundCreations.push({ pattern: m[0], line: lineNum });
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
if (foundCreations.length === 0)
|
|
1008
|
+
return results;
|
|
1009
|
+
// Check if cleanup exists in an afterEach/afterAll hook
|
|
1010
|
+
const hasLiteralCleanup = cleanupPatterns.some((p) => p.test(content));
|
|
1011
|
+
const hasHook = hookPattern.test(content);
|
|
1012
|
+
// Also check for variable-based cleanup: when a resource is assigned to a
|
|
1013
|
+
// variable (e.g., slPage = await slContext.newPage()) and that variable is
|
|
1014
|
+
// closed in a hook (e.g., slPage?.close() in afterAll).
|
|
1015
|
+
let hasVarCleanup = false;
|
|
1016
|
+
if (hasHook && !hasLiteralCleanup) {
|
|
1017
|
+
const resourceAssignRe = /(?:(?:const|let|var)\s+)?(\w+)\s*=\s*await\s+[\w.]+\.(?:newContext|newPage|launch)\s*\(/g;
|
|
1018
|
+
let am;
|
|
1019
|
+
while ((am = resourceAssignRe.exec(content)) !== null) {
|
|
1020
|
+
const varName = am[1];
|
|
1021
|
+
const closeRe = new RegExp(`\\b${varName}\\??\\s*\\.\\s*(?:close|dispose)\\s*\\(`);
|
|
1022
|
+
if (closeRe.test(content)) {
|
|
1023
|
+
hasVarCleanup = true;
|
|
1024
|
+
break;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
const cleanupInHook = hasHook && (hasLiteralCleanup || hasVarCleanup);
|
|
1029
|
+
if (!cleanupInHook) {
|
|
1030
|
+
results.push({
|
|
1031
|
+
file: filePath,
|
|
1032
|
+
creationPatterns: foundCreations.map((c) => c.pattern),
|
|
1033
|
+
line: foundCreations[0].line,
|
|
1034
|
+
});
|
|
1035
|
+
}
|
|
1036
|
+
return results;
|
|
1037
|
+
}
|
|
1038
|
+
function findSeleniumResourceLeaks(filePath, content) {
|
|
1039
|
+
const results = [];
|
|
1040
|
+
const creationPatterns = [
|
|
1041
|
+
/\bwebdriver\.Chrome\s*\(/g,
|
|
1042
|
+
/\bwebdriver\.Firefox\s*\(/g,
|
|
1043
|
+
/\bwebdriver\.Edge\s*\(/g,
|
|
1044
|
+
/\bwebdriver\.Safari\s*\(/g,
|
|
1045
|
+
/\bwebdriver\.Remote\s*\(/g,
|
|
1046
|
+
/\bnew\s+ChromeDriver\s*\(/g,
|
|
1047
|
+
/\bnew\s+FirefoxDriver\s*\(/g,
|
|
1048
|
+
/\bnew\s+RemoteWebDriver\s*\(/g,
|
|
1049
|
+
];
|
|
1050
|
+
const cleanupPattern = /\.quit\s*\(/;
|
|
1051
|
+
const teardownPattern = /\bteardown_method\b|\btearDown\b|\b@After\b|\bafterEach\b|\bafterAll\b/;
|
|
1052
|
+
const lines = content.split(/\r?\n/);
|
|
1053
|
+
const foundCreations = [];
|
|
1054
|
+
for (const re of creationPatterns) {
|
|
1055
|
+
let m;
|
|
1056
|
+
while ((m = re.exec(content)) !== null) {
|
|
1057
|
+
const lineNum = lineNumberAtIndex(content, m.index);
|
|
1058
|
+
const lineText = lines[lineNum - 1]?.trim() ?? "";
|
|
1059
|
+
if (!lineText.startsWith("#") && !lineText.startsWith("//") && !lineText.startsWith("*")) {
|
|
1060
|
+
foundCreations.push({ pattern: m[0], line: lineNum });
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
if (foundCreations.length === 0)
|
|
1065
|
+
return results;
|
|
1066
|
+
const hasCleanup = cleanupPattern.test(content);
|
|
1067
|
+
const hasTeardown = teardownPattern.test(content);
|
|
1068
|
+
const cleanupInTeardown = hasCleanup && hasTeardown;
|
|
1069
|
+
if (!cleanupInTeardown) {
|
|
1070
|
+
results.push({
|
|
1071
|
+
file: filePath,
|
|
1072
|
+
creationPatterns: foundCreations.map((c) => c.pattern),
|
|
1073
|
+
line: foundCreations[0].line,
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
return results;
|
|
1077
|
+
}
|
|
1078
|
+
function findOverAssertedTests(filePath, content) {
|
|
1079
|
+
const results = [];
|
|
1080
|
+
// Find test boundaries using test( or test.only(
|
|
1081
|
+
const testStartRe = /\btest(?:\.only)?\s*\(\s*(['"`])((?:(?!\1).)*)\1/g;
|
|
1082
|
+
let m;
|
|
1083
|
+
while ((m = testStartRe.exec(content)) !== null) {
|
|
1084
|
+
const testName = m[2];
|
|
1085
|
+
const testLine = lineNumberAtIndex(content, m.index);
|
|
1086
|
+
// Find the arrow function body opening brace (=> {), skipping destructuring braces
|
|
1087
|
+
const afterMatch = content.slice(m.index + m[0].length);
|
|
1088
|
+
const arrowMatch = afterMatch.match(/=>\s*\{/);
|
|
1089
|
+
if (!arrowMatch || arrowMatch.index === undefined)
|
|
1090
|
+
continue;
|
|
1091
|
+
let braceStart = m.index + m[0].length + arrowMatch.index + arrowMatch[0].length - 1;
|
|
1092
|
+
// Extract test body using brace-depth counting
|
|
1093
|
+
let depth = 1;
|
|
1094
|
+
let i = braceStart + 1;
|
|
1095
|
+
while (i < content.length && depth > 0) {
|
|
1096
|
+
if (content[i] === "{")
|
|
1097
|
+
depth++;
|
|
1098
|
+
else if (content[i] === "}")
|
|
1099
|
+
depth--;
|
|
1100
|
+
i++;
|
|
1101
|
+
}
|
|
1102
|
+
if (depth !== 0)
|
|
1103
|
+
continue;
|
|
1104
|
+
const testBody = content.slice(braceStart, i);
|
|
1105
|
+
// Skip tests with data-driven loops wrapping assertions
|
|
1106
|
+
if (/\bfor\s*\(/.test(testBody) || /\.forEach\s*\(/.test(testBody))
|
|
1107
|
+
continue;
|
|
1108
|
+
// Count expect() / expect.soft() / expect.poll() calls
|
|
1109
|
+
const assertionMatches = testBody.match(/\bexpect\s*(?:\.(?:soft|poll)\s*)?\(/g);
|
|
1110
|
+
const assertionCount = assertionMatches ? assertionMatches.length : 0;
|
|
1111
|
+
if (assertionCount > 5) {
|
|
1112
|
+
results.push({
|
|
1113
|
+
file: filePath,
|
|
1114
|
+
testName,
|
|
1115
|
+
assertionCount,
|
|
1116
|
+
line: testLine,
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
return results;
|
|
1121
|
+
}
|
|
1122
|
+
function findInlineTestDataInFile(filePath, content) {
|
|
1123
|
+
const matches = [];
|
|
1124
|
+
const lines = content.split(/\r?\n/);
|
|
1125
|
+
// Variable-name-anchored patterns
|
|
1126
|
+
const patterns = [
|
|
1127
|
+
{ re: /\b(?:username|userName|user_name)\s*=\s*['"`]/i, type: "username" },
|
|
1128
|
+
{ re: /\b(?:firstName|first_name|lastname|last_name)\s*=\s*['"`]/i, type: "name" },
|
|
1129
|
+
{ re: /\b(?:address|street|streetAddress|street_address)\s*=\s*['"`]/i, type: "address" },
|
|
1130
|
+
{ re: /\b(?:zipCode|zip_code|postalCode|postal_code)\s*=\s*['"`]\d/i, type: "zipCode" },
|
|
1131
|
+
{ re: /\b(?:birthDate|birth_date|dob|dateOfBirth)\s*=\s*['"`]\d/i, type: "date" },
|
|
1132
|
+
{ re: /\b(?:cardNumber|card_number|creditCard|credit_card)\s*=\s*['"`]\d/i, type: "creditCard" },
|
|
1133
|
+
{ re: /\b(?:price|amount|cost)\s*=\s*['"`]\$?\d/i, type: "price" },
|
|
1134
|
+
{ re: /\b(?:phoneNumber|phone_number|phone)\s*=\s*['"`]\+?\d/i, type: "phone" },
|
|
1135
|
+
{ re: /\b(?:city|town)\s*=\s*['"`][A-Z]/i, type: "city" },
|
|
1136
|
+
];
|
|
1137
|
+
// Skip patterns — environment variables, faker, fixtures, comments
|
|
1138
|
+
const skipPatterns = /process\.env|Cypress\.env|os\.environ|\bfaker\b|\bfixture\b/i;
|
|
1139
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1140
|
+
const line = lines[i].trim();
|
|
1141
|
+
// Skip comments
|
|
1142
|
+
if (line.startsWith("//") || line.startsWith("#") || line.startsWith("*") || line.startsWith("/*"))
|
|
1143
|
+
continue;
|
|
1144
|
+
// Skip lines with env/faker/fixture patterns
|
|
1145
|
+
if (skipPatterns.test(line))
|
|
1146
|
+
continue;
|
|
1147
|
+
for (const { re, type } of patterns) {
|
|
1148
|
+
if (re.test(line)) {
|
|
1149
|
+
matches.push({
|
|
1150
|
+
file: filePath,
|
|
1151
|
+
line: i + 1,
|
|
1152
|
+
snippet: line.slice(0, 120),
|
|
1153
|
+
type,
|
|
1154
|
+
});
|
|
1155
|
+
break; // One match per line is enough
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
return matches;
|
|
1160
|
+
}
|
|
1161
|
+
async function scanRepo(repoPath, options = {}) {
|
|
1162
|
+
if (!(0, fs_1.existsSync)(repoPath)) {
|
|
1163
|
+
console.warn(`[audit] Repo path does not exist: ${repoPath} — producing empty scan result.`);
|
|
1164
|
+
const repoName = path_1.default.basename(path_1.default.resolve(repoPath));
|
|
1165
|
+
return {
|
|
1166
|
+
repoPath,
|
|
1167
|
+
inventory: {
|
|
1168
|
+
totalFiles: 0,
|
|
1169
|
+
testFiles: 0,
|
|
1170
|
+
hasPlaywrightConfig: false,
|
|
1171
|
+
playwrightVersion: null,
|
|
1172
|
+
detectedFrameworks: [],
|
|
1173
|
+
},
|
|
1174
|
+
findings: [
|
|
1175
|
+
{
|
|
1176
|
+
findingId: "SCAN-MISSING-REPO",
|
|
1177
|
+
severity: "high",
|
|
1178
|
+
title: "Repository path not found",
|
|
1179
|
+
description: `The target repository path "${repoPath}" does not exist. This may indicate the repo was moved or deleted. No files were scanned.`,
|
|
1180
|
+
recommendation: "Verify that the repository path is correct and accessible, then re-run the audit.",
|
|
1181
|
+
evidence: [],
|
|
1182
|
+
},
|
|
1183
|
+
],
|
|
1184
|
+
gitBranch: undefined,
|
|
1185
|
+
gitCommit: undefined,
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
const git = (0, diffTracker_1.getGitInfo)(repoPath);
|
|
1189
|
+
if (options.branch)
|
|
1190
|
+
git.branch = options.branch;
|
|
1191
|
+
const allFiles = (await (0, fast_glob_1.default)(["**/*"], {
|
|
1192
|
+
cwd: repoPath,
|
|
1193
|
+
absolute: true,
|
|
1194
|
+
dot: true,
|
|
1195
|
+
onlyFiles: true,
|
|
1196
|
+
followSymbolicLinks: false,
|
|
1197
|
+
}));
|
|
1198
|
+
const files = allFiles.filter((f) => !isIgnored(f));
|
|
1199
|
+
// Match standard AND variant Playwright config names:
|
|
1200
|
+
// playwright.config.ts, playwright-ci.config.ts, playwright-local.config.js,
|
|
1201
|
+
// playwright-mobile.config.js, playwright.ct.config.ts, etc.
|
|
1202
|
+
const PW_CONFIG_RE = /playwright[-.]?[\w.-]*\.config\.(ts|js|mjs|cjs)$/i;
|
|
1203
|
+
const allConfigFiles = files.filter((f) => PW_CONFIG_RE.test(path_1.default.basename(f)));
|
|
1204
|
+
const inventory = {
|
|
1205
|
+
totalFiles: files.length,
|
|
1206
|
+
testFiles: files.filter((f) => /\.(spec|test)\.(ts|js|tsx|jsx)$/.test(f)).length,
|
|
1207
|
+
hasPlaywrightConfig: allConfigFiles.length > 0,
|
|
1208
|
+
playwrightConfigPaths: allConfigFiles.length
|
|
1209
|
+
? allConfigFiles.map((f) => path_1.default.relative(repoPath, f).replace(/\\/g, "/"))
|
|
1210
|
+
: undefined,
|
|
1211
|
+
playwrightVersion: null,
|
|
1212
|
+
detectedFrameworks: [],
|
|
1213
|
+
};
|
|
1214
|
+
const findings = [];
|
|
1215
|
+
// ── Locate package.json ────────────────────────────────────────────
|
|
1216
|
+
// Prefer root-level package.json (same directory as repoPath).
|
|
1217
|
+
// fast-glob does not guarantee ordering, so files.find() may return
|
|
1218
|
+
// a nested package.json first. Explicitly check the root first.
|
|
1219
|
+
const allPkgJsonPaths = files.filter((f) => path_1.default.basename(f) === "package.json");
|
|
1220
|
+
const rootPkgPath = allPkgJsonPaths.find((f) => normalize(path_1.default.dirname(f)) === normalize(path_1.default.resolve(repoPath)));
|
|
1221
|
+
let pkgPath = rootPkgPath || allPkgJsonPaths[0];
|
|
1222
|
+
let pkgJson;
|
|
1223
|
+
if (pkgPath) {
|
|
1224
|
+
try {
|
|
1225
|
+
const pkg = JSON.parse(await promises_1.default.readFile(pkgPath, "utf8"));
|
|
1226
|
+
pkgJson = pkg;
|
|
1227
|
+
inventory.playwrightVersion =
|
|
1228
|
+
pkg.devDependencies?.["@playwright/test"] ||
|
|
1229
|
+
pkg.dependencies?.["@playwright/test"] ||
|
|
1230
|
+
null;
|
|
1231
|
+
}
|
|
1232
|
+
catch {
|
|
1233
|
+
inventory.playwrightVersion = null;
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
// If the root package.json didn't contain @playwright/test, look for it
|
|
1237
|
+
// in the package.json co-located with the Playwright config, then in any
|
|
1238
|
+
// other package.json across the repo.
|
|
1239
|
+
if (!inventory.playwrightVersion && allPkgJsonPaths.length > 1) {
|
|
1240
|
+
// 1. Check package.json next to the Playwright config
|
|
1241
|
+
const configDir = allConfigFiles.length
|
|
1242
|
+
? normalize(path_1.default.dirname(allConfigFiles[0]))
|
|
1243
|
+
: null;
|
|
1244
|
+
const configPkgPath = configDir
|
|
1245
|
+
? allPkgJsonPaths.find((f) => normalize(path_1.default.dirname(f)) === configDir && f !== pkgPath)
|
|
1246
|
+
: undefined;
|
|
1247
|
+
// 2. Collect candidates: config-adjacent first, then everything else
|
|
1248
|
+
const candidates = [
|
|
1249
|
+
...(configPkgPath ? [configPkgPath] : []),
|
|
1250
|
+
...allPkgJsonPaths.filter((f) => f !== pkgPath && f !== configPkgPath),
|
|
1251
|
+
];
|
|
1252
|
+
for (const candidatePath of candidates) {
|
|
1253
|
+
try {
|
|
1254
|
+
const pkg = JSON.parse(await promises_1.default.readFile(candidatePath, "utf8"));
|
|
1255
|
+
const ver = pkg.devDependencies?.["@playwright/test"] ||
|
|
1256
|
+
pkg.dependencies?.["@playwright/test"] ||
|
|
1257
|
+
null;
|
|
1258
|
+
if (ver) {
|
|
1259
|
+
inventory.playwrightVersion = ver;
|
|
1260
|
+
// Update pkgJson + pkgPath so downstream dep checks use the right package.json
|
|
1261
|
+
pkgJson = pkg;
|
|
1262
|
+
pkgPath = candidatePath;
|
|
1263
|
+
break;
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
catch {
|
|
1267
|
+
continue;
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
try {
|
|
1272
|
+
// Pick the "primary" config for deep extraction:
|
|
1273
|
+
// 1. Prefer the standard name (playwright.config.{ts,js,...}) at repo root
|
|
1274
|
+
// 2. Then standard name anywhere
|
|
1275
|
+
// 3. Then first variant found
|
|
1276
|
+
const isStandardName = (f) => /^playwright\.config\.(ts|js|mjs|cjs)$/.test(path_1.default.basename(f));
|
|
1277
|
+
const primaryConfig = allConfigFiles.find((f) => isStandardName(f) && path_1.default.dirname(f) === repoPath) ||
|
|
1278
|
+
allConfigFiles.find((f) => isStandardName(f)) ||
|
|
1279
|
+
allConfigFiles[0];
|
|
1280
|
+
const pwConfig = primaryConfig
|
|
1281
|
+
? await (0, configExtractor_1.extractPlaywrightConfig)(repoPath, primaryConfig)
|
|
1282
|
+
: undefined;
|
|
1283
|
+
if (pwConfig)
|
|
1284
|
+
inventory.playwrightConfigSummary = pwConfig;
|
|
1285
|
+
}
|
|
1286
|
+
catch {
|
|
1287
|
+
inventory.playwrightConfigSummary = undefined;
|
|
1288
|
+
}
|
|
1289
|
+
await maybeAddOutdatedDependencyFinding({
|
|
1290
|
+
repoPath,
|
|
1291
|
+
pkgPath,
|
|
1292
|
+
pkg: pkgJson,
|
|
1293
|
+
pwConfig: inventory.playwrightConfigSummary,
|
|
1294
|
+
findings,
|
|
1295
|
+
enabled: Boolean(options.checkOutdatedDependencies),
|
|
1296
|
+
});
|
|
1297
|
+
try {
|
|
1298
|
+
const ciSummary = await (0, ciExtractor_1.extractCiSummary)(repoPath);
|
|
1299
|
+
inventory.ciSummary = ciSummary;
|
|
1300
|
+
}
|
|
1301
|
+
catch (e) {
|
|
1302
|
+
inventory.ciSummary = {
|
|
1303
|
+
files: [],
|
|
1304
|
+
detectedAzurePipelines: false,
|
|
1305
|
+
detectedGitHubActions: false,
|
|
1306
|
+
publishesPlaywrightReport: false,
|
|
1307
|
+
publishesJUnit: false,
|
|
1308
|
+
publishesTracesOrTestResultsDir: false,
|
|
1309
|
+
publishesArtifactsOnFailure: false,
|
|
1310
|
+
usesCache: false,
|
|
1311
|
+
usesSharding: false,
|
|
1312
|
+
shardingFiles: [],
|
|
1313
|
+
setsWorkersEnv: false,
|
|
1314
|
+
workersEnvName: undefined,
|
|
1315
|
+
workersEnvFiles: [],
|
|
1316
|
+
setsHeadlessEnv: false,
|
|
1317
|
+
headlessEnvName: undefined,
|
|
1318
|
+
mentionsNodeSetup: false,
|
|
1319
|
+
mentionsNpmInstall: false,
|
|
1320
|
+
mentionsPlaywrightInstall: false,
|
|
1321
|
+
playwrightInstallMethod: undefined,
|
|
1322
|
+
usesSelfHostedPool: false,
|
|
1323
|
+
usesMicrosoftHostedPool: false,
|
|
1324
|
+
mentionsBrowsersPreinstalled: false,
|
|
1325
|
+
hasFailureHandling: false,
|
|
1326
|
+
failureHandlingMethods: [],
|
|
1327
|
+
setsCiEnvTrue: false,
|
|
1328
|
+
ciEnvTrueFiles: [],
|
|
1329
|
+
hasPipelineTimeout: false,
|
|
1330
|
+
pipelineTimeoutMinutes: undefined,
|
|
1331
|
+
hasReporterInPlaywrightCommand: false,
|
|
1332
|
+
notes: [`CI extraction failed: ${String(e)}`],
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
// ============================================
|
|
1336
|
+
// Framework detection: Playwright, Cypress, Selenium
|
|
1337
|
+
// ============================================
|
|
1338
|
+
if (inventory.hasPlaywrightConfig || inventory.playwrightVersion) {
|
|
1339
|
+
inventory.detectedFrameworks.push("playwright");
|
|
1340
|
+
}
|
|
1341
|
+
// Cypress detection
|
|
1342
|
+
try {
|
|
1343
|
+
const cypressConfig = await (0, cypressExtractor_1.extractCypressConfig)(repoPath);
|
|
1344
|
+
if (cypressConfig) {
|
|
1345
|
+
inventory.detectedFrameworks.push("cypress");
|
|
1346
|
+
inventory.cypressConfigSummary = cypressConfig;
|
|
1347
|
+
// Count Cypress test files
|
|
1348
|
+
const cypressTestGlobs = [
|
|
1349
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
1350
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
1351
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
1352
|
+
];
|
|
1353
|
+
const cypressTestFiles = (await (0, fast_glob_1.default)(cypressTestGlobs, {
|
|
1354
|
+
cwd: repoPath,
|
|
1355
|
+
absolute: true,
|
|
1356
|
+
dot: true,
|
|
1357
|
+
onlyFiles: true,
|
|
1358
|
+
followSymbolicLinks: false,
|
|
1359
|
+
}));
|
|
1360
|
+
inventory.cypressTestFiles = cypressTestFiles.filter((f) => !isIgnored(f)).length;
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
catch {
|
|
1364
|
+
// Cypress detection failed silently
|
|
1365
|
+
}
|
|
1366
|
+
// Selenium detection (scan for selenium imports/dependencies)
|
|
1367
|
+
try {
|
|
1368
|
+
const seleniumIndicators = [];
|
|
1369
|
+
// Check package.json for selenium-webdriver
|
|
1370
|
+
if (pkgJson) {
|
|
1371
|
+
const allDeps = { ...(pkgJson.dependencies ?? {}), ...(pkgJson.devDependencies ?? {}) };
|
|
1372
|
+
if (allDeps["selenium-webdriver"] || allDeps["webdriverio"] || allDeps["nightwatch"]) {
|
|
1373
|
+
seleniumIndicators.push("npm");
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
// Check for requirements.txt with selenium
|
|
1377
|
+
const reqTxt = files.find((f) => path_1.default.basename(f) === "requirements.txt");
|
|
1378
|
+
if (reqTxt) {
|
|
1379
|
+
const reqContent = await promises_1.default.readFile(reqTxt, "utf8");
|
|
1380
|
+
if (/\bselenium\b/i.test(reqContent))
|
|
1381
|
+
seleniumIndicators.push("pip");
|
|
1382
|
+
}
|
|
1383
|
+
// Check for pom.xml with selenium
|
|
1384
|
+
const pomXml = files.find((f) => path_1.default.basename(f) === "pom.xml");
|
|
1385
|
+
if (pomXml) {
|
|
1386
|
+
const pomContent = await promises_1.default.readFile(pomXml, "utf8");
|
|
1387
|
+
if (/selenium/i.test(pomContent))
|
|
1388
|
+
seleniumIndicators.push("maven");
|
|
1389
|
+
}
|
|
1390
|
+
// Quick scan: look for Selenium imports in source files
|
|
1391
|
+
if (!seleniumIndicators.length) {
|
|
1392
|
+
const seleniumFileGlobs = ["**/*.py", "**/*.java", "**/*.ts", "**/*.js"];
|
|
1393
|
+
const candidateFiles = (await (0, fast_glob_1.default)(seleniumFileGlobs, {
|
|
1394
|
+
cwd: repoPath,
|
|
1395
|
+
absolute: true,
|
|
1396
|
+
dot: true,
|
|
1397
|
+
onlyFiles: true,
|
|
1398
|
+
followSymbolicLinks: false,
|
|
1399
|
+
}));
|
|
1400
|
+
const filtered = candidateFiles.filter((f) => !isIgnored(f)).slice(0, 50);
|
|
1401
|
+
let seTestCount = 0;
|
|
1402
|
+
for (const f of filtered) {
|
|
1403
|
+
try {
|
|
1404
|
+
const content = await promises_1.default.readFile(f, "utf8");
|
|
1405
|
+
if (/from\s+selenium\b/.test(content) ||
|
|
1406
|
+
/import\s+org\.openqa\.selenium/.test(content) ||
|
|
1407
|
+
/require\s*\(\s*['"]selenium-webdriver['"]/.test(content) ||
|
|
1408
|
+
/from\s+['"]selenium-webdriver['"]/.test(content)) {
|
|
1409
|
+
seTestCount++;
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
catch { /* skip */ }
|
|
1413
|
+
}
|
|
1414
|
+
if (seTestCount > 0) {
|
|
1415
|
+
seleniumIndicators.push("imports");
|
|
1416
|
+
inventory.seleniumTestFiles = seTestCount;
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
if (seleniumIndicators.length > 0) {
|
|
1420
|
+
inventory.detectedFrameworks.push("selenium");
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
catch {
|
|
1424
|
+
// Selenium detection failed silently
|
|
1425
|
+
}
|
|
1426
|
+
for (const rule of RULES) {
|
|
1427
|
+
// Skip framework-specific rules when that framework isn't detected
|
|
1428
|
+
if (rule.framework && !inventory.detectedFrameworks.includes(rule.framework)) {
|
|
1429
|
+
continue;
|
|
1430
|
+
}
|
|
1431
|
+
const targetFiles = (await (0, fast_glob_1.default)(rule.fileGlobs, {
|
|
1432
|
+
cwd: repoPath,
|
|
1433
|
+
absolute: true,
|
|
1434
|
+
dot: true,
|
|
1435
|
+
onlyFiles: true,
|
|
1436
|
+
followSymbolicLinks: false,
|
|
1437
|
+
}));
|
|
1438
|
+
const scopedFiles = targetFiles.filter((f) => !isIgnored(f));
|
|
1439
|
+
let evidence = [];
|
|
1440
|
+
const max = rule.maxEvidence ?? 20;
|
|
1441
|
+
let totalOccurrences = 0;
|
|
1442
|
+
const affectedFileSet = new Set();
|
|
1443
|
+
for (const f of scopedFiles) {
|
|
1444
|
+
const content = await promises_1.default.readFile(f, "utf8");
|
|
1445
|
+
// Skip test.fixme/test.describe.fixme files for all rules except PW-FLAKE-006
|
|
1446
|
+
if (rule.id !== "PW-FLAKE-006" && /\btest\.(?:describe\.)?fixme\s*\(/.test(content))
|
|
1447
|
+
continue;
|
|
1448
|
+
// Skip non-test files for rules that only target test content
|
|
1449
|
+
if (rule.testContentOnly && !hasTestContent(content))
|
|
1450
|
+
continue;
|
|
1451
|
+
if (evidence.length < max) {
|
|
1452
|
+
const ev = await collectEvidence(normalize(f), content, rule.patterns, max - evidence.length, rule.id);
|
|
1453
|
+
if (ev.length) {
|
|
1454
|
+
evidence = evidence.concat(ev);
|
|
1455
|
+
totalOccurrences += ev.length;
|
|
1456
|
+
affectedFileSet.add(normalize(f));
|
|
1457
|
+
// If evidence filled up mid-file, count remaining matches in this file
|
|
1458
|
+
if (evidence.length >= max) {
|
|
1459
|
+
const remaining = countMatches(content, rule.patterns, rule.id) - ev.length;
|
|
1460
|
+
if (remaining > 0)
|
|
1461
|
+
totalOccurrences += remaining;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
else {
|
|
1466
|
+
// Evidence cap reached — count only
|
|
1467
|
+
const fileCount = countMatches(content, rule.patterns, rule.id);
|
|
1468
|
+
if (fileCount > 0) {
|
|
1469
|
+
totalOccurrences += fileCount;
|
|
1470
|
+
affectedFileSet.add(normalize(f));
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (evidence.length) {
|
|
1475
|
+
findings.push({
|
|
1476
|
+
findingId: rule.id,
|
|
1477
|
+
severity: rule.severity,
|
|
1478
|
+
title: rule.title,
|
|
1479
|
+
description: rule.description,
|
|
1480
|
+
recommendation: rule.recommendation,
|
|
1481
|
+
evidence,
|
|
1482
|
+
totalOccurrences,
|
|
1483
|
+
affectedFiles: affectedFileSet.size,
|
|
1484
|
+
});
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
// Get retry configuration early (needed for correlation checks)
|
|
1488
|
+
const retriesExpr = inventory.playwrightConfigSummary?.retries;
|
|
1489
|
+
const retriesEnabled = retriesLikelyEnabled(retriesExpr);
|
|
1490
|
+
// ============================================
|
|
1491
|
+
// Cypress-specific config-based findings
|
|
1492
|
+
// ============================================
|
|
1493
|
+
if (inventory.cypressConfigSummary) {
|
|
1494
|
+
const cyConfig = inventory.cypressConfigSummary;
|
|
1495
|
+
// CY-PERF-001: Missing baseUrl
|
|
1496
|
+
if (!cyConfig.baseUrl) {
|
|
1497
|
+
findings.push({
|
|
1498
|
+
findingId: "CY-PERF-001",
|
|
1499
|
+
severity: "medium",
|
|
1500
|
+
title: "Missing baseUrl in Cypress configuration",
|
|
1501
|
+
description: "Cypress config does not define a baseUrl. Without baseUrl, every cy.visit() must use a full URL, making it hard to switch between environments (dev, staging, prod) and violating DRY.",
|
|
1502
|
+
recommendation: "Add baseUrl to your cypress.config.ts e2e block: baseUrl: process.env.CYPRESS_BASE_URL || 'http://localhost:3000'. Then use cy.visit('/path') instead of cy.visit('http://localhost:3000/path').\nhttps://docs.cypress.io/guides/references/best-practices#Setting-a-Global-baseUrl",
|
|
1503
|
+
evidence: [{
|
|
1504
|
+
file: normalize(cyConfig.configPath),
|
|
1505
|
+
line: 1,
|
|
1506
|
+
snippet: "baseUrl not configured in Cypress config",
|
|
1507
|
+
}],
|
|
1508
|
+
});
|
|
1509
|
+
}
|
|
1510
|
+
// CY-PERF-002: Default command timeout too high
|
|
1511
|
+
if (cyConfig.defaultCommandTimeout && cyConfig.defaultCommandTimeout > 30000) {
|
|
1512
|
+
findings.push({
|
|
1513
|
+
findingId: "CY-PERF-002",
|
|
1514
|
+
severity: "medium",
|
|
1515
|
+
title: `Excessive defaultCommandTimeout (${cyConfig.defaultCommandTimeout}ms)`,
|
|
1516
|
+
description: `Cypress defaultCommandTimeout is set to ${cyConfig.defaultCommandTimeout}ms (${Math.round(cyConfig.defaultCommandTimeout / 1000)}s). High timeouts mask slow application responses and make failing tests hang for too long.`,
|
|
1517
|
+
recommendation: "Set defaultCommandTimeout to 10-15s maximum. If specific commands need longer, use cy.get(selector, { timeout: 30000 }) for those specific cases.\nhttps://docs.cypress.io/guides/references/configuration#Timeouts",
|
|
1518
|
+
evidence: [{
|
|
1519
|
+
file: normalize(cyConfig.configPath),
|
|
1520
|
+
line: 1,
|
|
1521
|
+
snippet: `defaultCommandTimeout: ${cyConfig.defaultCommandTimeout}`,
|
|
1522
|
+
}],
|
|
1523
|
+
});
|
|
1524
|
+
}
|
|
1525
|
+
// CY-STABILITY-001: No cy.intercept() in test files (no network mocking)
|
|
1526
|
+
if (inventory.cypressTestFiles && inventory.cypressTestFiles >= 5) {
|
|
1527
|
+
const cypressTestGlobs = [
|
|
1528
|
+
"**/*.cy.ts", "**/*.cy.js", "**/*.cy.tsx", "**/*.cy.jsx",
|
|
1529
|
+
"cypress/e2e/**/*.ts", "cypress/e2e/**/*.js",
|
|
1530
|
+
"cypress/integration/**/*.ts", "cypress/integration/**/*.js",
|
|
1531
|
+
];
|
|
1532
|
+
const cyTestFiles = (await (0, fast_glob_1.default)(cypressTestGlobs, {
|
|
1533
|
+
cwd: repoPath,
|
|
1534
|
+
absolute: true,
|
|
1535
|
+
dot: true,
|
|
1536
|
+
onlyFiles: true,
|
|
1537
|
+
followSymbolicLinks: false,
|
|
1538
|
+
}));
|
|
1539
|
+
const filteredCyTests = cyTestFiles.filter((f) => !isIgnored(f));
|
|
1540
|
+
let hasIntercept = false;
|
|
1541
|
+
for (const f of filteredCyTests.slice(0, 100)) {
|
|
1542
|
+
try {
|
|
1543
|
+
const content = await promises_1.default.readFile(f, "utf8");
|
|
1544
|
+
if (/\bcy\.intercept\s*\(/.test(content)) {
|
|
1545
|
+
hasIntercept = true;
|
|
1546
|
+
break;
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
catch { /* skip */ }
|
|
1550
|
+
}
|
|
1551
|
+
if (!hasIntercept) {
|
|
1552
|
+
findings.push({
|
|
1553
|
+
findingId: "CY-STABILITY-001",
|
|
1554
|
+
severity: "low",
|
|
1555
|
+
title: "No cy.intercept() usage detected in Cypress tests",
|
|
1556
|
+
description: `No cy.intercept() patterns found across ${filteredCyTests.length} Cypress test files. Network interception enables faster, more reliable tests by mocking API responses and controlling network behavior.`,
|
|
1557
|
+
recommendation: "Use cy.intercept() to mock API responses for faster, deterministic tests. This allows testing error states, slow responses, and edge cases without backend dependencies.\nhttps://docs.cypress.io/api/commands/intercept",
|
|
1558
|
+
evidence: [{
|
|
1559
|
+
file: normalize(cyConfig.configPath),
|
|
1560
|
+
line: 1,
|
|
1561
|
+
snippet: `${filteredCyTests.length} Cypress test files; no cy.intercept() found`,
|
|
1562
|
+
}],
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
// Test file structure analysis for stability checks
|
|
1568
|
+
const testFiles = (await (0, fast_glob_1.default)(["**/*.spec.ts", "**/*.spec.js", "**/*.test.ts", "**/*.test.js"], {
|
|
1569
|
+
cwd: repoPath,
|
|
1570
|
+
absolute: true,
|
|
1571
|
+
dot: true,
|
|
1572
|
+
onlyFiles: true,
|
|
1573
|
+
followSymbolicLinks: false,
|
|
1574
|
+
}));
|
|
1575
|
+
const filteredTestFiles = testFiles.filter((f) => !isIgnored(f));
|
|
1576
|
+
// Files containing test.fixme() are excluded from all rules except PW-FLAKE-006.
|
|
1577
|
+
// Inventory counting (test files, test cases) uses the full filteredTestFiles.
|
|
1578
|
+
const fixmeFileSet = new Set();
|
|
1579
|
+
for (const f of filteredTestFiles) {
|
|
1580
|
+
try {
|
|
1581
|
+
const content = await promises_1.default.readFile(f, "utf8");
|
|
1582
|
+
if (/\btest\.(?:describe\.)?fixme\s*\(/.test(content)) {
|
|
1583
|
+
fixmeFileSet.add(f);
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
catch { /* skip unreadable */ }
|
|
1587
|
+
}
|
|
1588
|
+
const nonFixmeTestFiles = filteredTestFiles.filter((f) => !fixmeFileSet.has(f));
|
|
1589
|
+
// Accumulate individual test case count across all Playwright test files
|
|
1590
|
+
let totalTestCases = 0;
|
|
1591
|
+
// PW-STABILITY-001: Shared state / missing cleanup hooks
|
|
1592
|
+
// Categorize files by risk level
|
|
1593
|
+
const filesNoHooksAtAll = [];
|
|
1594
|
+
const filesOnlyBeforeAfterAll = [];
|
|
1595
|
+
// PW-STABILITY-002: Missing organization
|
|
1596
|
+
const filesWithoutOrganization = [];
|
|
1597
|
+
// PW-STABILITY-003: Using page fixture in beforeAll (shared state risk)
|
|
1598
|
+
const beforeAllUsesPage = [];
|
|
1599
|
+
const beforeAllUsesPagePatterns = [
|
|
1600
|
+
/\btest\.beforeAll\s*\(\s*(?:async\s*)?\(\s*\{\s*[^}]*\bpage\b[^}]*\}\s*\)\s*=>/gi,
|
|
1601
|
+
/\bbeforeAll\s*\(\s*(?:async\s*)?\(\s*\{\s*[^}]*\bpage\b[^}]*\}\s*\)\s*=>/gi,
|
|
1602
|
+
];
|
|
1603
|
+
for (const testFile of filteredTestFiles) {
|
|
1604
|
+
try {
|
|
1605
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
1606
|
+
const analysis = analyzeTestFile(content);
|
|
1607
|
+
totalTestCases += analysis.testCount;
|
|
1608
|
+
// Skip rule analysis for test.fixme files (inventory counting above still applies)
|
|
1609
|
+
if (fixmeFileSet.has(testFile))
|
|
1610
|
+
continue;
|
|
1611
|
+
// STABILITY-003: Flag any beforeAll using the Playwright page fixture
|
|
1612
|
+
// This commonly causes shared browser state and flaky cascades.
|
|
1613
|
+
const beforeAllPageEvidence = await collectEvidence(normalize(testFile), content, beforeAllUsesPagePatterns, 1, "PW-STABILITY-003");
|
|
1614
|
+
if (beforeAllPageEvidence.length > 0) {
|
|
1615
|
+
beforeAllUsesPage.push(beforeAllPageEvidence[0]);
|
|
1616
|
+
}
|
|
1617
|
+
// STABILITY-001: Multiple tests without per-test cleanup hooks
|
|
1618
|
+
// Skip files that use Playwright fixtures (they handle state via fixture lifecycle)
|
|
1619
|
+
if (!analysis.usesFixtures && analysis.testCount >= 3 && !analysis.hasBeforeEach && !analysis.hasAfterEach) {
|
|
1620
|
+
const hasAnyAllHooks = analysis.hasBeforeAll || analysis.hasAfterAll;
|
|
1621
|
+
if (hasAnyAllHooks) {
|
|
1622
|
+
// Has beforeAll/afterAll but no beforeEach/afterEach
|
|
1623
|
+
// This is less risky - only flag if many tests (5+) since beforeAll/afterAll
|
|
1624
|
+
// indicates intentional shared setup (e.g., login once, run multiple tests)
|
|
1625
|
+
if (analysis.testCount >= 5) {
|
|
1626
|
+
filesOnlyBeforeAfterAll.push({
|
|
1627
|
+
file: normalize(testFile),
|
|
1628
|
+
testCount: analysis.testCount,
|
|
1629
|
+
analysis
|
|
1630
|
+
});
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
else {
|
|
1634
|
+
// No hooks at all - higher risk
|
|
1635
|
+
filesNoHooksAtAll.push({
|
|
1636
|
+
file: normalize(testFile),
|
|
1637
|
+
testCount: analysis.testCount,
|
|
1638
|
+
analysis
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
// STABILITY-002: 10+ tests without describe blocks
|
|
1643
|
+
if (analysis.testCount >= 10 && analysis.describeCount === 0) {
|
|
1644
|
+
filesWithoutOrganization.push({
|
|
1645
|
+
file: normalize(testFile),
|
|
1646
|
+
testCount: analysis.testCount
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
catch {
|
|
1651
|
+
// Skip unreadable files
|
|
1652
|
+
continue;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
inventory.testCases = totalTestCases;
|
|
1656
|
+
// STABILITY-003 finding
|
|
1657
|
+
if (beforeAllUsesPage.length > 0) {
|
|
1658
|
+
findings.push({
|
|
1659
|
+
findingId: "PW-STABILITY-003",
|
|
1660
|
+
severity: "high",
|
|
1661
|
+
title: "Test isolation risk: page fixture used in beforeAll",
|
|
1662
|
+
description: `${beforeAllUsesPage.length} test file(s) use the Playwright page fixture inside beforeAll. This pattern often creates shared browser state across tests, leading to hard-to-debug flakiness and cascading failures (especially under retries or parallel execution).`,
|
|
1663
|
+
recommendation: "Move UI setup and navigation that relies on the test 'page' into test.beforeEach(). Reserve test.beforeAll() for external setup that does not depend on the page fixture (e.g., API calls). If you truly need one-time UI setup, create your own context/page via the 'browser' fixture in beforeAll and close it in afterAll. \nhttps://playwright.dev/docs/api/class-test#test-before-all",
|
|
1664
|
+
evidence: beforeAllUsesPage.slice(0, 25),
|
|
1665
|
+
totalOccurrences: beforeAllUsesPage.length,
|
|
1666
|
+
affectedFiles: new Set(beforeAllUsesPage.map(e => e.file)).size,
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
// STABILITY-001 finding - split into two severity levels
|
|
1670
|
+
const totalFilesWithIssues = filesNoHooksAtAll.length + filesOnlyBeforeAfterAll.length;
|
|
1671
|
+
if (totalFilesWithIssues > 0) {
|
|
1672
|
+
// High-risk: files with NO hooks at all
|
|
1673
|
+
if (filesNoHooksAtAll.length > 0) {
|
|
1674
|
+
const topFiles = filesNoHooksAtAll
|
|
1675
|
+
.sort((a, b) => b.testCount - a.testCount)
|
|
1676
|
+
.slice(0, 20);
|
|
1677
|
+
const severity = retriesEnabled ? "high" : "medium";
|
|
1678
|
+
findings.push({
|
|
1679
|
+
findingId: "PW-STABILITY-001",
|
|
1680
|
+
severity,
|
|
1681
|
+
title: retriesEnabled
|
|
1682
|
+
? "Shared state detected (no lifecycle hooks + retries enabled)"
|
|
1683
|
+
: "Shared state risk (no lifecycle hooks)",
|
|
1684
|
+
description: retriesEnabled
|
|
1685
|
+
? `${filesNoHooksAtAll.length} test file(s) contain multiple tests but have NO lifecycle hooks (beforeEach/afterEach/beforeAll/afterAll). Playwright gives each test its own BrowserContext automatically, but custom state (global variables, API sessions, database records) still needs explicit cleanup. Combined with retries, leftover state from a first attempt can affect retries.`
|
|
1686
|
+
: `${filesNoHooksAtAll.length} test file(s) contain multiple tests but have NO lifecycle hooks. Playwright isolates each test with its own BrowserContext, but custom state (global variables, external services, database records) may still leak without explicit cleanup hooks.`,
|
|
1687
|
+
recommendation: "Add lifecycle hooks (beforeEach/afterEach) for custom state cleanup (API sessions, databases, global variables). Note: Playwright's built-in page/context fixtures already handle browser isolation automatically — hooks are needed for non-browser state. \nhttps://playwright.dev/docs/test-fixtures\nhttps://playwright.dev/docs/api/class-test#test-before-each",
|
|
1688
|
+
evidence: topFiles.map(({ file, testCount }) => ({
|
|
1689
|
+
file,
|
|
1690
|
+
line: 1,
|
|
1691
|
+
snippet: `${testCount} tests, no lifecycle hooks at all`,
|
|
1692
|
+
})),
|
|
1693
|
+
totalOccurrences: filesNoHooksAtAll.length,
|
|
1694
|
+
affectedFiles: filesNoHooksAtAll.length,
|
|
1695
|
+
});
|
|
1696
|
+
}
|
|
1697
|
+
// Medium-risk: files with beforeAll/afterAll but no beforeEach/afterEach (only if many such files)
|
|
1698
|
+
if (filesOnlyBeforeAfterAll.length >= 3) {
|
|
1699
|
+
const topFiles = filesOnlyBeforeAfterAll
|
|
1700
|
+
.sort((a, b) => b.testCount - a.testCount)
|
|
1701
|
+
.slice(0, 15);
|
|
1702
|
+
findings.push({
|
|
1703
|
+
findingId: "PW-STABILITY-001a",
|
|
1704
|
+
severity: "low",
|
|
1705
|
+
title: "Test isolation could be improved (beforeAll/afterAll only)",
|
|
1706
|
+
description: `${filesOnlyBeforeAfterAll.length} test file(s) use beforeAll/afterAll for shared setup but lack beforeEach/afterEach for per-test cleanup. While shared setup is valid for performance, per-test cleanup helps ensure test independence and easier debugging.`,
|
|
1707
|
+
recommendation: "Consider adding test.beforeEach() for state that should be fresh per test (e.g., clearing localStorage, resetting mocks). Keep beforeAll for expensive one-time setup (login, data seeding). This is lower priority than files with no hooks at all. \nhttps://playwright.dev/docs/api/class-test#test-before-each",
|
|
1708
|
+
evidence: topFiles.map(({ file, testCount }) => ({
|
|
1709
|
+
file,
|
|
1710
|
+
line: 1,
|
|
1711
|
+
snippet: `${testCount} tests with beforeAll/afterAll only`,
|
|
1712
|
+
})),
|
|
1713
|
+
totalOccurrences: filesOnlyBeforeAfterAll.length,
|
|
1714
|
+
affectedFiles: filesOnlyBeforeAfterAll.length,
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
// STABILITY-002 finding
|
|
1719
|
+
if (filesWithoutOrganization.length > 0) {
|
|
1720
|
+
const topFiles = filesWithoutOrganization
|
|
1721
|
+
.sort((a, b) => b.testCount - a.testCount)
|
|
1722
|
+
.slice(0, 25);
|
|
1723
|
+
findings.push({
|
|
1724
|
+
findingId: "PW-STABILITY-002",
|
|
1725
|
+
severity: "medium",
|
|
1726
|
+
title: "Poor test organization (large files without describe blocks)",
|
|
1727
|
+
description: `${filesWithoutOrganization.length} test file(s) contain 10+ tests but have no test.describe() blocks for organization. This makes test failures harder to categorize, impacts reporting clarity, and increases maintenance burden.`,
|
|
1728
|
+
recommendation: "Organize tests into logical groups using test.describe() blocks. Group related tests (e.g., 'Login flow', 'User profile', 'Error handling'). This improves test reports, makes failures easier to understand, and helps with targeted test execution using --grep.",
|
|
1729
|
+
evidence: topFiles.map(({ file, testCount }) => ({
|
|
1730
|
+
file,
|
|
1731
|
+
line: 1,
|
|
1732
|
+
snippet: `${testCount} tests with no test.describe() organization`,
|
|
1733
|
+
})),
|
|
1734
|
+
totalOccurrences: filesWithoutOrganization.length,
|
|
1735
|
+
affectedFiles: filesWithoutOrganization.length,
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
// ============================================
|
|
1739
|
+
// PW-STABILITY-004: Tests without assertions
|
|
1740
|
+
// ============================================
|
|
1741
|
+
const testsWithoutAssertions = [];
|
|
1742
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
1743
|
+
try {
|
|
1744
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
1745
|
+
const analysis = analyzeTestFile(content);
|
|
1746
|
+
if (analysis.testCount === 0)
|
|
1747
|
+
continue;
|
|
1748
|
+
// Count assertions by counting expect() / expect.soft() / expect.poll() calls.
|
|
1749
|
+
// Previously this also counted individual matchers (.toBeVisible, .toHaveText, etc.)
|
|
1750
|
+
// which double-counted: `expect(loc).toBeVisible()` matched BOTH expect( AND .toBeVisible(,
|
|
1751
|
+
// inflating the count ~2x and making the threshold too forgiving.
|
|
1752
|
+
// Every Playwright assertion chains off one of these entry points, so they are
|
|
1753
|
+
// the single reliable anchors.
|
|
1754
|
+
const assertionPattern = /\bexpect\s*(?:\.(?:soft|poll)\s*)?\(/g;
|
|
1755
|
+
const assertionMatches = content.match(assertionPattern);
|
|
1756
|
+
const assertionCount = assertionMatches ? assertionMatches.length : 0;
|
|
1757
|
+
// Flag files where assertion count is very low relative to test count
|
|
1758
|
+
// Heuristic: fewer than 1 assertion (expect call) per test indicates likely missing checks
|
|
1759
|
+
if (analysis.testCount >= 2 && assertionCount < analysis.testCount) {
|
|
1760
|
+
testsWithoutAssertions.push({
|
|
1761
|
+
file: normalize(testFile),
|
|
1762
|
+
testCount: analysis.testCount,
|
|
1763
|
+
assertionCount,
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
catch {
|
|
1768
|
+
continue;
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
if (testsWithoutAssertions.length > 0) {
|
|
1772
|
+
const sorted = testsWithoutAssertions.sort((a, b) => b.testCount - a.testCount);
|
|
1773
|
+
findings.push({
|
|
1774
|
+
findingId: "PW-STABILITY-004",
|
|
1775
|
+
severity: "high",
|
|
1776
|
+
title: "Tests without assertions (expect()) detected",
|
|
1777
|
+
description: `${testsWithoutAssertions.length} test file(s) contain tests but have very few or no assertions. Tests that navigate but never assert are 'green noise', they always pass and provide zero quality signal.`,
|
|
1778
|
+
recommendation: "Every test should contain at least one expect() assertion that verifies observable behavior. Add assertions for page content, URLs, API responses, or element states. Focus on the most critical user flows first.",
|
|
1779
|
+
evidence: sorted.slice(0, 20).map(({ file, testCount, assertionCount }) => ({
|
|
1780
|
+
file,
|
|
1781
|
+
line: 1,
|
|
1782
|
+
snippet: `${testCount} tests but only ${assertionCount} assertion(s)`,
|
|
1783
|
+
})),
|
|
1784
|
+
totalOccurrences: testsWithoutAssertions.length,
|
|
1785
|
+
affectedFiles: new Set(testsWithoutAssertions.map(t => t.file)).size,
|
|
1786
|
+
});
|
|
1787
|
+
}
|
|
1788
|
+
// ============================================
|
|
1789
|
+
// PW-STABILITY-005: Browser resource leak risk
|
|
1790
|
+
// ============================================
|
|
1791
|
+
const resourceLeakEvidence = [];
|
|
1792
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
1793
|
+
try {
|
|
1794
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
1795
|
+
const leaks = findBrowserResourceLeaks(normalize(testFile), content);
|
|
1796
|
+
resourceLeakEvidence.push(...leaks);
|
|
1797
|
+
}
|
|
1798
|
+
catch {
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
if (resourceLeakEvidence.length > 0) {
|
|
1803
|
+
findings.push({
|
|
1804
|
+
findingId: "PW-STABILITY-005",
|
|
1805
|
+
severity: "medium",
|
|
1806
|
+
title: "Browser/request resource leak risk",
|
|
1807
|
+
description: `${resourceLeakEvidence.length} test file(s) manually create browser/context/page/request resources without proper cleanup in afterEach/afterAll hooks. This can leak browser processes and API connections, cause memory pressure, and leave zombie processes on CI runners.`,
|
|
1808
|
+
recommendation: "Either use Playwright's built-in fixtures (({ page, request }) => ...) which auto-cleanup, or add explicit browser.close()/context.close()/page.close()/requestContext.dispose() calls inside afterEach/afterAll hooks.",
|
|
1809
|
+
evidence: resourceLeakEvidence.slice(0, 10).map((r) => ({
|
|
1810
|
+
file: r.file,
|
|
1811
|
+
line: r.line,
|
|
1812
|
+
snippet: `Manual resource creation: ${r.creationPatterns.join(", ")} — no cleanup in afterEach/afterAll`,
|
|
1813
|
+
})),
|
|
1814
|
+
totalOccurrences: resourceLeakEvidence.length,
|
|
1815
|
+
affectedFiles: new Set(resourceLeakEvidence.map(r => r.file)).size,
|
|
1816
|
+
});
|
|
1817
|
+
}
|
|
1818
|
+
// ============================================
|
|
1819
|
+
// SE-STABILITY-001: WebDriver resource leak risk
|
|
1820
|
+
// ============================================
|
|
1821
|
+
if (inventory.detectedFrameworks.includes("selenium")) {
|
|
1822
|
+
const seleniumLeakEvidence = [];
|
|
1823
|
+
const seleniumFileGlobs = ["**/*.py", "**/*.java", "**/*.ts", "**/*.js"];
|
|
1824
|
+
const seleniumCandidateFiles = (await (0, fast_glob_1.default)(seleniumFileGlobs, {
|
|
1825
|
+
cwd: repoPath,
|
|
1826
|
+
absolute: true,
|
|
1827
|
+
dot: true,
|
|
1828
|
+
onlyFiles: true,
|
|
1829
|
+
followSymbolicLinks: false,
|
|
1830
|
+
}));
|
|
1831
|
+
const filteredSeleniumFiles = seleniumCandidateFiles.filter((f) => !isIgnored(f)).slice(0, 100);
|
|
1832
|
+
for (const f of filteredSeleniumFiles) {
|
|
1833
|
+
try {
|
|
1834
|
+
const content = await promises_1.default.readFile(f, "utf8");
|
|
1835
|
+
// Only check files with selenium imports
|
|
1836
|
+
if (/from\s+selenium\b/.test(content) ||
|
|
1837
|
+
/import\s+org\.openqa\.selenium/.test(content) ||
|
|
1838
|
+
/require\s*\(\s*['"]selenium-webdriver['"]/.test(content) ||
|
|
1839
|
+
/from\s+['"]selenium-webdriver['"]/.test(content)) {
|
|
1840
|
+
const leaks = findSeleniumResourceLeaks(normalize(f), content);
|
|
1841
|
+
seleniumLeakEvidence.push(...leaks);
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
catch {
|
|
1845
|
+
continue;
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
if (seleniumLeakEvidence.length > 0) {
|
|
1849
|
+
findings.push({
|
|
1850
|
+
findingId: "SE-STABILITY-001",
|
|
1851
|
+
severity: "medium",
|
|
1852
|
+
title: "WebDriver resource leak risk",
|
|
1853
|
+
description: `${seleniumLeakEvidence.length} Selenium test file(s) create WebDriver instances without driver.quit() in a teardown hook. Unlike Playwright, Selenium has no auto-cleanup — missing quit() leaks browser processes.`,
|
|
1854
|
+
recommendation: "Add driver.quit() in a teardown hook (teardown_method for Python, @After/@AfterEach for Java/JS). This ensures cleanup even when tests fail.",
|
|
1855
|
+
evidence: seleniumLeakEvidence.slice(0, 10).map((r) => ({
|
|
1856
|
+
file: r.file,
|
|
1857
|
+
line: r.line,
|
|
1858
|
+
snippet: `WebDriver creation: ${r.creationPatterns.join(", ")} — no quit() in teardown`,
|
|
1859
|
+
})),
|
|
1860
|
+
totalOccurrences: seleniumLeakEvidence.length,
|
|
1861
|
+
affectedFiles: new Set(seleniumLeakEvidence.map(r => r.file)).size,
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
// ============================================
|
|
1866
|
+
// PW-STABILITY-006: Too many assertions per test
|
|
1867
|
+
// ============================================
|
|
1868
|
+
const allOverAsserted = [];
|
|
1869
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
1870
|
+
try {
|
|
1871
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
1872
|
+
const analysis = analyzeTestFile(content);
|
|
1873
|
+
// Only scan files with ≥2 tests
|
|
1874
|
+
if (analysis.testCount < 2)
|
|
1875
|
+
continue;
|
|
1876
|
+
const overAsserted = findOverAssertedTests(normalize(testFile), content);
|
|
1877
|
+
allOverAsserted.push(...overAsserted);
|
|
1878
|
+
}
|
|
1879
|
+
catch {
|
|
1880
|
+
continue;
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
// Only emit if ≥3 tests are over-asserted (don't flag a single outlier)
|
|
1884
|
+
if (allOverAsserted.length >= 3) {
|
|
1885
|
+
const sorted = allOverAsserted.sort((a, b) => b.assertionCount - a.assertionCount);
|
|
1886
|
+
findings.push({
|
|
1887
|
+
findingId: "PW-STABILITY-006",
|
|
1888
|
+
severity: "low",
|
|
1889
|
+
title: "Too many assertions per test",
|
|
1890
|
+
description: `${allOverAsserted.length} test(s) have more than 5 expect() assertions each. Tests that verify too many behaviors at once are harder to debug when they fail and indicate the test may be doing too much.`,
|
|
1891
|
+
recommendation: "Split over-asserted tests into focused tests that each verify one behavior. Use test.describe() to group related assertions. Consider page object methods that encapsulate multiple checks.",
|
|
1892
|
+
evidence: sorted.slice(0, 10).map((t) => ({
|
|
1893
|
+
file: t.file,
|
|
1894
|
+
line: t.line,
|
|
1895
|
+
snippet: `"${t.testName}" — ${t.assertionCount} assertions (threshold: 5)`,
|
|
1896
|
+
})),
|
|
1897
|
+
totalOccurrences: allOverAsserted.length,
|
|
1898
|
+
affectedFiles: new Set(allOverAsserted.map(t => t.file)).size,
|
|
1899
|
+
});
|
|
1900
|
+
}
|
|
1901
|
+
// ── PW-FLAKE-001 pre-scan: discover all sleep wrappers globally ──
|
|
1902
|
+
const NON_TEST_FILE_CAP = 500;
|
|
1903
|
+
const SLEEP_PREFILTER_RE = /setTimeout|waitForTimeout|waitForNetworkIdle|waitForLoadState/;
|
|
1904
|
+
const allSourceFiles = (await (0, fast_glob_1.default)(["**/*.{ts,js}"], {
|
|
1905
|
+
cwd: repoPath, absolute: true, dot: true, onlyFiles: true,
|
|
1906
|
+
followSymbolicLinks: false,
|
|
1907
|
+
ignore: ["**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**",
|
|
1908
|
+
"**/playwright-report/**", "**/test-results/**", "**/coverage/**"],
|
|
1909
|
+
}));
|
|
1910
|
+
const nonTestSourceFiles = allSourceFiles
|
|
1911
|
+
.filter((f) => !isIgnored(f) && !f.includes(".spec.") && !f.includes(".test."))
|
|
1912
|
+
.slice(0, NON_TEST_FILE_CAP);
|
|
1913
|
+
const globalWrapperNames = new Map(); // funcName → relPath
|
|
1914
|
+
const globalSleepMethodNames = new Map(); // methodName → relPath
|
|
1915
|
+
const nonTestContentMap = new Map(); // absolute path → file content
|
|
1916
|
+
for (const srcFile of nonTestSourceFiles) {
|
|
1917
|
+
try {
|
|
1918
|
+
const content = await promises_1.default.readFile(srcFile, "utf8");
|
|
1919
|
+
nonTestContentMap.set(srcFile, content);
|
|
1920
|
+
if (!SLEEP_PREFILTER_RE.test(content))
|
|
1921
|
+
continue;
|
|
1922
|
+
const relPath = normalize(srcFile);
|
|
1923
|
+
for (const name of (0, astRuleHelpers_1.findSleepWrapperExports)(srcFile, content)) {
|
|
1924
|
+
globalWrapperNames.set(name, relPath);
|
|
1925
|
+
}
|
|
1926
|
+
for (const cls of (0, astRuleHelpers_1.findClassSleepMethodExports)(srcFile, content)) {
|
|
1927
|
+
for (const method of cls.methodNames) {
|
|
1928
|
+
globalSleepMethodNames.set(method, relPath);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
catch {
|
|
1933
|
+
continue;
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
// ============================================
|
|
1937
|
+
// PW-STABILITY-007 + PW-FLAKE-007 + PW-STABILITY-008 + PW-FLAKE-010 + PW-FLAKE-001 disguised waits (AST-based)
|
|
1938
|
+
// Combined loop: read each file once, run all AST analyses
|
|
1939
|
+
// ============================================
|
|
1940
|
+
const allShadowing = [];
|
|
1941
|
+
const allCustomAssertions = [];
|
|
1942
|
+
const allManualContexts = [];
|
|
1943
|
+
const allMissingAwaits = [];
|
|
1944
|
+
const allUnguardedRetrievals = [];
|
|
1945
|
+
const allDisguisedWaits = [];
|
|
1946
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
1947
|
+
try {
|
|
1948
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
1949
|
+
const relFile = normalize(testFile);
|
|
1950
|
+
// PW-STABILITY-007: Variable shadowing
|
|
1951
|
+
const shadows = (0, astRuleHelpers_1.findVariableShadowing)(testFile, content);
|
|
1952
|
+
for (const s of shadows) {
|
|
1953
|
+
allShadowing.push({
|
|
1954
|
+
file: relFile,
|
|
1955
|
+
evidence: `'${s.name}' declared at line ${s.outerLine} (${s.outerKind}), shadowed by ${s.innerKind} at line ${s.innerLine}`,
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
// PW-FLAKE-007: Custom assertion wrappers
|
|
1959
|
+
const assertions = (0, astRuleHelpers_1.findCustomAssertionWrappers)(testFile, content);
|
|
1960
|
+
for (const a of assertions) {
|
|
1961
|
+
allCustomAssertions.push({
|
|
1962
|
+
file: relFile,
|
|
1963
|
+
evidence: `${a.importName}() called ${a.callCount}× vs ${a.expectCount} expect() call(s) (${a.testCount} test(s)) — from ${a.importPath}`,
|
|
1964
|
+
});
|
|
1965
|
+
}
|
|
1966
|
+
// PW-STABILITY-008: Manual browser/request context in test bodies
|
|
1967
|
+
const contexts = (0, astRuleHelpers_1.findManualContextInTestBodies)(testFile, content);
|
|
1968
|
+
for (const c of contexts) {
|
|
1969
|
+
allManualContexts.push({
|
|
1970
|
+
file: relFile,
|
|
1971
|
+
line: c.line,
|
|
1972
|
+
evidence: `Test '${c.testName}' creates ${c.count} context(s) manually — use fixtures for automatic lifecycle`,
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
// PW-FLAKE-008: Missing await on async Playwright APIs
|
|
1976
|
+
const missingAwaits = (0, astRuleHelpers_1.findMissingPlaywrightAwaits)(testFile, content);
|
|
1977
|
+
for (const m of missingAwaits) {
|
|
1978
|
+
allMissingAwaits.push({
|
|
1979
|
+
file: relFile,
|
|
1980
|
+
line: m.line,
|
|
1981
|
+
evidence: `Test '${m.testName}': unawaited ${m.snippet}`,
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
// PW-FLAKE-010: Unguarded retrieval methods
|
|
1985
|
+
const unguarded = (0, astRuleHelpers_1.findUnguardedRetrievalMethods)(testFile, content);
|
|
1986
|
+
for (const u of unguarded) {
|
|
1987
|
+
allUnguardedRetrievals.push({
|
|
1988
|
+
file: relFile,
|
|
1989
|
+
line: u.line,
|
|
1990
|
+
evidence: `Test '${u.testName}': ${u.snippet} without preceding content guard`,
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
// PW-FLAKE-001: Disguised hard wait detection (global name scan)
|
|
1994
|
+
if (globalWrapperNames.size > 0 || globalSleepMethodNames.size > 0) {
|
|
1995
|
+
const disguisedCalls = (0, astRuleHelpers_1.findDisguisedHardWaitCalls)(testFile, content, globalWrapperNames, globalSleepMethodNames);
|
|
1996
|
+
for (const d of disguisedCalls) {
|
|
1997
|
+
allDisguisedWaits.push({
|
|
1998
|
+
file: relFile,
|
|
1999
|
+
line: d.line,
|
|
2000
|
+
evidence: `Disguised hard wait: ${d.calledFunction}() is a sleep wrapper defined in '${d.importPath}'`,
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
catch {
|
|
2006
|
+
continue;
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
// PW-FLAKE-001 (continued): Scan non-test source files for disguised wait calls
|
|
2010
|
+
if (globalWrapperNames.size > 0 || globalSleepMethodNames.size > 0) {
|
|
2011
|
+
for (const [srcFile, content] of nonTestContentMap) {
|
|
2012
|
+
try {
|
|
2013
|
+
const relFile = normalize(srcFile);
|
|
2014
|
+
const disguisedCalls = (0, astRuleHelpers_1.findDisguisedHardWaitCalls)(srcFile, content, globalWrapperNames, globalSleepMethodNames);
|
|
2015
|
+
for (const d of disguisedCalls) {
|
|
2016
|
+
allDisguisedWaits.push({
|
|
2017
|
+
file: relFile,
|
|
2018
|
+
line: d.line,
|
|
2019
|
+
evidence: `Disguised hard wait: ${d.calledFunction}() is a sleep wrapper (defined in '${d.importPath}')`,
|
|
2020
|
+
});
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
catch {
|
|
2024
|
+
continue;
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
if (allShadowing.length > 0) {
|
|
2029
|
+
findings.push({
|
|
2030
|
+
findingId: "PW-STABILITY-007",
|
|
2031
|
+
severity: "high",
|
|
2032
|
+
title: "Variable shadowing between describe/test scopes",
|
|
2033
|
+
description: `${allShadowing.length} variable(s) declared with let/var in a describe scope are shadowed by a declaration inside a nested test or hook. The outer variable stays undefined, causing runtime crashes (typically in afterAll cleanup).`,
|
|
2034
|
+
recommendation: "Remove the inner declaration keyword (const/let/var) so the assignment targets the outer variable. Or use distinct names if intentional. Example: change `const homePageInternal = ...` inside test() to `homePageInternal = ...` (assignment, not declaration).",
|
|
2035
|
+
evidence: allShadowing.slice(0, 10).map((s) => ({
|
|
2036
|
+
file: s.file,
|
|
2037
|
+
line: 1,
|
|
2038
|
+
snippet: s.evidence,
|
|
2039
|
+
})),
|
|
2040
|
+
totalOccurrences: allShadowing.length,
|
|
2041
|
+
affectedFiles: new Set(allShadowing.map(s => s.file)).size,
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
2044
|
+
if (allCustomAssertions.length > 0) {
|
|
2045
|
+
findings.push({
|
|
2046
|
+
findingId: "PW-FLAKE-007",
|
|
2047
|
+
severity: "high",
|
|
2048
|
+
title: "Custom assertion wrappers bypass auto-retrying expect()",
|
|
2049
|
+
description: `${allCustomAssertions.length} file(s) use custom assertion functions imported from local utility modules instead of Playwright's built-in expect(). Custom wrappers do not auto-retry, leading to flaky assertions on async DOM changes.`,
|
|
2050
|
+
recommendation: "Replace custom assertion wrappers with Playwright's auto-retrying expect() matchers: expect(locator).toHaveText(), expect(locator).toBeVisible(), etc. If custom comparison logic is needed, wrap it inside expect.poll() or expect.toPass() for retry behavior.\nhttps://playwright.dev/docs/test-assertions",
|
|
2051
|
+
evidence: allCustomAssertions.slice(0, 10).map((a) => ({
|
|
2052
|
+
file: a.file,
|
|
2053
|
+
line: 1,
|
|
2054
|
+
snippet: a.evidence,
|
|
2055
|
+
})),
|
|
2056
|
+
totalOccurrences: allCustomAssertions.length,
|
|
2057
|
+
affectedFiles: new Set(allCustomAssertions.map(a => a.file)).size,
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
if (allManualContexts.length > 0) {
|
|
2061
|
+
findings.push({
|
|
2062
|
+
findingId: "PW-STABILITY-008",
|
|
2063
|
+
severity: "medium",
|
|
2064
|
+
title: "Manual browser.newContext()/request.newContext() in test bodies instead of fixtures",
|
|
2065
|
+
description: `${allManualContexts.length} test(s) manually create browser or request context(s) via browser.newContext()/request.newContext(). Manual lifecycle management is fragile and can leave contexts unclosed or undisposed on test failures (context is inaccessible in afterEach/afterAll if declared inside the test), causing resource leaks, connection exhaustion, and port exhaustion.`,
|
|
2066
|
+
recommendation: "Use Playwright fixtures (test.extend) to manage browser and request contexts. Fixtures automatically handle setup and teardown, even when tests fail. For multi-context scenarios, create a custom fixture that provides pre-configured contexts. For API request contexts, use the built-in request fixture or dispose manually in afterEach/afterAll.\nhttps://playwright.dev/docs/test-fixtures\nhttps://playwright.dev/docs/api-testing",
|
|
2067
|
+
evidence: allManualContexts.slice(0, 10).map((c) => ({
|
|
2068
|
+
file: c.file,
|
|
2069
|
+
line: c.line,
|
|
2070
|
+
snippet: c.evidence,
|
|
2071
|
+
})),
|
|
2072
|
+
totalOccurrences: allManualContexts.length,
|
|
2073
|
+
affectedFiles: new Set(allManualContexts.map(c => c.file)).size,
|
|
2074
|
+
});
|
|
2075
|
+
}
|
|
2076
|
+
if (allMissingAwaits.length > 0) {
|
|
2077
|
+
findings.push({
|
|
2078
|
+
findingId: "PW-FLAKE-008",
|
|
2079
|
+
severity: "high",
|
|
2080
|
+
title: "Missing await on async Playwright assertions or operations",
|
|
2081
|
+
description: `${allMissingAwaits.length} async Playwright call(s) are not awaited. Unawaited expect() matchers, expect.poll(), and test.step() silently resolve as fire-and-forget promises — the assertion never blocks the test flow, causing false passes or non-deterministic failures.`,
|
|
2082
|
+
recommendation: "Add `await` before every async Playwright assertion and operation. All Playwright expect() matchers (toBeVisible, toHaveText, etc.), expect.poll(), expect.soft() matchers, and test.step() return promises that must be awaited.\nhttps://playwright.dev/docs/test-assertions",
|
|
2083
|
+
evidence: allMissingAwaits.slice(0, 10).map((m) => ({
|
|
2084
|
+
file: m.file,
|
|
2085
|
+
line: m.line,
|
|
2086
|
+
snippet: m.evidence,
|
|
2087
|
+
})),
|
|
2088
|
+
totalOccurrences: allMissingAwaits.length,
|
|
2089
|
+
affectedFiles: new Set(allMissingAwaits.map(m => m.file)).size,
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
if (allUnguardedRetrievals.length > 0) {
|
|
2093
|
+
if (!options.auditConfig || !(0, auditConfig_1.isRuleDisabled)(options.auditConfig, "PW-FLAKE-010")) {
|
|
2094
|
+
findings.push({
|
|
2095
|
+
findingId: "PW-FLAKE-010",
|
|
2096
|
+
severity: options.auditConfig
|
|
2097
|
+
? (0, auditConfig_1.getEffectiveSeverity)(options.auditConfig, "PW-FLAKE-010", "medium")
|
|
2098
|
+
: "medium",
|
|
2099
|
+
title: "Retrieval methods used without content guard",
|
|
2100
|
+
description: "Retrieval method calls (innerText, textContent, innerHTML, inputValue, etc.) " +
|
|
2101
|
+
"are not preceded by a content guard. These methods only wait for the element to exist in the DOM, " +
|
|
2102
|
+
"not for its content to be populated. This can return empty or stale values when text is loaded dynamically.",
|
|
2103
|
+
recommendation: "If you need to assert text on the page, prefer expect(locator).toHaveText() or other auto-retrying matchers that include built-in waiting. \n" +
|
|
2104
|
+
"For complex assertions, guard retrieval calls with a web-first assertion or custom poll before extracting text.\n" +
|
|
2105
|
+
" await expect(locator).not.toHaveText('');\n" +
|
|
2106
|
+
" const text = await locator.innerText();\n" +
|
|
2107
|
+
"Or use expect().toPass() for custom polling:\n" +
|
|
2108
|
+
" let text = '';\n" +
|
|
2109
|
+
" await expect(async () => { text = (await locator.innerText()).trim(); expect(text).not.toBe(''); }).toPass();\n" +
|
|
2110
|
+
"https://playwright.dev/docs/actionability",
|
|
2111
|
+
evidence: allUnguardedRetrievals.slice(0, 25).map((u) => ({
|
|
2112
|
+
file: u.file,
|
|
2113
|
+
line: u.line,
|
|
2114
|
+
snippet: u.evidence,
|
|
2115
|
+
})),
|
|
2116
|
+
totalOccurrences: allUnguardedRetrievals.length,
|
|
2117
|
+
affectedFiles: new Set(allUnguardedRetrievals.map(u => u.file)).size,
|
|
2118
|
+
});
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
// Merge disguised hard wait evidence into PW-FLAKE-001
|
|
2122
|
+
if (allDisguisedWaits.length > 0) {
|
|
2123
|
+
const existingFlake001 = findings.find((f) => f.findingId === "PW-FLAKE-001");
|
|
2124
|
+
const disguisedEvidence = allDisguisedWaits.map((d) => ({
|
|
2125
|
+
file: d.file,
|
|
2126
|
+
line: d.line,
|
|
2127
|
+
snippet: d.evidence,
|
|
2128
|
+
}));
|
|
2129
|
+
if (existingFlake001) {
|
|
2130
|
+
const remaining = 25 - existingFlake001.evidence.length;
|
|
2131
|
+
if (remaining > 0) {
|
|
2132
|
+
existingFlake001.evidence.push(...disguisedEvidence.slice(0, remaining));
|
|
2133
|
+
}
|
|
2134
|
+
// Update counts to include disguised waits
|
|
2135
|
+
existingFlake001.totalOccurrences = (existingFlake001.totalOccurrences ?? existingFlake001.evidence.length) + allDisguisedWaits.length;
|
|
2136
|
+
existingFlake001.affectedFiles = new Set([
|
|
2137
|
+
...existingFlake001.evidence.map(e => e.file),
|
|
2138
|
+
...allDisguisedWaits.map(d => d.file),
|
|
2139
|
+
]).size;
|
|
2140
|
+
}
|
|
2141
|
+
else if (!options.auditConfig || !(0, auditConfig_1.isRuleDisabled)(options.auditConfig, "PW-FLAKE-001")) {
|
|
2142
|
+
findings.push({
|
|
2143
|
+
findingId: "PW-FLAKE-001",
|
|
2144
|
+
severity: options.auditConfig
|
|
2145
|
+
? (0, auditConfig_1.getEffectiveSeverity)(options.auditConfig, "PW-FLAKE-001", "high")
|
|
2146
|
+
: "high",
|
|
2147
|
+
title: "Hard waits or debug pauses detected (waitForTimeout / setTimeout / page.pause)",
|
|
2148
|
+
description: "Hard waits increase flakiness and slow tests. Disguised sleep wrappers imported from utility modules are equivalent to direct waitForTimeout/setTimeout calls.",
|
|
2149
|
+
recommendation: "Replace hard waits with Playwright auto-waiting + explicit expect(). Remove all sleep wrapper utilities and use proper wait strategies.\nhttps://playwright.dev/docs/api/class-page#page-wait-for-timeout",
|
|
2150
|
+
evidence: disguisedEvidence.slice(0, 25),
|
|
2151
|
+
totalOccurrences: allDisguisedWaits.length,
|
|
2152
|
+
affectedFiles: new Set(allDisguisedWaits.map(d => d.file)).size,
|
|
2153
|
+
});
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
// ============================================
|
|
2157
|
+
// PW-LOC-002: CSS selector nesting depth > 3
|
|
2158
|
+
// ============================================
|
|
2159
|
+
const deepCssEvidence = [];
|
|
2160
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2161
|
+
if (deepCssEvidence.length >= 25)
|
|
2162
|
+
break;
|
|
2163
|
+
try {
|
|
2164
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2165
|
+
const lines = content.split(/\r?\n/);
|
|
2166
|
+
// Find locator('css-selector') calls
|
|
2167
|
+
const locatorArgRegex = /\blocator\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/g;
|
|
2168
|
+
let m;
|
|
2169
|
+
while ((m = locatorArgRegex.exec(content)) !== null) {
|
|
2170
|
+
const selector = m[1];
|
|
2171
|
+
// Count combinators: >, space (descendant), +, ~ indicate nesting
|
|
2172
|
+
const combinators = selector.split(/\s*[>+~]\s*|\s+/).filter(Boolean);
|
|
2173
|
+
if (combinators.length > 3) {
|
|
2174
|
+
const lineNum = lineNumberAtIndex(content, m.index);
|
|
2175
|
+
const lineText = (lines[lineNum - 1] ?? "").trim();
|
|
2176
|
+
if (!lineText.startsWith("//") && !lineText.startsWith("*")) {
|
|
2177
|
+
deepCssEvidence.push({
|
|
2178
|
+
file: normalize(testFile),
|
|
2179
|
+
line: lineNum,
|
|
2180
|
+
snippet: lineText.slice(0, 150),
|
|
2181
|
+
});
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
if (deepCssEvidence.length >= 25)
|
|
2185
|
+
break;
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
catch {
|
|
2189
|
+
continue;
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
if (deepCssEvidence.length > 0) {
|
|
2193
|
+
findings.push({
|
|
2194
|
+
findingId: "PW-LOC-002",
|
|
2195
|
+
severity: "low",
|
|
2196
|
+
title: "Deeply nested CSS selectors (>3 levels)",
|
|
2197
|
+
description: `${deepCssEvidence.length} instance(s) of deeply nested CSS selectors detected. Selectors like 'div > span > .foo > .bar > input' are fragile and break when DOM structure changes, even if the target element is unchanged.`,
|
|
2198
|
+
recommendation: "Replace deep CSS selectors with semantic locators: getByRole(), getByTestId(), getByLabel(). If you must use CSS, keep selectors to 1-2 levels. Add data-testid attributes where needed.\nhttps://playwright.dev/docs/locators#quick-guide",
|
|
2199
|
+
evidence: deepCssEvidence,
|
|
2200
|
+
});
|
|
2201
|
+
}
|
|
2202
|
+
// ============================================
|
|
2203
|
+
// ARCH-008: Large inline test data (AST-based)
|
|
2204
|
+
// ============================================
|
|
2205
|
+
const largeInlineDataFiles = [];
|
|
2206
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2207
|
+
try {
|
|
2208
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2209
|
+
const literals = (0, astRuleHelpers_1.findLargeInlineDataLiterals)(testFile, content, 50);
|
|
2210
|
+
if (literals.length > 0) {
|
|
2211
|
+
// Pick the largest literal in the file
|
|
2212
|
+
const largest = literals.reduce((a, b) => a.lineCount > b.lineCount ? a : b);
|
|
2213
|
+
largeInlineDataFiles.push({
|
|
2214
|
+
file: normalize(testFile),
|
|
2215
|
+
line: largest.line,
|
|
2216
|
+
endLine: largest.endLine,
|
|
2217
|
+
largestBlock: largest.lineCount,
|
|
2218
|
+
symbolName: largest.symbolName,
|
|
2219
|
+
nodeType: largest.nodeType,
|
|
2220
|
+
});
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
catch {
|
|
2224
|
+
continue;
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
if (largeInlineDataFiles.length >= 2) {
|
|
2228
|
+
findings.push({
|
|
2229
|
+
findingId: "ARCH-008",
|
|
2230
|
+
severity: "medium",
|
|
2231
|
+
title: "Large inline test data detected",
|
|
2232
|
+
description: `${largeInlineDataFiles.length} test file(s) contain large inline data blocks (50+ lines). Embedding test data in test files reduces readability, makes data reuse difficult, and bloats test files.`,
|
|
2233
|
+
recommendation: "Extract large data into separate fixture files (e.g., /fixtures/testData.json or /data/ folder). Use test.use() or custom fixtures to load data. This improves readability and enables data sharing across tests.",
|
|
2234
|
+
evidence: largeInlineDataFiles.slice(0, 10).map(({ file, line, endLine, largestBlock, symbolName, nodeType }) => ({
|
|
2235
|
+
file,
|
|
2236
|
+
line,
|
|
2237
|
+
snippet: `${symbolName}: ${nodeType} literal (${largestBlock} lines, L${line}–L${endLine})`,
|
|
2238
|
+
})),
|
|
2239
|
+
totalOccurrences: largeInlineDataFiles.length,
|
|
2240
|
+
affectedFiles: largeInlineDataFiles.length,
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2243
|
+
// ============================================
|
|
2244
|
+
// ARCH-009: Missing test.use() for project-specific overrides
|
|
2245
|
+
// ============================================
|
|
2246
|
+
const inlineConfigOverrides = [];
|
|
2247
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2248
|
+
if (inlineConfigOverrides.length >= 20)
|
|
2249
|
+
break;
|
|
2250
|
+
try {
|
|
2251
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2252
|
+
const lines = content.split(/\r?\n/);
|
|
2253
|
+
// Detect inline overrides like { timeout: N } or { retries: N } in test() calls
|
|
2254
|
+
// but NOT in test.use() blocks (those are fine)
|
|
2255
|
+
const inlineOverrideRegex = /\btest\s*\(\s*['"`][^'"`]+['"`]\s*,\s*\{[^}]*(timeout|retries)\s*:/g;
|
|
2256
|
+
let match;
|
|
2257
|
+
while ((match = inlineOverrideRegex.exec(content)) !== null) {
|
|
2258
|
+
const lineNum = lineNumberAtIndex(content, match.index);
|
|
2259
|
+
const lineText = (lines[lineNum - 1] ?? "").trim();
|
|
2260
|
+
if (!lineText.startsWith("//")) {
|
|
2261
|
+
inlineConfigOverrides.push({
|
|
2262
|
+
file: normalize(testFile),
|
|
2263
|
+
line: lineNum,
|
|
2264
|
+
snippet: lineText.slice(0, 150),
|
|
2265
|
+
});
|
|
2266
|
+
}
|
|
2267
|
+
if (inlineConfigOverrides.length >= 20)
|
|
2268
|
+
break;
|
|
2269
|
+
}
|
|
2270
|
+
}
|
|
2271
|
+
catch {
|
|
2272
|
+
continue;
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
if (inlineConfigOverrides.length >= 3) {
|
|
2276
|
+
findings.push({
|
|
2277
|
+
findingId: "ARCH-009",
|
|
2278
|
+
severity: "low",
|
|
2279
|
+
title: "Inline test configuration overrides (consider test.use())",
|
|
2280
|
+
description: `${inlineConfigOverrides.length} test(s) override configuration (timeout/retries) inline. This scatters config across test files instead of centralizing it in playwright.config.ts or test.use() blocks.`,
|
|
2281
|
+
recommendation: "Use test.describe(() => { test.use({ ... }) }) to set config for groups of tests, or configure per-project settings in playwright.config.ts. Reserve inline overrides for genuinely exceptional cases.\nhttps://playwright.dev/docs/test-use-options",
|
|
2282
|
+
evidence: inlineConfigOverrides.slice(0, 10),
|
|
2283
|
+
totalOccurrences: inlineConfigOverrides.length,
|
|
2284
|
+
affectedFiles: new Set(inlineConfigOverrides.map(e => e.file)).size,
|
|
2285
|
+
});
|
|
2286
|
+
}
|
|
2287
|
+
// ============================================
|
|
2288
|
+
// ARCHITECTURAL ANALYSIS (ARCH-*)
|
|
2289
|
+
// ============================================
|
|
2290
|
+
// Collect POM files for analysis
|
|
2291
|
+
const pomFiles = (await (0, fast_glob_1.default)(["**/POMs/**/*.ts", "**/pages/**/*.ts", "**/page-objects/**/*.ts", "**/*Page.ts", "**/*Page.js"], {
|
|
2292
|
+
cwd: repoPath,
|
|
2293
|
+
absolute: true,
|
|
2294
|
+
dot: true,
|
|
2295
|
+
onlyFiles: true,
|
|
2296
|
+
followSymbolicLinks: false,
|
|
2297
|
+
}));
|
|
2298
|
+
const filteredPomFiles = pomFiles.filter((f) => !isIgnored(f) && !f.includes('.spec.') && !f.includes('.test.'));
|
|
2299
|
+
// Analyze POM architecture
|
|
2300
|
+
const pomAnalyses = [];
|
|
2301
|
+
const pomContentsMap = new Map();
|
|
2302
|
+
for (const pomFile of filteredPomFiles) {
|
|
2303
|
+
try {
|
|
2304
|
+
const content = await promises_1.default.readFile(pomFile, "utf8");
|
|
2305
|
+
pomContentsMap.set(pomFile, content);
|
|
2306
|
+
const analysis = (0, astRuleHelpers_1.analyzePomClassStructure)(pomFile, content);
|
|
2307
|
+
if (analysis) {
|
|
2308
|
+
pomAnalyses.push({ ...analysis, file: pomFile });
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
catch {
|
|
2312
|
+
// Skip files that can't be read
|
|
2313
|
+
}
|
|
2314
|
+
}
|
|
2315
|
+
// PW-FLAKE-010 (continued): Scan POM/page class files for unguarded retrieval methods
|
|
2316
|
+
if (!options.auditConfig || !(0, auditConfig_1.isRuleDisabled)(options.auditConfig, "PW-FLAKE-010")) {
|
|
2317
|
+
const pomRetrievals = [];
|
|
2318
|
+
for (const [pomFile, content] of pomContentsMap) {
|
|
2319
|
+
try {
|
|
2320
|
+
const relFile = normalize(pomFile);
|
|
2321
|
+
const hits = (0, astRuleHelpers_1.findUnguardedRetrievalMethodsInFunctions)(pomFile, content);
|
|
2322
|
+
for (const h of hits) {
|
|
2323
|
+
pomRetrievals.push({
|
|
2324
|
+
file: relFile,
|
|
2325
|
+
line: h.line,
|
|
2326
|
+
evidence: `${h.testName}(): ${h.snippet} without preceding content guard`,
|
|
2327
|
+
});
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
catch {
|
|
2331
|
+
continue;
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
if (pomRetrievals.length > 0) {
|
|
2335
|
+
const existing010 = findings.find((f) => f.findingId === "PW-FLAKE-010");
|
|
2336
|
+
const pomEvidence = pomRetrievals.map((p) => ({
|
|
2337
|
+
file: p.file,
|
|
2338
|
+
line: p.line,
|
|
2339
|
+
snippet: p.evidence,
|
|
2340
|
+
}));
|
|
2341
|
+
if (existing010) {
|
|
2342
|
+
const remaining = 25 - existing010.evidence.length;
|
|
2343
|
+
if (remaining > 0) {
|
|
2344
|
+
existing010.evidence.push(...pomEvidence.slice(0, remaining));
|
|
2345
|
+
}
|
|
2346
|
+
existing010.totalOccurrences = (existing010.totalOccurrences ?? existing010.evidence.length) + pomRetrievals.length;
|
|
2347
|
+
existing010.affectedFiles = new Set([
|
|
2348
|
+
...existing010.evidence.map(e => e.file),
|
|
2349
|
+
...pomRetrievals.map(p => p.file),
|
|
2350
|
+
]).size;
|
|
2351
|
+
// Refresh description with combined total
|
|
2352
|
+
existing010.description =
|
|
2353
|
+
"Retrieval method calls (innerText, textContent, innerHTML, inputValue, etc.) " +
|
|
2354
|
+
"are not preceded by a content guard. These methods only wait for the element to exist in the DOM, " +
|
|
2355
|
+
"not for its content to be populated. This can return empty or stale values when text is loaded dynamically.";
|
|
2356
|
+
}
|
|
2357
|
+
else {
|
|
2358
|
+
findings.push({
|
|
2359
|
+
findingId: "PW-FLAKE-010",
|
|
2360
|
+
severity: options.auditConfig
|
|
2361
|
+
? (0, auditConfig_1.getEffectiveSeverity)(options.auditConfig, "PW-FLAKE-010", "medium")
|
|
2362
|
+
: "medium",
|
|
2363
|
+
title: "Retrieval methods used without content guard",
|
|
2364
|
+
description: "Retrieval method calls (innerText, textContent, innerHTML, inputValue, etc.) " +
|
|
2365
|
+
"are not preceded by a content guard. These methods only wait for the element to exist in the DOM, " +
|
|
2366
|
+
"not for its content to be populated. This can return empty or stale values when text is loaded dynamically.",
|
|
2367
|
+
recommendation: "Guard retrieval calls with a web-first assertion or custom poll before extracting text:\n" +
|
|
2368
|
+
" await expect(locator).toBeVisible();\n" +
|
|
2369
|
+
" const text = await locator.innerText();\n" +
|
|
2370
|
+
"https://playwright.dev/docs/actionability",
|
|
2371
|
+
evidence: pomEvidence.slice(0, 25),
|
|
2372
|
+
totalOccurrences: pomRetrievals.length,
|
|
2373
|
+
affectedFiles: new Set(pomRetrievals.map(p => p.file)).size,
|
|
2374
|
+
});
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
// ARCH-001: Missing BasePage pattern
|
|
2379
|
+
const pomsWithoutBase = pomAnalyses.filter(p => !p.extendsClass);
|
|
2380
|
+
if (pomsWithoutBase.length >= 3 && pomAnalyses.length >= 5) {
|
|
2381
|
+
// Check if there's a clear BasePage pattern (some POMs extend a common class)
|
|
2382
|
+
const baseClasses = pomAnalyses.filter(p => p.extendsClass).map(p => p.extendsClass);
|
|
2383
|
+
const hasEstablishedBasePattern = new Set(baseClasses).size <= 2 && baseClasses.length >= 2;
|
|
2384
|
+
if (hasEstablishedBasePattern || pomsWithoutBase.length > pomAnalyses.length * 0.5) {
|
|
2385
|
+
findings.push({
|
|
2386
|
+
findingId: "ARCH-001",
|
|
2387
|
+
severity: "low",
|
|
2388
|
+
title: "No shared base pattern across Page Objects",
|
|
2389
|
+
description: `${pomsWithoutBase.length} of ${pomAnalyses.length} Page Object classes don't extend a common base class. A shared base can provide common helper methods and consistent initialization. Note: Playwright's fixture model is the primary composition mechanism — consider fixtures for shared setup logic rather than requiring class inheritance.`,
|
|
2390
|
+
recommendation: "Consider using Playwright fixtures for shared setup logic (recommended approach). If using class inheritance, create a BasePage with common helpers. Either approach ensures consistency — fixtures are preferred per Playwright docs.\nhttps://playwright.dev/docs/test-fixtures\nhttps://playwright.dev/docs/pom",
|
|
2391
|
+
evidence: pomsWithoutBase.slice(0, 10).map(p => ({
|
|
2392
|
+
file: normalize(p.file),
|
|
2393
|
+
line: 1,
|
|
2394
|
+
snippet: `class ${p.className} (no base class)`,
|
|
2395
|
+
})),
|
|
2396
|
+
totalOccurrences: pomsWithoutBase.length,
|
|
2397
|
+
affectedFiles: pomsWithoutBase.length,
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
// ARCH-002: Locator-bag POMs (many public locators, few/no action methods)
|
|
2402
|
+
const locatorBagPoms = pomAnalyses.filter(p => p.isLocatorBag);
|
|
2403
|
+
if (locatorBagPoms.length >= 2) {
|
|
2404
|
+
findings.push({
|
|
2405
|
+
findingId: "ARCH-002",
|
|
2406
|
+
severity: "low",
|
|
2407
|
+
title: "Locator-bag Page Objects lack action methods",
|
|
2408
|
+
description: `${locatorBagPoms.length} Page Object(s) expose many public locators without wrapping them in semantic action methods. ` +
|
|
2409
|
+
`POMs with public readonly locators AND action methods that use them (e.g., login(), addToCart()) are fine — only "locator bag" classes that are mostly bare locator lists are flagged.`,
|
|
2410
|
+
recommendation: "Add action methods that use the locators to encapsulate multi-step interactions (e.g., 'async login(user, pass) { ... }'). " +
|
|
2411
|
+
"Public readonly locators are acceptable per Playwright docs — this rule only flags classes where locators vastly outnumber the methods that use them.\nhttps://playwright.dev/docs/pom",
|
|
2412
|
+
evidence: locatorBagPoms.slice(0, 10).map(p => ({
|
|
2413
|
+
file: normalize(p.file),
|
|
2414
|
+
line: 1,
|
|
2415
|
+
snippet: `${p.publicLocatorCount} public locators, ${p.actionMethodCount} action methods`,
|
|
2416
|
+
})),
|
|
2417
|
+
totalOccurrences: locatorBagPoms.length,
|
|
2418
|
+
affectedFiles: locatorBagPoms.length,
|
|
2419
|
+
});
|
|
2420
|
+
}
|
|
2421
|
+
// ARCH-003: Test coupling (tests importing from other tests)
|
|
2422
|
+
const testCouplingIssues = [];
|
|
2423
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2424
|
+
try {
|
|
2425
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2426
|
+
const analysis = analyzeTestCoupling(testFile, content);
|
|
2427
|
+
if (analysis.importsFromTestFiles.length > 0) {
|
|
2428
|
+
testCouplingIssues.push(analysis);
|
|
2429
|
+
}
|
|
2430
|
+
}
|
|
2431
|
+
catch {
|
|
2432
|
+
// Skip files that can't be read
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
if (testCouplingIssues.length > 0) {
|
|
2436
|
+
findings.push({
|
|
2437
|
+
findingId: "ARCH-003",
|
|
2438
|
+
severity: "high",
|
|
2439
|
+
title: "Test coupling detected (tests importing from other tests)",
|
|
2440
|
+
description: `${testCouplingIssues.length} test file(s) import from other test files (.spec.ts/.test.ts). This creates tight coupling between tests, makes them harder to run in isolation, and can cause confusing failures when the imported test changes.`,
|
|
2441
|
+
recommendation: "Extract shared test utilities, fixtures, and helpers into dedicated utility files (e.g., /utils, /helpers, /fixtures folders). Tests should only import from non-test modules. This improves test isolation and makes the test suite more maintainable.",
|
|
2442
|
+
evidence: testCouplingIssues.slice(0, 10).map(c => ({
|
|
2443
|
+
file: normalize(c.file),
|
|
2444
|
+
line: 1,
|
|
2445
|
+
snippet: `imports: ${c.importsFromTestFiles.join(', ')}`,
|
|
2446
|
+
})),
|
|
2447
|
+
totalOccurrences: testCouplingIssues.length,
|
|
2448
|
+
affectedFiles: testCouplingIssues.length,
|
|
2449
|
+
});
|
|
2450
|
+
}
|
|
2451
|
+
// ARCH-004: Missing fixture typing (any types in fixtures)
|
|
2452
|
+
const fixtureFiles = (await (0, fast_glob_1.default)(["**/fixtures/**/*.ts", "**/fixtures.ts", "**/*.fixtures.ts"], {
|
|
2453
|
+
cwd: repoPath,
|
|
2454
|
+
absolute: true,
|
|
2455
|
+
dot: true,
|
|
2456
|
+
onlyFiles: true,
|
|
2457
|
+
followSymbolicLinks: false,
|
|
2458
|
+
}));
|
|
2459
|
+
const filteredFixtureFiles = fixtureFiles.filter((f) => !isIgnored(f));
|
|
2460
|
+
const fixturesWithAnyType = [];
|
|
2461
|
+
for (const fixtureFile of filteredFixtureFiles) {
|
|
2462
|
+
try {
|
|
2463
|
+
const content = await promises_1.default.readFile(fixtureFile, "utf8");
|
|
2464
|
+
const analysis = analyzeFixtureTyping(fixtureFile, content);
|
|
2465
|
+
if (analysis.hasAnyTypes && analysis.anyTypeCount >= 2) {
|
|
2466
|
+
fixturesWithAnyType.push(analysis);
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
catch {
|
|
2470
|
+
// Skip files that can't be read
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
if (fixturesWithAnyType.length > 0) {
|
|
2474
|
+
const totalAnyCount = fixturesWithAnyType.reduce((sum, f) => sum + f.anyTypeCount, 0);
|
|
2475
|
+
findings.push({
|
|
2476
|
+
findingId: "ARCH-004",
|
|
2477
|
+
severity: "medium",
|
|
2478
|
+
title: "Missing fixture typing (any types in fixtures)",
|
|
2479
|
+
description: `${fixturesWithAnyType.length} fixture file(s) contain 'any' type annotations (${totalAnyCount} total). Untyped fixtures lose TypeScript's type safety benefits, making it easy to pass incorrect data to tests and harder to catch errors at compile time.`,
|
|
2480
|
+
recommendation: "Define explicit types for all fixtures using test.extend<{ myFixture: MyType }>. Create interfaces for complex fixture data. This enables IDE autocomplete, catches type errors early, and serves as documentation for fixture contracts.",
|
|
2481
|
+
evidence: fixturesWithAnyType.slice(0, 10).map(f => ({
|
|
2482
|
+
file: normalize(f.file),
|
|
2483
|
+
line: 1,
|
|
2484
|
+
snippet: `${f.anyTypeCount} 'any' type(s)${f.hasProperTyping ? ' (has some proper typing)' : ' (no test.extend<> typing)'}`,
|
|
2485
|
+
})),
|
|
2486
|
+
totalOccurrences: fixturesWithAnyType.length,
|
|
2487
|
+
affectedFiles: fixturesWithAnyType.length,
|
|
2488
|
+
});
|
|
2489
|
+
}
|
|
2490
|
+
// ARCH-005: Hardcoded test data (credentials, emails, etc.)
|
|
2491
|
+
const allHardcodedData = [];
|
|
2492
|
+
const allTestAndPomFiles = [...nonFixmeTestFiles, ...filteredPomFiles];
|
|
2493
|
+
for (const file of allTestAndPomFiles.slice(0, 100)) { // Limit to avoid performance issues
|
|
2494
|
+
try {
|
|
2495
|
+
const content = await promises_1.default.readFile(file, "utf8");
|
|
2496
|
+
const matches = findHardcodedData(file, content);
|
|
2497
|
+
allHardcodedData.push(...matches);
|
|
2498
|
+
}
|
|
2499
|
+
catch {
|
|
2500
|
+
// Skip files that can't be read
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
// Group by type and only report if significant
|
|
2504
|
+
const dataByType = new Map();
|
|
2505
|
+
for (const match of allHardcodedData) {
|
|
2506
|
+
const existing = dataByType.get(match.type) || [];
|
|
2507
|
+
existing.push(match);
|
|
2508
|
+
dataByType.set(match.type, existing);
|
|
2509
|
+
}
|
|
2510
|
+
// Only report if we find passwords, API keys, or multiple email patterns
|
|
2511
|
+
const sensitiveData = [...(dataByType.get('password') || []), ...(dataByType.get('api_key') || [])];
|
|
2512
|
+
const emailData = dataByType.get('email') || [];
|
|
2513
|
+
if (sensitiveData.length > 0) {
|
|
2514
|
+
findings.push({
|
|
2515
|
+
findingId: "ARCH-005",
|
|
2516
|
+
severity: "high",
|
|
2517
|
+
title: "Hardcoded credentials or API keys detected",
|
|
2518
|
+
description: `${sensitiveData.length} instance(s) of hardcoded passwords or API keys found in test code. Hardcoded credentials are a security risk, make environment switching difficult, and can leak sensitive data in version control.`,
|
|
2519
|
+
recommendation: "Move all credentials to environment variables or a secure secrets manager. Use process.env.TEST_PASSWORD, Azure Key Vault, or similar. For test data, consider using a data factory pattern or test data generators.",
|
|
2520
|
+
evidence: sensitiveData.slice(0, 10).map(d => ({
|
|
2521
|
+
file: normalize(d.file),
|
|
2522
|
+
line: d.line,
|
|
2523
|
+
snippet: d.snippet.replace(/(['"`])(?=.*(?:password|secret|key|token))([^'"`]+)\1/gi, '$1[REDACTED]$1'),
|
|
2524
|
+
})),
|
|
2525
|
+
totalOccurrences: sensitiveData.length,
|
|
2526
|
+
affectedFiles: new Set(sensitiveData.map(d => d.file)).size,
|
|
2527
|
+
});
|
|
2528
|
+
}
|
|
2529
|
+
if (emailData.length >= 5) {
|
|
2530
|
+
findings.push({
|
|
2531
|
+
findingId: "ARCH-005a",
|
|
2532
|
+
severity: "low",
|
|
2533
|
+
title: "Hardcoded email addresses in tests",
|
|
2534
|
+
description: `${emailData.length} hardcoded email addresses found in test code. While not a security risk, hardcoded emails make it harder to run tests in different environments and can cause conflicts in parallel test runs.`,
|
|
2535
|
+
recommendation: "Use a test data factory or faker library to generate unique email addresses per test run. For example: 'test-${Date.now()}@example.com' or use @faker-js/faker. This prevents test data conflicts and makes tests more reliable.",
|
|
2536
|
+
evidence: emailData.slice(0, 10).map(d => ({
|
|
2537
|
+
file: normalize(d.file),
|
|
2538
|
+
line: d.line,
|
|
2539
|
+
snippet: d.snippet,
|
|
2540
|
+
})),
|
|
2541
|
+
totalOccurrences: emailData.length,
|
|
2542
|
+
affectedFiles: new Set(emailData.map(d => d.file)).size,
|
|
2543
|
+
});
|
|
2544
|
+
}
|
|
2545
|
+
// ============================================
|
|
2546
|
+
// ARCH-010: Hardcoded inline test data
|
|
2547
|
+
// ============================================
|
|
2548
|
+
{
|
|
2549
|
+
const inlineDataFiles = [];
|
|
2550
|
+
// Scope 1: Playwright test files (already have nonFixmeTestFiles + filteredPomFiles)
|
|
2551
|
+
for (const file of [...nonFixmeTestFiles, ...filteredPomFiles]) {
|
|
2552
|
+
// Skip files in fixtures/data/testdata directories (use repo-relative path to avoid
|
|
2553
|
+
// false matches on parent directories like tests/fixtures/bad-repo)
|
|
2554
|
+
const normalized = normalize(file);
|
|
2555
|
+
const relPath = normalize(path_1.default.relative(repoPath, file));
|
|
2556
|
+
if (/(?:^|[/\\])(?:fixtures|data|testdata)[/\\]/i.test(relPath))
|
|
2557
|
+
continue;
|
|
2558
|
+
try {
|
|
2559
|
+
const content = await promises_1.default.readFile(file, "utf8");
|
|
2560
|
+
const matches = findInlineTestDataInFile(normalized, content);
|
|
2561
|
+
if (matches.length >= 3) {
|
|
2562
|
+
inlineDataFiles.push({ file: normalized, matches });
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2565
|
+
catch {
|
|
2566
|
+
continue;
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
// Scope 2: Cypress test files (if detected)
|
|
2570
|
+
if (inventory.detectedFrameworks.includes("cypress")) {
|
|
2571
|
+
const cypressGlobs = ["**/cypress/e2e/**/*.{ts,js,tsx,jsx}", "**/*.cy.{ts,js,tsx,jsx}"];
|
|
2572
|
+
const cypressFiles = (await (0, fast_glob_1.default)(cypressGlobs, {
|
|
2573
|
+
cwd: repoPath, absolute: true, dot: true, onlyFiles: true, followSymbolicLinks: false,
|
|
2574
|
+
}));
|
|
2575
|
+
const filteredCypressFiles = cypressFiles.filter((f) => !isIgnored(f));
|
|
2576
|
+
for (const file of filteredCypressFiles.slice(0, 50)) {
|
|
2577
|
+
const normalized = normalize(file);
|
|
2578
|
+
const relPath = normalize(path_1.default.relative(repoPath, file));
|
|
2579
|
+
if (/(?:^|[/\\])(?:fixtures|data|testdata)[/\\]/i.test(relPath))
|
|
2580
|
+
continue;
|
|
2581
|
+
// Avoid double-counting files already scanned
|
|
2582
|
+
if (inlineDataFiles.some((d) => d.file === normalized))
|
|
2583
|
+
continue;
|
|
2584
|
+
try {
|
|
2585
|
+
const content = await promises_1.default.readFile(file, "utf8");
|
|
2586
|
+
const matches = findInlineTestDataInFile(normalized, content);
|
|
2587
|
+
if (matches.length >= 3) {
|
|
2588
|
+
inlineDataFiles.push({ file: normalized, matches });
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
catch {
|
|
2592
|
+
continue;
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
// Scope 3: Selenium test files (if detected)
|
|
2597
|
+
if (inventory.detectedFrameworks.includes("selenium")) {
|
|
2598
|
+
const seGlobs = ["**/*.py", "**/*.java"];
|
|
2599
|
+
const seFiles = (await (0, fast_glob_1.default)(seGlobs, {
|
|
2600
|
+
cwd: repoPath, absolute: true, dot: true, onlyFiles: true, followSymbolicLinks: false,
|
|
2601
|
+
}));
|
|
2602
|
+
const filteredSeFiles = seFiles.filter((f) => !isIgnored(f));
|
|
2603
|
+
for (const file of filteredSeFiles.slice(0, 50)) {
|
|
2604
|
+
const normalized = normalize(file);
|
|
2605
|
+
const relPath = normalize(path_1.default.relative(repoPath, file));
|
|
2606
|
+
if (/(?:^|[/\\])(?:fixtures|data|testdata)[/\\]/i.test(relPath))
|
|
2607
|
+
continue;
|
|
2608
|
+
if (inlineDataFiles.some((d) => d.file === normalized))
|
|
2609
|
+
continue;
|
|
2610
|
+
try {
|
|
2611
|
+
const content = await promises_1.default.readFile(file, "utf8");
|
|
2612
|
+
// Only check selenium files
|
|
2613
|
+
if (!/from\s+selenium\b|import\s+org\.openqa\.selenium|require\s*\(\s*['"]selenium-webdriver['"]/.test(content))
|
|
2614
|
+
continue;
|
|
2615
|
+
const matches = findInlineTestDataInFile(normalized, content);
|
|
2616
|
+
if (matches.length >= 3) {
|
|
2617
|
+
inlineDataFiles.push({ file: normalized, matches });
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
catch {
|
|
2621
|
+
continue;
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
// Require ≥2 files with matches before emitting
|
|
2626
|
+
if (inlineDataFiles.length >= 2) {
|
|
2627
|
+
const totalMatches = inlineDataFiles.reduce((sum, f) => sum + f.matches.length, 0);
|
|
2628
|
+
const allMatches = inlineDataFiles.flatMap((f) => f.matches);
|
|
2629
|
+
findings.push({
|
|
2630
|
+
findingId: "ARCH-010",
|
|
2631
|
+
severity: "low",
|
|
2632
|
+
title: "Hardcoded inline test data",
|
|
2633
|
+
description: `${totalMatches} inline test data values found across ${inlineDataFiles.length} test file(s). Hardcoded usernames, addresses, dates, prices, and other test data makes tests brittle and harder to maintain. Extract to fixture files or use data factories.`,
|
|
2634
|
+
recommendation: "Move hardcoded test data to JSON fixture files, TypeScript data factories, or use a library like @faker-js/faker to generate test data. This makes data reusable, easier to update, and supports parameterized testing.",
|
|
2635
|
+
evidence: allMatches.slice(0, 15).map((m) => ({
|
|
2636
|
+
file: m.file,
|
|
2637
|
+
line: m.line,
|
|
2638
|
+
snippet: m.snippet,
|
|
2639
|
+
})),
|
|
2640
|
+
totalOccurrences: totalMatches,
|
|
2641
|
+
affectedFiles: inlineDataFiles.length,
|
|
2642
|
+
});
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2645
|
+
// ARCH-006: Missing API mocking capability
|
|
2646
|
+
// Also check test files
|
|
2647
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2648
|
+
try {
|
|
2649
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2650
|
+
pomContentsMap.set(testFile, content);
|
|
2651
|
+
}
|
|
2652
|
+
catch {
|
|
2653
|
+
// Skip
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
const fullMockingAnalysis = analyzeApiMocking([...nonFixmeTestFiles, ...filteredPomFiles], pomContentsMap);
|
|
2657
|
+
// Only flag if we have a meaningful number of test files but no mocking
|
|
2658
|
+
if (nonFixmeTestFiles.length >= 10 && !fullMockingAnalysis.hasRouteMocking && !fullMockingAnalysis.hasRequestInterception) {
|
|
2659
|
+
findings.push({
|
|
2660
|
+
findingId: "ARCH-006",
|
|
2661
|
+
severity: "low",
|
|
2662
|
+
title: "No API mocking detected",
|
|
2663
|
+
description: `No page.route() or request interception patterns found across ${nonFixmeTestFiles.length} test files. While not always necessary, API mocking enables faster tests, better isolation from backend changes, and easier testing of error scenarios.`,
|
|
2664
|
+
recommendation: "Consider using Playwright's page.route() for API mocking in integration tests. This allows testing error states, slow responses, and edge cases without backend dependencies. Create a helper function for common mock patterns. See: https://playwright.dev/docs/mock",
|
|
2665
|
+
evidence: [
|
|
2666
|
+
{
|
|
2667
|
+
file: `${repoPath} (analysis summary)`,
|
|
2668
|
+
line: 1,
|
|
2669
|
+
snippet: `${nonFixmeTestFiles.length} test files analyzed, no route mocking found`,
|
|
2670
|
+
},
|
|
2671
|
+
],
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
// ARCH-007: No custom error types
|
|
2675
|
+
const errorAnalyses = [];
|
|
2676
|
+
for (const pomFile of filteredPomFiles) {
|
|
2677
|
+
try {
|
|
2678
|
+
const content = pomContentsMap.get(pomFile) || await promises_1.default.readFile(pomFile, "utf8");
|
|
2679
|
+
const analysis = analyzeErrorHandling(pomFile, content);
|
|
2680
|
+
if (analysis.genericThrowCount >= 2 && !analysis.hasCustomErrors) {
|
|
2681
|
+
errorAnalyses.push(analysis);
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
catch {
|
|
2685
|
+
// Skip
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
if (errorAnalyses.length >= 3) {
|
|
2689
|
+
const totalGenericThrows = errorAnalyses.reduce((sum, e) => sum + e.genericThrowCount, 0);
|
|
2690
|
+
findings.push({
|
|
2691
|
+
findingId: "ARCH-007",
|
|
2692
|
+
severity: "low",
|
|
2693
|
+
title: "No custom error types (generic Error throws)",
|
|
2694
|
+
description: `${errorAnalyses.length} Page Object(s) use generic 'throw new Error()' (${totalGenericThrows} total instances) without custom error classes. Custom error types enable better error categorization, easier debugging, and programmatic error handling in tests.`,
|
|
2695
|
+
recommendation: "Create custom error classes that extend Error for different failure scenarios (e.g., ElementNotFoundError, NavigationError, ValidationError). This makes test failures more descriptive and enables better error handling in helper functions.",
|
|
2696
|
+
evidence: errorAnalyses.slice(0, 10).map(e => ({
|
|
2697
|
+
file: normalize(e.file),
|
|
2698
|
+
line: 1,
|
|
2699
|
+
snippet: `${e.genericThrowCount} generic throw(s), no custom error types`,
|
|
2700
|
+
})),
|
|
2701
|
+
totalOccurrences: errorAnalyses.length,
|
|
2702
|
+
affectedFiles: errorAnalyses.length,
|
|
2703
|
+
});
|
|
2704
|
+
}
|
|
2705
|
+
// ============================================
|
|
2706
|
+
// ARCH-012: Mutating imported test data objects
|
|
2707
|
+
// ============================================
|
|
2708
|
+
const allDataMutations = [];
|
|
2709
|
+
for (const testFile of nonFixmeTestFiles) {
|
|
2710
|
+
try {
|
|
2711
|
+
const content = await promises_1.default.readFile(testFile, "utf8");
|
|
2712
|
+
const mutations = (0, astRuleHelpers_1.findTestDataMutations)(testFile, content);
|
|
2713
|
+
for (const m of mutations) {
|
|
2714
|
+
allDataMutations.push({
|
|
2715
|
+
file: normalize(testFile),
|
|
2716
|
+
line: m.line,
|
|
2717
|
+
evidence: m.snippet,
|
|
2718
|
+
});
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
catch {
|
|
2722
|
+
continue;
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2725
|
+
if (allDataMutations.length > 0) {
|
|
2726
|
+
findings.push({
|
|
2727
|
+
findingId: "ARCH-012",
|
|
2728
|
+
severity: "high",
|
|
2729
|
+
title: "Mutating imported test data objects breaks parallel execution",
|
|
2730
|
+
description: `${allDataMutations.length} file(s) directly mutate properties of imported test-data objects. Node.js modules are singletons — mutations in one test pollute the shared object for all other tests running in the same worker, causing order-dependent failures and broken parallel execution.`,
|
|
2731
|
+
recommendation: "Deep-clone or spread imported data in each test instead of mutating the original: `const payload = { ...testData.assetPayload, ParentCustomerId: myId }`. For complex objects, use structuredClone() or a factory function that returns fresh copies.",
|
|
2732
|
+
evidence: allDataMutations.slice(0, 10).map((m) => ({
|
|
2733
|
+
file: m.file,
|
|
2734
|
+
line: m.line,
|
|
2735
|
+
snippet: m.evidence,
|
|
2736
|
+
})),
|
|
2737
|
+
totalOccurrences: allDataMutations.length,
|
|
2738
|
+
affectedFiles: new Set(allDataMutations.map(m => m.file)).size,
|
|
2739
|
+
});
|
|
2740
|
+
}
|
|
2741
|
+
// ============================================
|
|
2742
|
+
// END ARCHITECTURAL ANALYSIS
|
|
2743
|
+
// ============================================
|
|
2744
|
+
// Derived premium insight
|
|
2745
|
+
const hardWaits = hasFinding(findings, "PW-FLAKE-001");
|
|
2746
|
+
const forceClicks = hasFinding(findings, "PW-FLAKE-002");
|
|
2747
|
+
if (retriesEnabled && hardWaits && forceClicks) {
|
|
2748
|
+
findings.push({
|
|
2749
|
+
findingId: "PW-INSIGHT-001",
|
|
2750
|
+
severity: "high",
|
|
2751
|
+
title: "Masked flakiness risk (retries + hard waits + force clicks)",
|
|
2752
|
+
description: "Retries are enabled while the codebase also contains hard waits and force clicks. This combination often indicates timing instability being masked rather than fixed, which can inflate CI time and make failures harder to reproduce.",
|
|
2753
|
+
recommendation: "Prioritize removing hard waits and force-click patterns in the top hotspot files first. Then re-evaluate retry policy (keep retries minimal and rely on trace/video artifacts for diagnosis).",
|
|
2754
|
+
evidence: [
|
|
2755
|
+
{
|
|
2756
|
+
file: inventory.playwrightConfigSummary?.configPath ?? "playwright.config",
|
|
2757
|
+
line: 1,
|
|
2758
|
+
snippet: `retries: ${retriesExpr ?? "unknown"}; detected PW-FLAKE-001 + PW-FLAKE-002`,
|
|
2759
|
+
},
|
|
2760
|
+
],
|
|
2761
|
+
});
|
|
2762
|
+
}
|
|
2763
|
+
// INSIGHT-002: Missing or risky timeout configuration
|
|
2764
|
+
const pw = inventory.playwrightConfigSummary;
|
|
2765
|
+
const testTimeoutMs = pw?.timeoutMs;
|
|
2766
|
+
const expectTimeoutMs = pw?.expectTimeoutMs;
|
|
2767
|
+
const actionTimeoutMs = pw?.use?.actionTimeoutMs;
|
|
2768
|
+
if (testTimeoutMs && testTimeoutMs > 120000) {
|
|
2769
|
+
findings.push({
|
|
2770
|
+
findingId: "PW-INSIGHT-002",
|
|
2771
|
+
severity: "medium",
|
|
2772
|
+
title: "Excessively high test timeout detected",
|
|
2773
|
+
description: `Test timeout is set to ${testTimeoutMs}ms (${Math.round(testTimeoutMs / 1000)}s). Very high timeouts can hide performance issues and make failing tests hang for too long.`,
|
|
2774
|
+
recommendation: "Review whether tests genuinely need such long timeouts. Consider setting appropriate expect/action timeouts instead of relying on the global test timeout. Aim for 30-60s test timeouts unless you have specific heavy integration tests.",
|
|
2775
|
+
evidence: [
|
|
2776
|
+
{
|
|
2777
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2778
|
+
line: 1,
|
|
2779
|
+
snippet: `timeout: ${testTimeoutMs} ms`,
|
|
2780
|
+
},
|
|
2781
|
+
],
|
|
2782
|
+
});
|
|
2783
|
+
}
|
|
2784
|
+
const isPlaywright = inventory.detectedFrameworks.includes("playwright");
|
|
2785
|
+
if (isPlaywright && !expectTimeoutMs && !actionTimeoutMs) {
|
|
2786
|
+
findings.push({
|
|
2787
|
+
findingId: "PW-INSIGHT-003",
|
|
2788
|
+
severity: "low",
|
|
2789
|
+
title: "No explicit expect or action timeout configured",
|
|
2790
|
+
description: "Neither expect.timeout nor use.actionTimeout are explicitly configured. This relies on Playwright defaults (5s for expect, 0 for actions which means they use test timeout). Explicit timeouts improve test clarity.",
|
|
2791
|
+
recommendation: "Set expect.timeout (e.g., 10-20s) and use.actionTimeout (e.g., 30s) explicitly in playwright.config.ts to make timeout behavior predictable and easier to tune.",
|
|
2792
|
+
evidence: [
|
|
2793
|
+
{
|
|
2794
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2795
|
+
line: 1,
|
|
2796
|
+
snippet: "expect.timeout and use.actionTimeout not explicitly set",
|
|
2797
|
+
},
|
|
2798
|
+
],
|
|
2799
|
+
});
|
|
2800
|
+
}
|
|
2801
|
+
// INSIGHT-004: Excessive retries
|
|
2802
|
+
// Skip when retries are driven by env vars or parseInt() — the extracted
|
|
2803
|
+
// numbers are arguments/radix, not the actual retry count.
|
|
2804
|
+
if (retriesEnabled && retriesExpr && !isEnvDriven(retriesExpr)) {
|
|
2805
|
+
const retriesMatch = retriesExpr.match(/\b(\d+)\b/g);
|
|
2806
|
+
const maxRetries = retriesMatch ? Math.max(...retriesMatch.map(Number)) : 0;
|
|
2807
|
+
if (maxRetries > 2) {
|
|
2808
|
+
const isHigh = maxRetries >= 5;
|
|
2809
|
+
findings.push({
|
|
2810
|
+
findingId: "PW-INSIGHT-004",
|
|
2811
|
+
severity: isHigh ? "high" : "medium",
|
|
2812
|
+
title: `Excessive retry count detected (${maxRetries} retries)`,
|
|
2813
|
+
description: isHigh
|
|
2814
|
+
? `Retries are set to ${maxRetries}, far above the recommended maximum (2). At this level, retries are compensating for systemic test instability and each flaky test can run up to ${maxRetries + 1} times, dramatically inflating CI duration and masking root causes. This degrades the signal-to-noise ratio of CI; both pass and fail results should be interpreted with caution.`
|
|
2815
|
+
: `Retries are set to ${maxRetries}, which is higher than the recommended maximum (2). Excessive retries can mask underlying flakiness and make CI runs significantly longer.`,
|
|
2816
|
+
recommendation: "Reduce retries to 1-2 maximum. Use retries to handle rare environmental flakiness, not to work around test instability. Prioritize fixing the root cause of flaky tests.",
|
|
2817
|
+
evidence: [
|
|
2818
|
+
{
|
|
2819
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2820
|
+
line: 1,
|
|
2821
|
+
snippet: `retries: ${retriesExpr}`,
|
|
2822
|
+
},
|
|
2823
|
+
],
|
|
2824
|
+
});
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
// INSIGHT-005: Missing trace/video on failure
|
|
2828
|
+
const trace = pw?.use?.trace;
|
|
2829
|
+
const video = pw?.use?.video;
|
|
2830
|
+
if (isPlaywright && (!trace || trace === "off")) {
|
|
2831
|
+
findings.push({
|
|
2832
|
+
findingId: "PW-INSIGHT-005",
|
|
2833
|
+
severity: "high",
|
|
2834
|
+
title: "Trace capture not configured or disabled",
|
|
2835
|
+
description: "Playwright trace is not enabled or is set to 'off'. The Playwright team explicitly recommends 'trace: on-first-retry' as a best practice. Traces are the primary debugging mechanism for CI failures, providing DOM snapshots, network logs, and action timeline.",
|
|
2836
|
+
recommendation: "Enable traces with 'on-first-retry' (recommended by Playwright best practices) or 'retain-on-failure' in playwright.config.ts. This provides rich debugging context with minimal storage overhead.\nhttps://playwright.dev/docs/trace-viewer-intro",
|
|
2837
|
+
evidence: [
|
|
2838
|
+
{
|
|
2839
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2840
|
+
line: 1,
|
|
2841
|
+
snippet: trace ? `trace: ${trace}` : "trace not configured",
|
|
2842
|
+
},
|
|
2843
|
+
],
|
|
2844
|
+
});
|
|
2845
|
+
}
|
|
2846
|
+
if (isPlaywright && (!video || video === "off")) {
|
|
2847
|
+
findings.push({
|
|
2848
|
+
findingId: "PW-INSIGHT-006",
|
|
2849
|
+
severity: "low",
|
|
2850
|
+
title: "Video capture not configured or disabled",
|
|
2851
|
+
description: "Video recording is not enabled or is set to 'off'. Videos can be helpful for debugging, though less detailed than traces.",
|
|
2852
|
+
recommendation: "Consider enabling video with 'on-first-retry' or 'retain-on-failure'. Note that traces are more useful for debugging, so if storage is a concern, prioritize traces over videos.",
|
|
2853
|
+
evidence: [
|
|
2854
|
+
{
|
|
2855
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2856
|
+
line: 1,
|
|
2857
|
+
snippet: video ? `video: ${video}` : "video not configured",
|
|
2858
|
+
},
|
|
2859
|
+
],
|
|
2860
|
+
});
|
|
2861
|
+
}
|
|
2862
|
+
// INSIGHT-007: High parallelism without env tuning
|
|
2863
|
+
const workers = pw?.workers;
|
|
2864
|
+
if (workers && /^\d+$/.test(workers) && parseInt(workers) > 8) {
|
|
2865
|
+
findings.push({
|
|
2866
|
+
findingId: "PW-INSIGHT-007",
|
|
2867
|
+
severity: "low",
|
|
2868
|
+
title: `High fixed worker count detected (${workers} workers)`,
|
|
2869
|
+
description: `Worker count is hardcoded to ${workers}. High parallelism can cause resource contention and may not be suitable for all environments (e.g., CI, developer laptops).`,
|
|
2870
|
+
recommendation: "Make workers environment-aware (e.g., process.env.WORKERS or process.env.CI ? 4 : 2). This allows tuning parallelism per environment and prevents resource exhaustion.",
|
|
2871
|
+
evidence: [
|
|
2872
|
+
{
|
|
2873
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2874
|
+
line: 1,
|
|
2875
|
+
snippet: `workers: ${workers}`,
|
|
2876
|
+
},
|
|
2877
|
+
],
|
|
2878
|
+
});
|
|
2879
|
+
}
|
|
2880
|
+
// ── New config-based insights (from Playwright docs audit) ────────────
|
|
2881
|
+
// INSIGHT-014: forbidOnly not configured (per best-practices doc)
|
|
2882
|
+
if (pw && inventory.ciSummary?.files?.length) {
|
|
2883
|
+
// Read the config file to check for forbidOnly
|
|
2884
|
+
try {
|
|
2885
|
+
const configContent = pw.configPath
|
|
2886
|
+
? await promises_1.default.readFile(pw.configPath, "utf8")
|
|
2887
|
+
: "";
|
|
2888
|
+
if (!configContent.includes("forbidOnly")) {
|
|
2889
|
+
findings.push({
|
|
2890
|
+
findingId: "PW-INSIGHT-014",
|
|
2891
|
+
severity: "high",
|
|
2892
|
+
title: "Missing forbidOnly in Playwright config (test.only can leak to CI)",
|
|
2893
|
+
description: "The Playwright config does not include forbidOnly. Without this guard, a test.only() accidentally left in committed code will silently run only one test in CI, making pass rates meaningless. The Playwright docs recommend 'forbidOnly: !!process.env.CI' as a best practice.",
|
|
2894
|
+
recommendation: "Add forbidOnly to your playwright.config.ts:\n forbidOnly: !!process.env.CI\nThis will cause CI runs to fail immediately if test.only() is found.\nhttps://playwright.dev/docs/api/class-testconfig#test-config-forbid-only",
|
|
2895
|
+
evidence: [{
|
|
2896
|
+
file: pw.configPath,
|
|
2897
|
+
line: 1,
|
|
2898
|
+
snippet: "forbidOnly not found in config",
|
|
2899
|
+
}],
|
|
2900
|
+
});
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
catch {
|
|
2904
|
+
// Skip if config can't be read
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
// INSIGHT-015: Retries not CI-conditional (per test-configuration doc)
|
|
2908
|
+
if (pw && retriesEnabled && retriesExpr && !isEnvDriven(retriesExpr)) {
|
|
2909
|
+
findings.push({
|
|
2910
|
+
findingId: "PW-INSIGHT-015",
|
|
2911
|
+
severity: "medium",
|
|
2912
|
+
title: "Retries are always on (not CI-conditional)",
|
|
2913
|
+
description: `Retries are set to a fixed value (${retriesExpr}) rather than being conditional on CI. The Playwright docs recommend 'retries: process.env.CI ? 2 : 0' so that retries are only enabled in CI — locally, developers see failures immediately without retry masking.`,
|
|
2914
|
+
recommendation: "Make retries CI-conditional:\n retries: process.env.CI ? 2 : 0\nThis helps developers catch flakiness locally while still having CI resilience.\nhttps://playwright.dev/docs/test-configuration",
|
|
2915
|
+
evidence: [{
|
|
2916
|
+
file: pw?.configPath ?? "playwright.config",
|
|
2917
|
+
line: 1,
|
|
2918
|
+
snippet: `retries: ${retriesExpr}`,
|
|
2919
|
+
}],
|
|
2920
|
+
});
|
|
2921
|
+
}
|
|
2922
|
+
// CI findings
|
|
2923
|
+
const ci = inventory.ciSummary;
|
|
2924
|
+
// CI-001 only: exclude pipeline files whose filename/path (relative to the
|
|
2925
|
+
// repo root) contains "scantrix" (case-insensitive). Those are Scantrix
|
|
2926
|
+
// audit pipelines, not the target repo's product CI pipeline, and must not
|
|
2927
|
+
// satisfy the "CI exists" check.
|
|
2928
|
+
const nonScantrixCiFiles = ci.files.filter((f) => {
|
|
2929
|
+
const rel = normalize(path_1.default.relative(repoPath, f.file)).toLowerCase();
|
|
2930
|
+
return !rel.includes("scantrix");
|
|
2931
|
+
});
|
|
2932
|
+
if (!nonScantrixCiFiles.length) {
|
|
2933
|
+
findings.push({
|
|
2934
|
+
findingId: "CI-001",
|
|
2935
|
+
severity: "medium",
|
|
2936
|
+
title: "No CI pipeline YAML detected",
|
|
2937
|
+
description: "No Azure Pipelines or GitHub Actions workflow YAML files were detected. CI signal checks (artifact publishing, caching, workers env wiring) could not be validated.",
|
|
2938
|
+
recommendation: "If CI exists elsewhere, point the audit tool at the repository that contains the pipeline YAML. Otherwise, add a CI workflow that runs Playwright and publishes artifacts.",
|
|
2939
|
+
evidence: [{ file: repoPath, line: 1, snippet: "No pipeline YAML files found." }],
|
|
2940
|
+
});
|
|
2941
|
+
return { repoPath, inventory, findings, gitBranch: git.branch, gitCommit: git.commit, repoDisplayName: options.repoName };
|
|
2942
|
+
}
|
|
2943
|
+
// CI-002
|
|
2944
|
+
if (!ci.publishesPlaywrightReport) {
|
|
2945
|
+
findings.push({
|
|
2946
|
+
findingId: "CI-002",
|
|
2947
|
+
severity: "high",
|
|
2948
|
+
title: "CI may not publish Playwright HTML report artifacts",
|
|
2949
|
+
description: "Pipeline YAML was detected but there is no strong evidence that the Playwright HTML report folder (playwright-report) is uploaded as a build artifact. This reduces debuggability on failures.",
|
|
2950
|
+
recommendation: "Upload the playwright-report folder as a pipeline artifact (ADO: PublishPipelineArtifact; GHA: actions/upload-artifact).",
|
|
2951
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
2952
|
+
file: f.file,
|
|
2953
|
+
line: 1,
|
|
2954
|
+
snippet: `Detected CI file (${f.kind}); playwright-report publish not confirmed.`,
|
|
2955
|
+
})),
|
|
2956
|
+
});
|
|
2957
|
+
}
|
|
2958
|
+
// CI-003
|
|
2959
|
+
if (!ci.publishesJUnit) {
|
|
2960
|
+
findings.push({
|
|
2961
|
+
findingId: "CI-003",
|
|
2962
|
+
severity: "medium",
|
|
2963
|
+
title: "CI may not publish JUnit test results",
|
|
2964
|
+
description: "Playwright is configured to output JUnit (test-results.xml) in CI, but the pipeline YAML does not clearly publish test results. This makes test reporting harder.",
|
|
2965
|
+
recommendation: "Publish test-results.xml as test results (ADO: PublishTestResults task; GHA: upload artifact + test reporter integration if used).",
|
|
2966
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
2967
|
+
file: f.file,
|
|
2968
|
+
line: 1,
|
|
2969
|
+
snippet: `Detected CI file (${f.kind}); JUnit publish not confirmed.`,
|
|
2970
|
+
})),
|
|
2971
|
+
});
|
|
2972
|
+
}
|
|
2973
|
+
// CI-004
|
|
2974
|
+
const pwWorkersExpr = inventory.playwrightConfigSummary?.workers ?? "";
|
|
2975
|
+
const envHints = inventory.playwrightConfigSummary?.envHints ?? {};
|
|
2976
|
+
const hintedWorkers = Boolean(envHints.workersEnvVar);
|
|
2977
|
+
const pwUsesWorkersEnv = hintedWorkers || /process\.env\.WORKERS|\bWORKERS\b/.test(pwWorkersExpr);
|
|
2978
|
+
;
|
|
2979
|
+
if (pwUsesWorkersEnv && !ci.setsWorkersEnv) {
|
|
2980
|
+
findings.push({
|
|
2981
|
+
findingId: "CI-004",
|
|
2982
|
+
severity: "medium",
|
|
2983
|
+
title: "Playwright workers appear env-driven but CI does not set WORKERS",
|
|
2984
|
+
description: "Playwright config suggests workers are controlled via an environment variable (WORKERS), but the pipeline YAML does not clearly set WORKERS. This can lead to unexpected default parallelism.",
|
|
2985
|
+
recommendation: "Explicitly set WORKERS in the pipeline YAML (and consider per-environment tuning).",
|
|
2986
|
+
evidence: [
|
|
2987
|
+
{
|
|
2988
|
+
file: inventory.playwrightConfigSummary?.configPath ?? "playwright.config",
|
|
2989
|
+
line: 1,
|
|
2990
|
+
snippet: `workers: ${pwWorkersExpr || "unknown"}`,
|
|
2991
|
+
},
|
|
2992
|
+
...ci.files.slice(0, 2).map((f) => ({
|
|
2993
|
+
file: f.file,
|
|
2994
|
+
line: 1,
|
|
2995
|
+
snippet: "WORKERS not detected in pipeline YAML.",
|
|
2996
|
+
})),
|
|
2997
|
+
],
|
|
2998
|
+
});
|
|
2999
|
+
}
|
|
3000
|
+
// CI-006 — inconsistent WORKERS usage across pipelines
|
|
3001
|
+
const totalCiFiles = ci.files.length;
|
|
3002
|
+
const workersFilesCount = ci.workersEnvFiles.length;
|
|
3003
|
+
if (pwUsesWorkersEnv &&
|
|
3004
|
+
totalCiFiles > 1 &&
|
|
3005
|
+
workersFilesCount > 0 &&
|
|
3006
|
+
workersFilesCount < totalCiFiles) {
|
|
3007
|
+
// Normalize for comparison
|
|
3008
|
+
const workersSet = new Set(ci.workersEnvFiles.map((f) => normalize(f).toLowerCase()));
|
|
3009
|
+
const allCiFiles = ci.files.map((f) => normalize(f.file));
|
|
3010
|
+
const missingWorkersFiles = allCiFiles.filter((file) => !workersSet.has(file.toLowerCase()));
|
|
3011
|
+
// Dynamic severity based on both pipeline count AND coverage percentage
|
|
3012
|
+
const coveragePercent = (workersFilesCount / totalCiFiles) * 100;
|
|
3013
|
+
let severity;
|
|
3014
|
+
// High severity if: many pipelines AND low coverage (<40%)
|
|
3015
|
+
// Medium severity if: moderate inconsistency
|
|
3016
|
+
// Low severity if: mostly consistent (>70% coverage) or few pipelines
|
|
3017
|
+
if (totalCiFiles >= 5 && coveragePercent < 40) {
|
|
3018
|
+
severity = "high"; // Majority of pipelines missing WORKERS
|
|
3019
|
+
}
|
|
3020
|
+
else if (totalCiFiles >= 4 || coveragePercent < 60) {
|
|
3021
|
+
severity = "medium";
|
|
3022
|
+
}
|
|
3023
|
+
else {
|
|
3024
|
+
severity = "low";
|
|
3025
|
+
}
|
|
3026
|
+
findings.push({
|
|
3027
|
+
findingId: "CI-006",
|
|
3028
|
+
severity,
|
|
3029
|
+
title: "WORKERS env is set inconsistently across CI pipelines",
|
|
3030
|
+
description: `Some pipeline YAML files set WORKERS (${Math.round(coveragePercent)}% coverage), while others rely on Playwright defaults. This can cause inconsistent parallelism, execution time, and stability between pipelines.`,
|
|
3031
|
+
recommendation: "Standardize WORKERS across all pipelines (or explicitly document per-pipeline intent). If different parallelism is desired, set WORKERS in every pipeline so behavior is intentional.",
|
|
3032
|
+
evidence: [
|
|
3033
|
+
{
|
|
3034
|
+
file: `${repoPath} (CI summary)`,
|
|
3035
|
+
line: 1,
|
|
3036
|
+
snippet: `WORKERS referenced in ${workersFilesCount}/${totalCiFiles} pipeline file(s); missing in ${missingWorkersFiles.length}/${totalCiFiles}.`,
|
|
3037
|
+
},
|
|
3038
|
+
// Files that DO set WORKERS
|
|
3039
|
+
...ci.workersEnvFiles.slice(0, 8).map((file) => ({
|
|
3040
|
+
file,
|
|
3041
|
+
line: 1,
|
|
3042
|
+
snippet: "WORKERS referenced here.",
|
|
3043
|
+
})),
|
|
3044
|
+
// Files that DO NOT set WORKERS
|
|
3045
|
+
...missingWorkersFiles.slice(0, 8).map((file) => ({
|
|
3046
|
+
file,
|
|
3047
|
+
line: 1,
|
|
3048
|
+
snippet: "WORKERS NOT set in this pipeline file.",
|
|
3049
|
+
})),
|
|
3050
|
+
...(missingWorkersFiles.length > 8
|
|
3051
|
+
? [
|
|
3052
|
+
{
|
|
3053
|
+
file: `${repoPath} (CI summary)`,
|
|
3054
|
+
line: 1,
|
|
3055
|
+
snippet: `...and ${missingWorkersFiles.length - 8} more pipeline file(s) without WORKERS.`,
|
|
3056
|
+
},
|
|
3057
|
+
]
|
|
3058
|
+
: []),
|
|
3059
|
+
],
|
|
3060
|
+
});
|
|
3061
|
+
}
|
|
3062
|
+
// CI-005
|
|
3063
|
+
if (!ci.usesCache) {
|
|
3064
|
+
const isMicrosoftHosted = ci.usesMicrosoftHostedPool && !ci.usesSelfHostedPool;
|
|
3065
|
+
findings.push({
|
|
3066
|
+
findingId: "CI-005",
|
|
3067
|
+
severity: isMicrosoftHosted ? "medium" : "low",
|
|
3068
|
+
title: isMicrosoftHosted
|
|
3069
|
+
? "No dependency caching detected in CI (Microsoft-hosted agents)"
|
|
3070
|
+
: "No dependency caching detected in CI (may be optional on self-hosted agents)",
|
|
3071
|
+
description: isMicrosoftHosted
|
|
3072
|
+
? "No dependency caching was detected (node_modules, npm cache). On Microsoft-hosted agents this increases CI duration and cost because each run starts from a clean machine. Note: Playwright docs recommend NOT caching browser binaries — download time is comparable to cache restore time."
|
|
3073
|
+
: "No dependency caching was detected (node_modules, npm cache). On self-hosted/on-prem agents this may be less critical if dependencies are already present, but caching can still reduce install time and variability. Note: Playwright docs recommend NOT caching browser binaries.",
|
|
3074
|
+
recommendation: ci.mentionsBrowsersPreinstalled
|
|
3075
|
+
? "Consider caching node_modules via ADO Cache@2 or actions/cache. Do NOT cache Playwright browser binaries — per Playwright docs, download time is comparable to cache restore time.\nhttps://playwright.dev/docs/ci#caching-browsers"
|
|
3076
|
+
: "Add caching for node_modules / npm cache (ADO Cache@2 or actions/cache). Do NOT cache Playwright browser binaries — per Playwright docs, download time is comparable to cache restore time.\nhttps://playwright.dev/docs/ci#caching-browsers",
|
|
3077
|
+
evidence: [
|
|
3078
|
+
{
|
|
3079
|
+
file: `${repoPath} (CI summary)`,
|
|
3080
|
+
line: 1,
|
|
3081
|
+
snippet: `No Cache@2 / actions/cache detected across ${ci.files.length} pipeline YAML file(s).`,
|
|
3082
|
+
},
|
|
3083
|
+
...ci.files.slice(0, 8).map((f) => ({
|
|
3084
|
+
file: f.file,
|
|
3085
|
+
line: 1,
|
|
3086
|
+
snippet: "No cache detected in this pipeline file.",
|
|
3087
|
+
})),
|
|
3088
|
+
],
|
|
3089
|
+
});
|
|
3090
|
+
}
|
|
3091
|
+
// CI-007: Missing sharding for large test suites
|
|
3092
|
+
const testFileCount = inventory.testFiles;
|
|
3093
|
+
if (testFileCount > 100 && !ci.usesSharding) {
|
|
3094
|
+
// Estimate time savings with sharding
|
|
3095
|
+
// Assume ~30s average per test file (conservative), and 4-shard parallelism
|
|
3096
|
+
const estimatedCurrentMinutes = Math.round((testFileCount * 30) / 60);
|
|
3097
|
+
const recommendedShards = testFileCount > 500 ? 8 : testFileCount > 200 ? 4 : 2;
|
|
3098
|
+
const estimatedShardedMinutes = Math.round(estimatedCurrentMinutes / recommendedShards);
|
|
3099
|
+
const timeSavingsPercent = Math.round((1 - (1 / recommendedShards)) * 100);
|
|
3100
|
+
findings.push({
|
|
3101
|
+
findingId: "CI-007",
|
|
3102
|
+
severity: testFileCount > 500 ? "high" : "medium",
|
|
3103
|
+
title: `Large test suite (${testFileCount} test files) without sharding`,
|
|
3104
|
+
description: `This repository has ${testFileCount} test files but CI pipelines don't appear to use sharding (--shard=). Without sharding, all tests run in a single job, leading to long CI times. **Estimated impact:** With ${recommendedShards}-way sharding, CI time could reduce from ~${estimatedCurrentMinutes} min to ~${estimatedShardedMinutes} min (${timeSavingsPercent}% faster).`,
|
|
3105
|
+
recommendation: `Implement test sharding in CI by running ${recommendedShards} parallel jobs with --shard=1/${recommendedShards}, --shard=2/${recommendedShards}, etc. This distributes tests across multiple machines/containers for faster feedback. In Azure Pipelines, use a matrix strategy; in GitHub Actions, use a matrix with shard indices. https://playwright.dev/docs/test-sharding#introduction`,
|
|
3106
|
+
evidence: [
|
|
3107
|
+
{
|
|
3108
|
+
file: `${repoPath} (CI summary)`,
|
|
3109
|
+
line: 1,
|
|
3110
|
+
snippet: `${testFileCount} test files; estimated ${estimatedCurrentMinutes} min → ${estimatedShardedMinutes} min with ${recommendedShards} shards`,
|
|
3111
|
+
},
|
|
3112
|
+
],
|
|
3113
|
+
});
|
|
3114
|
+
}
|
|
3115
|
+
// CI-008: Playwright install without --with-deps
|
|
3116
|
+
if (ci.mentionsPlaywrightInstall && ci.playwrightInstallMethod === "npx playwright install") {
|
|
3117
|
+
const isMicrosoftHosted = ci.usesMicrosoftHostedPool && !ci.usesSelfHostedPool;
|
|
3118
|
+
if (isMicrosoftHosted) {
|
|
3119
|
+
findings.push({
|
|
3120
|
+
findingId: "CI-008",
|
|
3121
|
+
severity: "medium",
|
|
3122
|
+
title: "Playwright browsers installed without system dependencies (--with-deps)",
|
|
3123
|
+
description: "CI uses 'npx playwright install' without the --with-deps flag. On Microsoft-hosted agents, this may result in missing system dependencies for browsers, causing cryptic failures.",
|
|
3124
|
+
recommendation: "Use 'npx playwright install --with-deps' to ensure all required system dependencies are installed. This is especially important on clean Microsoft-hosted agents. https://playwright.dev/docs/ci-intro#introduction",
|
|
3125
|
+
evidence: ci.files.filter(f => {
|
|
3126
|
+
// Filter to files that likely contain the install command
|
|
3127
|
+
return true; // Simplified for now
|
|
3128
|
+
}).slice(0, 3).map((f) => ({
|
|
3129
|
+
file: f.file,
|
|
3130
|
+
line: 1,
|
|
3131
|
+
snippet: "Uses 'npx playwright install' (consider adding --with-deps)",
|
|
3132
|
+
})),
|
|
3133
|
+
});
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
// CI-009: Retries enabled but test-results/traces not published on failure
|
|
3137
|
+
if (retriesEnabled && ci.files.length > 0) {
|
|
3138
|
+
const publishesTraces = ci.publishesTracesOrTestResultsDir;
|
|
3139
|
+
const publishesOnFailure = ci.publishesArtifactsOnFailure;
|
|
3140
|
+
// Only trigger if artifacts aren't published, OR if they're published but not on failure condition
|
|
3141
|
+
if (!publishesTraces) {
|
|
3142
|
+
findings.push({
|
|
3143
|
+
findingId: "CI-009",
|
|
3144
|
+
severity: "high",
|
|
3145
|
+
title: "Retries enabled but test artifacts not published on failure",
|
|
3146
|
+
description: `Playwright is configured to retry failed tests (retries: ${retriesExpr}), but CI pipelines don't appear to publish test-results directories or trace files. When retried tests eventually fail, you have no artifacts to debug why.`,
|
|
3147
|
+
recommendation: "Publish the test-results folder (which contains traces, videos, screenshots) as a build artifact. In Azure Pipelines, use PublishPipelineArtifact@1 with PathtoPublish: 'test-results'. In GitHub Actions, use actions/upload-artifact@v3. Configure to run even on failure (condition: always() or failed()).",
|
|
3148
|
+
evidence: [
|
|
3149
|
+
{
|
|
3150
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3151
|
+
line: 1,
|
|
3152
|
+
snippet: `retries: ${retriesExpr}; trace: ${trace ?? 'not set'}`,
|
|
3153
|
+
},
|
|
3154
|
+
{
|
|
3155
|
+
file: `${repoPath} (CI summary)`,
|
|
3156
|
+
line: 1,
|
|
3157
|
+
snippet: `${ci.files.length} CI pipeline(s) detected; no test-results artifact publishing found`,
|
|
3158
|
+
},
|
|
3159
|
+
...ci.files.slice(0, 3).map((f) => ({
|
|
3160
|
+
file: f.file,
|
|
3161
|
+
line: 1,
|
|
3162
|
+
snippet: "No test-results or trace artifact publishing detected",
|
|
3163
|
+
})),
|
|
3164
|
+
],
|
|
3165
|
+
});
|
|
3166
|
+
}
|
|
3167
|
+
else if (publishesTraces && !publishesOnFailure) {
|
|
3168
|
+
// Artifacts are published but may not run on failure
|
|
3169
|
+
findings.push({
|
|
3170
|
+
findingId: "CI-010",
|
|
3171
|
+
severity: "medium",
|
|
3172
|
+
title: "Test artifacts may not publish on test failure",
|
|
3173
|
+
description: `Playwright is configured to retry failed tests (retries: ${retriesExpr}) and CI appears to publish test artifacts, but no 'always()' or 'failed()' condition was detected. If the artifact publishing step only runs on success, you won't have debug artifacts when tests fail.`,
|
|
3174
|
+
recommendation: "Ensure artifact publishing runs even when tests fail. In Azure Pipelines, add 'condition: always()' or 'condition: failed()' to the PublishPipelineArtifact step. In GitHub Actions, add 'if: always()' or 'if: failure()' to the upload-artifact step.",
|
|
3175
|
+
evidence: [
|
|
3176
|
+
{
|
|
3177
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3178
|
+
line: 1,
|
|
3179
|
+
snippet: `retries: ${retriesExpr}`,
|
|
3180
|
+
},
|
|
3181
|
+
{
|
|
3182
|
+
file: `${repoPath} (CI summary)`,
|
|
3183
|
+
line: 1,
|
|
3184
|
+
snippet: `Artifacts published but no 'always()' or 'failed()' condition detected`,
|
|
3185
|
+
},
|
|
3186
|
+
],
|
|
3187
|
+
});
|
|
3188
|
+
}
|
|
3189
|
+
}
|
|
3190
|
+
// CI-011: Missing `npx playwright install` entirely
|
|
3191
|
+
if (ci.files.length > 0 && !ci.mentionsPlaywrightInstall && !ci.mentionsBrowsersPreinstalled) {
|
|
3192
|
+
findings.push({
|
|
3193
|
+
findingId: "CI-011",
|
|
3194
|
+
severity: "medium",
|
|
3195
|
+
title: "No 'npx playwright install' detected in CI pipelines",
|
|
3196
|
+
description: "CI pipeline YAML was found but there is no evidence that Playwright browsers are installed (npx playwright install). Unless browsers are pre-baked into the agent image, tests will fail with missing browser errors.",
|
|
3197
|
+
recommendation: "Add 'npx playwright install --with-deps' (or 'npx playwright install chromium --with-deps' for a single browser) to your CI pipeline before running tests. If browsers are pre-installed on agents, add a comment to document this.",
|
|
3198
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
3199
|
+
file: f.file,
|
|
3200
|
+
line: 1,
|
|
3201
|
+
snippet: "No playwright install command found in this pipeline",
|
|
3202
|
+
})),
|
|
3203
|
+
});
|
|
3204
|
+
}
|
|
3205
|
+
// CI-012: CI env variable mismatch — config uses process.env.CI but pipeline doesn't set it
|
|
3206
|
+
const pwRetries = inventory.playwrightConfigSummary?.retries ?? "";
|
|
3207
|
+
const pwUsesEnvCI = inventory.playwrightConfigSummary?.envHints?.ciEnvVar === "CI"
|
|
3208
|
+
|| pwRetries.includes("process.env.CI")
|
|
3209
|
+
|| (inventory.playwrightConfigSummary?.workers ?? "").includes("process.env.CI");
|
|
3210
|
+
if (pwUsesEnvCI && ci.files.length > 0) {
|
|
3211
|
+
// Only flag Azure Pipelines (GitHub Actions sets CI=true automatically)
|
|
3212
|
+
const azureFiles = ci.files.filter((f) => f.kind === "azure_pipelines");
|
|
3213
|
+
const missingCiEnvFiles = azureFiles.filter((f) => !ci.ciEnvTrueFiles.includes(f.file));
|
|
3214
|
+
if (missingCiEnvFiles.length > 0) {
|
|
3215
|
+
const total = azureFiles.length;
|
|
3216
|
+
const missing = missingCiEnvFiles.length;
|
|
3217
|
+
findings.push({
|
|
3218
|
+
findingId: "CI-012",
|
|
3219
|
+
severity: "medium",
|
|
3220
|
+
title: "Playwright config depends on CI env var, but pipelines may not set it",
|
|
3221
|
+
description: `Playwright config references process.env.CI (e.g., retries: process.env.CI ? 2 : 0) but ${missing} of ${total} Azure Pipeline${total > 1 ? "s" : ""} do not set CI=true. Azure Pipelines do not set CI=true by default, so CI-specific configuration (retries, workers, headless) may not activate.`,
|
|
3222
|
+
recommendation: "Add 'CI: true' to the environment variables section of your Azure Pipeline YAML (map-style `CI: true` or list-style `- name: CI / value: true`). GitHub Actions sets CI=true automatically.",
|
|
3223
|
+
evidence: [
|
|
3224
|
+
{
|
|
3225
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3226
|
+
line: 1,
|
|
3227
|
+
snippet: `retries: ${pwRetries || "(uses process.env.CI)"}`,
|
|
3228
|
+
},
|
|
3229
|
+
...missingCiEnvFiles.slice(0, 5).map((f) => ({
|
|
3230
|
+
file: f.file,
|
|
3231
|
+
line: 1,
|
|
3232
|
+
snippet: "CI env var not explicitly set in this pipeline",
|
|
3233
|
+
})),
|
|
3234
|
+
],
|
|
3235
|
+
});
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
// CI-013: Pipeline timeout not set or too generous
|
|
3239
|
+
if (ci.files.length > 0 && !ci.hasPipelineTimeout) {
|
|
3240
|
+
findings.push({
|
|
3241
|
+
findingId: "CI-013",
|
|
3242
|
+
severity: "low",
|
|
3243
|
+
title: "No pipeline timeout guard detected",
|
|
3244
|
+
description: "CI pipelines don't appear to set a job/step timeout (timeoutInMinutes for Azure, timeout-minutes for GitHub Actions). Without a timeout, a stuck test run can consume CI agent time indefinitely.",
|
|
3245
|
+
recommendation: "Set a timeout on the test job: timeoutInMinutes: 60 (Azure Pipelines) or timeout-minutes: 60 (GitHub Actions). Adjust based on your expected suite duration with a reasonable buffer.",
|
|
3246
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
3247
|
+
file: f.file,
|
|
3248
|
+
line: 1,
|
|
3249
|
+
snippet: "No timeout guard detected in this pipeline",
|
|
3250
|
+
})),
|
|
3251
|
+
});
|
|
3252
|
+
}
|
|
3253
|
+
else if (ci.hasPipelineTimeout && ci.pipelineTimeoutMinutes && ci.pipelineTimeoutMinutes > 120) {
|
|
3254
|
+
findings.push({
|
|
3255
|
+
findingId: "CI-013",
|
|
3256
|
+
severity: "low",
|
|
3257
|
+
title: `Pipeline timeout is very generous (${ci.pipelineTimeoutMinutes} min)`,
|
|
3258
|
+
description: `The CI pipeline timeout is set to ${ci.pipelineTimeoutMinutes} minutes. Very long timeouts can waste CI resources when tests are stuck. Most Playwright suites should complete within 30-60 minutes even for large repos.`,
|
|
3259
|
+
recommendation: "Review whether the timeout aligns with your actual suite duration. Set it to expected_duration + 50% buffer. If tests genuinely need this long, consider sharding to reduce per-job duration.",
|
|
3260
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
3261
|
+
file: f.file,
|
|
3262
|
+
line: 1,
|
|
3263
|
+
snippet: `Pipeline timeout: ${ci.pipelineTimeoutMinutes} minutes`,
|
|
3264
|
+
})),
|
|
3265
|
+
});
|
|
3266
|
+
}
|
|
3267
|
+
// CI-014: No --reporter override in CI command
|
|
3268
|
+
if (ci.files.length > 0 && !ci.hasReporterInPlaywrightCommand) {
|
|
3269
|
+
// Only flag if there are multiple reporters in config (suggesting differentiation is intended)
|
|
3270
|
+
const reporters = inventory.playwrightConfigSummary?.reporters;
|
|
3271
|
+
const hasMultipleReporters = reporters && reporters.length > 1;
|
|
3272
|
+
if (!hasMultipleReporters) {
|
|
3273
|
+
findings.push({
|
|
3274
|
+
findingId: "CI-014",
|
|
3275
|
+
severity: "medium",
|
|
3276
|
+
title: "No --reporter override in CI playwright test command",
|
|
3277
|
+
description: "The CI pipeline runs Playwright tests without specifying a reporter override (--reporter). Playwright best-practices docs recommend CI-conditional reporters — CI environments typically benefit from machine-readable reporters (junit, html) that differ from local development preferences (list, line).",
|
|
3278
|
+
recommendation: "Add --reporter=html,junit to your CI test command, or configure reporters conditionally in playwright.config.ts using process.env.CI. This ensures CI produces both human-readable and machine-parseable output.\nhttps://playwright.dev/docs/best-practices",
|
|
3279
|
+
evidence: ci.files.slice(0, 3).map((f) => ({
|
|
3280
|
+
file: f.file,
|
|
3281
|
+
line: 1,
|
|
3282
|
+
snippet: "No --reporter flag in playwright test command",
|
|
3283
|
+
})),
|
|
3284
|
+
});
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
// INSIGHT-008: The "Debugging Black Hole" - retry debugging gap
|
|
3288
|
+
if (retriesEnabled) {
|
|
3289
|
+
const hasTraceConfig = trace && trace !== "off";
|
|
3290
|
+
const hasVideoConfig = video && video !== "off";
|
|
3291
|
+
const publishesArtifacts = ci.publishesTracesOrTestResultsDir;
|
|
3292
|
+
// Calculate severity based on how many debugging mechanisms are missing
|
|
3293
|
+
let debuggingGaps = 0;
|
|
3294
|
+
const gaps = [];
|
|
3295
|
+
if (!hasTraceConfig) {
|
|
3296
|
+
debuggingGaps++;
|
|
3297
|
+
gaps.push("trace not enabled");
|
|
3298
|
+
}
|
|
3299
|
+
if (!hasVideoConfig) {
|
|
3300
|
+
debuggingGaps++;
|
|
3301
|
+
gaps.push("video not enabled");
|
|
3302
|
+
}
|
|
3303
|
+
if (!publishesArtifacts && ci.files.length > 0) {
|
|
3304
|
+
debuggingGaps++;
|
|
3305
|
+
gaps.push("artifacts not published in CI");
|
|
3306
|
+
}
|
|
3307
|
+
// Only trigger if at least 2 debugging mechanisms are missing
|
|
3308
|
+
if (debuggingGaps >= 2) {
|
|
3309
|
+
const severity = debuggingGaps === 3 ? "high" : "medium";
|
|
3310
|
+
findings.push({
|
|
3311
|
+
findingId: "PW-INSIGHT-008",
|
|
3312
|
+
severity,
|
|
3313
|
+
title: "Debugging black hole detected (retries enabled without debug artifacts)",
|
|
3314
|
+
description: `Tests are configured to retry on failure (retries: ${retriesExpr}), but critical debugging mechanisms are missing: ${gaps.join(", ")}. When tests fail after retries, you'll have no way to understand what went wrong. This creates a "debugging black hole" where flaky tests are invisible until they become blocking incidents.`,
|
|
3315
|
+
recommendation: "Enable the full debugging stack: (1) Set trace: 'retain-on-failure' or 'on-first-retry' in playwright.config.ts; (2) Set video: 'on-first-retry'; (3) Publish the test-results folder as a CI artifact (use condition: always() so it runs even on failure). This ensures every retry failure is debuggable.",
|
|
3316
|
+
evidence: [
|
|
3317
|
+
{
|
|
3318
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3319
|
+
line: 1,
|
|
3320
|
+
snippet: `retries: ${retriesExpr}; debugging gaps: ${gaps.join(", ")}`,
|
|
3321
|
+
},
|
|
3322
|
+
...(!hasTraceConfig
|
|
3323
|
+
? [
|
|
3324
|
+
{
|
|
3325
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3326
|
+
line: 1,
|
|
3327
|
+
snippet: trace ? `trace: ${trace} (insufficient)` : "trace: not configured",
|
|
3328
|
+
},
|
|
3329
|
+
]
|
|
3330
|
+
: []),
|
|
3331
|
+
...(!publishesArtifacts && ci.files.length > 0
|
|
3332
|
+
? [
|
|
3333
|
+
{
|
|
3334
|
+
file: `${repoPath} (CI summary)`,
|
|
3335
|
+
line: 1,
|
|
3336
|
+
snippet: "No test-results artifact publishing found in CI pipelines",
|
|
3337
|
+
},
|
|
3338
|
+
]
|
|
3339
|
+
: []),
|
|
3340
|
+
],
|
|
3341
|
+
});
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3344
|
+
// PERF-001 Insight: Correlate missing network waits with hard waits
|
|
3345
|
+
const networkIdleGoto = hasFinding(findings, "PW-PERF-001");
|
|
3346
|
+
if (networkIdleGoto && hardWaits) {
|
|
3347
|
+
findings.push({
|
|
3348
|
+
findingId: "PW-INSIGHT-009",
|
|
3349
|
+
severity: "high",
|
|
3350
|
+
title: "Unreliable networkidle wait compounded by hard waits",
|
|
3351
|
+
description: "Tests use page.goto() with waitUntil: 'networkidle' AND contain hard waits (waitForTimeout/setTimeout). The combination of unreliable networkidle detection with arbitrary delays suggests deep timing instabilities that should be resolved with proper wait strategies.",
|
|
3352
|
+
recommendation: "Remove waitUntil: 'networkidle' from page.goto() calls (the default 'load' is sufficient). Replace hard waits with page.waitForResponse() for specific API calls, or expect(locator).toBeVisible() for UI elements. Remove setTimeout/waitForTimeout delays once proper waits are in place.\nhttps://playwright.dev/docs/api/class-frame#frame-wait-for-url",
|
|
3353
|
+
evidence: [
|
|
3354
|
+
{
|
|
3355
|
+
file: `${repoPath} (correlation analysis)`,
|
|
3356
|
+
line: 1,
|
|
3357
|
+
snippet: "Detected PW-PERF-001 + PW-FLAKE-001: unreliable waits compounding",
|
|
3358
|
+
},
|
|
3359
|
+
],
|
|
3360
|
+
});
|
|
3361
|
+
}
|
|
3362
|
+
// PERF-002 Insight: Correlate hardcoded URLs with missing baseURL and high parallelism
|
|
3363
|
+
const hardcodedUrls = hasFinding(findings, "PW-PERF-002");
|
|
3364
|
+
const noBaseUrl = !pw?.use?.baseURL;
|
|
3365
|
+
// PW-PERF-002 severity downgrade: if baseURL IS configured, hardcoded URLs are less
|
|
3366
|
+
// critical (they may be specific API endpoints, not base navigation).
|
|
3367
|
+
if (hardcodedUrls && pw?.use?.baseURL) {
|
|
3368
|
+
const perfFinding = findings.find(f => f.findingId === "PW-PERF-002");
|
|
3369
|
+
if (perfFinding)
|
|
3370
|
+
perfFinding.severity = "low";
|
|
3371
|
+
}
|
|
3372
|
+
if (hardcodedUrls && noBaseUrl) {
|
|
3373
|
+
const workersNum = workers && /^\d+$/.test(workers) ? parseInt(workers) : 0;
|
|
3374
|
+
const severity = workersNum > 4 ? "high" : "medium";
|
|
3375
|
+
findings.push({
|
|
3376
|
+
findingId: "PW-INSIGHT-010",
|
|
3377
|
+
severity,
|
|
3378
|
+
title: noBaseUrl
|
|
3379
|
+
? "Hardcoded URLs without baseURL configuration"
|
|
3380
|
+
: "Potential environment flexibility issue",
|
|
3381
|
+
description: noBaseUrl
|
|
3382
|
+
? `Tests contain hardcoded URLs (http://, https://) but playwright.config.ts does not define baseURL. This makes it difficult to run tests against different environments (dev, staging, production) and violates DRY principles.${workersNum > 4 ? ` With ${workers} workers configured, environment setup becomes even more critical for avoiding test interference.` : ""}`
|
|
3383
|
+
: `Tests contain hardcoded URLs. Consider using baseURL for better environment management.`,
|
|
3384
|
+
recommendation: noBaseUrl
|
|
3385
|
+
? "Add baseURL to playwright.config.ts use block and refactor tests to use relative paths. For example: baseURL: process.env.BASE_URL || 'https://staging.example.com', then use page.goto('/login') instead of page.goto('https://example.com/login'). This enables easy environment switching via environment variables. https://playwright.dev/docs/api/class-testoptions#test-options-base-url"
|
|
3386
|
+
: "Refactor tests to use baseURL and relative paths for better maintainability.",
|
|
3387
|
+
evidence: [
|
|
3388
|
+
{
|
|
3389
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3390
|
+
line: 1,
|
|
3391
|
+
snippet: noBaseUrl
|
|
3392
|
+
? `baseURL not configured; hardcoded URLs detected${workersNum > 4 ? `; workers: ${workers}` : ""}`
|
|
3393
|
+
: "baseURL present but hardcoded URLs still used",
|
|
3394
|
+
},
|
|
3395
|
+
],
|
|
3396
|
+
});
|
|
3397
|
+
}
|
|
3398
|
+
// ============================================
|
|
3399
|
+
// PW-PERF-003: networkidle combined with high timeout
|
|
3400
|
+
// ============================================
|
|
3401
|
+
const hasNetworkIdle = hasFinding(findings, "PW-FLAKE-004");
|
|
3402
|
+
if (hasNetworkIdle && testTimeoutMs && testTimeoutMs > 60000) {
|
|
3403
|
+
findings.push({
|
|
3404
|
+
findingId: "PW-PERF-003",
|
|
3405
|
+
severity: "medium",
|
|
3406
|
+
title: "networkidle combined with high timeout — hidden slow tests",
|
|
3407
|
+
description: `Tests use waitForLoadState('networkidle') while the test timeout is set to ${Math.round(testTimeoutMs / 1000)}s. networkidle waits for zero network activity for 500ms, and combined with a generous timeout, tests may silently wait much longer than necessary without failing.`,
|
|
3408
|
+
recommendation: "Replace networkidle with specific waits (waitForResponse, expect(locator).toBeVisible()). Reduce test timeout to 30-60s to surface slow tests earlier. networkidle is unreliable in SPAs with background requests.\nhttps://playwright.dev/docs/api/class-page#page-wait-for-load-state",
|
|
3409
|
+
evidence: [
|
|
3410
|
+
{
|
|
3411
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3412
|
+
line: 1,
|
|
3413
|
+
snippet: `timeout: ${testTimeoutMs}ms + networkidle usage detected`,
|
|
3414
|
+
},
|
|
3415
|
+
],
|
|
3416
|
+
});
|
|
3417
|
+
}
|
|
3418
|
+
// ============================================
|
|
3419
|
+
// PW-INSIGHT-011: Passes without explicit checks
|
|
3420
|
+
// ============================================
|
|
3421
|
+
const noAssertions = hasFinding(findings, "PW-STABILITY-004");
|
|
3422
|
+
if (noAssertions && retriesEnabled) {
|
|
3423
|
+
// Pull evidence from PW-STABILITY-004, then remove it — INSIGHT-011 supersedes it
|
|
3424
|
+
const stability004Idx = findings.findIndex((f) => f.findingId === "PW-STABILITY-004");
|
|
3425
|
+
const stability004 = stability004Idx >= 0 ? findings[stability004Idx] : undefined;
|
|
3426
|
+
const testFileEvidence = (stability004?.evidence ?? []).slice(0, 10).map((e) => ({
|
|
3427
|
+
file: e.file,
|
|
3428
|
+
line: e.line,
|
|
3429
|
+
snippet: e.snippet,
|
|
3430
|
+
}));
|
|
3431
|
+
// Remove PW-STABILITY-004 — the insight is strictly more informative
|
|
3432
|
+
if (stability004Idx >= 0)
|
|
3433
|
+
findings.splice(stability004Idx, 1);
|
|
3434
|
+
findings.push({
|
|
3435
|
+
findingId: "PW-INSIGHT-011",
|
|
3436
|
+
severity: "high",
|
|
3437
|
+
title: "Passes without explicit checks (no in-test assertions + retries)",
|
|
3438
|
+
description: `${stability004 ? stability004.description + " " : ""}Combined with enabled retries, these tests create an illusion of quality, they may never fail, but count toward pass rates. This is worse than having no tests and creates false confidence.`,
|
|
3439
|
+
recommendation: "Add meaningful expect() assertions to every test. Keep Page Objects focused on actions and state accessors; put assertions in the test (or in small, domain-named assertion helpers) so intent remains explicit. A test without an assertion provides no quality signal. Add a linter rule to require at least one assertion per test",
|
|
3440
|
+
evidence: [
|
|
3441
|
+
...testFileEvidence,
|
|
3442
|
+
{
|
|
3443
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3444
|
+
line: 1,
|
|
3445
|
+
snippet: `retries: ${retriesExpr ?? "unknown"} (amplifies the risk from assertion-free tests above)`,
|
|
3446
|
+
},
|
|
3447
|
+
],
|
|
3448
|
+
totalOccurrences: (stability004?.totalOccurrences ?? stability004?.evidence.length ?? 0) + 1,
|
|
3449
|
+
affectedFiles: (stability004?.affectedFiles ?? new Set(stability004?.evidence.map(e => e.file)).size ?? 0) + 1,
|
|
3450
|
+
});
|
|
3451
|
+
}
|
|
3452
|
+
// ============================================
|
|
3453
|
+
// PW-INSIGHT-012: Suite scale vs CI parallelism mismatch
|
|
3454
|
+
// ============================================
|
|
3455
|
+
const testCount = inventory.testFiles;
|
|
3456
|
+
const workersValue = pw?.workers;
|
|
3457
|
+
const workersNum = workersValue && /^\d+$/.test(workersValue) ? parseInt(workersValue) : undefined;
|
|
3458
|
+
const effectiveWorkers = workersNum ?? (pwUsesWorkersEnv && ci.setsWorkersEnv ? undefined : 1);
|
|
3459
|
+
if (testCount >= 500 && effectiveWorkers !== undefined && effectiveWorkers <= 2) {
|
|
3460
|
+
const estimatedMinutes = Math.round((testCount * 30) / 60 / effectiveWorkers);
|
|
3461
|
+
findings.push({
|
|
3462
|
+
findingId: "PW-INSIGHT-012",
|
|
3463
|
+
severity: "high",
|
|
3464
|
+
title: `Suite scale vs. CI parallelism mismatch (${testCount} tests, ${effectiveWorkers} worker${effectiveWorkers > 1 ? "s" : ""})`,
|
|
3465
|
+
description: `This repository has ${testCount} test files but workers appear to be set to ${effectiveWorkers}. At this scale, tests would take an estimated ~${estimatedMinutes} minutes serially. This is almost certainly too slow for practical CI feedback.`,
|
|
3466
|
+
recommendation: `Increase workers to at least 4 (or use process.env.WORKERS for environment flexibility). Combined with sharding across ${testCount > 500 ? 4 : 2} CI jobs, this could reduce CI time by 75%+. Use 'workers: process.env.CI ? 4 : 2' for balanced local/CI behavior.`,
|
|
3467
|
+
evidence: [
|
|
3468
|
+
{
|
|
3469
|
+
file: pw?.configPath ?? "playwright.config",
|
|
3470
|
+
line: 1,
|
|
3471
|
+
snippet: `${testCount} test files; workers: ${workersValue ?? "default (1)"}; estimated ${estimatedMinutes} min`,
|
|
3472
|
+
},
|
|
3473
|
+
],
|
|
3474
|
+
});
|
|
3475
|
+
}
|
|
3476
|
+
// ============================================
|
|
3477
|
+
// PW-INSIGHT-013: Over-targeted test scope (E2E only)
|
|
3478
|
+
// ============================================
|
|
3479
|
+
// If all/most tests use page.goto() to full URLs, it suggests an E2E-only strategy
|
|
3480
|
+
// with no component-level or API-level testing.
|
|
3481
|
+
const gotoFinding = findings.find(f => f.findingId === "PW-PERF-002");
|
|
3482
|
+
const gotoEvidenceCount = gotoFinding?.evidence?.length ?? 0;
|
|
3483
|
+
if (testCount >= 20 && gotoEvidenceCount >= Math.min(testCount * 0.7, 15) && noBaseUrl) {
|
|
3484
|
+
findings.push({
|
|
3485
|
+
findingId: "PW-INSIGHT-013",
|
|
3486
|
+
severity: "low",
|
|
3487
|
+
title: "E2E-only test strategy detected (no component/API test layer)",
|
|
3488
|
+
description: `Most tests (${gotoEvidenceCount}+ instances) navigate to full URLs, suggesting an end-to-end-only testing strategy. While E2E tests are valuable, they are slow and fragile. A testing pyramid with API-level and component-level tests provides faster feedback and better isolation.`,
|
|
3489
|
+
recommendation: "Consider adding API-level tests (using request context) for business logic validation and component tests for UI behavior. This creates a testing pyramid: many fast API tests, some component tests, fewer E2E tests. Playwright supports all three.\nhttps://playwright.dev/docs/api-testing",
|
|
3490
|
+
evidence: [
|
|
3491
|
+
{
|
|
3492
|
+
file: `${repoPath} (analysis summary)`,
|
|
3493
|
+
line: 1,
|
|
3494
|
+
snippet: `${testCount} test files with ${gotoEvidenceCount}+ full-URL navigations; no baseURL configured`,
|
|
3495
|
+
},
|
|
3496
|
+
],
|
|
3497
|
+
});
|
|
3498
|
+
}
|
|
3499
|
+
// ============================================
|
|
3500
|
+
// POST-PROCESSING: Apply .auditrc.json overrides
|
|
3501
|
+
// ============================================
|
|
3502
|
+
const auditConfig = options.auditConfig;
|
|
3503
|
+
if (auditConfig) {
|
|
3504
|
+
const disabled = auditConfig.rules?.disabled ?? [];
|
|
3505
|
+
// Remove disabled findings
|
|
3506
|
+
for (let i = findings.length - 1; i >= 0; i--) {
|
|
3507
|
+
if (disabled.includes(findings[i].findingId)) {
|
|
3508
|
+
findings.splice(i, 1);
|
|
3509
|
+
}
|
|
3510
|
+
}
|
|
3511
|
+
// Apply severity overrides
|
|
3512
|
+
for (const f of findings) {
|
|
3513
|
+
const override = auditConfig.rules?.severityOverrides?.[f.findingId];
|
|
3514
|
+
if (override)
|
|
3515
|
+
f.severity = override;
|
|
3516
|
+
}
|
|
3517
|
+
}
|
|
3518
|
+
return { repoPath, inventory, findings, gitBranch: git.branch, gitCommit: git.commit };
|
|
3519
|
+
}
|