@qaguardian/playwright-score 2.1.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +159 -0
  2. package/METHODOLOGY.md +42 -1
  3. package/README.md +1 -1
  4. package/dist/cli.js +61 -5
  5. package/dist/confidence.d.ts +30 -0
  6. package/dist/confidence.js +67 -0
  7. package/dist/eslint-runner.js +18 -0
  8. package/dist/finding-triage.d.ts +13 -0
  9. package/dist/finding-triage.js +470 -0
  10. package/dist/formatters/json.js +4 -1
  11. package/dist/formatters/markdown.d.ts +2 -0
  12. package/dist/formatters/markdown.js +17 -5
  13. package/dist/formatters/report-css.d.ts +2 -0
  14. package/dist/formatters/report-css.js +68 -0
  15. package/dist/formatters/report.d.ts +20 -0
  16. package/dist/formatters/report.js +279 -0
  17. package/dist/formatters/text.d.ts +6 -1
  18. package/dist/formatters/text.js +73 -27
  19. package/dist/index.js +21 -8
  20. package/dist/metrics.js +33 -2
  21. package/dist/playwright-api-names.d.ts +10 -0
  22. package/dist/playwright-api-names.js +35 -0
  23. package/dist/profiles.js +14 -0
  24. package/dist/report-model.d.ts +52 -0
  25. package/dist/report-model.js +162 -0
  26. package/dist/report-source.d.ts +11 -0
  27. package/dist/report-source.js +96 -0
  28. package/dist/rule-copy.d.ts +14 -0
  29. package/dist/rule-copy.js +215 -0
  30. package/dist/rules/ast-util.d.ts +60 -0
  31. package/dist/rules/ast-util.js +140 -0
  32. package/dist/rules/base-plugin.d.ts +19 -5
  33. package/dist/rules/base-plugin.js +245 -8
  34. package/dist/rules/no-brittle-selector.d.ts +3 -0
  35. package/dist/rules/no-brittle-selector.js +150 -0
  36. package/dist/rules/no-excessive-timeout.d.ts +2 -0
  37. package/dist/rules/no-excessive-timeout.js +87 -0
  38. package/dist/rules/no-fixme-test.d.ts +14 -0
  39. package/dist/rules/no-fixme-test.js +53 -0
  40. package/dist/rules/no-self-fulfilling-fallback.d.ts +29 -0
  41. package/dist/rules/no-self-fulfilling-fallback.js +221 -0
  42. package/dist/rules/no-shared-mutable-state.d.ts +4 -0
  43. package/dist/rules/no-shared-mutable-state.js +186 -0
  44. package/dist/rules/no-swallowed-assertion.d.ts +13 -0
  45. package/dist/rules/no-swallowed-assertion.js +113 -0
  46. package/dist/rules/no-unawaited-action.d.ts +2 -0
  47. package/dist/rules/no-unawaited-action.js +77 -0
  48. package/dist/rules/no-unawaited-expect-value.d.ts +20 -0
  49. package/dist/rules/no-unawaited-expect-value.js +46 -0
  50. package/dist/rules/no-weak-assertion.d.ts +19 -0
  51. package/dist/rules/no-weak-assertion.js +82 -0
  52. package/dist/rules/rule-util.d.ts +56 -0
  53. package/dist/rules/rule-util.js +253 -0
  54. package/dist/score-engine.d.ts +5 -0
  55. package/dist/score-engine.js +10 -9
  56. package/dist/suite-name.d.ts +11 -0
  57. package/dist/suite-name.js +36 -0
  58. package/dist/test-aliases.js +144 -3
  59. package/dist/types.d.ts +18 -0
  60. package/package.json +1 -1
@@ -0,0 +1,470 @@
1
+ /**
2
+ * Post-lint triage of two upstream rules whose own checks cannot see across
3
+ * files or tell a guard from real branching (QAG-287):
4
+ *
5
+ * - `playwright/expect-expect` only sees `expect(...)` written in the test
6
+ * body. A test that asserts through a page-object method, an injected
7
+ * fixture or an imported helper was reported as "no assertion" at error
8
+ * severity even though the assertion runs. Each such finding is resolved
9
+ * here: dropped when a fixture the test uses is known to assert, turned
10
+ * into a report-only "needs a human check" finding when the test calls
11
+ * something this static scan cannot read, and kept as the error it is
12
+ * when every call resolves to code we could read (or is plain Playwright
13
+ * API on `page`, `request`, ...).
14
+ * - `playwright/valid-expect` fires on a helper whose name ends in
15
+ * "Expect" (`clickSortableHeaderAndExpect(...)`) as if it were expect().
16
+ * - `playwright/no-conditional-in-test` fires on a guard whose only body is
17
+ * `throw`. That is an assertion in disguise, not branching test logic.
18
+ *
19
+ * Purely syntactic and name-based, like the rest of this package: a
20
+ * resolved-by-name match is the same trade-off `assertFunctionNames` makes.
21
+ */
22
+ import path from 'node:path';
23
+ import { parseForESLint } from '@typescript-eslint/parser';
24
+ import { resolveRelativeImport } from './import-graph.js';
25
+ import { PLAYWRIGHT_API_METHODS } from './playwright-api-names.js';
26
+ export const ASSERTION_UNVERIFIED_RULE = 'playwright-score/assertion-unverified';
27
+ /** Identifiers whose calls are Playwright API or the language runtime, never a hidden assertion helper. */
28
+ const NEUTRAL_ROOTS = new Set([
29
+ 'page', 'context', 'request', 'browser', 'browserName', 'baseURL', 'testInfo', 'playwright',
30
+ 'expect', 'test', 'it', 'describe', 'console', 'Promise', 'JSON', 'Math', 'Date', 'Object',
31
+ 'Array', 'String', 'Number', 'Boolean', 'process', 'setTimeout', 'setInterval', 'require',
32
+ 'URL', 'Buffer', 'fetch', 'undefined', 'globalThis', 'window', 'document',
33
+ ]);
34
+ // Word-boundary, case-sensitive (same idea as the assertFunctionPatterns in
35
+ // eslint-runner.ts): `checkout`, `toISOString`, `toLowerCase` or `shop.checkout()`
36
+ // are not assertions, `checkFoo`, `assert_ok` and `toBeVisible` are.
37
+ const ASSERTIONISH_PREFIX = /^(assert|verify|check|validate|expect|ensure|should|confirm)([A-Z_0-9]|$)/;
38
+ const ASSERTIONISH_TO = /^to(Be|Have|Contain|Match|Equal|Throw)/;
39
+ /** JS / Node built-ins: `new Date().toISOString()` is never a page-object call. */
40
+ const BUILTIN_CLASSES = new Set([
41
+ 'Date', 'Map', 'Set', 'WeakMap', 'WeakSet', 'URL', 'URLSearchParams', 'Array', 'Error', 'TypeError',
42
+ 'RangeError', 'RegExp', 'Promise', 'Object', 'String', 'Number', 'Boolean', 'Buffer', 'Uint8Array',
43
+ 'ArrayBuffer', 'Intl', 'AbortController', 'TextEncoder', 'TextDecoder', 'FormData', 'Headers', 'Blob',
44
+ ]);
45
+ /**
46
+ * Only a method whose NAME reads like an assertion may be a hidden one:
47
+ * `app.waitForLoad()` or `users.list()` on an unreadable object is just an
48
+ * unverified call, and the test stays an expect-expect error. A name that is
49
+ * a Playwright API verb (`check`, `fill`, `goto`) is an action whatever the
50
+ * receiver: `loginPage.rememberMe.check()` is a locator held in a property.
51
+ */
52
+ function isAssertionLikeCall(name) {
53
+ if (!ASSERTIONISH_PREFIX.test(name) && !ASSERTIONISH_TO.test(name))
54
+ return false;
55
+ return !PLAYWRIGHT_API_METHODS.has(name);
56
+ }
57
+ function walk(node, visit, parent) {
58
+ if (!node || typeof node !== 'object')
59
+ return;
60
+ if (Array.isArray(node)) {
61
+ for (const item of node)
62
+ walk(item, visit, parent);
63
+ return;
64
+ }
65
+ if (typeof node.type !== 'string')
66
+ return;
67
+ visit(node, parent);
68
+ for (const key of Object.keys(node)) {
69
+ if (key === 'parent')
70
+ continue;
71
+ const v = node[key];
72
+ if (v && typeof v === 'object')
73
+ walk(v, visit, node);
74
+ }
75
+ }
76
+ function parse(astCache, source, file) {
77
+ const hit = astCache.get(file);
78
+ if (hit && hit.source === source)
79
+ return hit.ast;
80
+ let ast;
81
+ try {
82
+ ast = parseForESLint(source, {
83
+ ecmaVersion: 2022,
84
+ sourceType: 'module',
85
+ ecmaFeatures: { jsx: /\.(tsx|jsx)$/.test(file) },
86
+ }).ast;
87
+ }
88
+ catch {
89
+ ast = undefined;
90
+ }
91
+ astCache.set(file, { source, ast });
92
+ return ast;
93
+ }
94
+ function isExpectCall(n) {
95
+ if (n?.type !== 'CallExpression')
96
+ return false;
97
+ const c = n.callee;
98
+ if (c?.type === 'Identifier' && c.name === 'expect')
99
+ return true;
100
+ return c?.type === 'MemberExpression' && c.object?.type === 'Identifier' && c.object.name === 'expect';
101
+ }
102
+ function containsExpect(node) {
103
+ let found = false;
104
+ walk(node, (n) => {
105
+ if (!found && isExpectCall(n))
106
+ found = true;
107
+ });
108
+ return found;
109
+ }
110
+ function isFn(n) {
111
+ return n?.type === 'ArrowFunctionExpression' || n?.type === 'FunctionExpression';
112
+ }
113
+ function propKeyName(p) {
114
+ if (p?.type !== 'Property' && p?.type !== 'ObjectProperty')
115
+ return undefined;
116
+ if (p.key?.type === 'Identifier' && !p.computed)
117
+ return p.key.name;
118
+ if (p.key?.type === 'Literal' && typeof p.key.value === 'string')
119
+ return p.key.value;
120
+ return undefined;
121
+ }
122
+ /** Does the fixture function hand a function containing expect(...) to use(...)? */
123
+ function usesAssertingCallable(fixtureFn) {
124
+ const p1 = fixtureFn.params?.[1];
125
+ const useName = p1?.type === 'Identifier' ? p1.name : 'use';
126
+ let found = false;
127
+ walk(fixtureFn.body, (n) => {
128
+ if (found || n.type !== 'CallExpression' || n.callee?.type !== 'Identifier' || n.callee.name !== useName)
129
+ return;
130
+ if ((n.arguments ?? []).some((a) => isFn(a) && containsExpect(a)))
131
+ found = true;
132
+ });
133
+ return found;
134
+ }
135
+ function gather(ctx, cache) {
136
+ const defined = new Set();
137
+ const assertingFixtures = new Map();
138
+ const fixtureKeys = new Map();
139
+ const all = [...ctx.sources, ...ctx.importedFiles];
140
+ for (const [file, source] of all) {
141
+ const ast = parse(cache, source, file);
142
+ if (!ast)
143
+ continue;
144
+ walk(ast, (n) => {
145
+ if (n.type === 'FunctionDeclaration' && n.id?.name)
146
+ defined.add(n.id.name);
147
+ else if (n.type === 'ClassDeclaration' && n.id?.name)
148
+ defined.add(n.id.name);
149
+ else if (n.type === 'VariableDeclarator' && n.id?.type === 'Identifier' && isFn(n.init)) {
150
+ defined.add(n.id.name);
151
+ }
152
+ else if (n.type === 'MethodDefinition' && !n.computed && n.key?.type === 'Identifier') {
153
+ defined.add(n.key.name);
154
+ }
155
+ else if (n.type === 'PropertyDefinition' && !n.computed && n.key?.type === 'Identifier' && isFn(n.value)) {
156
+ defined.add(n.key.name);
157
+ }
158
+ else if (n.type === 'Property' && isFn(n.value)) {
159
+ const k = propKeyName(n);
160
+ if (k)
161
+ defined.add(k);
162
+ }
163
+ else if (n.type === 'CallExpression' &&
164
+ n.callee?.type === 'MemberExpression' &&
165
+ n.callee.property?.name === 'extend' &&
166
+ n.arguments?.[0]?.type === 'ObjectExpression') {
167
+ for (const p of n.arguments[0].properties ?? []) {
168
+ const k = propKeyName(p);
169
+ if (k) {
170
+ if (!fixtureKeys.has(file))
171
+ fixtureKeys.set(file, new Set());
172
+ fixtureKeys.get(file).add(k);
173
+ }
174
+ if (!k || NEUTRAL_ROOTS.has(k) || !isFn(p.value) || !usesAssertingCallable(p.value))
175
+ continue;
176
+ if (!assertingFixtures.has(file))
177
+ assertingFixtures.set(file, new Set());
178
+ assertingFixtures.get(file).add(k);
179
+ }
180
+ }
181
+ });
182
+ }
183
+ return { defined, assertingFixtures, fixtureKeys, cache };
184
+ }
185
+ /** Names bound by an identifier, object or array pattern. */
186
+ function boundNames(pat, out = []) {
187
+ if (!pat)
188
+ return out;
189
+ if (pat.type === 'Identifier')
190
+ out.push(pat.name);
191
+ else if (pat.type === 'ObjectPattern')
192
+ for (const p of pat.properties ?? [])
193
+ boundNames(p.type === 'RestElement' ? p.argument : p.value, out);
194
+ else if (pat.type === 'ArrayPattern')
195
+ for (const e of pat.elements ?? [])
196
+ boundNames(e, out);
197
+ else if (pat.type === 'AssignmentPattern')
198
+ boundNames(pat.left, out);
199
+ else if (pat.type === 'RestElement')
200
+ boundNames(pat.argument, out);
201
+ return out;
202
+ }
203
+ /** Root identifier / expression of a call or member chain. */
204
+ function chainRoot(node) {
205
+ let cur = node;
206
+ for (let i = 0; i < 64 && cur; i++) {
207
+ if (cur.type === 'AwaitExpression')
208
+ cur = cur.argument;
209
+ else if (cur.type === 'TSNonNullExpression' || cur.type === 'ChainExpression')
210
+ cur = cur.expression;
211
+ else if (cur.type === 'TSAsExpression' || cur.type === 'TSTypeAssertion')
212
+ cur = cur.expression;
213
+ else if (cur.type === 'CallExpression')
214
+ cur = cur.callee;
215
+ else if (cur.type === 'MemberExpression')
216
+ cur = cur.object;
217
+ else
218
+ break;
219
+ }
220
+ return cur;
221
+ }
222
+ function importKind(specifier) {
223
+ if (specifier.startsWith('.') || /^(@\/|~\/|#|src\/|tests?\/|e2e\/|pages\/|fixtures\/|utils?\/|lib\/)/.test(specifier)) {
224
+ return 'external';
225
+ }
226
+ return 'lib';
227
+ }
228
+ /**
229
+ * Calls inside a test callback whose receiver lives somewhere this scan
230
+ * cannot read: names of those calls (empty when every call is plain
231
+ * Playwright API, a same-file/readable helper, or a utility library).
232
+ */
233
+ function unresolvedCalls(testCall, ast, know) {
234
+ const fn = [...(testCall.arguments ?? [])].reverse().find(isFn);
235
+ if (!fn)
236
+ return [];
237
+ const imports = new Map();
238
+ for (const stmt of ast.body ?? []) {
239
+ if (stmt.type !== 'ImportDeclaration' || typeof stmt.source?.value !== 'string')
240
+ continue;
241
+ for (const s of stmt.specifiers ?? []) {
242
+ if (s.local?.name)
243
+ imports.set(s.local.name, importKind(stmt.source.value));
244
+ }
245
+ }
246
+ const kinds = new Map();
247
+ const first = fn.params?.[0];
248
+ if (first?.type === 'ObjectPattern') {
249
+ for (const p of first.properties ?? []) {
250
+ const local = p.value?.type === 'AssignmentPattern' ? p.value.left : p.value;
251
+ const name = local?.type === 'Identifier' ? local.name : undefined;
252
+ if (name)
253
+ kinds.set(name, NEUTRAL_ROOTS.has(propKeyName(p) ?? name) ? 'neutral' : 'external');
254
+ }
255
+ }
256
+ const rootKind = (root) => {
257
+ if (root?.type === 'NewExpression') {
258
+ const cls = root.callee?.name;
259
+ if (cls && (BUILTIN_CLASSES.has(cls) || NEUTRAL_ROOTS.has(cls)))
260
+ return 'neutral';
261
+ return cls && know.defined.has(cls) ? 'neutral' : 'external';
262
+ }
263
+ if (root?.type !== 'Identifier')
264
+ return 'neutral';
265
+ const local = kinds.get(root.name);
266
+ if (local)
267
+ return local;
268
+ if (NEUTRAL_ROOTS.has(root.name))
269
+ return 'neutral';
270
+ return imports.get(root.name) ?? 'neutral';
271
+ };
272
+ const unresolved = [];
273
+ walk(fn.body, (n) => {
274
+ if (n.type === 'VariableDeclarator' && n.init) {
275
+ // `const x = ...` and destructuring (`const { a, b: [c] } = await ...`):
276
+ // every bound name takes the kind of the initializer's root.
277
+ const kind = rootKind(chainRoot(n.init));
278
+ for (const bound of boundNames(n.id))
279
+ kinds.set(bound, kind);
280
+ return;
281
+ }
282
+ if (n.type !== 'CallExpression')
283
+ return;
284
+ const c = n.callee;
285
+ let name;
286
+ if (c?.type === 'Identifier')
287
+ name = c.name;
288
+ else if (c?.type === 'MemberExpression' && !c.computed && c.property?.type === 'Identifier')
289
+ name = c.property.name;
290
+ if (!name || know.defined.has(name))
291
+ return;
292
+ const kind = rootKind(chainRoot(c));
293
+ if ((kind === 'external' || kind === 'lib') && isAssertionLikeCall(name))
294
+ unresolved.push(name);
295
+ });
296
+ return unresolved;
297
+ }
298
+ /** The spec file plus every file reachable from it through relative imports / re-exports. */
299
+ function reachableFiles(spec, ctx, know) {
300
+ const boundary = ctx.importBoundary ?? ctx.filesBase;
301
+ const seen = new Set([spec]);
302
+ const queue = [spec];
303
+ while (queue.length) {
304
+ const file = queue.pop();
305
+ const source = ctx.sources.get(file) ?? ctx.importedFiles.get(file);
306
+ const ast = source === undefined ? undefined : parse(know.cache, source, file);
307
+ for (const stmt of ast?.body ?? []) {
308
+ const spec2 = stmt.source?.value;
309
+ if (typeof spec2 !== 'string' || !spec2.startsWith('.'))
310
+ continue;
311
+ const target = resolveRelativeImport(file, spec2, boundary);
312
+ if (target && !seen.has(target)) {
313
+ seen.add(target);
314
+ queue.push(target);
315
+ }
316
+ }
317
+ }
318
+ return seen;
319
+ }
320
+ /** Local names of destructured test args that are asserting fixtures reachable from this spec. */
321
+ function assertingFixtureLocals(testFn, reachable, know) {
322
+ const locals = new Set();
323
+ for (const p of testFn?.params?.[0]?.properties ?? []) {
324
+ const key = propKeyName(p);
325
+ if (!key || NEUTRAL_ROOTS.has(key))
326
+ continue;
327
+ if (![...reachable].some((f) => know.assertingFixtures.get(f)?.has(key)))
328
+ continue;
329
+ const local = p.value?.type === 'AssignmentPattern' ? p.value.left : p.value;
330
+ if (local?.type === 'Identifier')
331
+ locals.add(local.name);
332
+ }
333
+ return locals;
334
+ }
335
+ /**
336
+ * Fixtures the test destructures AND calls (`a11y()`) that no reachable file
337
+ * defines, while its `test` comes from a non-relative module this scan does
338
+ * not resolve (a tsconfig `paths` alias such as `@fx/base`, a workspace
339
+ * package). The fixture may well assert there, so it is "unverified", not
340
+ * "no assertion". Relative imports are followed by reachableFiles instead.
341
+ */
342
+ function unresolvedFixtureCalls(call, ast, fn, reachable, know) {
343
+ const testRoot = chainRoot(call.callee);
344
+ if (testRoot?.type !== 'Identifier')
345
+ return [];
346
+ const fromUnresolvedModule = (ast.body ?? []).some((stmt) => stmt.type === 'ImportDeclaration' &&
347
+ typeof stmt.source?.value === 'string' &&
348
+ !stmt.source.value.startsWith('.') &&
349
+ !/^(@playwright\/|playwright|node:)/.test(stmt.source.value) &&
350
+ (stmt.specifiers ?? []).some((sp) => sp.local?.name === testRoot.name));
351
+ if (!fromUnresolvedModule)
352
+ return [];
353
+ const out = [];
354
+ for (const p of fn?.params?.[0]?.properties ?? []) {
355
+ const key = propKeyName(p);
356
+ if (!key || NEUTRAL_ROOTS.has(key))
357
+ continue;
358
+ if ([...reachable].some((f) => know.fixtureKeys.get(f)?.has(key)))
359
+ continue;
360
+ const local = p.value?.type === 'AssignmentPattern' ? p.value.left : p.value;
361
+ if (local?.type === 'Identifier' && callsAny(fn.body, new Set([local.name])))
362
+ out.push(key);
363
+ }
364
+ return out;
365
+ }
366
+ /** Does the body call `name(...)` directly (the fixture is invoked, not merely destructured)? */
367
+ function callsAny(body, names) {
368
+ let found = false;
369
+ walk(body, (n) => {
370
+ if (!found && n.type === 'CallExpression' && n.callee?.type === 'Identifier' && names.has(n.callee.name))
371
+ found = true;
372
+ });
373
+ return found;
374
+ }
375
+ function findCall(ast, line, col0) {
376
+ let hit;
377
+ walk(ast, (n) => {
378
+ if (!hit && n.type === 'CallExpression' && n.loc?.start.line === line && n.loc.start.column === col0)
379
+ hit = n;
380
+ });
381
+ return hit;
382
+ }
383
+ function findIf(ast, line, col0) {
384
+ let hit;
385
+ walk(ast, (n) => {
386
+ if (!hit && n.type === 'IfStatement' && n.loc?.start.line === line && n.loc.start.column === col0)
387
+ hit = n;
388
+ });
389
+ return hit;
390
+ }
391
+ function isThrowOnly(stmt) {
392
+ if (stmt?.type === 'ThrowStatement')
393
+ return true;
394
+ return stmt?.type === 'BlockStatement' && stmt.body?.length === 1 && stmt.body[0].type === 'ThrowStatement';
395
+ }
396
+ export function triageFindings(findings, ctx) {
397
+ const wants = findings.some((f) => f.rule === 'playwright/expect-expect' ||
398
+ f.rule === 'playwright/no-conditional-in-test' ||
399
+ f.rule === 'playwright/valid-expect');
400
+ if (!wants)
401
+ return findings;
402
+ const cache = new Map(); // per call: nothing leaks between scorePaths runs
403
+ const know = gather(ctx, cache);
404
+ const out = [];
405
+ for (const f of findings) {
406
+ if ((f.rule !== 'playwright/expect-expect' &&
407
+ f.rule !== 'playwright/no-conditional-in-test' &&
408
+ f.rule !== 'playwright/valid-expect') ||
409
+ f.line === undefined ||
410
+ f.column === undefined) {
411
+ out.push(f);
412
+ continue;
413
+ }
414
+ const abs = path.resolve(ctx.filesBase, f.file);
415
+ const source = ctx.sources.get(abs);
416
+ const ast = source === undefined ? undefined : parse(cache, source, abs);
417
+ if (!ast) {
418
+ out.push(f);
419
+ continue;
420
+ }
421
+ if (f.rule === 'playwright/valid-expect') {
422
+ // Upstream treats any callee whose name merely ends in "Expect"
423
+ // (`clickSortableHeaderAndExpect(header, 'ascending')`) as an expect()
424
+ // call missing its matcher. Only the real `expect` is one.
425
+ const node = findCall(ast, f.line, f.column - 1);
426
+ const callee = node?.callee;
427
+ const isRealExpect = callee?.type === 'Identifier'
428
+ ? callee.name === 'expect'
429
+ : callee?.type === 'MemberExpression' && callee.object?.name === 'expect';
430
+ if (node && !isRealExpect)
431
+ continue;
432
+ out.push(f);
433
+ continue;
434
+ }
435
+ if (f.rule === 'playwright/no-conditional-in-test') {
436
+ const node = findIf(ast, f.line, f.column - 1);
437
+ if (node && !node.alternate && isThrowOnly(node.consequent))
438
+ continue; // a guard, not branching
439
+ out.push(f);
440
+ continue;
441
+ }
442
+ const call = findCall(ast, f.line, f.column - 1);
443
+ if (!call) {
444
+ out.push(f);
445
+ continue;
446
+ }
447
+ const fn = [...(call.arguments ?? [])].reverse().find(isFn);
448
+ const reachable = reachableFiles(abs, ctx, know);
449
+ const fixtureLocals = assertingFixtureLocals(fn, reachable, know);
450
+ if (fixtureLocals.size > 0 && callsAny(fn.body, fixtureLocals))
451
+ continue; // a callable fixture that asserts is invoked
452
+ const calls = [...new Set([...unresolvedCalls(call, ast, know), ...unresolvedFixtureCalls(call, ast, fn, reachable, know)])];
453
+ if (calls.length === 0) {
454
+ out.push(f);
455
+ continue;
456
+ }
457
+ out.push({
458
+ ...f,
459
+ rule: ASSERTION_UNVERIFIED_RULE,
460
+ severity: 'info',
461
+ reportOnly: true,
462
+ message: `Needs a human check: no expect(...) in this test body, but it calls ${calls
463
+ .slice(0, 3)
464
+ .map((c) => `${c}()`)
465
+ .join(', ')}${calls.length > 3 ? ', ...' : ''} on an object defined elsewhere (page object, injected fixture or ` +
466
+ 'imported helper) that this scan cannot read, so the assertion may live there. Not counted against the score.',
467
+ });
468
+ }
469
+ return out;
470
+ }
@@ -1,3 +1,6 @@
1
1
  export function formatJson(result) {
2
- return JSON.stringify(result, null, 2);
2
+ // baseDir is an absolute local path used only by the report's excerpt
3
+ // reader; it must never reach a shareable artifact.
4
+ const { baseDir: _baseDir, ...shareable } = result;
5
+ return JSON.stringify(shareable, null, 2);
3
6
  }
@@ -1,2 +1,4 @@
1
1
  import type { ScoreResult } from '../types.js';
2
+ /** Make a value safe inside a Markdown table cell: no pipe or newline can end the cell or row. */
3
+ export declare function escapeCell(v: unknown): string;
2
4
  export declare function formatMarkdown(result: ScoreResult): string;
@@ -1,3 +1,9 @@
1
+ import { INCOMPLETE_NOTE, notScoredState } from '../report-model.js';
2
+ import { sortFindings } from '../confidence.js';
3
+ /** Make a value safe inside a Markdown table cell: no pipe or newline can end the cell or row. */
4
+ export function escapeCell(v) {
5
+ return String(v).replace(/\|/g, '\\|').replace(/\r?\n|\r/g, ' ');
6
+ }
1
7
  export function formatMarkdown(result) {
2
8
  const lines = [];
3
9
  lines.push(`# Playwright Spec Score (\`${result.scoreVersion}\`)`);
@@ -9,17 +15,23 @@ export function formatMarkdown(result) {
9
15
  for (const [k, v] of Object.entries(result.dimensions)) {
10
16
  if (v === undefined)
11
17
  continue;
12
- lines.push(`| ${k} | ${v} |`);
18
+ lines.push(`| ${escapeCell(k)} | ${escapeCell(v)} |`);
13
19
  }
14
20
  lines.push('');
15
- lines.push(`Files: ${result.summary.files} · Tests: ${result.summary.tests} · SLOC: ${result.summary.sloc} · Findings: ${result.summary.findings}`);
16
- if (result.findings.length) {
21
+ lines.push(`Files: ${result.summary.files} · Tests: ${result.summary.tests} · SLOC: ${result.summary.sloc} · Findings: ${result.summary.findings}` +
22
+ (result.summary.unverifiedTests ? ` · Unverified assertions: ${result.summary.unverifiedTests}` : ''));
23
+ if (result.ruleCrashes?.length) {
24
+ lines.push('');
25
+ lines.push(INCOMPLETE_NOTE);
26
+ }
27
+ // 'no-tests' hard-fail findings carry the absolute working directory as their file.
28
+ if (result.findings.length && notScoredState(result) !== 'no-tests') {
17
29
  lines.push('');
18
30
  lines.push('## Findings');
19
31
  lines.push('');
20
- for (const f of result.findings.slice(0, 40)) {
32
+ for (const f of sortFindings(result.findings).slice(0, 40)) {
21
33
  const loc = f.line ? `:${f.line}` : '';
22
- lines.push(`- **${f.severity}** \`${f.rule}\` \`${f.file}${loc}\` — ${f.message}`);
34
+ lines.push(`- **${f.severity}** \`${f.rule}\` \`${escapeCell(f.file)}${loc}\` — ${escapeCell(f.message)}`);
23
35
  }
24
36
  }
25
37
  if (result.skippedFiles?.length) {
@@ -0,0 +1,2 @@
1
+ /** Inline stylesheet for the suite report: light/dark, phone width, Letter/A4 print. */
2
+ export declare const REPORT_CSS = "\n:root{--bg:#fff;--fg:#1a1d23;--muted:#5b6472;--line:#e3e6eb;--card:#f6f7f9;--accent:#2f5bea;--hit:#fff1c2;--hitfg:#1a1d23;--good:#1a7f4b;--warn:#b26a00;--bad:#c62828;--track:#e8ebf0}\n@media (prefers-color-scheme:dark){:root{--bg:#14161a;--fg:#e8eaee;--muted:#9aa3b2;--line:#2a2e36;--card:#1c1f25;--accent:#7c9bff;--hit:#4a3d10;--hitfg:#fff;--good:#4cc38a;--warn:#f0a63a;--bad:#ff6b6b;--track:#2a2e36}}\n*{box-sizing:border-box}\nbody{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,-apple-system,\"Segoe UI\",Roboto,sans-serif}\nmain{max-width:880px;margin:0 auto;padding:20px 20px 40px}\nheader{border-bottom:1px solid var(--line);padding-bottom:14px}\n.hd{display:flex;gap:24px;align-items:center;flex-wrap:wrap}\n.eyebrow{margin:0;font-size:12px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted)}\n.suite{margin:2px 0 10px;font-size:15px;font-weight:600;overflow-wrap:anywhere}\n.explain,.check{font-size:14px;margin:4px 0}\n.check a{color:var(--fg)}\n.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}\nh1{font-size:22px;margin:0 0 4px}\nh2{font-size:17px;margin:20px 0 8px;padding-bottom:6px;border-bottom:1px solid var(--line)}\nh3{font-size:15px;margin:0 0 4px}\np{margin:4px 0}\ncode,pre{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12.5px}\ncode{overflow-wrap:anywhere}\n.muted{color:var(--muted);font-size:13px}\n.meta{color:var(--muted);font-size:13px;margin-top:8px}\n.sub{font-weight:600;margin:12px 0 4px}\n.score{display:flex;align-items:baseline;gap:2px;padding:10px 18px;border-radius:12px;background:var(--card);border:1px solid var(--line);border-left:4px solid var(--muted)}\n.score .big{font-size:46px;font-weight:700;line-height:1}\n.score .of{color:var(--muted);font-size:16px}\n.score .grade{margin-left:12px;font-size:28px;font-weight:700}\n.g-A,.g-B{border-left-color:var(--good)}.g-A .grade,.g-B .grade{color:var(--good)}\n.g-C{border-left-color:var(--warn)}.g-C .grade{color:var(--warn)}\n.g-D,.g-F{border-left-color:var(--bad)}.g-D .grade,.g-F .grade{color:var(--bad)}\n.notscored{border-left-color:var(--bad)}.notscored .grade{margin-left:12px;font-size:16px;color:var(--fg)}\n@media (min-width:720px){.dims{display:grid;grid-template-columns:1fr 1fr;column-gap:28px}}\n.dim{margin:8px 0}.dimhead{display:flex;justify-content:space-between}\n.bar{height:10px;background:var(--track);border-radius:5px;overflow:hidden;margin:4px 0}\n.bar span{display:block;height:100%;background:var(--accent);-webkit-print-color-adjust:exact;print-color-adjust:exact}\n.finding{background:var(--card);border:1px solid var(--line);border-left:4px solid var(--warn);border-radius:8px;padding:12px 14px;margin:12px 0;break-inside:avoid}\n.finding.sev-error{border-left-color:var(--bad)}\n.rank{display:inline-block;width:22px;height:22px;line-height:22px;text-align:center;border-radius:50%;background:var(--fg);color:var(--bg);font-size:12px;margin-right:8px}\n.tag{font-size:11px;font-weight:600;border:1px solid var(--line);border-radius:10px;padding:1px 8px;margin-left:6px;color:var(--fg);border-color:var(--muted);white-space:nowrap}\n.where{margin:2px 0 6px}\nol.top3{list-style:none;padding:0;margin:8px 0}\nli.top{display:flex;gap:10px;align-items:flex-start;margin:8px 0;break-inside:avoid}\nli.top .rank{flex:none;margin-top:1px}\nli.top h3{margin:0 0 2px}li.top p{margin:2px 0}li.top .why{font-size:14px}\n.legend{margin:4px 0 8px}\npre.code{margin:6px 0;padding:8px 0;background:var(--bg);border:1px solid var(--line);border-radius:6px;overflow-x:auto}\n.ln{display:block;white-space:pre;padding:0 10px}\n.ln i{display:inline-block;width:3.2em;color:var(--muted);font-style:normal;user-select:none}\n.ln.hit{box-shadow:inset 3px 0 0 var(--bad);background:var(--hit);color:var(--hitfg);-webkit-print-color-adjust:exact;print-color-adjust:exact}\n.why{margin:6px 0}\nul.plain{list-style:none;padding:0;margin:6px 0}ul.plain li{padding:5px 0;border-bottom:1px solid var(--line);overflow-wrap:anywhere}\n.note{background:var(--card);border-radius:6px;padding:8px 12px}\nol{padding-left:22px}ol li{margin:6px 0}\nfooter{margin-top:32px;padding-top:12px;border-top:1px solid var(--line);color:var(--muted);font-size:12px}\n@media (max-width:520px){main{padding:16px 12px}.score .big{font-size:44px}.tag{display:inline-block;margin:4px 0 0}pre.code{font-size:11.5px}}\n@media print{\n@page{margin:14mm}\nbody{font-size:11.5px}main{max-width:none;padding:0}\nh2{margin:12px 0 6px;break-after:avoid}\n.summary .dims{display:grid;grid-template-columns:1fr 1fr;column-gap:16px}\n.summary pre.code{display:none}\n.finding{padding:6px 10px;margin:6px 0;break-inside:avoid}\nli.top{margin:6px 0}li.top .why{font-size:inherit}.explain,.check{font-size:inherit}.suite{margin-bottom:8px}p{margin:2px 0}ol li{margin:3px 0}.legend{font-size:11px;margin:2px 0 4px}\n.score .big{font-size:40px}header{padding-bottom:10px}\n.details{break-before:page}\npre.code{overflow:visible;white-space:pre-wrap}\n}\n";
@@ -0,0 +1,68 @@
1
+ /** Inline stylesheet for the suite report: light/dark, phone width, Letter/A4 print. */
2
+ export const REPORT_CSS = `
3
+ :root{--bg:#fff;--fg:#1a1d23;--muted:#5b6472;--line:#e3e6eb;--card:#f6f7f9;--accent:#2f5bea;--hit:#fff1c2;--hitfg:#1a1d23;--good:#1a7f4b;--warn:#b26a00;--bad:#c62828;--track:#e8ebf0}
4
+ @media (prefers-color-scheme:dark){:root{--bg:#14161a;--fg:#e8eaee;--muted:#9aa3b2;--line:#2a2e36;--card:#1c1f25;--accent:#7c9bff;--hit:#4a3d10;--hitfg:#fff;--good:#4cc38a;--warn:#f0a63a;--bad:#ff6b6b;--track:#2a2e36}}
5
+ *{box-sizing:border-box}
6
+ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif}
7
+ main{max-width:880px;margin:0 auto;padding:20px 20px 40px}
8
+ header{border-bottom:1px solid var(--line);padding-bottom:14px}
9
+ .hd{display:flex;gap:24px;align-items:center;flex-wrap:wrap}
10
+ .eyebrow{margin:0;font-size:12px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted)}
11
+ .suite{margin:2px 0 10px;font-size:15px;font-weight:600;overflow-wrap:anywhere}
12
+ .explain,.check{font-size:14px;margin:4px 0}
13
+ .check a{color:var(--fg)}
14
+ .sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}
15
+ h1{font-size:22px;margin:0 0 4px}
16
+ h2{font-size:17px;margin:20px 0 8px;padding-bottom:6px;border-bottom:1px solid var(--line)}
17
+ h3{font-size:15px;margin:0 0 4px}
18
+ p{margin:4px 0}
19
+ code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12.5px}
20
+ code{overflow-wrap:anywhere}
21
+ .muted{color:var(--muted);font-size:13px}
22
+ .meta{color:var(--muted);font-size:13px;margin-top:8px}
23
+ .sub{font-weight:600;margin:12px 0 4px}
24
+ .score{display:flex;align-items:baseline;gap:2px;padding:10px 18px;border-radius:12px;background:var(--card);border:1px solid var(--line);border-left:4px solid var(--muted)}
25
+ .score .big{font-size:46px;font-weight:700;line-height:1}
26
+ .score .of{color:var(--muted);font-size:16px}
27
+ .score .grade{margin-left:12px;font-size:28px;font-weight:700}
28
+ .g-A,.g-B{border-left-color:var(--good)}.g-A .grade,.g-B .grade{color:var(--good)}
29
+ .g-C{border-left-color:var(--warn)}.g-C .grade{color:var(--warn)}
30
+ .g-D,.g-F{border-left-color:var(--bad)}.g-D .grade,.g-F .grade{color:var(--bad)}
31
+ .notscored{border-left-color:var(--bad)}.notscored .grade{margin-left:12px;font-size:16px;color:var(--fg)}
32
+ @media (min-width:720px){.dims{display:grid;grid-template-columns:1fr 1fr;column-gap:28px}}
33
+ .dim{margin:8px 0}.dimhead{display:flex;justify-content:space-between}
34
+ .bar{height:10px;background:var(--track);border-radius:5px;overflow:hidden;margin:4px 0}
35
+ .bar span{display:block;height:100%;background:var(--accent);-webkit-print-color-adjust:exact;print-color-adjust:exact}
36
+ .finding{background:var(--card);border:1px solid var(--line);border-left:4px solid var(--warn);border-radius:8px;padding:12px 14px;margin:12px 0;break-inside:avoid}
37
+ .finding.sev-error{border-left-color:var(--bad)}
38
+ .rank{display:inline-block;width:22px;height:22px;line-height:22px;text-align:center;border-radius:50%;background:var(--fg);color:var(--bg);font-size:12px;margin-right:8px}
39
+ .tag{font-size:11px;font-weight:600;border:1px solid var(--line);border-radius:10px;padding:1px 8px;margin-left:6px;color:var(--fg);border-color:var(--muted);white-space:nowrap}
40
+ .where{margin:2px 0 6px}
41
+ ol.top3{list-style:none;padding:0;margin:8px 0}
42
+ li.top{display:flex;gap:10px;align-items:flex-start;margin:8px 0;break-inside:avoid}
43
+ li.top .rank{flex:none;margin-top:1px}
44
+ li.top h3{margin:0 0 2px}li.top p{margin:2px 0}li.top .why{font-size:14px}
45
+ .legend{margin:4px 0 8px}
46
+ pre.code{margin:6px 0;padding:8px 0;background:var(--bg);border:1px solid var(--line);border-radius:6px;overflow-x:auto}
47
+ .ln{display:block;white-space:pre;padding:0 10px}
48
+ .ln i{display:inline-block;width:3.2em;color:var(--muted);font-style:normal;user-select:none}
49
+ .ln.hit{box-shadow:inset 3px 0 0 var(--bad);background:var(--hit);color:var(--hitfg);-webkit-print-color-adjust:exact;print-color-adjust:exact}
50
+ .why{margin:6px 0}
51
+ ul.plain{list-style:none;padding:0;margin:6px 0}ul.plain li{padding:5px 0;border-bottom:1px solid var(--line);overflow-wrap:anywhere}
52
+ .note{background:var(--card);border-radius:6px;padding:8px 12px}
53
+ ol{padding-left:22px}ol li{margin:6px 0}
54
+ footer{margin-top:32px;padding-top:12px;border-top:1px solid var(--line);color:var(--muted);font-size:12px}
55
+ @media (max-width:520px){main{padding:16px 12px}.score .big{font-size:44px}.tag{display:inline-block;margin:4px 0 0}pre.code{font-size:11.5px}}
56
+ @media print{
57
+ @page{margin:14mm}
58
+ body{font-size:11.5px}main{max-width:none;padding:0}
59
+ h2{margin:12px 0 6px;break-after:avoid}
60
+ .summary .dims{display:grid;grid-template-columns:1fr 1fr;column-gap:16px}
61
+ .summary pre.code{display:none}
62
+ .finding{padding:6px 10px;margin:6px 0;break-inside:avoid}
63
+ li.top{margin:6px 0}li.top .why{font-size:inherit}.explain,.check{font-size:inherit}.suite{margin-bottom:8px}p{margin:2px 0}ol li{margin:3px 0}.legend{font-size:11px;margin:2px 0 4px}
64
+ .score .big{font-size:40px}header{padding-bottom:10px}
65
+ .details{break-before:page}
66
+ pre.code{overflow:visible;white-space:pre-wrap}
67
+ }
68
+ `;
@@ -0,0 +1,20 @@
1
+ import type { ScoreResult } from '../types.js';
2
+ export { rankTopFindings, compareGroups, type RuleGroup } from '../report-model.js';
3
+ /** Returns the source text of a finding's file (path as in Finding.file), or undefined. */
4
+ export type SourceReader = (file: string) => string | undefined;
5
+ export interface ReportOptions {
6
+ /** Shown in the header, e.g. "2026-11-14". Injected so output is reproducible. */
7
+ date: string;
8
+ /** Scorer package version. */
9
+ version: string;
10
+ /** Suite name or path shown under the headline. */
11
+ suiteName: string;
12
+ /** True when --threshold was passed, so the bar reads "your bar" instead of "the default bar". */
13
+ customThreshold?: boolean;
14
+ /** Source lookup for code excerpts; without it the excerpts are omitted. */
15
+ readSource?: SourceReader;
16
+ }
17
+ /** Rules that make a test flaky rather than merely untidy. */
18
+ export declare const FLAKE_RULES: readonly string[];
19
+ export declare function escapeHtml(s: string): string;
20
+ export declare function formatReport(result: ScoreResult, opts: ReportOptions): string;