@qaguardian/playwright-score 0.1.0 → 0.1.9

package/dist/metrics.d.ts

@@ -1,13 +1,34 @@
 import type { Finding, LocatorCounts } from './types.js';
-/** Count native vs raw locator calls (simple statistical). */
-export declare function countLocators(source: string): LocatorCounts;
-/** Count test( / test.only( / test.skip( style declarations (approx). */
-export declare function countTests(source: string): number;
 /**
- * Detect test callbacks that appear to have zero expect( calls.
- * Heuristic: for each test( ... async (...) => { body }, check body for expect.
+ * Heuristic: does this file show positive evidence of being a *non*-
+ * Playwright test (Jest/Vitest/Mocha/Jasmine/React Testing Library), as
+ * opposed to a Playwright spec that a broad `**\/*.test.ts` glob would
+ * otherwise sweep in alongside real e2e specs? Used only to decide whether
+ * to auto-exclude a file discovered via directory/glob expansion — never
+ * applied to a file path the caller named explicitly.
+ */
+export declare function looksLikeNonPlaywrightTest(source: string): boolean;
+export declare function countLocators(source: string, file?: string): LocatorCounts;
+/**
+ * Names of same-file functions (function declarations, or a const bound to
+ * an arrow/function expression) whose own body contains an expect(...)-
+ * shaped call anywhere. Fed into playwright/expect-expect's own
+ * assertFunctionNames option (see eslint-runner.ts) so a test that
+ * delegates its assertion to a locally-defined helper — `audit(page,
+ * path)`, `checkFlashMessageVisibility(page, msg)`, `assertDashboardLoaded`
+ * — isn't flagged as having no assertions, regardless of what the helper
+ * happens to be named. expect-expect can only see expect() calls written
+ * directly in the test body, so without this, delegating to a helper at
+ * all — an extremely common way to dedupe near-identical specs — reads as
+ * "no assertions"; verified against several real production files using
+ * several different naming conventions for the exact same shape.
+ * Deliberately local-only: a helper imported from another file can't be
+ * resolved without following module specifiers and parsing that file too,
+ * well beyond what a fast, per-file syntactic check should attempt.
  */
-export declare function findEmptyExpectTests(source: string, file: string): Finding[];
+export declare function findLocalAssertionHelperNames(source: string, file?: string): string[];
+/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
+export declare function countTests(source: string): number;
 export declare function findOversizedFile(source: string, file: string, maxSloc?: number): Finding[];
 export declare function analyzeSource(source: string, file: string): {
     sloc: number;

package/dist/metrics.js

@@ -1,56 +1,237 @@
+import { parseForESLint } from '@typescript-eslint/parser';
 import { countSloc } from './sloc.js';
-const NATIVE_RE = /\.\s*getBy(?:Role|Label|TestId|Text|Placeholder)\s*\(/g;
-const RAW_RE = /(?:page|frame)\s*\.\s*locator\s*\(/g;
-/** Count native vs raw locator calls (simple statistical). */
-export function countLocators(source) {
-    const native = [...source.matchAll(NATIVE_RE)].length;
-    const raw = [...source.matchAll(RAW_RE)].length;
+// A positive "this IS a Playwright spec" check turned out to be unsafe:
+// Playwright's own custom-fixtures pattern (test/expect re-exported from a
+// project-local module built with base.extend()) is extremely common in
+// real suites, and the fixtures a test destructures can be named anything
+// (`bomPage`, `authedUser`, ...) — not necessarily the stock page/context/
+// browser/request names. A file that legitimately imports `test`/`expect`
+// from '../fixtures' and destructures `{ bomPage }` has zero syntactic
+// signal distinguishing it from an unrelated framework, so requiring
+// positive Playwright evidence false-negatives on real production specs
+// (verified against real code: it silently skipped 18 of 19 genuine specs
+// in one real suite). Denylisting known non-Playwright frameworks is much
+// safer — a missed rare/unusual file just scores harmlessly, where a
+// falsely-skipped real spec silently stops being scored at all.
+const NON_PLAYWRIGHT_IMPORT_RE = /from\s+['"](?:vitest|jest|@jest\/[a-z-]+|mocha|jasmine|chai|@testing-library\/[a-z-]+)['"]/;
+const RTL_API_RE = /\bscreen\.(?:getBy|queryBy|findBy|getAllBy|queryAllBy|findAllBy)\w*\(|\btoBeInTheDocument\(/;
+/**
+ * Heuristic: does this file show positive evidence of being a *non*-
+ * Playwright test (Jest/Vitest/Mocha/Jasmine/React Testing Library), as
+ * opposed to a Playwright spec that a broad `**\/*.test.ts` glob would
+ * otherwise sweep in alongside real e2e specs? Used only to decide whether
+ * to auto-exclude a file discovered via directory/glob expansion — never
+ * applied to a file path the caller named explicitly.
+ */
+export function looksLikeNonPlaywrightTest(source) {
+    return NON_PLAYWRIGHT_IMPORT_RE.test(source) || RTL_API_RE.test(source);
+}
+const NATIVE_METHODS = new Set([
+    'getByRole',
+    'getByLabel',
+    'getByTestId',
+    'getByText',
+    'getByPlaceholder',
+]);
+/**
+ * Generic ESTree walk (no visitor-keys dependency). `parent` is excluded so
+ * this stays a walk over a DAG even if a parent back-reference is ever
+ * attached to the tree upstream.
+ */
+function walk(node, visit) {
+    if (!node || typeof node !== 'object')
+        return;
+    if (Array.isArray(node)) {
+        for (const item of node)
+            walk(item, visit);
+        return;
+    }
+    const obj = node;
+    if (typeof obj.type === 'string')
+        visit(obj);
+    for (const key in obj) {
+        if (key === 'parent')
+            continue;
+        walk(obj[key], visit);
+    }
+}
+/**
+ * Parses source into an ESTree-compatible AST for our own lightweight
+ * AST-walk checks (locator counting, local assertion-helper discovery).
+ * Returns undefined on any parse failure — callers degrade to "found
+ * nothing" rather than throwing; ESLint's own lint pass over the same
+ * file surfaces the real parse error as a finding.
+ */
+function parseSource(source, file) {
+    try {
+        return parseForESLint(source, {
+            ecmaVersion: 2022,
+            sourceType: 'module',
+            // JSX must be extension-driven, not always-on: a bare .ts file can
+            // legally use the old-style `<Type>value` cast syntax, which is
+            // ambiguous with — and would be misparsed as — a JSX element if JSX
+            // were enabled unconditionally.
+            ecmaFeatures: { jsx: /\.(tsx|jsx)$/.test(file ?? '') },
+        }).ast;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Count native vs raw locator calls via the same AST the ESLint pass
+ * already parses (rather than a regex), so this ratio can't silently
+ * disagree with `no-raw-locators` findings. A regex anchored to literal
+ * `page`/`frame` receivers misses raw locators on any other receiver —
+ * e.g. `this.container.locator(...)` in a Page Object Model, or a second
+ * `.locator()` chained off a prior locator call — both common in
+ * real-world (and AI-generated) specs.
+ */
+// Playwright's older direct-action API takes a raw selector string as the
+// first argument — page.click('#foo') instead of page.locator('#foo').click()
+// or a native locator. This is arguably the single most common raw-selector
+// anti-pattern in naive/AI-generated code (verified against a realistic
+// sample: a file using this exclusively scored a perfect 100 on the
+// locators dimension before this fix). Restricted to methods a Locator
+// itself takes zero required arguments for (click/dblclick/hover/check/
+// uncheck/tap/focus) — real Playwright types never allow a string there on
+// a Locator, so a string first argument unambiguously means the legacy
+// page/frame form regardless of receiver name. fill/type/press/
+// selectOption are deliberately excluded: their first argument is
+// legitimately a string on a Locator too (the value/key, not a selector),
+// so counting them would risk new false positives on correct code.
+const LEGACY_SELECTOR_ACTION_METHODS = new Set([
+    'click',
+    'dblclick',
+    'hover',
+    'check',
+    'uncheck',
+    'tap',
+    'focus',
+]);
+export function countLocators(source, file) {
+    let native = 0;
+    let raw = 0;
+    const ast = parseSource(source, file);
+    if (!ast)
+        return { native, raw };
+    walk(ast, (node) => {
+        if (node.type !== 'CallExpression' ||
+            node.callee?.type !== 'MemberExpression') {
+            return;
+        }
+        const callee = node.callee;
+        if (callee.computed)
+            return;
+        const property = callee.property;
+        if (property?.type !== 'Identifier')
+            return;
+        if (NATIVE_METHODS.has(property.name ?? '')) {
+            native++;
+        }
+        else if (property.name === 'locator') {
+            raw++;
+        }
+        else if (LEGACY_SELECTOR_ACTION_METHODS.has(property.name ?? '')) {
+            const args = node.arguments;
+            const first = args?.[0];
+            if (first?.type === 'Literal' && typeof first.value === 'string')
+                raw++;
+        }
+    });
     return { native, raw };
 }
-/** Count test( / test.only( / test.skip( style declarations (approx). */
+function isExpectCallee(callee) {
+    if (!callee)
+        return false;
+    if (callee.type === 'Identifier')
+        return callee.name === 'expect';
+    if (callee.type === 'MemberExpression') {
+        // Walk down to the root of a member chain, e.g. expect.poll(fn).toBe(x)
+        // — the callee of the outer .toBe() call is `expect.poll(fn).toBe`,
+        // whose object is the `expect.poll(fn)` CallExpression.
+        let obj = callee.object;
+        while (obj?.type === 'MemberExpression')
+            obj = obj.object;
+        if (obj?.type === 'Identifier' && obj.name === 'expect')
+            return true;
+        if (obj?.type === 'CallExpression') {
+            return isExpectCallee(obj.callee);
+        }
+    }
+    return false;
+}
+function containsExpectCall(node) {
+    let found = false;
+    walk(node, (n) => {
+        if (found || n.type !== 'CallExpression')
+            return;
+        if (isExpectCallee(n.callee))
+            found = true;
+    });
+    return found;
+}
+/**
+ * Names of same-file functions (function declarations, or a const bound to
+ * an arrow/function expression) whose own body contains an expect(...)-
+ * shaped call anywhere. Fed into playwright/expect-expect's own
+ * assertFunctionNames option (see eslint-runner.ts) so a test that
+ * delegates its assertion to a locally-defined helper — `audit(page,
+ * path)`, `checkFlashMessageVisibility(page, msg)`, `assertDashboardLoaded`
+ * — isn't flagged as having no assertions, regardless of what the helper
+ * happens to be named. expect-expect can only see expect() calls written
+ * directly in the test body, so without this, delegating to a helper at
+ * all — an extremely common way to dedupe near-identical specs — reads as
+ * "no assertions"; verified against several real production files using
+ * several different naming conventions for the exact same shape.
+ * Deliberately local-only: a helper imported from another file can't be
+ * resolved without following module specifiers and parsing that file too,
+ * well beyond what a fast, per-file syntactic check should attempt.
+ */
+export function findLocalAssertionHelperNames(source, file) {
+    const ast = parseSource(source, file);
+    if (!ast)
+        return [];
+    const names = new Set();
+    walk(ast, (node) => {
+        let name;
+        let body;
+        if (node.type === 'FunctionDeclaration') {
+            const id = node.id;
+            if (id?.type === 'Identifier') {
+                name = id.name;
+                body = node;
+            }
+        }
+        else if (node.type === 'VariableDeclarator') {
+            const id = node.id;
+            const init = node.init;
+            if (id?.type === 'Identifier' &&
+                (init?.type === 'ArrowFunctionExpression' || init?.type === 'FunctionExpression')) {
+                name = id.name;
+                body = init;
+            }
+        }
+        if (name && body && containsExpectCall(body))
+            names.add(name);
+    });
+    return [...names];
+}
+const TEST_NON_DECL_RE = /\btest\.(describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info|expect)\b/;
+/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
 export function countTests(source) {
-    const re = /\btest(?:\.(?:only|skip|fix|slow))?\s*\(/g;
-    // Exclude test.describe, test.step, test.before*, test.after*, test.use
+    // Playwright's real modifier API is test.fixme() — not test.fix().
+    const re = /\btest(?:\.(?:only|skip|fixme|slow))?\s*\(/g;
     let count = 0;
     let m;
-    const exclude = /\btest\.(?:describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info|expect)\b/;
     while ((m = re.exec(source)) !== null) {
-        const start = Math.max(0, m.index - 0);
-        const snippet = source.slice(start, start + 40);
-        if (exclude.test(snippet.split('(')[0] + '(')) {
-            // more precise: check the matched prefix
-        }
         const before = source.slice(Math.max(0, m.index - 1), m.index + 20);
-        if (/\btest\.(describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info)\b/.test(before)) {
-            continue;
-        }
-        // test.expect is rare as call form
-        if (/\btest\.expect\b/.test(before))
+        if (TEST_NON_DECL_RE.test(before))
             continue;
         count++;
     }
     return count;
 }
-/**
- * Detect test callbacks that appear to have zero expect( calls.
- * Heuristic: for each test( ... async (...) => { body }, check body for expect.
- */
-export function findEmptyExpectTests(source, file) {
-    const findings = [];
-    // Rough scan: if file has test( but no expect( at all
-    const hasTest = /\btest(?:\.(?:only|skip|fix))?\s*\(/.test(source);
-    const hasExpect = /\bexpect\s*\(/.test(source);
-    if (hasTest && !hasExpect) {
-        findings.push({
-            rule: 'metrics/no-empty-test',
-            severity: 'error',
-            message: 'Test file has test() but no expect() assertions',
-            file,
-            dimension: 'assertions',
-        });
-    }
-    return findings;
-}
 export function findOversizedFile(source, file, maxSloc = 400) {
     const sloc = countSloc(source);
     if (sloc > maxSloc) {
@@ -70,10 +251,16 @@ export function analyzeSource(source, file) {
     return {
         sloc: countSloc(source),
         tests: countTests(source),
-        locators: countLocators(source),
-        findings: [
-            ...findEmptyExpectTests(source, file),
-            ...findOversizedFile(source, file),
-        ],
+        locators: countLocators(source, file),
+        // No homegrown "has no assertions" check here: playwright/expect-expect
+        // (a real AST-based community rule, always enabled — see
+        // eslint-runner.ts) already covers this correctly, including
+        // expect.poll(...)/expect.soft(...) chained forms that a source-text
+        // regex would need to special-case. A prior regex-based version of
+        // this check (metrics/no-empty-test) only matched a bare `expect(`
+        // call and so false-positived at error severity on any file whose only
+        // assertions were expect.poll()/expect.soft() — verified against real
+        // production code.
+        findings: [...findOversizedFile(source, file)],
     };
 }

package/dist/profiles.js

@@ -45,10 +45,9 @@ export const ESLINT_RULE_MAP = {
     },
     // Assertions
     'playwright/expect-expect': { dimension: 'assertions' },
-    'playwright/prefer-web-first-assertions': {
-        dimension: 'assertions',
-        severityOverride: 'warning',
-    },
+    // No severityOverride: matches eslint-plugin-playwright's own
+    // recommended severity ('error'), set via eslint-runner.ts's rule config.
+    'playwright/prefer-web-first-assertions': { dimension: 'assertions' },
     'playwright/valid-expect': { dimension: 'assertions' },
     'playwright/no-standalone-expect': { dimension: 'assertions' },
     // Locators — findings for repair, reportOnly so ratio owns locators score
@@ -69,10 +68,10 @@ export const ESLINT_RULE_MAP = {
     },
     // Structure
     'playwright/no-focused-test': { dimension: 'structure' },
-    'playwright/no-skipped-test': {
-        dimension: 'structure',
-        severityOverride: 'warning',
-    },
+    // playwright/no-skipped-test is turned off in eslint-runner.ts in favor
+    // of pwscore/no-skipped-test-declaration, which doesn't false-positive
+    // on Playwright's documented conditional test.skip(condition, reason).
+    'pwscore/no-skipped-test-declaration': { dimension: 'structure' },
     'playwright/max-nested-describe': {
         dimension: 'structure',
         severityOverride: 'warning',
@@ -101,17 +100,7 @@ export const ESLINT_RULE_MAP = {
         dimension: 'guardianConventions',
         severityOverride: 'warning',
     },
-    'guardian/prefer-uuid': {
-        dimension: 'guardianConventions',
-        severityOverride: 'warning',
-    },
-    'guardian/fixtures-import': {
-        dimension: 'guardianConventions',
-        severityOverride: 'info',
-        reportOnly: true,
-    },
     // Metrics injected as synthetic findings
-    'metrics/no-empty-test': { dimension: 'assertions' },
     'metrics/oversized-file': {
         dimension: 'structure',
         severityOverride: 'warning',

package/dist/rules/base-plugin.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Rules active for both `standard` and `guardian` profiles — corrections
+ * to upstream eslint-plugin-playwright behavior, not house style.
+ */
+import type { Rule } from 'eslint';
+export declare const basePlugin: {
+    meta: {
+        name: string;
+        version: string;
+    };
+    rules: {
+        'no-skipped-test-declaration': Rule.RuleModule;
+    };
+};

package/dist/rules/base-plugin.js

@@ -0,0 +1,77 @@
+function isSkipCallee(callee) {
+    if (!callee || callee.type !== 'MemberExpression' || callee.computed)
+        return false;
+    const obj = callee.object;
+    const prop = callee.property;
+    if (prop?.type !== 'Identifier' || prop.name !== 'skip')
+        return false;
+    // test.skip(...)
+    if (obj?.type === 'Identifier' && obj.name === 'test')
+        return true;
+    // test.describe.skip(...)
+    if (obj?.type === 'MemberExpression' &&
+        !obj.computed &&
+        obj.object?.type === 'Identifier' &&
+        obj.object.name === 'test' &&
+        obj.property?.type === 'Identifier' &&
+        obj.property.name === 'describe') {
+        return true;
+    }
+    return false;
+}
+const FUNCTION_TYPES = new Set(['FunctionExpression', 'ArrowFunctionExpression']);
+/**
+ * eslint-plugin-playwright's no-skipped-test flags every form of
+ * test.skip(...)/test.describe.skip(...) identically. That conflates two
+ * very different, both-real patterns:
+ *
+ *  - test.skip('name', fn) / test.describe.skip('name', fn) — a
+ *    DECLARATION whose test/suite never runs at all. Worth flagging: this
+ *    is the "test is broken, hide it" anti-pattern.
+ *  - test.skip(condition, reason) / test.skip() called inside a test/hook
+ *    body — Playwright's own documented API for conditionally skipping a
+ *    test at runtime (e.g. per-browser, per-env). This is the correct,
+ *    recommended way to handle exactly that case, not an anti-pattern —
+ *    see playwright.dev/docs/test-annotations#conditionally-skip-a-test.
+ *
+ * Verified against real-world code: eslint-plugin-playwright's rule flags
+ * both forms identically, generating warnings on entirely idiomatic
+ * cross-browser test suites. This rule replaces it (see eslint-runner.ts,
+ * where the upstream rule is turned off) with the same "declaration form"
+ * severity but without the false positive on the conditional form — the
+ * two are told apart by whether the trailing argument is a function (a
+ * declaration always supplies the test body as a function; a conditional
+ * skip's second argument, if any, is the string reason).
+ */
+const noSkippedTestDeclaration = {
+    meta: {
+        type: 'suggestion',
+        docs: {
+            description: 'Disallow test.skip()/test.describe.skip() used to declare an always-skipped test (conditional runtime skips are unaffected)',
+        },
+        messages: {
+            skipped: 'Unexpected use of the .skip() annotation to declare an always-skipped test.',
+        },
+        schema: [],
+    },
+    create(context) {
+        return {
+            CallExpression(node) {
+                if (!isSkipCallee(node.callee))
+                    return;
+                const args = node.arguments ?? [];
+                const lastArg = args[args.length - 1];
+                const isDeclaration = args.length >= 2 && FUNCTION_TYPES.has(lastArg?.type ?? '');
+                if (isDeclaration) {
+                    context.report({ node: node, messageId: 'skipped' });
+                }
+            },
+        };
+    },
+};
+export const basePlugin = {
+    meta: { name: 'pwscore-base', version: '0.1.0' },
+    rules: {
+        'no-skipped-test-declaration': noSkippedTestDeclaration,
+    },
+};