@qaguardian/playwright-score 0.1.10 → 0.1.12

package/dist/import-graph.d.ts

@@ -0,0 +1,29 @@
+/**
+ * A Page Object Model is common in real Playwright suites: spec files
+ * interact with the page entirely through methods on an imported class
+ * (`n8n.canvas.clickZoomToFitButton()`), and the actual `getByRole()` /
+ * `.locator()` calls live in a separate file the spec never touches
+ * directly. Verified against a real, large suite (n8n's own e2e tests,
+ * 256 spec files, 0 direct locator calls across all of them) where every
+ * interaction routes through exactly this pattern — the locator-ratio
+ * metric read a flat zero for a suite that, underneath, uses native
+ * locators almost exclusively (confirmed by hand in its page-object
+ * files).
+ *
+ * This follows only *relative* import specifiers (`./`, `../`) — bare
+ * specifiers (npm packages, workspace packages, tsconfig path aliases)
+ * are always left alone, since resolving them correctly needs a real
+ * module resolver and getting it wrong risks pulling in unrelated code.
+ * Under-counting by skipping an alias is a safe failure mode; guessing
+ * wrong isn't. Traversal is also confined to `boundaryDir` (the scored
+ * files' own common ancestor) and depth/file-count capped — see MAX_DEPTH
+ * / MAX_FILES above — so a large or adversarial import graph can't blow
+ * up cost.
+ *
+ * Returns dependency files only (never a file already present in
+ * `specSources`), keyed by absolute path, so callers can fold their
+ * locator counts (and, optionally, assertion-helper names) into the
+ * suite-level totals without treating them as scored test files —
+ * they contribute no SLOC/tests/findings of their own.
+ */
+export declare function collectLocallyImportedFiles(specSources: Map<string, string>, boundaryDir: string): Map<string, string>;

package/dist/import-graph.js

@@ -0,0 +1,124 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseForESLint } from '@typescript-eslint/parser';
+const RESOLVE_EXTENSIONS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'];
+const INDEX_FILES = ['index.ts', 'index.tsx', 'index.js', 'index.jsx'];
+// Bounds on the traversal so a large, deep, or circular import graph can't
+// blow up cost. Chosen generously relative to real page-object directory
+// sizes (verified against n8n's own packages/testing/playwright/pages,
+// ~90 files, deepest chain 3) while still being a hard ceiling.
+const MAX_DEPTH = 6;
+const MAX_FILES = 300;
+function extractRelativeImportSpecifiers(source, file) {
+    let ast;
+    try {
+        ast = parseForESLint(source, {
+            ecmaVersion: 2022,
+            sourceType: 'module',
+            ecmaFeatures: { jsx: /\.(tsx|jsx)$/.test(file) },
+        }).ast;
+    }
+    catch {
+        return [];
+    }
+    const specifiers = [];
+    for (const node of ast.body ?? []) {
+        if (node.type !== 'ImportDeclaration' &&
+            node.type !== 'ExportNamedDeclaration' &&
+            node.type !== 'ExportAllDeclaration') {
+            continue;
+        }
+        const source_ = node.source;
+        if (typeof source_?.value === 'string' && source_.value.startsWith('.')) {
+            specifiers.push(source_.value);
+        }
+    }
+    return specifiers;
+}
+function resolveRelativeImport(fromFile, specifier, boundaryDir) {
+    const base = path.resolve(path.dirname(fromFile), specifier);
+    const candidates = [];
+    for (const ext of RESOLVE_EXTENSIONS)
+        candidates.push(base + ext);
+    for (const idx of INDEX_FILES)
+        candidates.push(path.join(base, idx));
+    for (const candidate of candidates) {
+        // Never resolve outside the scanned suite's own directory tree — keeps
+        // the traversal bounded to the code actually being scored, and refuses
+        // to follow a relative import that climbs out to a sibling package or
+        // node_modules via a longer `../../..` chain.
+        const rel = path.relative(boundaryDir, candidate);
+        if (rel.startsWith('..') || path.isAbsolute(rel))
+            continue;
+        try {
+            if (fs.statSync(candidate).isFile())
+                return candidate;
+        }
+        catch {
+            // Candidate doesn't exist — try the next extension/index form.
+        }
+    }
+    return undefined;
+}
+/**
+ * A Page Object Model is common in real Playwright suites: spec files
+ * interact with the page entirely through methods on an imported class
+ * (`n8n.canvas.clickZoomToFitButton()`), and the actual `getByRole()` /
+ * `.locator()` calls live in a separate file the spec never touches
+ * directly. Verified against a real, large suite (n8n's own e2e tests,
+ * 256 spec files, 0 direct locator calls across all of them) where every
+ * interaction routes through exactly this pattern — the locator-ratio
+ * metric read a flat zero for a suite that, underneath, uses native
+ * locators almost exclusively (confirmed by hand in its page-object
+ * files).
+ *
+ * This follows only *relative* import specifiers (`./`, `../`) — bare
+ * specifiers (npm packages, workspace packages, tsconfig path aliases)
+ * are always left alone, since resolving them correctly needs a real
+ * module resolver and getting it wrong risks pulling in unrelated code.
+ * Under-counting by skipping an alias is a safe failure mode; guessing
+ * wrong isn't. Traversal is also confined to `boundaryDir` (the scored
+ * files' own common ancestor) and depth/file-count capped — see MAX_DEPTH
+ * / MAX_FILES above — so a large or adversarial import graph can't blow
+ * up cost.
+ *
+ * Returns dependency files only (never a file already present in
+ * `specSources`), keyed by absolute path, so callers can fold their
+ * locator counts (and, optionally, assertion-helper names) into the
+ * suite-level totals without treating them as scored test files —
+ * they contribute no SLOC/tests/findings of their own.
+ */
+export function collectLocallyImportedFiles(specSources, boundaryDir) {
+    const collected = new Map();
+    const visited = new Set(specSources.keys());
+    const queue = [];
+    for (const [file, source] of specSources) {
+        queue.push({ file, source, depth: 0 });
+    }
+    while (queue.length > 0 && collected.size < MAX_FILES) {
+        const next = queue.shift();
+        if (!next)
+            break;
+        const { file, source, depth } = next;
+        if (depth >= MAX_DEPTH)
+            continue;
+        for (const specifier of extractRelativeImportSpecifiers(source, file)) {
+            if (collected.size >= MAX_FILES)
+                break;
+            const resolved = resolveRelativeImport(file, specifier, boundaryDir);
+            if (!resolved || visited.has(resolved))
+                continue;
+            visited.add(resolved);
+            let depSource;
+            try {
+                depSource = fs.readFileSync(resolved, 'utf8');
+            }
+            catch {
+                continue;
+            }
+            collected.set(resolved, depSource);
+            queue.push({ file: resolved, source: depSource, depth: depth + 1 });
+        }
+    }
+    return collected;
+}

package/dist/index.js

@@ -3,7 +3,8 @@ import path from 'node:path';
 import { globSync } from 'glob';
 import { runEslint } from './eslint-runner.js';
 import { commonAncestorDir } from './fs-util.js';
-import { analyzeSource, findLocalAssertionHelperNames, looksLikeNonPlaywrightTest, } from './metrics.js';
+import { collectLocallyImportedFiles } from './import-graph.js';
+import { analyzeSource, countLocators, findLocalAssertionHelperNames, looksLikeNonPlaywrightTest, } from './metrics.js';
 import { DEFAULT_THRESHOLDS } from './profiles.js';
 import { computeScore } from './score-engine.js';
 export { computeScore, SQS_V1, locatorsScore, gradeFromScore } from './score-engine.js';
@@ -47,6 +48,7 @@ const DEFAULT_IGNORE_GLOBS = [
 function expandPaths(inputs, cwd) {
     const explicit = new Set();
     const expanded = new Set();
+    const scanRootDirs = new Set();
     for (const input of inputs) {
         const abs = path.isAbsolute(input) ? input : path.resolve(cwd, input);
         if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
@@ -54,6 +56,13 @@ function expandPaths(inputs, cwd) {
             continue;
         }
         if (fs.existsSync(abs) && fs.statSync(abs).isDirectory()) {
+            // The directory the caller actually pointed at — not just wherever a
+            // spec glob happened to match inside it. A Page Object Model's
+            // classes typically live in a sibling directory to the specs
+            // themselves (tests/ next to pages/, fixtures/), so import-graph.ts
+            // needs this wider boundary, not the narrower common ancestor of
+            // only the matched spec files.
+            scanRootDirs.add(abs);
             for (const g of SPEC_GLOBS) {
                 for (const f of globSync(g, {
                     cwd: abs,
@@ -80,7 +89,11 @@ function expandPaths(inputs, cwd) {
     // explicit path plus an overlapping glob); explicit intent wins.
     for (const f of explicit)
         expanded.delete(f);
-    return { explicit: [...explicit].sort(), expanded: [...expanded].sort() };
+    return {
+        explicit: [...explicit].sort(),
+        expanded: [...expanded].sort(),
+        scanRootDirs: [...scanRootDirs].sort(),
+    };
 }
 function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
     return {
@@ -119,7 +132,7 @@ export async function scorePaths(options) {
     const cwd = options.cwd ?? process.cwd();
     const profile = options.profile ?? 'standard';
     const threshold = options.threshold ?? DEFAULT_THRESHOLDS[profile] ?? 80;
-    const { explicit, expanded } = expandPaths(options.paths, cwd);
+    const { explicit, expanded, scanRootDirs } = expandPaths(options.paths, cwd);
     if (explicit.length === 0 && expanded.length === 0) {
         // Hard-fail: empty match must never look like a healthy suite (was ~99 PASS).
         return hardFail(profile, threshold, [
@@ -163,6 +176,15 @@ export async function scorePaths(options) {
     // may be unrelated) so findings stay portable across machines/CI and
     // formatters/sarif.ts can emit repo-relative artifactLocation URIs.
     const filesBase = commonAncestorDir(files);
+    // Wider than filesBase on purpose: a Page Object Model's classes
+    // typically live in a directory *sibling* to the specs themselves
+    // (tests/ next to pages/, fixtures/), so bounding import-graph.ts's
+    // traversal to just the matched spec files' own common ancestor would
+    // reject every import that climbs back out of tests/ — verified against
+    // a real suite (n8n) where this was the exact reason locator counts from
+    // its page objects were being discarded. Falls back to filesBase itself
+    // when every input was an explicit file (no directory to widen to).
+    const importBoundary = commonAncestorDir([...files, ...scanRootDirs]);
     // Read every file once, up front, so (a) we can feed ESLint's
     // expect-expect a per-run list of local assertion-helper names before
     // linting (see findLocalAssertionHelperNames) and (b) the metrics loop
@@ -176,6 +198,18 @@ export async function scorePaths(options) {
             assertFunctionNames.add(name);
         }
     }
+    // Page Object Model suites route locator calls through imported classes
+    // (`n8n.canvas.click(...)`) rather than calling `page.locator()`/
+    // `getByRole()` directly in the spec file — see import-graph.ts. These
+    // dependency files are never linted or counted toward sloc/tests/
+    // findings; they only contribute locator counts (and, the same way,
+    // any assertion-helper names) to the suite-level totals below.
+    const importedFiles = collectLocallyImportedFiles(sources, importBoundary);
+    for (const [file, source] of importedFiles) {
+        for (const name of findLocalAssertionHelperNames(source, file)) {
+            assertFunctionNames.add(name);
+        }
+    }
     const eslintFindings = await runEslint({
         files,
         profile,
@@ -198,6 +232,11 @@ export async function scorePaths(options) {
         // metrics/no-empty-test duplicates guardian/require-expect under guardian — keep both mild
         metricFindings.push(...m.findings);
     }
+    for (const [file, source] of importedFiles) {
+        const counts = countLocators(source, file);
+        native += counts.native;
+        raw += counts.raw;
+    }
     // Dedupe empty-test style: if guardian require-expect already fired, still ok
     const findings = [...eslintFindings, ...metricFindings];
     const parseErrors = findings.filter((f) => f.rule === 'playwright-score/parse-error');

package/dist/metrics.d.ts

@@ -10,21 +10,32 @@ import type { Finding, LocatorCounts } from './types.js';
 export declare function looksLikeNonPlaywrightTest(source: string): boolean;
 export declare function countLocators(source: string, file?: string): LocatorCounts;
 /**
- * Names of same-file functions (function declarations, or a const bound to
- * an arrow/function expression) whose own body contains an expect(...)-
- * shaped call anywhere. Fed into playwright/expect-expect's own
- * assertFunctionNames option (see eslint-runner.ts) so a test that
- * delegates its assertion to a locally-defined helper — `audit(page,
- * path)`, `checkFlashMessageVisibility(page, msg)`, `assertDashboardLoaded`
- * — isn't flagged as having no assertions, regardless of what the helper
- * happens to be named. expect-expect can only see expect() calls written
- * directly in the test body, so without this, delegating to a helper at
- * all — an extremely common way to dedupe near-identical specs — reads as
- * "no assertions"; verified against several real production files using
- * several different naming conventions for the exact same shape.
- * Deliberately local-only: a helper imported from another file can't be
- * resolved without following module specifiers and parsing that file too,
- * well beyond what a fast, per-file syntactic check should attempt.
+ * Names of same-file assertion helpers — function declarations, a const
+ * bound to an arrow/function expression, or a class method (`kind:
+ * 'method'`, e.g. a Page Object Model's `async expectNodeVisible(name) {
+ * await expect(...) }`) — whose own body contains an expect(...)-shaped
+ * call, a `.waitFor(...)` call (see containsWaitForCall), or a call to a
+ * name already recognized by an earlier round (so e.g.
+ * `waitForNotificationAndClose()` calling `this.waitForNotification()`,
+ * itself resolved to a direct `.waitFor()` call, is recognized too —
+ * verified against a real Page Object Model, n8n's NotificationsPage,
+ * where the assertion sits two calls deep). Fed into
+ * playwright/expect-expect's own assertFunctionNames option (see
+ * eslint-runner.ts) so a test that delegates its assertion to a helper —
+ * `audit(page, path)`, `canvas.expectNodeVisible(name)`,
+ * `n8n.notifications.waitForNotificationAndClose(text)` — isn't flagged as
+ * having no assertions, regardless of what the helper is named or how
+ * deep the delegation chain runs *within this one file*. Matching is by
+ * name only, any receiver — the same tradeoff eslint-plugin-playwright's
+ * own option already makes, so this doesn't introduce a new risk, just
+ * extends an already-accepted one from free functions to methods too; a
+ * coincidental name collision between an asserting and non-asserting
+ * function is the known, accepted failure mode, same as before.
+ * Deliberately single-file only: resolving `this.foo()` across file
+ * boundaries needs real type resolution, well beyond what a fast,
+ * syntactic per-file check should attempt (cross-*file* delegation is
+ * handled separately, at the call site in index.ts, by running this same
+ * function against each imported dependency file too).
  */
 export declare function findLocalAssertionHelperNames(source: string, file?: string): string[];
 /** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */

package/dist/metrics.js

@@ -172,27 +172,88 @@ function containsExpectCall(node) {
     return found;
 }
 /**
- * Names of same-file functions (function declarations, or a const bound to
- * an arrow/function expression) whose own body contains an expect(...)-
- * shaped call anywhere. Fed into playwright/expect-expect's own
- * assertFunctionNames option (see eslint-runner.ts) so a test that
- * delegates its assertion to a locally-defined helper — `audit(page,
- * path)`, `checkFlashMessageVisibility(page, msg)`, `assertDashboardLoaded`
- * — isn't flagged as having no assertions, regardless of what the helper
- * happens to be named. expect-expect can only see expect() calls written
- * directly in the test body, so without this, delegating to a helper at
- * all — an extremely common way to dedupe near-identical specs — reads as
- * "no assertions"; verified against several real production files using
- * several different naming conventions for the exact same shape.
- * Deliberately local-only: a helper imported from another file can't be
- * resolved without following module specifiers and parsing that file too,
- * well beyond what a fast, per-file syntactic check should attempt.
+ * `locator.waitFor({ state: ... })` (state defaults to `'visible'` when
+ * omitted) has no purpose other than verifying the element reaches a
+ * state — unlike an action method (`.click()`, `.fill()`) that also
+ * happens to throw on failure as a side effect, this call exists only to
+ * check something, functionally equivalent to
+ * `expect(locator).toBeVisible()`/`toBeHidden()`. Matched on the exact
+ * property name `waitFor` (not a prefix), so it can't be confused with
+ * Page's differently-named `waitForEvent`/`waitForURL`/`waitForResponse`/
+ * `waitForLoadState`/`waitForTimeout`, none of which are assertions.
+ */
+function containsWaitForCall(node) {
+    let found = false;
+    walk(node, (n) => {
+        if (found || n.type !== 'CallExpression')
+            return;
+        const callee = n.callee;
+        if (callee?.type !== 'MemberExpression' || callee.computed)
+            return;
+        const property = callee.property;
+        if (property?.type === 'Identifier' && property.name === 'waitFor')
+            found = true;
+    });
+    return found;
+}
+/** Does `node`'s body contain a call (any receiver, matched by name only —
+ * same name-based matching eslint-plugin-playwright's own
+ * assertFunctionNames option uses) to one of `names`? */
+function containsCallToAnyName(node, names) {
+    if (names.size === 0)
+        return false;
+    let found = false;
+    walk(node, (n) => {
+        if (found || n.type !== 'CallExpression')
+            return;
+        const callee = n.callee;
+        let name;
+        if (callee?.type === 'Identifier') {
+            name = callee.name;
+        }
+        else if (callee?.type === 'MemberExpression' && !callee.computed) {
+            const property = callee.property;
+            if (property?.type === 'Identifier')
+                name = property.name;
+        }
+        if (name && names.has(name))
+            found = true;
+    });
+    return found;
+}
+/**
+ * Names of same-file assertion helpers — function declarations, a const
+ * bound to an arrow/function expression, or a class method (`kind:
+ * 'method'`, e.g. a Page Object Model's `async expectNodeVisible(name) {
+ * await expect(...) }`) — whose own body contains an expect(...)-shaped
+ * call, a `.waitFor(...)` call (see containsWaitForCall), or a call to a
+ * name already recognized by an earlier round (so e.g.
+ * `waitForNotificationAndClose()` calling `this.waitForNotification()`,
+ * itself resolved to a direct `.waitFor()` call, is recognized too —
+ * verified against a real Page Object Model, n8n's NotificationsPage,
+ * where the assertion sits two calls deep). Fed into
+ * playwright/expect-expect's own assertFunctionNames option (see
+ * eslint-runner.ts) so a test that delegates its assertion to a helper —
+ * `audit(page, path)`, `canvas.expectNodeVisible(name)`,
+ * `n8n.notifications.waitForNotificationAndClose(text)` — isn't flagged as
+ * having no assertions, regardless of what the helper is named or how
+ * deep the delegation chain runs *within this one file*. Matching is by
+ * name only, any receiver — the same tradeoff eslint-plugin-playwright's
+ * own option already makes, so this doesn't introduce a new risk, just
+ * extends an already-accepted one from free functions to methods too; a
+ * coincidental name collision between an asserting and non-asserting
+ * function is the known, accepted failure mode, same as before.
+ * Deliberately single-file only: resolving `this.foo()` across file
+ * boundaries needs real type resolution, well beyond what a fast,
+ * syntactic per-file check should attempt (cross-*file* delegation is
+ * handled separately, at the call site in index.ts, by running this same
+ * function against each imported dependency file too).
  */
 export function findLocalAssertionHelperNames(source, file) {
     const ast = parseSource(source, file);
     if (!ast)
         return [];
-    const names = new Set();
+    const candidates = [];
     walk(ast, (node) => {
         let name;
         let body;
@@ -212,9 +273,35 @@ export function findLocalAssertionHelperNames(source, file) {
                 body = init;
             }
         }
-        if (name && body && containsExpectCall(body))
-            names.add(name);
+        else if (node.type === 'MethodDefinition') {
+            const key = node.key;
+            if (!node.computed && key?.type === 'Identifier' && node.kind === 'method') {
+                name = key.name;
+                body = node.value;
+            }
+        }
+        if (name && body)
+            candidates.push({ name, body });
     });
+    const names = new Set();
+    for (const c of candidates) {
+        if (containsExpectCall(c.body) || containsWaitForCall(c.body))
+            names.add(c.name);
+    }
+    // Fixed-point: naturally bounded by candidates.length rounds, since each
+    // round adds at least one name or the loop stops.
+    let added = true;
+    while (added) {
+        added = false;
+        for (const c of candidates) {
+            if (names.has(c.name))
+                continue;
+            if (containsCallToAnyName(c.body, names)) {
+                names.add(c.name);
+                added = true;
+            }
+        }
+    }
     return [...names];
 }
 const TEST_NON_DECL_RE = /\btest\.(describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info|expect)\b/;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qaguardian/playwright-score",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "description": "Deterministic, AI-free Playwright spec quality score (sqs-v1). Lint + score with standard and guardian profiles.",
   "type": "module",
   "main": "./dist/index.js",