@qaguardian/playwright-score 1.0.0 → 2.1.0

package/dist/metrics.d.ts

@@ -61,8 +61,14 @@ export interface TestSpans {
  * for `test.fixme()` used as a statement. Spans nested inside another test
  * span (rare, but e.g. a helper declaring a test inside a test would be
  * malformed anyway) are dropped so each line maps to at most one test.
+ *
+ * `testAliasNames` (see test-aliases.ts) are identifiers bound to a
+ * `test.extend(...)`-derived fixture (`const loggedTest = test.extend(...)`,
+ * possibly imported from another file) — treated exactly like the literal
+ * `test` identifier here, same as `it`. Without this, a suite that always
+ * calls its extended fixture under a custom name would count zero tests.
  */
-export declare function getTestSpans(source: string, file?: string): TestSpans;
+export declare function getTestSpans(source: string, file?: string, testAliasNames?: readonly string[]): TestSpans;
 /**
  * Model-v3 finding attribution (see score-engine.ts): a finding inside a
  * test span belongs to that test (`scope: 'test'` + a stable
@@ -75,10 +81,18 @@ export declare function getTestSpans(source: string, file?: string): TestSpans;
  * Mutates the findings in place and returns them.
  */
 export declare function attributeFindingScopes(findings: Finding[], spansByFile: Map<string, TestSpans>): Finding[];
-/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
-export declare function countTests(source: string): number;
+/**
+ * Count test( / test.only( / test.skip( / test.fixme( style declarations
+ * (approx) — the regex fallback used only when `parseSource` can't parse
+ * the file (see `getTestSpans`'s AST path, preferred whenever parsing
+ * succeeds). `testAliasNames` extends the matched identifier the same way
+ * `getTestSpans` does, so a parse-fallback file using an extended-fixture
+ * name isn't undercounted either.
+ */
+export declare function countTests(source: string, testAliasNames?: readonly string[]): number;
 export declare function findOversizedFile(source: string, file: string, maxSloc?: number): Finding[];
-export declare function analyzeSource(source: string, file: string): {
+export declare function findEslintDisableComments(source: string, file: string): Finding[];
+export declare function analyzeSource(source: string, file: string, testAliasNames?: readonly string[]): {
     sloc: number;
     tests: number;
     locators: LocatorCounts;

package/dist/metrics.js

@@ -334,13 +334,21 @@ function hasFunctionArg(node) {
  * for `test.fixme()` used as a statement. Spans nested inside another test
  * span (rare, but e.g. a helper declaring a test inside a test would be
  * malformed anyway) are dropped so each line maps to at most one test.
+ *
+ * `testAliasNames` (see test-aliases.ts) are identifiers bound to a
+ * `test.extend(...)`-derived fixture (`const loggedTest = test.extend(...)`,
+ * possibly imported from another file) — treated exactly like the literal
+ * `test` identifier here, same as `it`. Without this, a suite that always
+ * calls its extended fixture under a custom name would count zero tests.
  */
-export function getTestSpans(source, file) {
+export function getTestSpans(source, file, testAliasNames = []) {
     const tests = [];
     const hooks = [];
     const ast = parseSource(source, file);
     if (!ast)
         return { tests, hooks };
+    const aliasSet = new Set(testAliasNames);
+    const isTestName = (name) => name === 'test' || name === 'it' || aliasSet.has(name);
     walk(ast, (node) => {
         if (node.type !== 'CallExpression')
             return;
@@ -353,7 +361,7 @@ export function getTestSpans(source, file) {
         let isHook = false;
         if (callee?.type === 'Identifier') {
             const name = callee.name;
-            if (name === 'test' || name === 'it')
+            if (name && isTestName(name))
                 isTest = true;
             else if (name && HOOK_NAMES.has(name))
                 isHook = true;
@@ -361,7 +369,7 @@ export function getTestSpans(source, file) {
         else if (callee?.type === 'MemberExpression' && !callee.computed) {
             const obj = callee.object;
             const prop = callee.property;
-            if (obj?.type === 'Identifier' && (obj.name === 'test' || obj.name === 'it') && prop?.type === 'Identifier') {
+            if (obj?.type === 'Identifier' && obj.name && isTestName(obj.name) && prop?.type === 'Identifier') {
                 if (TEST_MODIFIERS.has(prop.name ?? ''))
                     isTest = true;
                 else if (HOOK_NAMES.has(prop.name ?? ''))
@@ -422,10 +430,21 @@ export function attributeFindingScopes(findings, spansByFile) {
     return findings;
 }
 const TEST_NON_DECL_RE = /\btest\.(describe|step|beforeEach|afterEach|beforeAll|afterAll|use|setTimeout|info|expect)\b/;
-/** Count test( / test.only( / test.skip( / test.fixme( style declarations (approx). */
-export function countTests(source) {
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * Count test( / test.only( / test.skip( / test.fixme( style declarations
+ * (approx) — the regex fallback used only when `parseSource` can't parse
+ * the file (see `getTestSpans`'s AST path, preferred whenever parsing
+ * succeeds). `testAliasNames` extends the matched identifier the same way
+ * `getTestSpans` does, so a parse-fallback file using an extended-fixture
+ * name isn't undercounted either.
+ */
+export function countTests(source, testAliasNames = []) {
     // Playwright's real modifier API is test.fixme() — not test.fix().
-    const re = /\btest(?:\.(?:only|skip|fixme|slow))?\s*\(/g;
+    const names = ['test', ...testAliasNames].map(escapeRegExp).join('|');
+    const re = new RegExp(`\\b(?:${names})(?:\\.(?:only|skip|fixme|slow))?\\s*\\(`, 'g');
     let count = 0;
     let m;
     while ((m = re.exec(source)) !== null) {
@@ -451,10 +470,41 @@ export function findOversizedFile(source, file, maxSloc = 400) {
     }
     return [];
 }
-export function analyzeSource(source, file) {
+/**
+ * Matches every ESLint suppression-directive comment: `// eslint-disable`,
+ * `// eslint-disable-line`, `// eslint-disable-next-line`, and their
+ * `/* ... *\/` block-comment equivalents. Deliberately does NOT parse which
+ * rule(s) it names or narrow to only our own rule ids — a pre-2.0.0 review
+ * found this project's own docs (VALIDATION.md) recommending
+ * `eslint-disable-next-line pwscore/no-coordinate-click` as "the correct
+ * escape hatch", while every one of this project's rules can in fact be
+ * silenced this way, defeating the score entirely for whatever it hides.
+ * Reporting the comment itself, without attempting to resolve exactly what
+ * it suppresses, is a known-coarse but honest signal — see VALIDATION.md's
+ * "Known limitations" for the plan to make it precise. Scored (a real
+ * demerit, not report-only) as of 2.0.0/model v4 — see CHANGELOG.md.
+ */
+const ESLINT_DISABLE_COMMENT_RE = /\/[/*]\s*eslint-disable(?:-next-line|-line)?\b/g;
+export function findEslintDisableComments(source, file) {
+    const findings = [];
+    ESLINT_DISABLE_COMMENT_RE.lastIndex = 0;
+    let m;
+    while ((m = ESLINT_DISABLE_COMMENT_RE.exec(source)) !== null) {
+        findings.push({
+            rule: 'pwscore/eslint-disable-comment',
+            severity: 'warning',
+            message: 'Inline ESLint suppression comment. Disabling a rule hides whatever finding it silences from every tool that runs it, including this scorer — fix the underlying issue, or if the finding is a genuine false positive, report it as a bug in this project rather than relying on a permanent local workaround.',
+            file,
+            line: source.slice(0, m.index).split('\n').length,
+            dimension: 'structure',
+        });
+    }
+    return findings;
+}
+export function analyzeSource(source, file, testAliasNames = []) {
     return {
         sloc: countSloc(source),
-        tests: countTests(source),
+        tests: countTests(source, testAliasNames),
         locators: countLocators(source, file),
         // No homegrown "has no assertions" check here: playwright/expect-expect
         // (a real AST-based community rule, always enabled — see
@@ -465,6 +515,6 @@ export function analyzeSource(source, file) {
         // call and so false-positived at error severity on any file whose only
         // assertions were expect.poll()/expect.soft() — verified against real
         // production code.
-        findings: [...findOversizedFile(source, file)],
+        findings: [...findOversizedFile(source, file), ...findEslintDisableComments(source, file)],
     };
 }

package/dist/playwright-config.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Walks upward from `startDir` to find this scan's repo boundary — the
+ * first ancestor directory that contains a `.git` entry (a directory for a
+ * normal clone, a file for a worktree/submodule — either counts as "this is
+ * a repo root").
+ *
+ * When no `.git` turns up within MAX_ANCESTOR_LOOKUP hops or before
+ * reaching the filesystem root, the *caller* decides what's safe to fall
+ * back to via `fallbackBoundary`:
+ *
+ * - `findPlaywrightConfig`'s own internal use (below) omits it, keeping
+ *   this function's legacy behavior of returning whatever directory the
+ *   climb stopped at. That's fine there — it only ever bounds a loop of
+ *   cheap `fs.statSync` existence checks for a handful of fixed filenames,
+ *   never a glob or a file read, so a wide bound doesn't let an untrusted
+ *   repo make this tool touch anything outside itself.
+ * - Every caller that uses the return value as an actual security
+ *   boundary — most importantly `resolveConfigScopedRoot`'s `repoRoot`,
+ *   which clamps a config's `testDir` before it becomes a glob root (see
+ *   its own doc comment) — MUST pass an explicit `fallbackBoundary`, and
+ *   it must never be the filesystem root or some arbitrary ancestor 20
+ *   levels up (that would make the "repo boundary" nearly unbounded and
+ *   defeat the escape checks it exists to support). index.ts passes the
+ *   scanned directory, or, when a config file was found above it, that
+ *   config file's own directory — whichever of the two is the ancestor
+ *   (the config file is always found by walking up from the scan root, so
+ *   its directory is always at-or-above it).
+ */
+export declare function findRepoBoundary(startDir: string, fallbackBoundary?: string): string;
+/**
+ * Walks upward from `scanRootAbs` (the directory the caller pointed at, or
+ * its own directory tree) looking for a playwright.config.* file "at or
+ * above" it — the common real-world shape is a monorepo root config one or
+ * more levels above the actual spec directory being scanned. Never searches
+ * above the repo boundary (see findRepoBoundary): a config living outside
+ * the scanned repo (e.g. a coincidental playwright.config.* sitting in some
+ * unrelated ancestor directory on a shared CI runner) must never be picked
+ * up and applied to this scan.
+ */
+export declare function findPlaywrightConfig(scanRootAbs: string): string | undefined;
+export interface ParsedPlaywrightConfig {
+    /** Resolved absolute path (config's `testDir`, default: the config's own directory). */
+    testDirAbs: string;
+    /** Glob patterns, relative to testDirAbs — undefined when the config didn't set testMatch (index.ts's own default SPEC_GLOBS still apply). */
+    testMatch?: string[];
+    /** Glob patterns, relative to testDirAbs — undefined when the config didn't set testIgnore. */
+    testIgnore?: string[];
+}
+/**
+ * Parses `testDir`/`testMatch`/`testIgnore` out of a playwright.config.*
+ * file's source, statically (AST-based, never executed — a scored repo's
+ * config is untrusted input, same reasoning as never running its tests).
+ * Returns undefined when the shape can't be read with confidence (a
+ * dynamic value from `process.env`, a spread, an unsupported top-level
+ * export shape, ...) — callers fall back to today's filename-suffix
+ * discovery for the whole scan, with a warning, rather than acting on a
+ * partial or guessed reading.
+ */
+export declare function parsePlaywrightConfig(configFile: string): ParsedPlaywrightConfig | undefined;
+/**
+ * True when `testDirAbs`, resolved through any symlinks, is NOT inside
+ * (or equal to) `repoRoot`, also resolved through symlinks. A scored
+ * repo's config is untrusted input, and a lexical ancestor check alone
+ * isn't enough — a `testDir` that lexically sits inside the repo can still
+ * be a symlink whose real target is elsewhere on disk (e.g. `tests/` ->
+ * `/tmp/somewhere-else`), which would otherwise let a crafted repo make
+ * this public tool glob outside its own boundary. Both sides are
+ * realpath'd (never just testDirAbs) because `repoRoot` itself could in
+ * principle be reached through a symlinked path.
+ */
+export declare function testDirEscapesRepo(testDirAbs: string, repoRoot: string): boolean;
+/**
+ * Chooses which directory to actually glob from, given the directory the
+ * caller pointed `scorePaths` at (`scanRootAbs`) and the config's resolved
+ * `testDir`. Always testDirAbs, never the caller's narrower `scanRootAbs`
+ * — Playwright itself unconditionally resolves testMatch/testIgnore
+ * relative to testDir, and a testMatch entry is not always a bare
+ * basename pattern; it can embed real path segments anchored at testDir
+ * (verified against a real config, PostHog/posthog's `playwright/
+ * playwright.config.ts`: `testDir: '..'` — the repo root — with
+ * `testMatch: ['playwright/e2e/**\/*.spec.ts', 'products/*\/frontend/e2e/
+ * **\/*.spec.ts']`; globbing from the caller's narrower `scanRootAbs`
+ * instead of testDirAbs made the `playwright/e2e/` segment never match
+ * anything, silently zeroing a real, correctly-discovered config). The
+ * caller (index.ts) is responsible for filtering the resulting absolute
+ * paths back down to its own requested subtree when `scanRootAbs` is
+ * narrower than `testDirAbs`, so scoping to testDir here never widens
+ * what actually gets scored beyond what the caller asked for.
+ *
+ * Returns undefined when testDir doesn't exist on disk, or doesn't
+ * overlap the requested scan root at all in either direction — the
+ * config doesn't describe this scan, so the caller should ignore it
+ * entirely rather than force a mismatch.
+ *
+ * `repoRoot` (see findRepoBoundary) clamps `testDirAbs` before any of the
+ * above, via `testDirEscapesRepo` (realpath'd on both sides, so a
+ * symlinked `testDir` can't launder an escape either): a scored repo's own
+ * config is untrusted input, and a `testDir` that resolves outside the
+ * repo (an absolute `'/'`, a relative `'../../..'` that climbs past the
+ * repo root, a symlink pointing elsewhere, ...) must never turn into a
+ * glob root — that would let a crafted config make this public tool read
+ * or scan arbitrary parts of the filesystem it's running on. When
+ * testDirAbs falls outside repoRoot, repoRoot itself is used as the
+ * effective testDir instead of rejecting the config outright, so a
+ * merely-overshooting `testDir` still gets *some* config-scoped result
+ * (bounded to the repo) rather than silently falling all the way back to
+ * unscoped filename discovery. index.ts additionally treats this
+ * escaped-and-clamped case as "the config doesn't cleanly describe this
+ * scan" and falls back to filename-based discovery with a configWarning,
+ * rather than applying the config's testMatch/testIgnore (written for the
+ * escaped testDir) to the clamped repoRoot instead.
+ */
+export declare function resolveConfigScopedRoot(scanRootAbs: string, testDirAbs: string, repoRoot: string): string | undefined;
+export declare function importsPlaywrightTestTransitively(file: string, source: string, boundaryDir: string): boolean;