@qaguardian/playwright-score 0.1.14 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/METHODOLOGY.md CHANGED
@@ -11,14 +11,11 @@ Any change to formulas, weights, SLOC definition, caps, or constants requires a
11
11
 
12
12
  ---
13
13
 
14
- ## Profiles
14
+ ## Profile
15
15
 
16
- | Profile | Purpose |
17
- |---|---|
18
- | `standard` | Community Playwright best practices (`eslint-plugin-playwright`) + suite metrics |
19
- | `guardian` | `standard` + QA Guardian house conventions (optional; clearly labeled) |
20
-
21
- House rules are **not** claimed to be official Playwright doctrine.
16
+ `standard` (the only profile): community Playwright best practices
17
+ (`eslint-plugin-playwright`) plus suite-level metrics (locator ratio,
18
+ assertion-delegation tracing, Page Object Model import resolution).
22
19
 
23
20
  ---
24
21
 
@@ -34,8 +31,6 @@ House rules are **not** claimed to be official Playwright doctrine.
34
31
 
35
32
  ## Dimension weights
36
33
 
37
- ### `standard`
38
-
39
34
  | Dimension | Weight |
40
35
  |---|---|
41
36
  | playwrightHygiene | 40 |
@@ -43,16 +38,6 @@ House rules are **not** claimed to be official Playwright doctrine.
43
38
  | locators | 20 |
44
39
  | structure | 15 |
45
40
 
46
- ### `guardian`
47
-
48
- | Dimension | Weight |
49
- |---|---|
50
- | playwrightHygiene | 30 |
51
- | assertions | 15 |
52
- | locators | 15 |
53
- | structure | 15 |
54
- | guardianConventions | 25 |
55
-
56
41
  ---
57
42
 
58
43
  ## Locators dimension (ratio)
@@ -88,7 +73,7 @@ Used for penalty density. JSDoc / decorative comments do not inflate slots.
88
73
 
89
74
  ## Penalty dimensions
90
75
 
91
- Applies to: `playwrightHygiene`, `assertions`, `structure`, `guardianConventions`.
76
+ Applies to: `playwrightHygiene`, `assertions`, `structure`.
92
77
 
93
78
  ### Constants (frozen sqs-v1)
94
79
 
@@ -159,11 +144,13 @@ These examples feel fair: tiny single error stays high; multi-error drops materi
159
144
 
160
145
  ## Default thresholds
161
146
 
162
- | Context | Profile | Threshold |
163
- |---|---|---|
164
- | CLI default | standard | 80 |
165
- | QAG dogfood (warn) | guardian | 75 → 80 |
166
- | QAG gate (later) | guardian | 80 |
147
+ | Context | Threshold |
148
+ |---|---|
149
+ | CLI default | 80 |
150
+
151
+ QA Guardian's own internal codegen pipeline (`playwright_runner`) layers a
152
+ private house-rules gate on top of `standard` — see the README's
153
+ "QA Guardian integration" section. That layer isn't part of this package.
167
154
 
168
155
  ---
169
156
 
@@ -174,5 +161,14 @@ See package rule docs / source `profiles.ts`. High level:
174
161
  - **playwrightHygiene:** waits, force, networkidle, missing await, handles, conditionals
175
162
  - **assertions:** expect-expect, empty tests, web-first preference
176
163
  - **locators:** ratio metric only (for score); raw-locator ESLint in findings
177
- - **structure:** focused/skipped, oversized file, describe shape
178
- - **guardianConventions:** load-state ban, Date.now ban, secrets, test.step, one-describe-one-test, etc.
164
+ - **structure:** focused/skipped, oversized file, describe shape
165
+
166
+ ### Active rule set (frozen)
167
+
168
+ The exact ESLint rules that produce sqs-v1 findings are enabled explicitly
169
+ in `eslint-runner.ts` with fixed severities — the score never inherits
170
+ eslint-plugin-playwright's `recommended` set at install time, so identical
171
+ code scores identically regardless of which plugin version npm resolves.
172
+ (Versions ≤0.2.0 intended to inherit `recommended` but a config-shape bug
173
+ dropped it entirely; 0.3.0 froze the intended set explicitly. See
174
+ CHANGELOG.)
package/README.md CHANGED
@@ -9,14 +9,51 @@
9
9
  **Website:** [qaguardian.com/open-source/playwright-score](https://qaguardian.com/open-source/playwright-score)
10
10
  **Built by:** [QA Guardian](https://qaguardian.com) — managed Playwright E2E (AI drafts, engineers verify, you own the code)
11
11
 
12
- Lint + score Playwright tests with:
13
-
14
- - **`standard`** — community best practices via [`eslint-plugin-playwright`](https://github.com/playwright-community/eslint-plugin-playwright) + suite metrics
15
- - **`guardian`** — optional house conventions (QA Guardian codegen gate); not official Playwright doctrine
12
+ Lint + score Playwright tests against community best practices via
13
+ [`eslint-plugin-playwright`](https://github.com/playwright-community/eslint-plugin-playwright)
14
+ plus suite-level metrics (locator ratio, assertion-delegation tracing,
15
+ Page Object Model import resolution, ...).
16
16
 
17
17
  The score never calls an LLM. AI may *generate* or *repair* code using findings; **rules grade code**.
18
18
 
19
- See [METHODOLOGY.md](./METHODOLOGY.md) for the frozen formula, [VALIDATION.md](./VALIDATION.md) for real-world results against public repos (cal.com, Grafana, Mattermost, ...) and a re-runnable script to reproduce them, or the full product write-up on the [landing page](https://qaguardian.com/open-source/playwright-score).
19
+ ## Real-world results
20
+
21
+ 17 public Playwright suites, scored with the published package against each
22
+ project's actual source — chosen to include both well-known,
23
+ heavily-engineered platforms and smaller, less mature projects. Not curated
24
+ to look good: 3 of 17 fail the default threshold, for real, verifiable
25
+ reasons (see [VALIDATION.md](./VALIDATION.md)).
26
+
27
+ **17 suites · 1,228 files · 6,037 tests · 14/17 passed (80% threshold)**
28
+
29
+ | Repo (source scanned) | Score | Grade | Result |
30
+ |---|---:|:-:|:-:|
31
+ | [Playwright (own TodoMVC example)](https://github.com/microsoft/playwright/tree/main/examples/todomvc) | 98/100 | A | PASS |
32
+ | [Supabase](https://github.com/supabase/supabase/tree/master/e2e) | 97/100 | A | PASS |
33
+ | [Documenso](https://github.com/documenso/documenso/tree/main/packages/app-tests) | 96/100 | A | PASS |
34
+ | [Storybook](https://github.com/storybookjs/storybook/tree/next/code/e2e-internal) | 96/100 | A | PASS |
35
+ | [freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp/tree/main/e2e) | 96/100 | A | PASS |
36
+ | [n8n](https://github.com/n8n-io/n8n/tree/master/packages/testing/playwright) | 95/100 | A | PASS |
37
+ | [dub](https://github.com/dubinc/dub/tree/main/apps/web/playwright) | 95/100 | A | PASS |
38
+ | [novu](https://github.com/novuhq/novu/tree/next/apps/dashboard/tests) | 93/100 | A | PASS |
39
+ | [Grafana](https://github.com/grafana/grafana/tree/main/e2e-playwright) | 91/100 | A | PASS |
40
+ | [PostHog](https://github.com/PostHog/posthog/tree/master/playwright) | 90/100 | A | PASS |
41
+ | [sencho](https://github.com/Studio-Saelix/sencho/tree/main/e2e) | 90/100 | A | PASS |
42
+ | [Mattermost](https://github.com/mattermost/mattermost/tree/master/e2e-tests/playwright) | 89/100 | B | PASS |
43
+ | [Immich](https://github.com/immich-app/immich/tree/main/e2e/src/specs) | 89/100 | B | PASS |
44
+ | [cal.com](https://github.com/calcom/cal.com/tree/main/apps/web/playwright) | 84/100 | B | PASS |
45
+ | [livecodes](https://github.com/live-codes/livecodes/tree/develop/e2e) | 78/100 | C | FAIL |
46
+ | [openplayerjs](https://github.com/openplayerjs/openplayerjs/tree/master/e2e) | 77/100 | C | FAIL |
47
+ | [TheCyberHub](https://github.com/th3cyb3rhub/TheCyberHub/tree/dev/e2e) | 68/100 | D | FAIL |
48
+
49
+ Repo names link to the exact source scanned. Full methodology, findings
50
+ breakdown, and `scripts/validate-corpus.sh` to reproduce every number here
51
+ yourself live in [VALIDATION.md](./VALIDATION.md). A self-contained visual
52
+ report of an earlier five-suite audit lives at
53
+ [`docs/scorecard.html`](./docs/scorecard.html) — host it wherever
54
+ (GitHub Pages, qaguardian.com, ...).
55
+
56
+ See [METHODOLOGY.md](./METHODOLOGY.md) for the frozen formula, or the full product write-up on the [landing page](https://qaguardian.com/open-source/playwright-score).
20
57
 
21
58
  ## Install
22
59
 
@@ -33,12 +70,12 @@ Package: [@qaguardian/playwright-score](https://www.npmjs.com/package/@qaguardia
33
70
  npx -p @qaguardian/playwright-score playwright-score ./tests --profile standard --threshold 80
34
71
 
35
72
  # After local install
36
- npx playwright-score ./flow.spec.ts --profile guardian --format json --out report.json
73
+ npx playwright-score ./flow.spec.ts --format json --out report.json
37
74
  ```
38
75
 
39
76
  | Flag | Description |
40
77
  |---|---|
41
- | `--profile standard\|guardian` | Scoring profile (default `standard`) |
78
+ | `--profile standard` | Scoring profile (default, and only, profile) |
42
79
  | `--threshold <n>` | Pass bar 0–100 |
43
80
  | `--format text\|json\|markdown\|sarif` | Output format |
44
81
  | `--out <file>` | Write report to file |
@@ -69,16 +106,15 @@ keep a sticky PR comment with the full findings up to date:
69
106
  - uses: qa-guardian/playwright-score@v1
70
107
  with:
71
108
  paths: tests e2e
72
- profile: standard # or guardian
73
- # threshold: 80 # defaults to the profile's default
109
+ # threshold: 80 # defaults to 80
74
110
  mode: gate # gate: fail CI below threshold · warn: report only
75
111
  ```
76
112
 
77
113
  | Input | Description | Default |
78
114
  |---|---|---|
79
115
  | `paths` | Space-separated paths/globs to score | `tests` |
80
- | `profile` | `standard` \| `guardian` | `standard` |
81
- | `threshold` | Pass bar 0–100 | profile default (80 / 75) |
116
+ | `profile` | `standard` (default, and only, profile) | `standard` |
117
+ | `threshold` | Pass bar 0–100 | `80` |
82
118
  | `mode` | `gate` (fail CI below threshold) \| `warn` (report only) | `gate` |
83
119
  | `comment` | Post/update a sticky PR comment | `true` |
84
120
  | `version` | `@qaguardian/playwright-score` version to run | `latest` |
@@ -97,13 +133,17 @@ section above — same score, same exit codes:
97
133
 
98
134
  ## QA Guardian integration
99
135
 
100
- When used from `playwright_runner`, set:
136
+ QA Guardian's own codegen pipeline (`playwright_runner`) dogfoods this
137
+ package for the `standard` profile, layering its own private house-rules
138
+ gate on top internally — that layer isn't part of this package (it's
139
+ product-specific, e.g. "timeouts must be exactly 2000 or 20000ms", not
140
+ Playwright best practice) and isn't published here. `playwright_runner`
141
+ sets:
101
142
 
102
143
  | Env | Values | Default |
103
144
  |---|---|---|
104
145
  | `SPEC_SCORE_MODE` | `off` \| `warn` \| `gate` | `warn` |
105
- | `SPEC_SCORE_PROFILE` | `standard` \| `guardian` | `guardian` |
106
- | `SPEC_SCORE_THRESHOLD` | `0`–`100` | `75` (guardian) / `80` (standard) |
146
+ | `SPEC_SCORE_THRESHOLD` | `0`–`100` | `80` |
107
147
 
108
148
  - **`warn`**: log score; never fail the run; findings still inject into heal/generate repair prompts
109
149
  - **`gate`**: fail the run when score &lt; threshold
@@ -114,21 +154,7 @@ When used from `playwright_runner`, set:
114
154
  # Local from monorepo
115
155
  cd playwright-score && npm run build
116
156
  cd ../playwright_runner && npm install
117
- SPEC_SCORE_MODE=warn SPEC_SCORE_PROFILE=guardian node ...
118
- ```
119
-
120
- ## Profiles
121
-
122
- | Profile | Use when |
123
- |---|---|
124
- | `standard` | Any Playwright repo — best practices + locator mix + empty-expect detection |
125
- | `guardian` | QA Guardian (or similar) codegen quality gate |
126
-
127
- Disable a guardian secrets false positive:
128
-
129
- ```ts
130
- // spec-score-disable-next-line guardian/no-hardcoded-secrets
131
- const password = 'intentional-fixture-value-xyz';
157
+ SPEC_SCORE_MODE=warn node ...
132
158
  ```
133
159
 
134
160
  ## Development
package/dist/cli.js CHANGED
@@ -18,8 +18,8 @@ Usage:
18
18
  npx playwright-score <paths...> [options]
19
19
 
20
20
  Options:
21
- --profile <standard|guardian> Scoring profile (default: standard)
22
- --threshold <n> Pass threshold 0-100 (default: 80 standard / 75 guardian)
21
+ --profile <standard> Scoring profile (default, and only: standard)
22
+ --threshold <n> Pass threshold 0-100 (default: 80)
23
23
  --format <text|json|markdown|sarif> Output format (default: text)
24
24
  --out <file> Write report to file
25
25
  --version Print version
@@ -70,12 +70,18 @@ async function main() {
70
70
  for (let i = 0; i < args.length; i++) {
71
71
  const a = args[i];
72
72
  if (a === '--profile') {
73
- profile = args[++i];
74
- if (profile !== 'standard' && profile !== 'guardian') {
75
- console.error(`Invalid profile: ${profile}`);
73
+ const p = args[++i];
74
+ if (p === 'guardian') {
75
+ console.error('Profile "guardian" is no longer public — it was QA Guardian\'s internal house-rules layer, never official Playwright doctrine, and doesn\'t belong in a general-purpose tool. See CHANGELOG.md. Use --profile standard.');
76
76
  process.exitCode = 2;
77
77
  return;
78
78
  }
79
+ if (p !== 'standard') {
80
+ console.error(`Invalid profile: ${p}`);
81
+ process.exitCode = 2;
82
+ return;
83
+ }
84
+ profile = p;
79
85
  }
80
86
  else if (a === '--threshold') {
81
87
  threshold = Number(args[++i]);
@@ -2,12 +2,9 @@ import path from 'node:path';
2
2
  import { ESLint } from 'eslint';
3
3
  import { commonAncestorDir } from './fs-util.js';
4
4
  import playwright from 'eslint-plugin-playwright';
5
- import { guardianPlugin, guardianRuleConfigs, } from './rules/guardian-plugin.js';
6
5
  import { basePlugin } from './rules/base-plugin.js';
7
6
  import { mapRule } from './profiles.js';
8
- function buildConfig(profile, assertFunctionNames) {
9
- const playwrightRecommended = playwright.configs?.['flat/recommended'] ??
10
- playwright.configs?.recommended;
7
+ function buildConfig(assertFunctionNames) {
11
8
  const base = [
12
9
  {
13
10
  files: ['**/*.{ts,tsx,js,jsx,mjs,cjs}'],
@@ -19,7 +16,6 @@ function buildConfig(profile, assertFunctionNames) {
19
16
  plugins: {
20
17
  playwright,
21
18
  pwscore: basePlugin,
22
- ...(profile === 'guardian' ? { guardian: guardianPlugin } : {}),
23
19
  },
24
20
  rules: {
25
21
  // Ensure key hygiene rules even if recommended set differs by
@@ -91,26 +87,34 @@ function buildConfig(profile, assertFunctionNames) {
91
87
  'playwright/prefer-native-locators': 'warn',
92
88
  // Matches eslint-plugin-playwright's own recommended severity.
93
89
  'playwright/prefer-web-first-assertions': 'error',
94
- ...(profile === 'guardian' ? { ...guardianRuleConfigs } : {}),
90
+ // The rules below complete the sqs-v1 rule set from
91
+ // profiles.ts's ESLINT_RULE_MAP. Earlier versions (<=0.2.0)
92
+ // expected them to arrive via eslint-plugin-playwright's
93
+ // flat/recommended config, but the spread only handled an array
94
+ // shape while the plugin exports a single config object — so the
95
+ // entire recommended set was silently dropped and these rules
96
+ // never ran (METHODOLOGY.md's hygiene dimension lists
97
+ // "conditionals" and "handles", yet neither could ever fire).
98
+ // They are enabled explicitly and frozen here on purpose:
99
+ // eslint-plugin-playwright is a ^range dependency, and inheriting
100
+ // whatever its recommended set says at install time would make
101
+ // identical code score differently across installs. Severities
102
+ // mirror upstream flat/recommended as of eslint-plugin-playwright
103
+ // 2.x. require-top-level-describe stays disabled deliberately —
104
+ // it is stylistic, not in upstream recommended, and would
105
+ // penalize idiomatic describe-less suites.
106
+ 'playwright/no-conditional-in-test': 'warn',
107
+ 'playwright/no-conditional-expect': 'warn',
108
+ 'playwright/no-element-handle': 'warn',
109
+ 'playwright/no-eval': 'warn',
110
+ 'playwright/no-page-pause': 'warn',
111
+ 'playwright/no-useless-await': 'warn',
112
+ 'playwright/valid-expect': 'error',
113
+ 'playwright/no-standalone-expect': 'error',
114
+ 'playwright/max-nested-describe': 'warn',
95
115
  },
96
116
  },
97
117
  ];
98
- // Prefer spreading official recommended flat config when available
99
- if (Array.isArray(playwrightRecommended)) {
100
- const configs = playwrightRecommended;
101
- if (profile === 'guardian') {
102
- return [
103
- ...configs,
104
- {
105
- plugins: { guardian: guardianPlugin },
106
- rules: { ...guardianRuleConfigs },
107
- },
108
- // our severity tweaks
109
- base[0],
110
- ];
111
- }
112
- return [...configs, base[0]];
113
- }
114
118
  return base;
115
119
  }
116
120
  function toSeverity(eslintSeverity, override) {
@@ -138,7 +142,7 @@ export async function runEslint(options) {
138
142
  const cwd = options.files.length > 0
139
143
  ? commonAncestorDir(options.files)
140
144
  : (options.cwd ?? process.cwd());
141
- const overrideConfig = buildConfig(options.profile, options.assertFunctionNames ?? []);
145
+ const overrideConfig = buildConfig(options.assertFunctionNames ?? []);
142
146
  // Try to attach typescript-eslint parser for .ts files
143
147
  try {
144
148
  const tsParser = await import('@typescript-eslint/parser');
package/dist/index.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { ScoreOptions, ScoreResult } from './types.js';
2
2
  export type { ScoreResult, ScoreOptions, Finding, ProfileName } from './types.js';
3
- export { computeScore, SQS_V1, locatorsScore, gradeFromScore } from './score-engine.js';
3
+ export { computeScore, SQS_V1, locatorsScore, gradeFromScore, penaltyDimensionScore, applyPerRuleCap, } from './score-engine.js';
4
4
  export { countSloc } from './sloc.js';
5
5
  export { countLocators, analyzeSource } from './metrics.js';
6
6
  export { formatJson } from './formatters/json.js';
package/dist/index.js CHANGED
@@ -7,7 +7,7 @@ import { collectLocallyImportedFiles } from './import-graph.js';
7
7
  import { analyzeSource, countLocators, findLocalAssertionHelperNames, looksLikeNonPlaywrightTest, } from './metrics.js';
8
8
  import { DEFAULT_THRESHOLDS } from './profiles.js';
9
9
  import { computeScore } from './score-engine.js';
10
- export { computeScore, SQS_V1, locatorsScore, gradeFromScore } from './score-engine.js';
10
+ export { computeScore, SQS_V1, locatorsScore, gradeFromScore, penaltyDimensionScore, applyPerRuleCap, } from './score-engine.js';
11
11
  export { countSloc } from './sloc.js';
12
12
  export { countLocators, analyzeSource } from './metrics.js';
13
13
  export { formatJson } from './formatters/json.js';
@@ -132,7 +132,6 @@ function hardFail(profile, threshold, findings, extra = {}, skippedFiles) {
132
132
  assertions: 0,
133
133
  locators: 0,
134
134
  structure: 0,
135
- ...(profile === 'guardian' ? { guardianConventions: 0 } : {}),
136
135
  },
137
136
  findings,
138
137
  ...(skippedFiles && skippedFiles.length > 0 ? { skippedFiles } : {}),
@@ -242,7 +241,6 @@ export async function scorePaths(options) {
242
241
  totalTests += m.tests;
243
242
  native += m.locators.native;
244
243
  raw += m.locators.raw;
245
- // metrics/no-empty-test duplicates guardian/require-expect under guardian — keep both mild
246
244
  metricFindings.push(...m.findings);
247
245
  }
248
246
  for (const [file, source] of importedFiles) {
@@ -250,7 +248,6 @@ export async function scorePaths(options) {
250
248
  native += counts.native;
251
249
  raw += counts.raw;
252
250
  }
253
- // Dedupe empty-test style: if guardian require-expect already fired, still ok
254
251
  const findings = [...eslintFindings, ...metricFindings];
255
252
  const parseErrors = findings.filter((f) => f.rule === 'playwright-score/parse-error');
256
253
  if (parseErrors.length > 0) {
package/dist/profiles.js CHANGED
@@ -5,17 +5,9 @@ export const PROFILE_WEIGHTS = {
5
5
  locators: 20,
6
6
  structure: 15,
7
7
  },
8
- guardian: {
9
- playwrightHygiene: 30,
10
- assertions: 15,
11
- locators: 15,
12
- structure: 15,
13
- guardianConventions: 25,
14
- },
15
8
  };
16
9
  export const DEFAULT_THRESHOLDS = {
17
10
  standard: 80,
18
- guardian: 75,
19
11
  };
20
12
  /**
21
13
  * ESLint rule → dimension map (plugin rule ids without prefix handled in runner).
@@ -80,26 +72,6 @@ export const ESLINT_RULE_MAP = {
80
72
  dimension: 'structure',
81
73
  severityOverride: 'warning',
82
74
  },
83
- // Guardian custom rules
84
- 'guardian/no-wait-for-load-state': { dimension: 'guardianConventions' },
85
- 'guardian/no-date-now-id': { dimension: 'guardianConventions' },
86
- 'guardian/require-expect': { dimension: 'guardianConventions' },
87
- 'guardian/no-hardcoded-secrets': {
88
- dimension: 'guardianConventions',
89
- severityOverride: 'warning',
90
- },
91
- 'guardian/require-test-step': {
92
- dimension: 'guardianConventions',
93
- severityOverride: 'warning',
94
- },
95
- 'guardian/one-describe-one-test': {
96
- dimension: 'guardianConventions',
97
- severityOverride: 'warning',
98
- },
99
- 'guardian/no-generic-long-timeout': {
100
- dimension: 'guardianConventions',
101
- severityOverride: 'warning',
102
- },
103
75
  // Metrics injected as synthetic findings
104
76
  'metrics/oversized-file': {
105
77
  dimension: 'structure',
@@ -109,9 +81,6 @@ export const ESLINT_RULE_MAP = {
109
81
  export function mapRule(ruleId) {
110
82
  if (ESLINT_RULE_MAP[ruleId])
111
83
  return ESLINT_RULE_MAP[ruleId];
112
- if (ruleId.startsWith('guardian/')) {
113
- return { dimension: 'guardianConventions', severityOverride: 'warning' };
114
- }
115
84
  if (ruleId.startsWith('playwright/')) {
116
85
  return { dimension: 'playwrightHygiene', severityOverride: 'warning' };
117
86
  }
@@ -1,6 +1,6 @@
1
1
  /**
2
- * Rules active for both `standard` and `guardian` profiles — corrections
3
- * to upstream eslint-plugin-playwright behavior, not house style.
2
+ * Corrections to upstream eslint-plugin-playwright behavior, not house
3
+ * style — always active.
4
4
  */
5
5
  import type { Rule } from 'eslint';
6
6
  export declare const basePlugin: {
@@ -10,12 +10,7 @@ export const SQS_V1 = {
10
10
  MIN_SLOTS: 4,
11
11
  K: 0.4,
12
12
  };
13
- const PENALTY_DIMENSIONS = [
14
- 'playwrightHygiene',
15
- 'assertions',
16
- 'structure',
17
- 'guardianConventions',
18
- ];
13
+ const PENALTY_DIMENSIONS = ['playwrightHygiene', 'assertions', 'structure'];
19
14
  export function gradeFromScore(score) {
20
15
  if (score >= 90)
21
16
  return 'A';
@@ -69,16 +64,18 @@ export function penaltyDimensionScore(findings, dimension, sloc) {
69
64
  return Math.min(100, Math.max(0, score));
70
65
  }
71
66
  export function computeScore(input) {
72
- const weights = PROFILE_WEIGHTS[input.profile];
67
+ // Falls back to `standard`'s weights for any unrecognized profile string
68
+ // rather than throwing — a defensive guard against exactly the failure
69
+ // mode a profile removal creates: a caller pinned to an old contract
70
+ // (e.g. `profile: 'guardian'`, removed in 0.2.0) degrades to a working
71
+ // score instead of a hard crash.
72
+ const weights = PROFILE_WEIGHTS[input.profile] ?? PROFILE_WEIGHTS.standard;
73
73
  const dims = {
74
74
  playwrightHygiene: penaltyDimensionScore(input.findings, 'playwrightHygiene', input.sloc),
75
75
  assertions: penaltyDimensionScore(input.findings, 'assertions', input.sloc),
76
76
  locators: locatorsScore(input.nativeLocators, input.rawLocators),
77
77
  structure: penaltyDimensionScore(input.findings, 'structure', input.sloc),
78
78
  };
79
- if (input.profile === 'guardian') {
80
- dims.guardianConventions = penaltyDimensionScore(input.findings, 'guardianConventions', input.sloc);
81
- }
82
79
  let score = 0;
83
80
  for (const [dim, weight] of Object.entries(weights)) {
84
81
  const key = dim;
package/dist/types.d.ts CHANGED
@@ -1,8 +1,8 @@
1
1
  export type ScoreVersion = 'sqs-v1';
2
- export type ProfileName = 'standard' | 'guardian';
2
+ export type ProfileName = 'standard';
3
3
  export type Grade = 'A' | 'B' | 'C' | 'D' | 'F';
4
4
  export type Severity = 'error' | 'warning' | 'info';
5
- export type DimensionName = 'playwrightHygiene' | 'assertions' | 'locators' | 'structure' | 'guardianConventions';
5
+ export type DimensionName = 'playwrightHygiene' | 'assertions' | 'locators' | 'structure';
6
6
  export interface Finding {
7
7
  rule: string;
8
8
  severity: Severity;
@@ -19,7 +19,6 @@ export interface ScoreDimensions {
19
19
  assertions: number;
20
20
  locators: number;
21
21
  structure: number;
22
- guardianConventions?: number;
23
22
  }
24
23
  export interface ScoreSummary {
25
24
  files: number;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@qaguardian/playwright-score",
3
- "version": "0.1.14",
4
- "description": "Deterministic, AI-free Playwright spec quality score (sqs-v1). Lint + score with standard and guardian profiles.",
3
+ "version": "0.3.0",
4
+ "description": "Deterministic, AI-free Playwright spec quality score (sqs-v1).",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
7
7
  "types": "./dist/index.d.ts",
@@ -26,7 +26,6 @@
26
26
  "build:tests": "tsc -p tsconfig.tests.json",
27
27
  "test": "npm run build && npm run build:tests && node --test dist-tests/tests/*.js",
28
28
  "score": "npm run build && node bin/playwright-score.js",
29
- "dogfood": "npm run build && bash scripts/dogfood.sh",
30
29
  "prepublishOnly": "npm run build"
31
30
  },
32
31
  "keywords": [
@@ -1,29 +0,0 @@
1
- /**
2
- * Guardian house-style ESLint rules for profile=guardian.
3
- * Embedded in playwright-score — not claimed as official Playwright doctrine.
4
- */
5
- import type { Rule } from 'eslint';
6
- export declare const guardianPlugin: {
7
- meta: {
8
- name: string;
9
- version: string;
10
- };
11
- rules: {
12
- 'no-wait-for-load-state': Rule.RuleModule;
13
- 'no-date-now-id': Rule.RuleModule;
14
- 'no-hardcoded-secrets': Rule.RuleModule;
15
- 'no-generic-long-timeout': Rule.RuleModule;
16
- 'one-describe-one-test': Rule.RuleModule;
17
- 'require-test-step': Rule.RuleModule;
18
- 'require-expect': Rule.RuleModule;
19
- };
20
- };
21
- export declare const guardianRuleConfigs: {
22
- readonly 'guardian/no-wait-for-load-state': "error";
23
- readonly 'guardian/no-date-now-id': "error";
24
- readonly 'guardian/no-hardcoded-secrets': "warn";
25
- readonly 'guardian/no-generic-long-timeout': "warn";
26
- readonly 'guardian/one-describe-one-test': "warn";
27
- readonly 'guardian/require-test-step': "warn";
28
- readonly 'guardian/require-expect': "error";
29
- };
@@ -1,386 +0,0 @@
1
- const DUMMY_SECRETS = new Set([
2
- 'password',
3
- 'password123',
4
- 'test',
5
- 'test123',
6
- 'secret',
7
- 'changeme',
8
- 'admin',
9
- 'admin123',
10
- 'pass',
11
- 'pwd',
12
- '123456',
13
- 'qwerty',
14
- '',
15
- ].map((s) => s.toLowerCase()));
16
- // Playwright's actual modifier API is test.fixme() — not test.fix().
17
- const TEST_MODIFIERS = new Set(['only', 'skip', 'fixme', 'slow']);
18
- const DESCRIBE_MODIFIERS = new Set(['serial', 'parallel', 'only', 'fixme']);
19
- /** Matches a bare test(...) call. */
20
- function isBareTestCall(node) {
21
- return node.callee?.type === 'Identifier' && node.callee.name === 'test';
22
- }
23
- /** Matches test.describe(...) and test.describe.serial/parallel/only/fixme(...). */
24
- function isDescribeCall(node) {
25
- const callee = node.callee;
26
- if (!callee || callee.type !== 'MemberExpression' || callee.computed)
27
- return false;
28
- const obj = callee.object;
29
- const prop = callee.property;
30
- if (!obj || !prop || prop.type !== 'Identifier')
31
- return false;
32
- // test.describe(...)
33
- if (obj.type === 'Identifier' && obj.name === 'test' && prop.name === 'describe') {
34
- return true;
35
- }
36
- // test.describe.serial/parallel/only/fixme(...)
37
- if (obj.type === 'MemberExpression' &&
38
- !obj.computed &&
39
- obj.object?.type === 'Identifier' &&
40
- obj.object.name === 'test' &&
41
- obj.property?.type === 'Identifier' &&
42
- obj.property.name === 'describe' &&
43
- DESCRIBE_MODIFIERS.has(prop.name ?? '')) {
44
- return true;
45
- }
46
- return false;
47
- }
48
- /** Matches specifically test.describe.serial(...). */
49
- function isSerialDescribeCall(node) {
50
- const callee = node.callee;
51
- if (!callee || callee.type !== 'MemberExpression' || callee.computed)
52
- return false;
53
- const obj = callee.object;
54
- const prop = callee.property;
55
- return (obj?.type === 'MemberExpression' &&
56
- !obj.computed &&
57
- obj.object?.type === 'Identifier' &&
58
- obj.object.name === 'test' &&
59
- obj.property?.type === 'Identifier' &&
60
- obj.property.name === 'describe' &&
61
- prop?.type === 'Identifier' &&
62
- prop.name === 'serial');
63
- }
64
- /** Matches test.only/skip/fixme/slow(...) (a modifier form of a real test declaration). */
65
- function isTestModifierCall(node) {
66
- const callee = node.callee;
67
- if (!callee || callee.type !== 'MemberExpression' || callee.computed)
68
- return false;
69
- const obj = callee.object;
70
- const prop = callee.property;
71
- return (obj?.type === 'Identifier' &&
72
- obj.name === 'test' &&
73
- prop?.type === 'Identifier' &&
74
- TEST_MODIFIERS.has(prop.name ?? ''));
75
- }
76
- /** Matches test.step(...). */
77
- function isTestStepCall(node) {
78
- const callee = node.callee;
79
- return (callee?.type === 'MemberExpression' &&
80
- !callee.computed &&
81
- callee.object?.type === 'Identifier' &&
82
- callee.object.name === 'test' &&
83
- callee.property?.type === 'Identifier' &&
84
- callee.property.name === 'step');
85
- }
86
- function hasDisableNextLine(sourceCode, line, ruleId) {
87
- const lines = sourceCode.getText().split(/\r?\n/);
88
- if (line < 2)
89
- return false;
90
- const prev = lines[line - 2] ?? '';
91
- return (prev.includes('spec-score-disable-next-line') &&
92
- (prev.includes(ruleId) || prev.includes('guardian/')));
93
- }
94
- const noWaitForLoadState = {
95
- meta: {
96
- type: 'problem',
97
- docs: { description: 'Disallow page.waitForLoadState()' },
98
- messages: {
99
- banned: 'waitForLoadState is banned under guardian profile. Use web-first assertions.',
100
- },
101
- schema: [],
102
- },
103
- create(context) {
104
- return {
105
- CallExpression(node) {
106
- const callee = node.callee;
107
- if (callee.type === 'MemberExpression' &&
108
- !callee.computed &&
109
- callee.property.type === 'Identifier' &&
110
- callee.property.name === 'waitForLoadState') {
111
- context.report({ node, messageId: 'banned' });
112
- }
113
- },
114
- };
115
- },
116
- };
117
- const noDateNowId = {
118
- meta: {
119
- type: 'problem',
120
- docs: { description: 'Disallow Date.now() for unique ids' },
121
- messages: {
122
- banned: 'Date.now() is banned for unique ids. Use crypto.randomUUID().split("-")[0].',
123
- },
124
- schema: [],
125
- },
126
- create(context) {
127
- return {
128
- CallExpression(node) {
129
- const callee = node.callee;
130
- if (callee.type === 'MemberExpression' &&
131
- !callee.computed &&
132
- callee.object.type === 'Identifier' &&
133
- callee.object.name === 'Date' &&
134
- callee.property.type === 'Identifier' &&
135
- callee.property.name === 'now') {
136
- context.report({ node, messageId: 'banned' });
137
- }
138
- },
139
- };
140
- },
141
- };
142
- // Whole words only (after splitting camelCase/snake_case/kebab-case), not a
143
- // bare substring match — a substring match on "auth" flags "authorName",
144
- // "isAuthenticated", every "Author"/"authenticate"/"authorize" field in a
145
- // typical app under test, which is a lot of unrelated real-world test data.
146
- const SENSITIVE_WORDS = new Set([
147
- 'password',
148
- 'passwd',
149
- 'secret',
150
- 'secrets',
151
- 'token',
152
- 'tokens',
153
- 'auth',
154
- 'authorization',
155
- 'credential',
156
- 'credentials',
157
- ]);
158
- function splitIdentifierWords(identifier) {
159
- return identifier
160
- .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
161
- .replace(/[_-]+/g, ' ')
162
- .split(/\s+/)
163
- .filter(Boolean)
164
- .map((w) => w.toLowerCase());
165
- }
166
- function looksLikeSensitiveKey(rawKey) {
167
- const words = splitIdentifierWords(rawKey);
168
- if (words.some((w) => SENSITIVE_WORDS.has(w)))
169
- return true;
170
- // apiKey / api_key / API-KEY — "key" alone is too generic (primaryKey,
171
- // sortKey, ...) so only flag it directly adjacent to "api".
172
- for (let i = 0; i < words.length - 1; i++) {
173
- if (words[i] === 'api' && words[i + 1] === 'key')
174
- return true;
175
- }
176
- return false;
177
- }
178
- const noHardcodedSecrets = {
179
- meta: {
180
- type: 'suggestion',
181
- docs: { description: 'Flag likely hardcoded secrets (with allowlist)' },
182
- messages: {
183
- secret: 'Possible hardcoded secret value. Use env/fixtures or allowlist dummy data.',
184
- },
185
- schema: [],
186
- },
187
- create(context) {
188
- const sourceCode = context.sourceCode ?? context.getSourceCode();
189
- function check(node, rawKey) {
190
- if (node.type !== 'Literal' || typeof node.value !== 'string')
191
- return;
192
- const value = String(node.value);
193
- if (DUMMY_SECRETS.has(value.toLowerCase()))
194
- return;
195
- if (value.length < 6)
196
- return;
197
- if (!looksLikeSensitiveKey(rawKey ?? ''))
198
- return;
199
- const line = node.loc?.start.line ?? 1;
200
- if (hasDisableNextLine(sourceCode, line, 'guardian/no-hardcoded-secrets'))
201
- return;
202
- context.report({ node: node, messageId: 'secret' });
203
- }
204
- return {
205
- Property(node) {
206
- if (node.key.type === 'Identifier') {
207
- check(node.value, node.key.name);
208
- }
209
- else if (node.key.type === 'Literal') {
210
- check(node.value, String(node.key.value));
211
- }
212
- },
213
- VariableDeclarator(node) {
214
- if (node.id.type === 'Identifier' && node.init) {
215
- check(node.init, node.id.name);
216
- }
217
- },
218
- };
219
- },
220
- };
221
- const noGenericLongTimeout = {
222
- meta: {
223
- type: 'suggestion',
224
- docs: { description: 'Only allow timeout 2000 or 20000' },
225
- messages: {
226
- timeout: 'Avoid generic timeouts; allowed values under guardian: 2000 (audit) or 20000 (backend).',
227
- },
228
- schema: [],
229
- },
230
- create(context) {
231
- const allowed = new Set([2000, 20000]);
232
- return {
233
- Property(node) {
234
- if (node.key.type !== 'Identifier' ||
235
- node.key.name !== 'timeout' ||
236
- node.value.type !== 'Literal' ||
237
- typeof node.value.value !== 'number' ||
238
- allowed.has(node.value.value)) {
239
- return;
240
- }
241
- // Only flag timeout options passed inline to a call, e.g.
242
- // locator.click({ timeout }), expect(x).toBeVisible({ timeout }),
243
- // test('x', { timeout }, fn) — not unrelated business data that
244
- // happens to have a field named "timeout" (session config, a
245
- // fixture payload, a mocked API response body, ...).
246
- const objectExpr = node.parent;
247
- const call = objectExpr?.parent;
248
- if (objectExpr?.type !== 'ObjectExpression' ||
249
- call?.type !== 'CallExpression' ||
250
- !call.arguments?.includes(objectExpr)) {
251
- return;
252
- }
253
- context.report({ node, messageId: 'timeout' });
254
- },
255
- };
256
- },
257
- };
258
- const oneDescribeOneTest = {
259
- meta: {
260
- type: 'suggestion',
261
- docs: { description: 'Prefer one test.describe and one test per file' },
262
- messages: {
263
- describes: 'Guardian profile prefers exactly one test.describe() per file (found {{count}}).',
264
- tests: 'Guardian profile prefers exactly one test() per file (found {{count}}).',
265
- },
266
- schema: [],
267
- },
268
- create(context) {
269
- let describes = 0;
270
- let tests = 0;
271
- let usesSerialDescribe = false;
272
- return {
273
- CallExpression(node) {
274
- if (isDescribeCall(node)) {
275
- describes++;
276
- if (isSerialDescribeCall(node))
277
- usesSerialDescribe = true;
278
- }
279
- if (isBareTestCall(node) || isTestModifierCall(node))
280
- tests++;
281
- },
282
- 'Program:exit'() {
283
- if (describes > 1) {
284
- context.report({
285
- loc: { line: 1, column: 0 },
286
- messageId: 'describes',
287
- data: { count: String(describes) },
288
- });
289
- }
290
- // test.describe.serial(...) is Playwright's documented mechanism
291
- // for one user journey split into multiple ordered, dependent
292
- // test() calls — the same "one scenario per file" intent as this
293
- // rule, expressed a different way. Don't nag a pattern the
294
- // framework itself recommends for exactly this case.
295
- if (tests > 1 && !usesSerialDescribe) {
296
- context.report({
297
- loc: { line: 1, column: 0 },
298
- messageId: 'tests',
299
- data: { count: String(tests) },
300
- });
301
- }
302
- },
303
- };
304
- },
305
- };
306
- const requireTestStep = {
307
- meta: {
308
- type: 'suggestion',
309
- docs: { description: 'Warn when file uses test() but no test.step()' },
310
- messages: {
311
- missing: 'Consider wrapping actions in test.step() for clearer structure.',
312
- },
313
- schema: [],
314
- },
315
- create(context) {
316
- let hasTest = false;
317
- let hasStep = false;
318
- return {
319
- CallExpression(node) {
320
- if (isBareTestCall(node) || isTestModifierCall(node))
321
- hasTest = true;
322
- if (isTestStepCall(node))
323
- hasStep = true;
324
- },
325
- 'Program:exit'() {
326
- if (hasTest && !hasStep) {
327
- context.report({ loc: { line: 1, column: 0 }, messageId: 'missing' });
328
- }
329
- },
330
- };
331
- },
332
- };
333
- const requireExpect = {
334
- meta: {
335
- type: 'problem',
336
- docs: { description: 'Require expect() in files with test()' },
337
- messages: {
338
- missing: 'Guardian profile requires at least one expect() in the spec.',
339
- },
340
- schema: [],
341
- },
342
- create(context) {
343
- let hasTest = false;
344
- let hasExpect = false;
345
- return {
346
- CallExpression(node) {
347
- if (isBareTestCall(node) || isTestModifierCall(node))
348
- hasTest = true;
349
- if (node.callee?.type === 'Identifier' && node.callee.name === 'expect') {
350
- hasExpect = true;
351
- }
352
- if (node.callee?.type === 'MemberExpression' &&
353
- node.callee.object?.type === 'Identifier' &&
354
- node.callee.object.name === 'expect') {
355
- hasExpect = true;
356
- }
357
- },
358
- 'Program:exit'() {
359
- if (hasTest && !hasExpect) {
360
- context.report({ loc: { line: 1, column: 0 }, messageId: 'missing' });
361
- }
362
- },
363
- };
364
- },
365
- };
366
- export const guardianPlugin = {
367
- meta: { name: 'guardian', version: '0.1.0' },
368
- rules: {
369
- 'no-wait-for-load-state': noWaitForLoadState,
370
- 'no-date-now-id': noDateNowId,
371
- 'no-hardcoded-secrets': noHardcodedSecrets,
372
- 'no-generic-long-timeout': noGenericLongTimeout,
373
- 'one-describe-one-test': oneDescribeOneTest,
374
- 'require-test-step': requireTestStep,
375
- 'require-expect': requireExpect,
376
- },
377
- };
378
- export const guardianRuleConfigs = {
379
- 'guardian/no-wait-for-load-state': 'error',
380
- 'guardian/no-date-now-id': 'error',
381
- 'guardian/no-hardcoded-secrets': 'warn',
382
- 'guardian/no-generic-long-timeout': 'warn',
383
- 'guardian/one-describe-one-test': 'warn',
384
- 'guardian/require-test-step': 'warn',
385
- 'guardian/require-expect': 'error',
386
- };