eslint-plugin-test-governance 0.1.0 → 0.2.0

package/dist/utils/options.js

@@ -1,6 +1,13 @@
+import { DEFAULT_COMMENT_KEYWORDS } from './constants.js';
+import { ConfigError } from './errors.js';
+import { optionsSchema } from './schema.js';
 import { compileTicketSpec, DEFAULT_PLACEHOLDERS } from './tickets.js';
+import { validate } from './validate.js';
 export const SETTINGS_KEY = 'test-governance';
-export const BUILTIN_STATES = {
+export function isTagState(state) {
+    return state.tag !== undefined;
+}
+const BUILTIN_STATES = {
     skip: { marker: 'SKIP', modifier: 'skip', enabled: true },
     fixme: { marker: 'FIXME', modifier: 'fixme', enabled: true },
     fail: { marker: 'FAIL', modifier: 'fail', enabled: false },
@@ -8,39 +15,30 @@ export const BUILTIN_STATES = {
     new: { marker: 'NEW', tag: '@new', lifecycle: true, enabled: false },
     unstable: { marker: 'UNSTABLE', tag: '@unstable', lifecycle: true, enabled: false },
 };
-export const MARKER_RE = /^[A-Z][A-Z0-9_-]*$/;
-export const TAG_RE = /^@[\w-]+$/;
-export class ConfigError extends Error {
-    constructor(message) {
-        super(`eslint-plugin-test-governance: ${message}`);
-        this.name = 'ConfigError';
-    }
-}
+export { ConfigError } from './errors.js';
 function toSpecs(spec) {
     return Array.isArray(spec) ? spec : [spec];
 }
 const cache = new WeakMap();
 const EMPTY = {};
 /**
- * Merges rule options over `settings['test-governance']` and compiles the result.
- * The result is cached per settings object, so all four rules share one compile.
+ * Compiles `settings['test-governance']`. Every rule reads the same settings, so they always agree,
+ * and the result is cached per settings object, so the rules share one compile.
  */
-export function resolveOptions(settings, ruleOptions) {
+export function resolveOptions(settings) {
     const shared = (settings?.[SETTINGS_KEY] ?? EMPTY);
-    const key = JSON.stringify(ruleOptions ?? null);
-    let byRule = cache.get(shared);
-    if (!byRule) {
-        byRule = new Map();
-        cache.set(shared, byRule);
-    }
-    let resolved = byRule.get(key);
+    let resolved = cache.get(shared);
     if (!resolved) {
-        resolved = compileOptions({ ...shared, ...ruleOptions });
-        byRule.set(key, resolved);
+        resolved = compileOptions(shared);
+        cache.set(shared, resolved);
     }
     return resolved;
 }
 export function compileOptions(options) {
+    const problems = validate(options, optionsSchema);
+    if (problems.length > 0) {
+        throw new ConfigError(`invalid options:\n${problems.map((problem) => `  - ${problem}`).join('\n')}`);
+    }
     const placeholders = options.placeholders ?? DEFAULT_PLACEHOLDERS;
     const defaultTicket = toSpecs(options.ticket ?? { preset: 'any' });
     const compile = (spec) => compileTicketSpec(spec === undefined ? defaultTicket : toSpecs(spec), placeholders);
@@ -53,21 +51,17 @@ export function compileOptions(options) {
         const enabled = override.enabled ?? (def.lifecycle ? (options.lifecycleTags ?? def.enabled) : def.enabled);
         if (!enabled)
             continue;
-        states.push({
+        const base = {
             name,
-            marker: checkMarker(override.marker ?? def.marker, name),
-            modifier: def.modifier,
-            tag: def.tag,
+            marker: override.marker ?? def.marker,
             ticket: override.ticket === undefined ? defaultMatcher : compile(override.ticket),
-        });
+        };
+        states.push(def.modifier === undefined ? { ...base, tag: def.tag } : { ...base, modifier: def.modifier });
     }
     for (const [name, custom] of Object.entries(options.customStates ?? {})) {
-        if (!TAG_RE.test(custom.when)) {
-            throw new ConfigError(`customStates.${name}.when must be a tag like "@${name}", got "${custom.when}".`);
-        }
         states.push({
             name,
-            marker: checkMarker(custom.marker, name),
+            marker: custom.marker,
             tag: custom.when,
             ticket: custom.ticket === undefined ? defaultMatcher : compile(custom.ticket),
         });
@@ -84,6 +78,7 @@ export function compileOptions(options) {
             tags.add(state.tag);
         }
     }
+    const commentKeywords = options.comments === false ? [] : (options.comments?.keywords ?? DEFAULT_COMMENT_KEYWORDS);
     return {
         testFunctions: new Set(options.testFunctions ?? ['test']),
         states,
@@ -91,12 +86,9 @@ export function compileOptions(options) {
         allowBlankLine: options.allowBlankLine ?? false,
         allowNotes: options.allowNotes ?? false,
         reportDynamicTitles: options.reportDynamicTitles ?? false,
+        commentKeywords,
+        workCommentKeywords: new Set([...DEFAULT_COMMENT_KEYWORDS, ...commentKeywords]),
+        commentTicket: defaultMatcher,
     };
 }
-function checkMarker(marker, state) {
-    if (!MARKER_RE.test(marker)) {
-        throw new ConfigError(`the marker for state "${state}" must be uppercase letters, digits, "-" or "_", got "${marker}".`);
-    }
-    return marker;
-}
 //# sourceMappingURL=options.js.map

package/dist/utils/schema.d.ts

@@ -1,4 +1,4 @@
-import type { JSONSchema4 } from '@typescript-eslint/utils/json-schema';
-/** JSON schema shared by every rule; the same object is accepted in `settings['test-governance']`. */
-export declare const optionsSchema: JSONSchema4[];
+import type { Schema } from './validate.js';
+/** The shape of `settings['test-governance']`, the options of `configure()`. */
+export declare const optionsSchema: Schema;
 //# sourceMappingURL=schema.d.ts.map

package/dist/utils/schema.js

@@ -1,11 +1,10 @@
-import { BUILTIN_STATES, MARKER_RE, TAG_RE } from './options.js';
+import { BUILTIN_STATE_NAMES, MARKER_RE, TAG_RE, TICKET_PRESETS } from './constants.js';
+const MARKER_HINT = 'an uppercase keyword such as NEEDS-DATA (letters, digits, "-" and "_")';
+const TAG_HINT = 'a tag such as @needs-data';
 const ticketSpecSchema = {
     type: 'object',
     properties: {
-        preset: {
-            type: 'string',
-            enum: ['any', 'jira', 'github', 'gitlab', 'linear', 'azure-devops', 'numeric', 'pattern'],
-        },
+        preset: { type: 'string', enum: [...TICKET_PRESETS] },
         projects: { type: 'array', items: { type: 'string' } },
         teams: { type: 'array', items: { type: 'string' } },
         host: { type: 'string' },
@@ -27,46 +26,56 @@ const stateOverrideSchema = {
             type: 'object',
             properties: {
                 enabled: { type: 'boolean' },
-                marker: { type: 'string', pattern: MARKER_RE.source },
+                marker: { type: 'string', pattern: MARKER_RE.source, patternHint: MARKER_HINT },
                 ticket: ticketSchema,
             },
             additionalProperties: false,
         },
     ],
 };
-/** JSON schema shared by every rule; the same object is accepted in `settings['test-governance']`. */
-export const optionsSchema = [
-    {
-        type: 'object',
-        properties: {
-            testFunctions: { type: 'array', items: { type: 'string' } },
-            ticket: ticketSchema,
-            placeholders: { type: 'array', items: { type: 'string' } },
-            lifecycleTags: { type: 'boolean' },
-            states: {
+/** The shape of `settings['test-governance']`, the options of `configure()`. */
+export const optionsSchema = {
+    type: 'object',
+    properties: {
+        testFunctions: { type: 'array', items: { type: 'string' } },
+        ticket: ticketSchema,
+        placeholders: { type: 'array', items: { type: 'string' } },
+        lifecycleTags: { type: 'boolean' },
+        states: {
+            type: 'object',
+            properties: Object.fromEntries(BUILTIN_STATE_NAMES.map((name) => [name, stateOverrideSchema])),
+            additionalProperties: false,
+        },
+        customStates: {
+            type: 'object',
+            additionalProperties: {
                 type: 'object',
-                properties: Object.fromEntries(Object.keys(BUILTIN_STATES).map((name) => [name, stateOverrideSchema])),
+                properties: {
+                    when: { type: 'string', pattern: TAG_RE.source, patternHint: TAG_HINT },
+                    marker: { type: 'string', pattern: MARKER_RE.source, patternHint: MARKER_HINT },
+                    ticket: ticketSchema,
+                },
+                required: ['when', 'marker'],
                 additionalProperties: false,
             },
-            customStates: {
-                type: 'object',
-                additionalProperties: {
+        },
+        comments: {
+            anyOf: [
+                { type: 'boolean', enum: [false] },
+                {
                     type: 'object',
                     properties: {
-                        when: { type: 'string', pattern: TAG_RE.source },
-                        marker: { type: 'string', pattern: MARKER_RE.source },
-                        ticket: ticketSchema,
+                        keywords: { type: 'array', items: { type: 'string', pattern: MARKER_RE.source, patternHint: MARKER_HINT } },
                     },
-                    required: ['when', 'marker'],
                     additionalProperties: false,
                 },
-            },
-            requireTicketForConditional: { type: 'boolean' },
-            allowBlankLine: { type: 'boolean' },
-            allowNotes: { type: 'boolean' },
-            reportDynamicTitles: { type: 'boolean' },
+            ],
         },
-        additionalProperties: false,
+        requireTicketForConditional: { type: 'boolean' },
+        allowBlankLine: { type: 'boolean' },
+        allowNotes: { type: 'boolean' },
+        reportDynamicTitles: { type: 'boolean' },
     },
-];
+    additionalProperties: false,
+};
 //# sourceMappingURL=schema.js.map

package/dist/utils/tickets.js

@@ -1,3 +1,4 @@
+import { ConfigError } from './errors.js';
 export const DEFAULT_PLACEHOLDERS = ['TODO', 'TBD', 'XXX-*', '0', '123', '1234', '12345'];
 // Every pattern below is anchored and has no nested or overlapping quantifiers, so matching is
 // linear in the token length. URLs are parsed with `new URL` rather than a regex.
@@ -122,6 +123,9 @@ function compilePreset(spec) {
         case 'numeric': {
             const min = spec.minLength ?? 1;
             const max = spec.maxLength ?? 20;
+            if (min > max) {
+                throw new ConfigError(`numeric ticket minLength (${min}) is greater than maxLength (${max}).`);
+            }
             return {
                 test: (t) => DIGITS_RE.test(t) && t.length >= min && t.length <= max,
                 expected: min === max ? `a ${min}-digit number` : `a number with ${min} to ${max} digits`,
@@ -129,12 +133,16 @@ function compilePreset(spec) {
             };
         }
         case 'pattern': {
+            // The schema can't tie `pattern` to this preset, so check it here.
+            if (spec.pattern === undefined) {
+                throw new ConfigError('the "pattern" ticket preset needs a `pattern` regex string.');
+            }
             let re;
             try {
                 re = new RegExp(`^(?:${spec.pattern})$`, spec.flags);
             }
             catch (error) {
-                throw new Error(`eslint-plugin-test-governance: invalid ticket pattern "${spec.pattern}": ${error.message}`);
+                throw new ConfigError(`invalid ticket pattern "${spec.pattern}": ${error.message}`);
             }
             return { test: (t) => re.test(t), expected: `a ticket matching /${spec.pattern}/`, example: 'TICKET' };
         }

package/dist/utils/validate.d.ts

@@ -0,0 +1,21 @@
+/** The subset of JSON Schema that the options schema uses. */
+export interface Schema {
+    type?: 'object' | 'array' | 'string' | 'boolean' | 'integer';
+    enum?: readonly unknown[];
+    pattern?: string;
+    /** How to describe `pattern` in messages, e.g. "an uppercase keyword such as NEEDS-DATA". */
+    patternHint?: string;
+    minimum?: number;
+    minItems?: number;
+    items?: Schema;
+    properties?: Record<string, Schema>;
+    additionalProperties?: boolean | Schema;
+    required?: readonly string[];
+    anyOf?: readonly Schema[];
+}
+/**
+ * Validates `value` against the subset of JSON Schema that the options schema uses and returns
+ * readable problems such as `ticket.preset must be one of "any", "jira", … (got "jria")`.
+ */
+export declare function validate(value: unknown, schema: Schema, path?: string): string[];
+//# sourceMappingURL=validate.d.ts.map

package/dist/utils/validate.js

@@ -0,0 +1,122 @@
+import { editDistance } from './distance.js';
+function kindOf(value) {
+    if (value === null)
+        return 'null';
+    if (Array.isArray(value))
+        return 'array';
+    if (typeof value === 'number')
+        return Number.isInteger(value) ? 'integer' : 'number';
+    if (typeof value === 'object')
+        return 'object';
+    if (typeof value === 'string')
+        return 'string';
+    if (typeof value === 'boolean')
+        return 'boolean';
+    return 'other';
+}
+function matchesType(value, type) {
+    return kindOf(value) === type;
+}
+const ARTICLES = { object: 'an object', array: 'an array', integer: 'an integer' };
+function describeType(schema) {
+    if (schema.enum?.length === 1)
+        return show(schema.enum[0]);
+    const type = String(schema.type);
+    return ARTICLES[type] ?? `a ${type}`;
+}
+/** `JSON.stringify` returns undefined for values JSON can't represent, such as symbols; its typings don't say so. */
+const toJson = JSON.stringify;
+function show(value) {
+    if (typeof value === 'function')
+        return 'a function';
+    const text = toJson(value) ?? String(value);
+    return text.length > 40 ? `${text.slice(0, 37)}...` : text;
+}
+function join(path, key) {
+    if (typeof key === 'number')
+        return `${path}[${key}]`;
+    return path ? `${path}.${key}` : key;
+}
+/** The closest known name to a misspelled one, if it's close enough to be a likely typo. */
+function suggest(name, known) {
+    const lower = name.toLowerCase();
+    let best;
+    let bestDistance = Infinity;
+    for (const candidate of known) {
+        const distance = editDistance(lower, candidate.toLowerCase());
+        if (distance < bestDistance) {
+            best = candidate;
+            bestDistance = distance;
+        }
+    }
+    return bestDistance <= Math.max(2, Math.floor(name.length / 3)) ? best : undefined;
+}
+/**
+ * Validates `value` against the subset of JSON Schema that the options schema uses and returns
+ * readable problems such as `ticket.preset must be one of "any", "jira", … (got "jria")`.
+ */
+export function validate(value, schema, path = '') {
+    const label = path || 'options';
+    if (schema.anyOf) {
+        const branches = schema.anyOf;
+        const results = branches.map((branch) => validate(value, branch, path));
+        if (results.some((problems) => problems.length === 0))
+            return [];
+        const sameType = branches.findIndex((branch) => matchesType(value, branch.type));
+        if (sameType !== -1)
+            return results[sameType];
+        return [`${label} must be ${branches.map(describeType).join(' or ')} (got ${show(value)})`];
+    }
+    if (schema.type !== undefined && !matchesType(value, schema.type)) {
+        return [`${label} must be ${describeType(schema)} (got ${show(value)})`];
+    }
+    if (schema.enum && !schema.enum.includes(value)) {
+        const allowed = schema.enum.map(show);
+        return [
+            allowed.length === 1
+                ? `${label} must be ${allowed[0]} (got ${show(value)})`
+                : `${label} must be one of ${allowed.join(', ')} (got ${show(value)})`,
+        ];
+    }
+    if (typeof value === 'string' && schema.pattern !== undefined && !new RegExp(schema.pattern).test(value)) {
+        return [`${label} must be ${schema.patternHint ?? `a string matching /${schema.pattern}/`} (got ${show(value)})`];
+    }
+    if (typeof value === 'number' && schema.minimum !== undefined && value < schema.minimum) {
+        return [`${label} must be at least ${schema.minimum} (got ${value})`];
+    }
+    if (Array.isArray(value)) {
+        const problems = [];
+        if (schema.minItems !== undefined && value.length < schema.minItems) {
+            problems.push(`${label} must have at least ${schema.minItems} item${schema.minItems === 1 ? '' : 's'}`);
+        }
+        const { items } = schema;
+        if (items)
+            value.forEach((item, index) => problems.push(...validate(item, items, join(path, index))));
+        return problems;
+    }
+    if (kindOf(value) === 'object') {
+        const object = value;
+        const properties = schema.properties ?? {};
+        const known = Object.keys(properties);
+        const problems = [];
+        for (const key of schema.required ?? []) {
+            if (object[key] === undefined)
+                problems.push(`${join(path, key)} is required`);
+        }
+        for (const [key, item] of Object.entries(object)) {
+            if (item === undefined)
+                continue;
+            const propertySchema = properties[key] ?? schema.additionalProperties;
+            if (typeof propertySchema === 'object') {
+                problems.push(...validate(item, propertySchema, join(path, key)));
+            }
+            else if (propertySchema === false) {
+                const hint = suggest(key, known);
+                problems.push(`${join(path, key)} is not a known option${hint ? `; did you mean "${hint}"?` : ` (expected one of: ${known.join(', ')})`}`);
+            }
+        }
+        return problems;
+    }
+    return [];
+}
+//# sourceMappingURL=validate.js.map

package/docs/rules/marker-matches-state.md

@@ -20,12 +20,14 @@ that reuses the same ticket.
 Incorrect:
 
 <!-- example: invalid -->
+
 ```js
 // SKIP: WEB-500
 test.fixme('splits shipping', async () => {});
 ```
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 // SKIP: WEB-500
 test.skip('splits shipping @unstable', async () => {});
@@ -34,12 +36,14 @@ test.skip('splits shipping @unstable', async () => {});
 Correct:
 
 <!-- example: valid -->
+
 ```js
 // FIXME: WEB-500
 test.fixme('splits shipping', async () => {});
 ```
 
 <!-- example: valid settings={"lifecycleTags":true} -->
+
 ```js
 // SKIP: WEB-500
 // UNSTABLE: WEB-500
@@ -49,4 +53,3 @@ test.skip('splits shipping @unstable', async () => {});
 ## Options
 
 Uses the shared options described in the [README](../../README.md#options).
-

package/docs/rules/no-conflicting-states.md

@@ -24,11 +24,13 @@ every tag state that is on, including custom states.
 Incorrect:
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 test('filters by category @new @unstable', async () => {});
 ```
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 // SKIP: WEB-9
 // NEW: WEB-9
@@ -36,11 +38,13 @@ test.skip('filters by category @new', async () => {});
 ```
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 test('invoice download @New', async () => {});
 ```
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 test('loyalty points @unstabel', async () => {});
 ```
@@ -48,6 +52,7 @@ test('loyalty points @unstabel', async () => {});
 Correct:
 
 <!-- example: valid settings={"lifecycleTags":true} -->
+
 ```js
 test('filters by category @unstable', async () => {});
 ```
@@ -55,4 +60,3 @@ test('filters by category @unstable', async () => {});
 ## Options
 
 Uses the shared options described in the [README](../../README.md#options).
-

package/docs/rules/no-orphaned-marker.md

@@ -14,6 +14,16 @@ A leftover marker makes searches for open tickets lie. This rule reports:
 - a marker above a test, describe or skip call that is not in that marker's state, and
 - a marker that is not directly above a test, describe or skip call at all, so it has no effect.
 
+`FIXME:` and `TODO:` are also everyday work comments, so this rule leaves them to
+[`require-ticket-in-comments`](require-ticket-in-comments.md), which requires a ticket in them anywhere
+in a test file. A `// FIXME: WEB-12` above a helper is a tracked work comment, not a stray marker. A
+`// FIXME: refactor this` gets one clear error from that rule rather than a confusing one from this.
+
+Other markers that read as a sentence, such as `// SKIP: this one is flaky`, are left alone too: the rule
+counts a comment as prose when its first word is not a valid ticket and more text follows.
+
+Files that don't use Playwright, such as application code, are not checked.
+
 The rule offers a suggestion to delete the comment.
 
 ## Examples
@@ -21,15 +31,17 @@ The rule offers a suggestion to delete the comment.
 Incorrect:
 
 <!-- example: invalid settings={"lifecycleTags":true} -->
+
 ```js
 // NEW: WEB-80 promoted last sprint
 test('saved addresses', async () => {});
 ```
 
 <!-- example: invalid -->
+
 ```js
 test('checkout', async ({ page }) => {
-  // FIXME: WEB-81
+  // SKIP: WEB-81
   await page.goto('/checkout');
 });
 ```
@@ -37,11 +49,22 @@ test('checkout', async ({ page }) => {
 Correct:
 
 <!-- example: valid settings={"lifecycleTags":true} -->
+
 ```js
 test('saved addresses', async () => {});
 ```
 
 <!-- example: valid -->
+
+```js
+test('checkout', async ({ page }) => {
+  // FIXME: the banner sometimes covers the button on slow machines
+  await page.goto('/checkout');
+});
+```
+
+<!-- example: valid -->
+
 ```js
 // FIXME: WEB-81
 test.fixme('checkout', async ({ page }) => {
@@ -52,4 +75,3 @@ test.fixme('checkout', async ({ page }) => {
 ## Options
 
 Uses the shared options described in the [README](../../README.md#options).
-

package/docs/rules/require-ticket-in-comments.md

@@ -0,0 +1,80 @@
+# Require FIXME and TODO comments in test files to start with a ticket (`test-governance/require-ticket-in-comments`)
+
+📝 Require FIXME and TODO comments in test files to start with a ticket.
+
+💼 This rule is enabled in the ✅ `recommended` [config](https://github.com/Huldoser/eslint-plugin-test-governance#usage).
+
+💡 This rule is manually fixable by [editor suggestions](https://eslint.org/docs/latest/use/core-concepts#rule-suggestions).
+
+<!-- end auto-generated rule header -->
+
+A `// TODO: clean this up` comment is a promise nobody tracks. This rule makes every `FIXME` and
+`TODO` comment in a test file start with the ticket that tracks the work, so the ticket stays the
+source of truth and nothing is forgotten in a comment.
+
+```ts
+// TODO: WEB-512                 ✓
+// FIXME: WEB-77, WEB-78         ✓ several tickets, separated by commas
+// TODO: refactor this           ✖ no ticket
+// FIXME: WEB-77 flaky locator   ✖ only tickets are allowed (allowNotes: true allows the note)
+```
+
+How comments are read:
+
+- The keyword starts the comment (or a line of a block comment). `TODO` and `FIXME` in capitals
+  count with or without a colon. Other casings, such as `todo:`, need the colon, so a sentence that
+  happens to start with "Todo" is left alone.
+- The tickets follow the same rules as markers: the shared ticket format, placeholders such as `TBD`
+  rejected, and no text after the tickets unless `allowNotes` is set.
+- A `// FIXME:` marker directly above a `test.fixme(...)` is checked by
+  [`require-ticket`](require-ticket.md) instead, so it is reported once.
+- Only files that use Playwright (a Playwright import or a call to a test function) are checked.
+  Application code is left alone, even if your config applies the plugin to it.
+
+## Examples
+
+Incorrect:
+
+<!-- example: invalid -->
+
+```js
+test('checkout', async ({ page }) => {
+  // FIXME: this locator breaks on mobile
+  await page.click('#pay');
+});
+```
+
+<!-- example: invalid -->
+
+```js
+// TODO: TBD
+test('refund', async ({ page }) => {});
+```
+
+Correct:
+
+<!-- example: valid -->
+
+```js
+test('checkout', async ({ page }) => {
+  // FIXME: WEB-81
+  await page.click('#pay');
+});
+```
+
+<!-- example: valid settings={"comments":{"keywords":["FIXME"]}} -->
+
+```js
+// TODO: add a refund test once the API is stable
+test('checkout', async ({ page }) => {});
+```
+
+## Options
+
+This rule has no options of its own. It reads the shared options described in the
+[README](../../README.md#options):
+
+- `comments`: the keywords to check, `{ keywords: ['FIXME', 'TODO'] }` by default. Use
+  `{ keywords: ['FIXME'] }` to allow free-form `TODO`s, add your own such as `HACK`, or set
+  `comments: false` to turn the check off. Turning the rule off in your config works too.
+- `ticket`, `placeholders` and `allowNotes` work as they do for markers.