eslint 8.57.0 → 9.2.0

package/lib/rule-tester/rule-tester.js

@@ -1,68 +1,42 @@
 /**
- * @fileoverview Mocha test wrapper
+ * @fileoverview Mocha/Jest test wrapper
  * @author Ilya Volodin
  */
 "use strict";
 
 /* globals describe, it -- Mocha globals */
 
-/*
- * This is a wrapper around mocha to allow for DRY unittests for eslint
- * Format:
- * RuleTester.run("{ruleName}", {
- *      valid: [
- *          "{code}",
- *          { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings} }
- *      ],
- *      invalid: [
- *          { code: "{code}", errors: {numErrors} },
- *          { code: "{code}", errors: ["{errorMessage}"] },
- *          { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings}, errors: [{ message: "{errorMessage}", type: "{errorNodeType}"}] }
- *      ]
- *  });
- *
- * Variables:
- * {code} - String that represents the code to be tested
- * {options} - Arguments that are passed to the configurable rules.
- * {globals} - An object representing a list of variables that are
- *             registered as globals
- * {parser} - String representing the parser to use
- * {settings} - An object representing global settings for all rules
- * {numErrors} - If failing case doesn't need to check error message,
- *               this integer will specify how many errors should be
- *               received
- * {errorMessage} - Message that is returned by the rule on failure
- * {errorNodeType} - AST node type that is returned by they rule as
- *                   a cause of the failure.
- */
-
 //------------------------------------------------------------------------------
 // Requirements
 //------------------------------------------------------------------------------
 
 const
     assert = require("assert"),
-    path = require("path"),
     util = require("util"),
-    merge = require("lodash.merge"),
+    path = require("path"),
     equal = require("fast-deep-equal"),
-    Traverser = require("../../lib/shared/traverser"),
-    { getRuleOptionsSchema, validate } = require("../shared/config-validator"),
-    { Linter, SourceCodeFixer, interpolate } = require("../linter"),
-    CodePath = require("../linter/code-path-analysis/code-path");
+    Traverser = require("../shared/traverser"),
+    { getRuleOptionsSchema } = require("../config/flat-config-helpers"),
+    { Linter, SourceCodeFixer } = require("../linter"),
+    { interpolate, getPlaceholderMatcher } = require("../linter/interpolate"),
+    stringify = require("json-stable-stringify-without-jsonify");
+
+const { FlatConfigArray } = require("../config/flat-config-array");
+const { defaultConfig } = require("../config/default-config");
 
 const ajv = require("../shared/ajv")({ strictDefaults: true });
 
-const espreePath = require.resolve("espree");
 const parserSymbol = Symbol.for("eslint.RuleTester.parser");
-
 const { SourceCode } = require("../source-code");
+const { ConfigArraySymbol } = require("@humanwhocodes/config-array");
+const { isSerializable } = require("../shared/serialization");
 
 //------------------------------------------------------------------------------
 // Typedefs
 //------------------------------------------------------------------------------
 
 /** @typedef {import("../shared/types").Parser} Parser */
+/** @typedef {import("../shared/types").LanguageOptions} LanguageOptions */
 /** @typedef {import("../shared/types").Rule} Rule */
 
 
@@ -72,12 +46,9 @@ const { SourceCode } = require("../source-code");
  * @property {string} [name] Name for the test case.
  * @property {string} code Code for the test case.
  * @property {any[]} [options] Options for the test case.
+ * @property {LanguageOptions} [languageOptions] The language options to use in the test case.
  * @property {{ [name: string]: any }} [settings] Settings for the test case.
  * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames.
- * @property {string} [parser] The absolute path for the parser.
- * @property {{ [name: string]: any }} [parserOptions] Options for the parser.
- * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables.
- * @property {{ [name: string]: boolean }} [env] Environments for the test case.
  * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
  */
 
@@ -91,10 +62,7 @@ const { SourceCode } = require("../source-code");
  * @property {any[]} [options] Options for the test case.
  * @property {{ [name: string]: any }} [settings] Settings for the test case.
  * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames.
- * @property {string} [parser] The absolute path for the parser.
- * @property {{ [name: string]: any }} [parserOptions] Options for the parser.
- * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables.
- * @property {{ [name: string]: boolean }} [env] Environments for the test case.
+ * @property {LanguageOptions} [languageOptions] The language options to use in the test case.
  * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
  */
 
@@ -120,7 +88,12 @@ const { SourceCode } = require("../source-code");
  * the initial default configuration
  */
 const testerDefaultConfig = { rules: {} };
-let defaultConfig = { rules: {} };
+
+/*
+ * RuleTester uses this config as its default. This can be overwritten via
+ * setDefaultConfig().
+ */
+let sharedDefaultConfig = { rules: {} };
 
 /*
  * List every parameters possible on a test case that are not related to eslint
@@ -163,42 +136,25 @@ const suggestionObjectParameters = new Set([
 ]);
 const friendlySuggestionObjectParameterList = `[${[...suggestionObjectParameters].map(key => `'${key}'`).join(", ")}]`;
 
+/*
+ * Ignored test case properties when checking for test case duplicates.
+ */
+const duplicationIgnoredParameters = new Set([
+    "name",
+    "errors",
+    "output"
+]);
+
 const forbiddenMethods = [
     "applyInlineConfig",
     "applyLanguageOptions",
     "finalize"
 ];
 
-const hasOwnProperty = Function.call.bind(Object.hasOwnProperty);
+/** @type {Map<string,WeakSet>} */
+const forbiddenMethodCalls = new Map(forbiddenMethods.map(methodName => ([methodName, new WeakSet()])));
 
-const DEPRECATED_SOURCECODE_PASSTHROUGHS = {
-    getSource: "getText",
-    getSourceLines: "getLines",
-    getAllComments: "getAllComments",
-    getNodeByRangeIndex: "getNodeByRangeIndex",
-
-    // getComments: "getComments", -- already handled by a separate error
-    getCommentsBefore: "getCommentsBefore",
-    getCommentsAfter: "getCommentsAfter",
-    getCommentsInside: "getCommentsInside",
-    getJSDocComment: "getJSDocComment",
-    getFirstToken: "getFirstToken",
-    getFirstTokens: "getFirstTokens",
-    getLastToken: "getLastToken",
-    getLastTokens: "getLastTokens",
-    getTokenAfter: "getTokenAfter",
-    getTokenBefore: "getTokenBefore",
-    getTokenByRangeStart: "getTokenByRangeStart",
-    getTokens: "getTokens",
-    getTokensAfter: "getTokensAfter",
-    getTokensBefore: "getTokensBefore",
-    getTokensBetween: "getTokensBetween",
-
-    getScope: "getScope",
-    getAncestors: "getAncestors",
-    getDeclaredVariables: "getDeclaredVariables",
-    markVariableAsUsed: "markVariableAsUsed"
-};
+const hasOwnProperty = Function.call.bind(Object.hasOwnProperty);
 
 /**
  * Clones a given value deeply.
@@ -331,23 +287,27 @@ function wrapParser(parser) {
 }
 
 /**
- * Function to replace `SourceCode.prototype.getComments`.
- * @returns {void}
- * @throws {Error} Deprecation message.
- */
-function getCommentsDeprecation() {
-    throw new Error(
-        "`SourceCode#getComments()` is deprecated and will be removed in a future major version. Use `getCommentsBefore()`, `getCommentsAfter()`, and `getCommentsInside()` instead."
-    );
-}
-
-/**
- * Function to replace forbidden `SourceCode` methods.
+ * Function to replace forbidden `SourceCode` methods. Allows just one call per method.
  * @param {string} methodName The name of the method to forbid.
+ * @param {Function} prototype The prototype with the original method to call.
  * @returns {Function} The function that throws the error.
  */
-function throwForbiddenMethodError(methodName) {
-    return () => {
+function throwForbiddenMethodError(methodName, prototype) {
+
+    const original = prototype[methodName];
+
+    return function(...args) {
+
+        const called = forbiddenMethodCalls.get(methodName);
+
+        /* eslint-disable no-invalid-this -- needed to operate as a method. */
+        if (!called.has(this)) {
+            called.add(this);
+
+            return original.apply(this, args);
+        }
+        /* eslint-enable no-invalid-this -- not needed past this point */
+
         throw new Error(
             `\`SourceCode#${methodName}()\` cannot be called inside a rule.`
         );
@@ -355,81 +315,45 @@ function throwForbiddenMethodError(methodName) {
 }
 
 /**
- * Emit a deprecation warning if function-style format is being used.
- * @param {string} ruleName Name of the rule.
- * @returns {void}
+ * Extracts names of {{ placeholders }} from the reported message.
+ * @param   {string} message Reported message
+ * @returns {string[]} Array of placeholder names
  */
-function emitLegacyRuleAPIWarning(ruleName) {
-    if (!emitLegacyRuleAPIWarning[`warned-${ruleName}`]) {
-        emitLegacyRuleAPIWarning[`warned-${ruleName}`] = true;
-        process.emitWarning(
-            `"${ruleName}" rule is using the deprecated function-style format and will stop working in ESLint v9. Please use object-style format: https://eslint.org/docs/latest/extend/custom-rules`,
-            "DeprecationWarning"
-        );
-    }
-}
+function getMessagePlaceholders(message) {
+    const matcher = getPlaceholderMatcher();
 
-/**
- * Emit a deprecation warning if rule has options but is missing the "meta.schema" property
- * @param {string} ruleName Name of the rule.
- * @returns {void}
- */
-function emitMissingSchemaWarning(ruleName) {
-    if (!emitMissingSchemaWarning[`warned-${ruleName}`]) {
-        emitMissingSchemaWarning[`warned-${ruleName}`] = true;
-        process.emitWarning(
-            `"${ruleName}" rule has options but is missing the "meta.schema" property and will stop working in ESLint v9. Please add a schema: https://eslint.org/docs/latest/extend/custom-rules#options-schemas`,
-            "DeprecationWarning"
-        );
-    }
+    return Array.from(message.matchAll(matcher), ([, name]) => name.trim());
 }
 
 /**
- * Emit a deprecation warning if a rule uses a deprecated `context` method.
- * @param {string} ruleName Name of the rule.
- * @param {string} methodName The name of the method on `context` that was used.
- * @returns {void}
+ * Returns the placeholders in the reported messages but
+ * only includes the placeholders available in the raw message and not in the provided data.
+ * @param {string} message The reported message
+ * @param {string} raw The raw message specified in the rule meta.messages
+ * @param {undefined|Record<unknown, unknown>} data The passed
+ * @returns {string[]} Missing placeholder names
  */
-function emitDeprecatedContextMethodWarning(ruleName, methodName) {
-    if (!emitDeprecatedContextMethodWarning[`warned-${ruleName}-${methodName}`]) {
-        emitDeprecatedContextMethodWarning[`warned-${ruleName}-${methodName}`] = true;
-        process.emitWarning(
-            `"${ruleName}" rule is using \`context.${methodName}()\`, which is deprecated and will be removed in ESLint v9. Please use \`sourceCode.${DEPRECATED_SOURCECODE_PASSTHROUGHS[methodName]}()\` instead.`,
-            "DeprecationWarning"
-        );
-    }
-}
+function getUnsubstitutedMessagePlaceholders(message, raw, data = {}) {
+    const unsubstituted = getMessagePlaceholders(message);
 
-/**
- * Emit a deprecation warning if rule uses CodePath#currentSegments.
- * @param {string} ruleName Name of the rule.
- * @returns {void}
- */
-function emitCodePathCurrentSegmentsWarning(ruleName) {
-    if (!emitCodePathCurrentSegmentsWarning[`warned-${ruleName}`]) {
-        emitCodePathCurrentSegmentsWarning[`warned-${ruleName}`] = true;
-        process.emitWarning(
-            `"${ruleName}" rule uses CodePath#currentSegments and will stop working in ESLint v9. Please read the documentation for how to update your code: https://eslint.org/docs/latest/extend/code-path-analysis#usage-examples`,
-            "DeprecationWarning"
-        );
+    if (unsubstituted.length === 0) {
+        return [];
     }
-}
 
-/**
- * Emit a deprecation warning if `context.parserServices` is used.
- * @param {string} ruleName Name of the rule.
- * @returns {void}
- */
-function emitParserServicesWarning(ruleName) {
-    if (!emitParserServicesWarning[`warned-${ruleName}`]) {
-        emitParserServicesWarning[`warned-${ruleName}`] = true;
-        process.emitWarning(
-            `"${ruleName}" rule is using \`context.parserServices\`, which is deprecated and will be removed in ESLint v9. Please use \`sourceCode.parserServices\` instead.`,
-            "DeprecationWarning"
-        );
-    }
+    // Remove false positives by only counting placeholders in the raw message, which were not provided in the data matcher or added with a data property
+    const known = getMessagePlaceholders(raw);
+    const provided = Object.keys(data);
+
+    return unsubstituted.filter(name => known.includes(name) && !provided.includes(name));
 }
 
+const metaSchemaDescription = `
+\t- If the rule has options, set \`meta.schema\` to an array or non-empty object to enable options validation.
+\t- If the rule doesn't have options, omit \`meta.schema\` to enforce that no options can be passed to the rule.
+\t- You can also set \`meta.schema\` to \`false\` to opt-out of options validation (not recommended).
+
+\thttps://eslint.org/docs/latest/extend/custom-rules#options-schemas
+`;
 
 //------------------------------------------------------------------------------
 // Public Interface
@@ -479,26 +403,20 @@ class RuleTester {
      * Creates a new instance of RuleTester.
      * @param {Object} [testerConfig] Optional, extra configuration for the tester
      */
-    constructor(testerConfig) {
+    constructor(testerConfig = {}) {
 
         /**
          * The configuration to use for this tester. Combination of the tester
          * configuration and the default configuration.
          * @type {Object}
          */
-        this.testerConfig = merge(
-            {},
-            defaultConfig,
+        this.testerConfig = [
+            sharedDefaultConfig,
             testerConfig,
             { rules: { "rule-tester/validate-ast": "error" } }
-        );
+        ];
 
-        /**
-         * Rule definitions to define before tests.
-         * @type {Object}
-         */
-        this.rules = {};
-        this.linter = new Linter();
+        this.linter = new Linter({ configType: "flat" });
     }
 
     /**
@@ -511,10 +429,10 @@ class RuleTester {
         if (typeof config !== "object" || config === null) {
             throw new TypeError("RuleTester.setDefaultConfig: config must be an object");
         }
-        defaultConfig = config;
+        sharedDefaultConfig = config;
 
         // Make sure the rules object exists since it is assumed to exist later
-        defaultConfig.rules = defaultConfig.rules || {};
+        sharedDefaultConfig.rules = sharedDefaultConfig.rules || {};
     }
 
     /**
@@ -522,7 +440,7 @@ class RuleTester {
      * @returns {Object} the current configuration
      */
     static getDefaultConfig() {
-        return defaultConfig;
+        return sharedDefaultConfig;
     }
 
     /**
@@ -531,7 +449,11 @@ class RuleTester {
      * @returns {void}
      */
     static resetDefaultConfig() {
-        defaultConfig = merge({}, testerDefaultConfig);
+        sharedDefaultConfig = {
+            rules: {
+                ...testerDefaultConfig.rules
+            }
+        };
     }
 
 
@@ -602,29 +524,17 @@ class RuleTester {
         this[IT_ONLY] = value;
     }
 
-    /**
-     * Define a rule for one particular run of tests.
-     * @param {string} name The name of the rule to define.
-     * @param {Function | Rule} rule The rule definition.
-     * @returns {void}
-     */
-    defineRule(name, rule) {
-        if (typeof rule === "function") {
-            emitLegacyRuleAPIWarning(name);
-        }
-        this.rules[name] = rule;
-    }
 
     /**
      * Adds a new rule test to execute.
      * @param {string} ruleName The name of the rule to run.
-     * @param {Function | Rule} rule The rule to test.
+     * @param {Rule} rule The rule to test.
      * @param {{
      *   valid: (ValidTestCase | string)[],
      *   invalid: InvalidTestCase[]
      * }} test The collection of tests to run.
-     * @throws {TypeError|Error} If non-object `test`, or if a required
-     * scenario of the given type is missing.
+     * @throws {TypeError|Error} If `rule` is not an object with a `create` method,
+     * or if non-object `test`, or if a required scenario of the given type is missing.
      * @returns {void}
      */
     run(ruleName, rule, test) {
@@ -632,7 +542,15 @@ class RuleTester {
         const testerConfig = this.testerConfig,
             requiredScenarios = ["valid", "invalid"],
             scenarioErrors = [],
-            linter = this.linter;
+            linter = this.linter,
+            ruleId = `rule-to-test/${ruleName}`;
+
+        const seenValidTestCases = new Set();
+        const seenInvalidTestCases = new Set();
+
+        if (!rule || typeof rule !== "object" || typeof rule.create !== "function") {
+            throw new TypeError("Rule must be an object with a `create` method");
+        }
 
         if (!test || typeof test !== "object") {
             throw new TypeError(`Test Scenarios for rule ${ruleName} : Could not find test scenario object`);
@@ -650,54 +568,55 @@ class RuleTester {
             ].concat(scenarioErrors).join("\n"));
         }
 
-        if (typeof rule === "function") {
-            emitLegacyRuleAPIWarning(ruleName);
-        }
+        const baseConfig = [
+            { files: ["**"] }, // Make sure the default config matches for all files
+            {
+                plugins: {
 
-        linter.defineRule(ruleName, Object.assign({}, rule, {
-
-            // Create a wrapper rule that freezes the `context` properties.
-            create(context) {
-                freezeDeeply(context.options);
-                freezeDeeply(context.settings);
-                freezeDeeply(context.parserOptions);
-
-                // wrap all deprecated methods
-                const newContext = Object.create(
-                    context,
-                    Object.fromEntries(Object.keys(DEPRECATED_SOURCECODE_PASSTHROUGHS).map(methodName => [
-                        methodName,
-                        {
-                            value(...args) {
-
-                                // emit deprecation warning
-                                emitDeprecatedContextMethodWarning(ruleName, methodName);
-
-                                // call the original method
-                                return context[methodName].call(this, ...args);
-                            },
-                            enumerable: true
-                        }
-                    ]))
-                );
+                    // copy root plugin over
+                    "@": {
 
-                // emit warning about context.parserServices
-                const parserServices = context.parserServices;
+                        /*
+                         * Parsers are wrapped to detect more errors, so this needs
+                         * to be a new object for each call to run(), otherwise the
+                         * parsers will be wrapped multiple times.
+                         */
+                        parsers: {
+                            ...defaultConfig[0].plugins["@"].parsers
+                        },
 
-                Object.defineProperty(newContext, "parserServices", {
-                    get() {
-                        emitParserServicesWarning(ruleName);
-                        return parserServices;
-                    }
-                });
+                        /*
+                         * The rules key on the default plugin is a proxy to lazy-load
+                         * just the rules that are needed. So, don't create a new object
+                         * here, just use the default one to keep that performance
+                         * enhancement.
+                         */
+                        rules: defaultConfig[0].plugins["@"].rules
+                    },
+                    "rule-to-test": {
+                        rules: {
+                            [ruleName]: Object.assign({}, rule, {
 
-                Object.freeze(newContext);
+                                // Create a wrapper rule that freezes the `context` properties.
+                                create(context) {
+                                    freezeDeeply(context.options);
+                                    freezeDeeply(context.settings);
+                                    freezeDeeply(context.parserOptions);
 
-                return (typeof rule === "function" ? rule : rule.create)(newContext);
-            }
-        }));
+                                    // freezeDeeply(context.languageOptions);
 
-        linter.defineRules(this.rules);
+                                    return rule.create(context);
+                                }
+                            })
+                        }
+                    }
+                },
+                languageOptions: {
+                    ...defaultConfig[0].languageOptions
+                }
+            },
+            ...defaultConfig.slice(1)
+        ];
 
         /**
          * Run the rule for the given item
@@ -707,8 +626,34 @@ class RuleTester {
          * @private
          */
         function runRuleForItem(item) {
-            let config = merge({}, testerConfig),
-                code, filename, output, beforeAST, afterAST;
+            const flatConfigArrayOptions = {
+                baseConfig
+            };
+
+            if (item.filename) {
+                flatConfigArrayOptions.basePath = path.parse(item.filename).root;
+            }
+
+            const configs = new FlatConfigArray(testerConfig, flatConfigArrayOptions);
+
+            /*
+             * Modify the returned config so that the parser is wrapped to catch
+             * access of the start/end properties. This method is called just
+             * once per code snippet being tested, so each test case gets a clean
+             * parser.
+             */
+            configs[ConfigArraySymbol.finalizeConfig] = function(...args) {
+
+                // can't do super here :(
+                const proto = Object.getPrototypeOf(this);
+                const calculatedConfig = proto[ConfigArraySymbol.finalizeConfig].apply(this, args);
+
+                // wrap the parser to catch start/end property access
+                calculatedConfig.languageOptions.parser = wrapParser(calculatedConfig.languageOptions.parser);
+                return calculatedConfig;
+            };
+
+            let code, filename, output, beforeAST, afterAST;
 
             if (typeof item === "string") {
                 code = item;
@@ -725,64 +670,97 @@ class RuleTester {
                     delete itemConfig[parameter];
                 }
 
+                // wrap any parsers
+                if (itemConfig.languageOptions && itemConfig.languageOptions.parser) {
+
+                    const parser = itemConfig.languageOptions.parser;
+
+                    if (parser && typeof parser !== "object") {
+                        throw new Error("Parser must be an object with a parse() or parseForESLint() method.");
+                    }
+
+                }
+
                 /*
                  * Create the config object from the tester config and this item
                  * specific configurations.
                  */
-                config = merge(
-                    config,
-                    itemConfig
-                );
+                configs.push(itemConfig);
             }
 
-            if (item.filename) {
+            if (hasOwnProperty(item, "only")) {
+                assert.ok(typeof item.only === "boolean", "Optional test case property 'only' must be a boolean");
+            }
+            if (hasOwnProperty(item, "filename")) {
+                assert.ok(typeof item.filename === "string", "Optional test case property 'filename' must be a string");
                 filename = item.filename;
             }
 
+            let ruleConfig = 1;
+
             if (hasOwnProperty(item, "options")) {
                 assert(Array.isArray(item.options), "options must be an array");
-                if (
-                    item.options.length > 0 &&
-                    typeof rule === "object" &&
-                    (
-                        !rule.meta || (rule.meta && (typeof rule.meta.schema === "undefined" || rule.meta.schema === null))
-                    )
-                ) {
-                    emitMissingSchemaWarning(ruleName);
+                ruleConfig = [1, ...item.options];
+            }
+
+            configs.push({
+                rules: {
+                    [ruleId]: ruleConfig
                 }
-                config.rules[ruleName] = [1].concat(item.options);
-            } else {
-                config.rules[ruleName] = 1;
+            });
+
+            let schema;
+
+            try {
+                schema = getRuleOptionsSchema(rule);
+            } catch (err) {
+                err.message += metaSchemaDescription;
+                throw err;
             }
 
-            const schema = getRuleOptionsSchema(rule);
+            /*
+             * Check and throw an error if the schema is an empty object (`schema:{}`), because such schema
+             * doesn't validate or enforce anything and is therefore considered a possible error. If the intent
+             * was to skip options validation, `schema:false` should be set instead (explicit opt-out).
+             *
+             * For this purpose, a schema object is considered empty if it doesn't have any own enumerable string-keyed
+             * properties. While `ajv.compile()` does use enumerable properties from the prototype chain as well,
+             * it caches compiled schemas by serializing only own enumerable properties, so it's generally not a good idea
+             * to use inherited properties in schemas because schemas that differ only in inherited properties would end up
+             * having the same cache entry that would be correct for only one of them.
+             *
+             * At this point, `schema` can only be an object or `null`.
+             */
+            if (schema && Object.keys(schema).length === 0) {
+                throw new Error(`\`schema: {}\` is a no-op${metaSchemaDescription}`);
+            }
 
             /*
              * Setup AST getters.
              * The goal is to check whether or not AST was modified when
              * running the rule under test.
              */
-            linter.defineRule("rule-tester/validate-ast", {
-                create() {
-                    return {
-                        Program(node) {
-                            beforeAST = cloneDeeplyExcludesParent(node);
-                        },
-                        "Program:exit"(node) {
-                            afterAST = node;
+            configs.push({
+                plugins: {
+                    "rule-tester": {
+                        rules: {
+                            "validate-ast": {
+                                create() {
+                                    return {
+                                        Program(node) {
+                                            beforeAST = cloneDeeplyExcludesParent(node);
+                                        },
+                                        "Program:exit"(node) {
+                                            afterAST = node;
+                                        }
+                                    };
+                                }
+                            }
                         }
-                    };
+                    }
                 }
             });
 
-            if (typeof config.parser === "string") {
-                assert(path.isAbsolute(config.parser), "Parsers provided as strings to RuleTester must be absolute paths");
-            } else {
-                config.parser = espreePath;
-            }
-
-            linter.defineParser(config.parser, wrapParser(require(config.parser)));
-
             if (schema) {
                 ajv.validateSchema(schema);
 
@@ -809,35 +787,32 @@ class RuleTester {
                 }
             }
 
-            validate(config, "rule-tester", id => (id === ruleName ? rule : null));
+            // check for validation errors
+            try {
+                configs.normalizeSync();
+                configs.getConfig("test.js");
+            } catch (error) {
+                error.message = `ESLint configuration in rule-tester is invalid: ${error.message}`;
+                throw error;
+            }
 
             // Verify the code.
-            const { getComments, applyLanguageOptions, applyInlineConfig, finalize } = SourceCode.prototype;
-            const originalCurrentSegments = Object.getOwnPropertyDescriptor(CodePath.prototype, "currentSegments");
+            const { applyLanguageOptions, applyInlineConfig, finalize } = SourceCode.prototype;
             let messages;
 
             try {
-                SourceCode.prototype.getComments = getCommentsDeprecation;
-                Object.defineProperty(CodePath.prototype, "currentSegments", {
-                    get() {
-                        emitCodePathCurrentSegmentsWarning(ruleName);
-                        return originalCurrentSegments.get.call(this);
-                    }
-                });
-
                 forbiddenMethods.forEach(methodName => {
-                    SourceCode.prototype[methodName] = throwForbiddenMethodError(methodName);
+                    SourceCode.prototype[methodName] = throwForbiddenMethodError(methodName, SourceCode.prototype);
                 });
 
-                messages = linter.verify(code, config, filename);
+                messages = linter.verify(code, configs, filename);
             } finally {
-                SourceCode.prototype.getComments = getComments;
-                Object.defineProperty(CodePath.prototype, "currentSegments", originalCurrentSegments);
                 SourceCode.prototype.applyInlineConfig = applyInlineConfig;
                 SourceCode.prototype.applyLanguageOptions = applyLanguageOptions;
                 SourceCode.prototype.finalize = finalize;
             }
 
+
             const fatalErrorMessage = messages.find(m => m.fatal);
 
             assert(!fatalErrorMessage, `A fatal parsing error occurred: ${fatalErrorMessage && fatalErrorMessage.message}`);
@@ -845,7 +820,7 @@ class RuleTester {
             // Verify if autofix makes a syntax error or not.
             if (messages.some(m => m.fix)) {
                 output = SourceCodeFixer.applyFixes(code, messages).output;
-                const errorMessageInFix = linter.verify(output, config, filename).find(m => m.fatal);
+                const errorMessageInFix = linter.verify(output, configs, filename).find(m => m.fatal);
 
                 assert(!errorMessageInFix, [
                     "A fatal parsing error occurred in autofix.",
@@ -861,7 +836,9 @@ class RuleTester {
                 messages,
                 output,
                 beforeAST,
-                afterAST: cloneDeeplyExcludesParent(afterAST)
+                afterAST: cloneDeeplyExcludesParent(afterAST),
+                configs,
+                filename
             };
         }
 
@@ -878,6 +855,39 @@ class RuleTester {
             }
         }
 
+        /**
+         * Check if this test case is a duplicate of one we have seen before.
+         * @param {string|Object} item test case object
+         * @param {Set<string>} seenTestCases set of serialized test cases we have seen so far (managed by this function)
+         * @returns {void}
+         * @private
+         */
+        function checkDuplicateTestCase(item, seenTestCases) {
+            if (!isSerializable(item)) {
+
+                /*
+                 * If we can't serialize a test case (because it contains a function, RegExp, etc), skip the check.
+                 * This might happen with properties like: options, plugins, settings, languageOptions.parser, languageOptions.parserOptions.
+                 */
+                return;
+            }
+
+            const normalizedItem = typeof item === "string" ? { code: item } : item;
+            const serializedTestCase = stringify(normalizedItem, {
+                replacer(key, value) {
+
+                    // "this" is the currently stringified object --> only ignore top-level properties
+                    return (normalizedItem !== this || !duplicationIgnoredParameters.has(key)) ? value : void 0;
+                }
+            });
+
+            assert(
+                !seenTestCases.has(serializedTestCase),
+                "detected duplicate test case"
+            );
+            seenTestCases.add(serializedTestCase);
+        }
+
         /**
          * Check if the template is valid or not
          * all valid cases go through this
@@ -893,6 +903,8 @@ class RuleTester {
                 assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string");
             }
 
+            checkDuplicateTestCase(item, seenValidTestCases);
+
             const result = runRuleForItem(item);
             const messages = result.messages;
 
@@ -944,12 +956,30 @@ class RuleTester {
                 assert.fail("Invalid cases must have at least one error");
             }
 
+            checkDuplicateTestCase(item, seenInvalidTestCases);
+
             const ruleHasMetaMessages = hasOwnProperty(rule, "meta") && hasOwnProperty(rule.meta, "messages");
             const friendlyIDList = ruleHasMetaMessages ? `[${Object.keys(rule.meta.messages).map(key => `'${key}'`).join(", ")}]` : null;
 
             const result = runRuleForItem(item);
             const messages = result.messages;
 
+            for (const message of messages) {
+                if (hasOwnProperty(message, "suggestions")) {
+
+                    /** @type {Map<string, number>} */
+                    const seenMessageIndices = new Map();
+
+                    for (let i = 0; i < message.suggestions.length; i += 1) {
+                        const suggestionMessage = message.suggestions[i].desc;
+                        const previous = seenMessageIndices.get(suggestionMessage);
+
+                        assert.ok(!seenMessageIndices.has(suggestionMessage), `Suggestion message '${suggestionMessage}' reported from suggestion ${i} was previously reported by suggestion ${previous}. Suggestion messages should be unique within an error.`);
+                        seenMessageIndices.set(suggestionMessage, i);
+                    }
+                }
+            }
+
             if (typeof item.errors === "number") {
 
                 if (item.errors === 0) {
@@ -972,7 +1002,7 @@ class RuleTester {
                     )
                 );
 
-                const hasMessageOfThisRule = messages.some(m => m.ruleId === ruleName);
+                const hasMessageOfThisRule = messages.some(m => m.ruleId === ruleId);
 
                 for (let i = 0, l = item.errors.length; i < l; i++) {
                     const error = item.errors[i];
@@ -984,6 +1014,7 @@ class RuleTester {
 
                         // Just an error message.
                         assertMessageMatches(message.message, error);
+                        assert.ok(message.suggestions === void 0, `Error at index ${i} has suggestions. Please convert the test error into an object and specify 'suggestions' property on it to test suggestions.`);
                     } else if (typeof error === "object" && error !== null) {
 
                         /*
@@ -1016,6 +1047,18 @@ class RuleTester {
                                 error.messageId,
                                 `messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`
                             );
+
+                            const unsubstitutedPlaceholders = getUnsubstitutedMessagePlaceholders(
+                                message.message,
+                                rule.meta.messages[message.messageId],
+                                error.data
+                            );
+
+                            assert.ok(
+                                unsubstitutedPlaceholders.length === 0,
+                                `The reported message has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property in the context.report() call.`
+                            );
+
                             if (hasOwnProperty(error, "data")) {
 
                                 /*
@@ -1032,13 +1075,10 @@ class RuleTester {
                                     `Hydrated message "${rehydratedMessage}" does not match "${message.message}"`
                                 );
                             }
+                        } else {
+                            assert.fail("Test error must specify either a 'messageId' or 'message'.");
                         }
 
-                        assert.ok(
-                            hasOwnProperty(error, "data") ? hasOwnProperty(error, "messageId") : true,
-                            "Error must specify 'messageId' if 'data' is used."
-                        );
-
                         if (error.type) {
                             assert.strictEqual(message.nodeType, error.type, `Error type should be ${error.type}, found ${message.nodeType}`);
                         }
@@ -1059,81 +1099,117 @@ class RuleTester {
                             assert.strictEqual(message.endColumn, error.endColumn, `Error endColumn should be ${error.endColumn}`);
                         }
 
+                        assert.ok(!message.suggestions || hasOwnProperty(error, "suggestions"), `Error at index ${i} has suggestions. Please specify 'suggestions' property on the test error object.`);
                         if (hasOwnProperty(error, "suggestions")) {
 
                             // Support asserting there are no suggestions
-                            if (!error.suggestions || (Array.isArray(error.suggestions) && error.suggestions.length === 0)) {
-                                if (Array.isArray(message.suggestions) && message.suggestions.length > 0) {
-                                    assert.fail(`Error should have no suggestions on error with message: "${message.message}"`);
-                                }
-                            } else {
-                                assert.strictEqual(Array.isArray(message.suggestions), true, `Error should have an array of suggestions. Instead received "${message.suggestions}" on error with message: "${message.message}"`);
-                                assert.strictEqual(message.suggestions.length, error.suggestions.length, `Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`);
-
-                                error.suggestions.forEach((expectedSuggestion, index) => {
-                                    assert.ok(
-                                        typeof expectedSuggestion === "object" && expectedSuggestion !== null,
-                                        "Test suggestion in 'suggestions' array must be an object."
-                                    );
-                                    Object.keys(expectedSuggestion).forEach(propertyName => {
-                                        assert.ok(
-                                            suggestionObjectParameters.has(propertyName),
-                                            `Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.`
-                                        );
-                                    });
-
-                                    const actualSuggestion = message.suggestions[index];
-                                    const suggestionPrefix = `Error Suggestion at index ${index} :`;
-
-                                    if (hasOwnProperty(expectedSuggestion, "desc")) {
+                            const expectsSuggestions = Array.isArray(error.suggestions) ? error.suggestions.length > 0 : Boolean(error.suggestions);
+                            const hasSuggestions = message.suggestions !== void 0;
+
+                            if (!hasSuggestions && expectsSuggestions) {
+                                assert.ok(!error.suggestions, `Error should have suggestions on error with message: "${message.message}"`);
+                            } else if (hasSuggestions) {
+                                assert.ok(expectsSuggestions, `Error should have no suggestions on error with message: "${message.message}"`);
+                                if (typeof error.suggestions === "number") {
+                                    assert.strictEqual(message.suggestions.length, error.suggestions, `Error should have ${error.suggestions} suggestions. Instead found ${message.suggestions.length} suggestions`);
+                                } else if (Array.isArray(error.suggestions)) {
+                                    assert.strictEqual(message.suggestions.length, error.suggestions.length, `Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`);
+
+                                    error.suggestions.forEach((expectedSuggestion, index) => {
                                         assert.ok(
-                                            !hasOwnProperty(expectedSuggestion, "data"),
-                                            `${suggestionPrefix} Test should not specify both 'desc' and 'data'.`
-                                        );
-                                        assert.strictEqual(
-                                            actualSuggestion.desc,
-                                            expectedSuggestion.desc,
-                                            `${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.`
+                                            typeof expectedSuggestion === "object" && expectedSuggestion !== null,
+                                            "Test suggestion in 'suggestions' array must be an object."
                                         );
-                                    }
+                                        Object.keys(expectedSuggestion).forEach(propertyName => {
+                                            assert.ok(
+                                                suggestionObjectParameters.has(propertyName),
+                                                `Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.`
+                                            );
+                                        });
 
-                                    if (hasOwnProperty(expectedSuggestion, "messageId")) {
-                                        assert.ok(
-                                            ruleHasMetaMessages,
-                                            `${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.`
-                                        );
-                                        assert.ok(
-                                            hasOwnProperty(rule.meta.messages, expectedSuggestion.messageId),
-                                            `${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.`
-                                        );
-                                        assert.strictEqual(
-                                            actualSuggestion.messageId,
-                                            expectedSuggestion.messageId,
-                                            `${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`
-                                        );
-                                        if (hasOwnProperty(expectedSuggestion, "data")) {
-                                            const unformattedMetaMessage = rule.meta.messages[expectedSuggestion.messageId];
-                                            const rehydratedDesc = interpolate(unformattedMetaMessage, expectedSuggestion.data);
+                                        const actualSuggestion = message.suggestions[index];
+                                        const suggestionPrefix = `Error Suggestion at index ${index}:`;
 
+                                        if (hasOwnProperty(expectedSuggestion, "desc")) {
+                                            assert.ok(
+                                                !hasOwnProperty(expectedSuggestion, "data"),
+                                                `${suggestionPrefix} Test should not specify both 'desc' and 'data'.`
+                                            );
+                                            assert.ok(
+                                                !hasOwnProperty(expectedSuggestion, "messageId"),
+                                                `${suggestionPrefix} Test should not specify both 'desc' and 'messageId'.`
+                                            );
                                             assert.strictEqual(
                                                 actualSuggestion.desc,
-                                                rehydratedDesc,
-                                                `${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".`
+                                                expectedSuggestion.desc,
+                                                `${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.`
+                                            );
+                                        } else if (hasOwnProperty(expectedSuggestion, "messageId")) {
+                                            assert.ok(
+                                                ruleHasMetaMessages,
+                                                `${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.`
+                                            );
+                                            assert.ok(
+                                                hasOwnProperty(rule.meta.messages, expectedSuggestion.messageId),
+                                                `${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.`
+                                            );
+                                            assert.strictEqual(
+                                                actualSuggestion.messageId,
+                                                expectedSuggestion.messageId,
+                                                `${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`
+                                            );
+
+                                            const unsubstitutedPlaceholders = getUnsubstitutedMessagePlaceholders(
+                                                actualSuggestion.desc,
+                                                rule.meta.messages[expectedSuggestion.messageId],
+                                                expectedSuggestion.data
+                                            );
+
+                                            assert.ok(
+                                                unsubstitutedPlaceholders.length === 0,
+                                                `The message of the suggestion has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property for the suggestion in the context.report() call.`
+                                            );
+
+                                            if (hasOwnProperty(expectedSuggestion, "data")) {
+                                                const unformattedMetaMessage = rule.meta.messages[expectedSuggestion.messageId];
+                                                const rehydratedDesc = interpolate(unformattedMetaMessage, expectedSuggestion.data);
+
+                                                assert.strictEqual(
+                                                    actualSuggestion.desc,
+                                                    rehydratedDesc,
+                                                    `${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".`
+                                                );
+                                            }
+                                        } else if (hasOwnProperty(expectedSuggestion, "data")) {
+                                            assert.fail(
+                                                `${suggestionPrefix} Test must specify 'messageId' if 'data' is used.`
+                                            );
+                                        } else {
+                                            assert.fail(
+                                                `${suggestionPrefix} Test must specify either 'messageId' or 'desc'.`
                                             );
                                         }
-                                    } else {
-                                        assert.ok(
-                                            !hasOwnProperty(expectedSuggestion, "data"),
-                                            `${suggestionPrefix} Test must specify 'messageId' if 'data' is used.`
-                                        );
-                                    }
 
-                                    if (hasOwnProperty(expectedSuggestion, "output")) {
+                                        assert.ok(hasOwnProperty(expectedSuggestion, "output"), `${suggestionPrefix} The "output" property is required.`);
                                         const codeWithAppliedSuggestion = SourceCodeFixer.applyFixes(item.code, [actualSuggestion]).output;
 
+                                        // Verify if suggestion fix makes a syntax error or not.
+                                        const errorMessageInSuggestion =
+                                                linter.verify(codeWithAppliedSuggestion, result.configs, result.filename).find(m => m.fatal);
+
+                                        assert(!errorMessageInSuggestion, [
+                                            "A fatal parsing error occurred in suggestion fix.",
+                                            `Error: ${errorMessageInSuggestion && errorMessageInSuggestion.message}`,
+                                            "Suggestion output:",
+                                            codeWithAppliedSuggestion
+                                        ].join("\n"));
+
                                         assert.strictEqual(codeWithAppliedSuggestion, expectedSuggestion.output, `Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`);
-                                    }
-                                });
+                                        assert.notStrictEqual(expectedSuggestion.output, item.code, `The output of a suggestion should differ from the original source code for suggestion at index: ${index} on error with message: "${message.message}"`);
+                                    });
+                                } else {
+                                    assert.fail("Test error object property 'suggestions' should be an array or a number");
+                                }
                             }
                         }
                     } else {
@@ -1153,6 +1229,7 @@ class RuleTester {
                     );
                 } else {
                     assert.strictEqual(result.output, item.output, "Output is incorrect.");
+                    assert.notStrictEqual(item.code, item.output, "Test property 'output' matches 'code'. If no autofix is expected, then omit the 'output' property or set it to null.");
                 }
             } else {
                 assert.strictEqual(