eslint 10.0.0-alpha.1 → 10.0.0-rc.0

package/lib/rule-tester/rule-tester.js

@@ -11,6 +11,7 @@
 //------------------------------------------------------------------------------
 
 const assert = require("node:assert"),
+	{ existsSync, readFileSync } = require("node:fs"),
 	util = require("node:util"),
 	path = require("node:path"),
 	equal = require("fast-deep-equal"),
@@ -387,9 +388,10 @@ function normalizeTestCase(item) {
  * Asserts that the `errors` property of an invalid test case is valid.
  * @param {number | string[]} errors The `errors` property of the invalid test case.
  * @param {string} ruleName The name of the rule being tested.
+ * @param {Object} [assertionOptions] The assertion options for the test case.
  * @returns {void}
  */
-function assertErrorsProperty(errors, ruleName) {
+function assertErrorsProperty(errors, ruleName, assertionOptions = {}) {
 	const isNumber = typeof errors === "number";
 	const isArray = Array.isArray(errors);
 
@@ -407,12 +409,75 @@ function assertErrorsProperty(errors, ruleName) {
 		}
 	}
 
+	const { requireMessage = false, requireLocation = false } =
+		assertionOptions;
+
 	if (isArray) {
 		assert.ok(
 			errors.length !== 0,
 			"Invalid cases must have at least one error",
 		);
+
+		for (const [number, error] of errors.entries()) {
+			if (typeof error === "string" || error instanceof RegExp) {
+				// Just an error message.
+				assert.ok(
+					requireMessage !== "messageId" && !requireLocation,
+					`errors[${number}] should be an object when 'assertionOptions.requireMessage' is 'messageId' or 'assertionOptions.requireLocation' is true.`,
+				);
+			} else if (typeof error === "object" && error !== null) {
+				/*
+				 * Error object.
+				 * This may have a message, messageId, data, line, and/or column.
+				 */
+
+				for (const propertyName of Object.keys(error)) {
+					assert.ok(
+						errorObjectParameters.has(propertyName),
+						`Invalid error property name '${propertyName}'. Expected one of ${friendlyErrorObjectParameterList}.`,
+					);
+				}
+
+				if (requireMessage === "message") {
+					assert.ok(
+						!hasOwnProperty(error, "messageId") &&
+							hasOwnProperty(error, "message"),
+						`errors[${number}] should specify 'message' (and not 'messageId') when 'assertionOptions.requireMessage' is 'message'.`,
+					);
+				} else if (requireMessage === "messageId") {
+					assert.ok(
+						!hasOwnProperty(error, "message") &&
+							hasOwnProperty(error, "messageId"),
+						`errors[${number}] should specify 'messageId' (and not 'message') when 'assertionOptions.requireMessage' is 'messageId'.`,
+					);
+				}
+
+				if (hasOwnProperty(error, "message")) {
+					assert.ok(
+						!hasOwnProperty(error, "messageId"),
+						`errors[${number}] should not specify both 'message' and 'messageId'.`,
+					);
+					assert.ok(
+						!hasOwnProperty(error, "data"),
+						`errors[${number}] should not specify both 'data' and 'message'.`,
+					);
+				} else {
+					assert.ok(
+						hasOwnProperty(error, "messageId"),
+						`errors[${number}] must specify either 'messageId' or 'message'.`,
+					);
+				}
+			} else {
+				assert.fail(
+					`errors[${number}] must be a string, RegExp, or an object.`,
+				);
+			}
+		}
 	} else {
+		assert.ok(
+			!requireMessage && !requireLocation,
+			"Invalid cases must have 'errors' value as an array",
+		);
 		assert.ok(
 			errors > 0,
 			"Invalid cases must have 'error' value greater than 0",
@@ -564,13 +629,19 @@ function assertValidTestCase(item, seenTestCases) {
  * @param {Object} item The invalid test case object to check.
  * @param {Set<string>} seenTestCases Set of serialized test cases to check for duplicates.
  * @param {string} ruleName The name of the rule being tested.
+ * @param {Object} [assertionOptions] The assertion options for the test case.
  * @returns {void}
  * @throws {AssertionError} If the test case is not valid.
  */
-function assertInvalidTestCase(item, seenTestCases, ruleName) {
+function assertInvalidTestCase(
+	item,
+	seenTestCases,
+	ruleName,
+	assertionOptions = {},
+) {
 	assertTestCommonProperties(item);
 
-	assertErrorsProperty(item.errors, ruleName);
+	assertErrorsProperty(item.errors, ruleName, assertionOptions);
 
 	// 'output' is optional, but if it exists it must be a string or null
 	if (hasOwnProperty(item, "output")) {
@@ -583,6 +654,190 @@ function assertInvalidTestCase(item, seenTestCases, ruleName) {
 	checkDuplicateTestCase(item, seenTestCases);
 }
 
+/**
+ * Gets the invocation location from the stack trace for later use.
+ * @param {Function} relative The function before the invocation point.
+ * @returns {string} The invocation location.
+ */
+function getInvocationLocation(relative = getInvocationLocation) {
+	const dummyObject = {};
+	Error.captureStackTrace(dummyObject, relative);
+	const { stack } = dummyObject;
+
+	return stack.split("\n")[1].replace(/.*?\(/u, "").replace(/\)$/u, "");
+}
+
+/**
+ * Estimates the location of the test case in the source file.
+ * @param {Function} invoker The method that runs the tests.
+ * @returns {(key: string) => string} The lazy resolver for the estimated location of the test case.
+ */
+function buildLazyTestLocationEstimator(invoker) {
+	const invocationLocation = getInvocationLocation(invoker);
+	let testLocations = null;
+	return key => {
+		if (testLocations === null) {
+			let [sourceFile, sourceLine = "1", sourceColumn = "1"] =
+				invocationLocation
+					.replace(/:\\/u, "\\\\") // Windows workaround for C:\\
+					.split(":");
+			sourceFile = sourceFile.replace(/\\\\/u, ":\\");
+			sourceLine = Number(sourceLine);
+			sourceColumn = Number(sourceColumn);
+			testLocations = { root: invocationLocation };
+
+			if (existsSync(sourceFile)) {
+				let content = readFileSync(sourceFile, "utf8")
+					.split("\n")
+					.slice(sourceLine - 1);
+				content[0] = content[0].slice(Math.max(0, sourceColumn - 1));
+				content = content.map(
+					l =>
+						l
+							.trim() // Remove whitespace
+							.replace(/\s*\/\/.*$(?<!,)/u, ""), // and trailing in-line comments that aren't part of the test `code`
+				);
+
+				// Roots
+				const validStartIndex = content.findIndex(line =>
+					/\bvalid:/u.test(line),
+				);
+				const invalidStartIndex = content.findIndex(line =>
+					/\binvalid:/u.test(line),
+				);
+
+				testLocations.valid = `${sourceFile}:${
+					sourceLine + validStartIndex
+				}`;
+				testLocations.invalid = `${sourceFile}:${
+					sourceLine + invalidStartIndex
+				}`;
+
+				// Scenario basics
+				const validEndIndex =
+					validStartIndex < invalidStartIndex
+						? invalidStartIndex
+						: content.length;
+				const invalidEndIndex =
+					validStartIndex < invalidStartIndex
+						? content.length
+						: validStartIndex;
+
+				const validLines = content.slice(
+					validStartIndex,
+					validEndIndex,
+				);
+				const invalidLines = content.slice(
+					invalidStartIndex,
+					invalidEndIndex,
+				);
+
+				let objectDepth = 0;
+				const validLineIndexes = validLines
+					.map((l, i) => {
+						// matches `key: {` and `{`
+						if (/^(?:\w+\s*:\s*)?\{/u.test(l)) {
+							objectDepth++;
+						}
+
+						if (objectDepth > 0) {
+							if (l.endsWith("}") || l.endsWith("},")) {
+								objectDepth--;
+							}
+
+							return objectDepth <= 1 && l.includes("code:")
+								? i
+								: null;
+						}
+
+						return l.endsWith(",") ? i : null;
+					})
+					.filter(Boolean);
+				const invalidLineIndexes = invalidLines
+					.map((l, i) =>
+						l.trimStart().startsWith("errors:") ? i : null,
+					)
+					.filter(Boolean);
+
+				Object.assign(
+					testLocations,
+					{
+						[`valid[0]`]: `${sourceFile}:${
+							sourceLine + validStartIndex
+						}`,
+					},
+					Object.fromEntries(
+						validLineIndexes.map((location, validIndex) => [
+							`valid[${validIndex}]`,
+							`${sourceFile}:${
+								sourceLine + validStartIndex + location
+							}`,
+						]),
+					),
+					Object.fromEntries(
+						invalidLineIndexes.map((location, invalidIndex) => [
+							`invalid[${invalidIndex}]`,
+							`${sourceFile}:${
+								sourceLine + invalidStartIndex + location
+							}`,
+						]),
+					),
+				);
+
+				// Indexes for errors inside each invalid test case
+				invalidLineIndexes.push(invalidLines.length);
+
+				for (let i = 0; i < invalidLineIndexes.length - 1; i++) {
+					const start = invalidLineIndexes[i];
+					const end = invalidLineIndexes[i + 1];
+					const errorLines = invalidLines.slice(start, end);
+					let errorObjectDepth = 0;
+					const errorLineIndexes = errorLines
+						.map((l, j) => {
+							if (l.startsWith("{") || l.endsWith("{")) {
+								errorObjectDepth++;
+
+								if (l.endsWith("}") || l.endsWith("},")) {
+									errorObjectDepth--;
+								}
+
+								return errorObjectDepth <= 1 ? j : null;
+							}
+
+							if (errorObjectDepth > 0) {
+								if (l.endsWith("}") || l.endsWith("},")) {
+									errorObjectDepth--;
+								}
+
+								return null;
+							}
+
+							return l.endsWith(",") ? j : null;
+						})
+						.filter(Boolean);
+
+					Object.assign(
+						testLocations,
+						Object.fromEntries(
+							errorLineIndexes.map((line, errorIndex) => [
+								`invalid[${i}].errors[${errorIndex}]`,
+								`${sourceFile}:${
+									sourceLine +
+									invalidStartIndex +
+									start +
+									line
+								}`,
+							]),
+						),
+					);
+				}
+			}
+		}
+
+		return testLocations[key] || "unknown source";
+	};
+}
+
 //------------------------------------------------------------------------------
 // Public Interface
 //------------------------------------------------------------------------------
@@ -763,6 +1018,11 @@ class RuleTester {
 	 * @param {string} ruleName The name of the rule to run.
 	 * @param {RuleDefinition} rule The rule to test.
 	 * @param {{
+	 *   assertionOptions?: {
+	 *     requireMessage?: boolean | "message" | "messageId",
+	 *     requireLocation?: boolean
+	 *     requireData?: boolean | "error" | "suggestion"
+	 *   },
 	 *   valid: (ValidTestCase | string)[],
 	 *   invalid: InvalidTestCase[]
 	 * }} test The collection of tests to run.
@@ -778,6 +1038,8 @@ class RuleTester {
 		assertRule(rule, ruleName);
 		assertTest(test, ruleName);
 
+		const estimateTestLocation = buildLazyTestLocationEstimator(this.run);
+
 		const baseConfig = [
 			{
 				plugins: {
@@ -1141,8 +1403,15 @@ class RuleTester {
 		 * @param {Object} item Item to run the rule against
 		 * @returns {void}
 		 * @private
+		 * @throws {Error} If the test case is invalid or has an invalid error.
 		 */
 		function testInvalidTemplate(item) {
+			const {
+				requireMessage = false,
+				requireLocation = false,
+				requireData = false,
+			} = test.assertionOptions ?? {};
+
 			const ruleHasMetaMessages =
 				hasOwnProperty(rule, "meta") &&
 				hasOwnProperty(rule.meta, "messages");
@@ -1152,6 +1421,11 @@ class RuleTester {
 						.join(", ")}]`
 				: null;
 
+			assert.ok(
+				ruleHasMetaMessages || requireMessage !== "messageId",
+				`Assertion options can not use 'requireMessage: "messageId"' if rule under test doesn't define 'meta.messages'.`,
+			);
+
 			const result = runRuleForItem(item);
 			const messages = result.messages;
 
@@ -1204,357 +1478,384 @@ class RuleTester {
 				);
 
 				for (let i = 0, l = item.errors.length; i < l; i++) {
-					const error = item.errors[i];
-					const message = messages[i];
+					try {
+						const error = item.errors[i];
+						const message = messages[i];
 
-					assert(
-						hasMessageOfThisRule,
-						"Error rule name should be the same as the name of the rule being tested",
-					);
-
-					if (typeof error === "string" || error instanceof RegExp) {
-						// Just an error message.
-						assertMessageMatches(message.message, error);
-						assert.ok(
-							message.suggestions === void 0,
-							`Error at index ${i} has suggestions. Please convert the test error into an object and specify 'suggestions' property on it to test suggestions.`,
+						assert(
+							hasMessageOfThisRule,
+							"Error rule name should be the same as the name of the rule being tested",
 						);
-					} else if (typeof error === "object" && error !== null) {
-						/*
-						 * Error object.
-						 * This may have a message, messageId, data, line, and/or column.
-						 */
-
-						Object.keys(error).forEach(propertyName => {
-							assert.ok(
-								errorObjectParameters.has(propertyName),
-								`Invalid error property name '${propertyName}'. Expected one of ${friendlyErrorObjectParameterList}.`,
-							);
-						});
 
-						if (hasOwnProperty(error, "message")) {
-							assert.ok(
-								!hasOwnProperty(error, "messageId"),
-								"Error should not specify both 'message' and a 'messageId'.",
-							);
+						if (
+							typeof error === "string" ||
+							error instanceof RegExp
+						) {
+							// Just an error message.
+							assertMessageMatches(message.message, error);
 							assert.ok(
-								!hasOwnProperty(error, "data"),
-								"Error should not specify both 'data' and 'message'.",
+								message.suggestions === void 0,
+								`Error at index ${i} has suggestions. Please convert the test error into an object and specify 'suggestions' property on it to test suggestions.`,
 							);
-							assertMessageMatches(
-								message.message,
-								error.message,
-							);
-						} else if (hasOwnProperty(error, "messageId")) {
-							assert.ok(
-								ruleHasMetaMessages,
-								"Error can not use 'messageId' if rule under test doesn't define 'meta.messages'.",
-							);
-							if (
-								!hasOwnProperty(
-									rule.meta.messages,
-									error.messageId,
-								)
-							) {
-								assert(
-									false,
-									`Invalid messageId '${error.messageId}'. Expected one of ${friendlyIDList}.`,
-								);
-							}
-							assert.strictEqual(
-								message.messageId,
-								error.messageId,
-								`messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`,
-							);
-
-							const unsubstitutedPlaceholders =
-								getUnsubstitutedMessagePlaceholders(
+						} else if (
+							typeof error === "object" &&
+							error !== null
+						) {
+							/*
+							 * Error object.
+							 * This may have a message, messageId, data, line, and/or column.
+							 */
+
+							if (hasOwnProperty(error, "message")) {
+								assertMessageMatches(
 									message.message,
-									rule.meta.messages[message.messageId],
-									error.data,
+									error.message,
 								);
-
-							assert.ok(
-								unsubstitutedPlaceholders.length === 0,
-								`The reported message has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property in the context.report() call.`,
-							);
-
-							if (hasOwnProperty(error, "data")) {
-								/*
-								 *  if data was provided, then directly compare the returned message to a synthetic
-								 *  interpolated message using the same message ID and data provided in the test.
-								 *  See https://github.com/eslint/eslint/issues/9890 for context.
-								 */
-								const unformattedOriginalMessage =
-									rule.meta.messages[error.messageId];
-								const rehydratedMessage = interpolate(
-									unformattedOriginalMessage,
-									error.data,
+							} else if (hasOwnProperty(error, "messageId")) {
+								assert.ok(
+									ruleHasMetaMessages,
+									"Error can not use 'messageId' if rule under test doesn't define 'meta.messages'.",
 								);
-
+								if (
+									!hasOwnProperty(
+										rule.meta.messages,
+										error.messageId,
+									)
+								) {
+									assert(
+										false,
+										`Invalid messageId '${error.messageId}'. Expected one of ${friendlyIDList}.`,
+									);
+								}
 								assert.strictEqual(
-									message.message,
-									rehydratedMessage,
-									`Hydrated message "${rehydratedMessage}" does not match "${message.message}"`,
+									message.messageId,
+									error.messageId,
+									`messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`,
 								);
-							}
-						} else {
-							assert.fail(
-								"Test error must specify either a 'messageId' or 'message'.",
-							);
-						}
-
-						const actualLocation = {};
-						const expectedLocation = {};
 
-						if (hasOwnProperty(error, "line")) {
-							actualLocation.line = message.line;
-							expectedLocation.line = error.line;
-						}
+								const unsubstitutedPlaceholders =
+									getUnsubstitutedMessagePlaceholders(
+										message.message,
+										rule.meta.messages[message.messageId],
+										error.data,
+									);
 
-						if (hasOwnProperty(error, "column")) {
-							actualLocation.column = message.column;
-							expectedLocation.column = error.column;
-						}
+								assert.ok(
+									unsubstitutedPlaceholders.length === 0,
+									`The reported message has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property in the context.report() call.`,
+								);
 
-						if (hasOwnProperty(error, "endLine")) {
-							actualLocation.endLine = message.endLine;
-							expectedLocation.endLine = error.endLine;
-						}
+								if (hasOwnProperty(error, "data")) {
+									/*
+									 *  if data was provided, then directly compare the returned message to a synthetic
+									 *  interpolated message using the same message ID and data provided in the test.
+									 *  See https://github.com/eslint/eslint/issues/9890 for context.
+									 */
+									const unformattedOriginalMessage =
+										rule.meta.messages[error.messageId];
+									const rehydratedMessage = interpolate(
+										unformattedOriginalMessage,
+										error.data,
+									);
 
-						if (hasOwnProperty(error, "endColumn")) {
-							actualLocation.endColumn = message.endColumn;
-							expectedLocation.endColumn = error.endColumn;
-						}
+									assert.strictEqual(
+										message.message,
+										rehydratedMessage,
+										`Hydrated message "${rehydratedMessage}" does not match "${message.message}"`,
+									);
+								} else {
+									const requiresDataProperty =
+										requireData === true ||
+										requireData === "error";
+									const hasPlaceholders =
+										getMessagePlaceholders(
+											rule.meta.messages[error.messageId],
+										).length > 0;
+									assert.ok(
+										!requiresDataProperty ||
+											!hasPlaceholders,
+										`Error should specify the 'data' property as the referenced message has placeholders.`,
+									);
+								}
+							}
 
-						if (Object.keys(expectedLocation).length > 0) {
-							assert.deepStrictEqual(
-								actualLocation,
-								expectedLocation,
-								"Actual error location does not match expected error location.",
-							);
-						}
+							const locationProperties = [
+								"line",
+								"column",
+								"endLine",
+								"endColumn",
+							];
+							const actualLocation = {};
+							const expectedLocation = {};
+
+							for (const key of locationProperties) {
+								if (hasOwnProperty(error, key)) {
+									actualLocation[key] = message[key];
+									expectedLocation[key] = error[key];
+								}
+							}
 
-						assert.ok(
-							!message.suggestions ||
-								hasOwnProperty(error, "suggestions"),
-							`Error at index ${i} has suggestions. Please specify 'suggestions' property on the test error object.`,
-						);
-						if (hasOwnProperty(error, "suggestions")) {
-							// Support asserting there are no suggestions
-							const expectsSuggestions = Array.isArray(
-								error.suggestions,
-							)
-								? error.suggestions.length > 0
-								: Boolean(error.suggestions);
-							const hasSuggestions =
-								message.suggestions !== void 0;
-
-							if (!hasSuggestions && expectsSuggestions) {
-								assert.ok(
-									!error.suggestions,
-									`Error should have suggestions on error with message: "${message.message}"`,
+							if (requireLocation) {
+								const missingKeys = locationProperties.filter(
+									key =>
+										!hasOwnProperty(error, key) &&
+										hasOwnProperty(message, key),
 								);
-							} else if (hasSuggestions) {
 								assert.ok(
-									expectsSuggestions,
-									`Error should have no suggestions on error with message: "${message.message}"`,
+									missingKeys.length === 0,
+									`Error is missing expected location properties: ${missingKeys.join(", ")}`,
 								);
-								if (typeof error.suggestions === "number") {
-									assert.strictEqual(
-										message.suggestions.length,
-										error.suggestions,
-										`Error should have ${error.suggestions} suggestions. Instead found ${message.suggestions.length} suggestions`,
+							}
+
+							if (Object.keys(expectedLocation).length > 0) {
+								assert.deepStrictEqual(
+									actualLocation,
+									expectedLocation,
+									"Actual error location does not match expected error location.",
+								);
+							}
+
+							assert.ok(
+								!message.suggestions ||
+									hasOwnProperty(error, "suggestions"),
+								`Error at index ${i} has suggestions. Please specify 'suggestions' property on the test error object.`,
+							);
+							if (hasOwnProperty(error, "suggestions")) {
+								// Support asserting there are no suggestions
+								const expectsSuggestions = Array.isArray(
+									error.suggestions,
+								)
+									? error.suggestions.length > 0
+									: Boolean(error.suggestions);
+								const hasSuggestions =
+									message.suggestions !== void 0;
+
+								if (!hasSuggestions && expectsSuggestions) {
+									assert.ok(
+										!error.suggestions,
+										`Error should have suggestions on error with message: "${message.message}"`,
 									);
-								} else if (Array.isArray(error.suggestions)) {
-									assert.strictEqual(
-										message.suggestions.length,
-										error.suggestions.length,
-										`Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`,
+								} else if (hasSuggestions) {
+									assert.ok(
+										expectsSuggestions,
+										`Error should have no suggestions on error with message: "${message.message}"`,
 									);
-
-									error.suggestions.forEach(
-										(expectedSuggestion, index) => {
-											assert.ok(
-												typeof expectedSuggestion ===
-													"object" &&
-													expectedSuggestion !== null,
-												"Test suggestion in 'suggestions' array must be an object.",
-											);
-											Object.keys(
-												expectedSuggestion,
-											).forEach(propertyName => {
+									if (typeof error.suggestions === "number") {
+										assert.strictEqual(
+											message.suggestions.length,
+											error.suggestions,
+											`Error should have ${error.suggestions} suggestions. Instead found ${message.suggestions.length} suggestions`,
+										);
+									} else if (
+										Array.isArray(error.suggestions)
+									) {
+										assert.strictEqual(
+											message.suggestions.length,
+											error.suggestions.length,
+											`Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`,
+										);
+
+										error.suggestions.forEach(
+											(expectedSuggestion, index) => {
 												assert.ok(
-													suggestionObjectParameters.has(
-														propertyName,
-													),
-													`Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.`,
+													typeof expectedSuggestion ===
+														"object" &&
+														expectedSuggestion !==
+															null,
+													"Test suggestion in 'suggestions' array must be an object.",
 												);
-											});
+												Object.keys(
+													expectedSuggestion,
+												).forEach(propertyName => {
+													assert.ok(
+														suggestionObjectParameters.has(
+															propertyName,
+														),
+														`Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.`,
+													);
+												});
 
-											const actualSuggestion =
-												message.suggestions[index];
-											const suggestionPrefix = `Error Suggestion at index ${index}:`;
+												const actualSuggestion =
+													message.suggestions[index];
+												const suggestionPrefix = `Error Suggestion at index ${index}:`;
 
-											if (
-												hasOwnProperty(
-													expectedSuggestion,
-													"desc",
-												)
-											) {
-												assert.ok(
-													!hasOwnProperty(
+												if (
+													hasOwnProperty(
 														expectedSuggestion,
-														"data",
-													),
-													`${suggestionPrefix} Test should not specify both 'desc' and 'data'.`,
-												);
-												assert.ok(
-													!hasOwnProperty(
+														"desc",
+													)
+												) {
+													assert.ok(
+														!hasOwnProperty(
+															expectedSuggestion,
+															"data",
+														),
+														`${suggestionPrefix} Test should not specify both 'desc' and 'data'.`,
+													);
+													assert.ok(
+														!hasOwnProperty(
+															expectedSuggestion,
+															"messageId",
+														),
+														`${suggestionPrefix} Test should not specify both 'desc' and 'messageId'.`,
+													);
+													assert.strictEqual(
+														actualSuggestion.desc,
+														expectedSuggestion.desc,
+														`${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.`,
+													);
+												} else if (
+													hasOwnProperty(
 														expectedSuggestion,
 														"messageId",
-													),
-													`${suggestionPrefix} Test should not specify both 'desc' and 'messageId'.`,
-												);
-												assert.strictEqual(
-													actualSuggestion.desc,
-													expectedSuggestion.desc,
-													`${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.`,
-												);
-											} else if (
-												hasOwnProperty(
-													expectedSuggestion,
-													"messageId",
-												)
-											) {
-												assert.ok(
-													ruleHasMetaMessages,
-													`${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.`,
-												);
-												assert.ok(
-													hasOwnProperty(
-														rule.meta.messages,
+													)
+												) {
+													assert.ok(
+														ruleHasMetaMessages,
+														`${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.`,
+													);
+													assert.ok(
+														hasOwnProperty(
+															rule.meta.messages,
+															expectedSuggestion.messageId,
+														),
+														`${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.`,
+													);
+													assert.strictEqual(
+														actualSuggestion.messageId,
 														expectedSuggestion.messageId,
-													),
-													`${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.`,
-												);
-												assert.strictEqual(
-													actualSuggestion.messageId,
-													expectedSuggestion.messageId,
-													`${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`,
-												);
+														`${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`,
+													);
 
-												const unsubstitutedPlaceholders =
-													getUnsubstitutedMessagePlaceholders(
-														actualSuggestion.desc,
+													const rawSuggestionMessage =
 														rule.meta.messages[
 															expectedSuggestion
 																.messageId
-														],
-														expectedSuggestion.data,
-													);
+														];
+													const unsubstitutedPlaceholders =
+														getUnsubstitutedMessagePlaceholders(
+															actualSuggestion.desc,
+															rawSuggestionMessage,
+															expectedSuggestion.data,
+														);
 
-												assert.ok(
-													unsubstitutedPlaceholders.length ===
-														0,
-													`The message of the suggestion has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property for the suggestion in the context.report() call.`,
-												);
+													assert.ok(
+														unsubstitutedPlaceholders.length ===
+															0,
+														`The message of the suggestion has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide the missing ${unsubstitutedPlaceholders.length > 1 ? "values" : "value"} via the 'data' property for the suggestion in the context.report() call.`,
+													);
 
-												if (
+													if (
+														hasOwnProperty(
+															expectedSuggestion,
+															"data",
+														)
+													) {
+														const unformattedMetaMessage =
+															rule.meta.messages[
+																expectedSuggestion
+																	.messageId
+															];
+														const rehydratedDesc =
+															interpolate(
+																unformattedMetaMessage,
+																expectedSuggestion.data,
+															);
+
+														assert.strictEqual(
+															actualSuggestion.desc,
+															rehydratedDesc,
+															`${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".`,
+														);
+													} else {
+														const requiresDataProperty =
+															requireData ===
+																true ||
+															requireData ===
+																"suggestion";
+														const hasPlaceholders =
+															getMessagePlaceholders(
+																rawSuggestionMessage,
+															).length > 0;
+														assert.ok(
+															!requiresDataProperty ||
+																!hasPlaceholders,
+															`${suggestionPrefix} Suggestion should specify the 'data' property as the referenced message has placeholders.`,
+														);
+													}
+												} else if (
 													hasOwnProperty(
 														expectedSuggestion,
 														"data",
 													)
 												) {
-													const unformattedMetaMessage =
-														rule.meta.messages[
-															expectedSuggestion
-																.messageId
-														];
-													const rehydratedDesc =
-														interpolate(
-															unformattedMetaMessage,
-															expectedSuggestion.data,
-														);
-
-													assert.strictEqual(
-														actualSuggestion.desc,
-														rehydratedDesc,
-														`${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".`,
+													assert.fail(
+														`${suggestionPrefix} Test must specify 'messageId' if 'data' is used.`,
+													);
+												} else {
+													assert.fail(
+														`${suggestionPrefix} Test must specify either 'messageId' or 'desc'.`,
 													);
 												}
-											} else if (
-												hasOwnProperty(
-													expectedSuggestion,
-													"data",
-												)
-											) {
-												assert.fail(
-													`${suggestionPrefix} Test must specify 'messageId' if 'data' is used.`,
+
+												assert.ok(
+													hasOwnProperty(
+														expectedSuggestion,
+														"output",
+													),
+													`${suggestionPrefix} The "output" property is required.`,
 												);
-											} else {
-												assert.fail(
-													`${suggestionPrefix} Test must specify either 'messageId' or 'desc'.`,
+												const codeWithAppliedSuggestion =
+													SourceCodeFixer.applyFixes(
+														item.code,
+														[actualSuggestion],
+													).output;
+
+												// Verify if suggestion fix makes a syntax error or not.
+												const errorMessageInSuggestion =
+													linter
+														.verify(
+															codeWithAppliedSuggestion,
+															result.configs,
+															result.filename,
+														)
+														.find(m => m.fatal);
+
+												assert(
+													!errorMessageInSuggestion,
+													[
+														"A fatal parsing error occurred in suggestion fix.",
+														`Error: ${errorMessageInSuggestion && errorMessageInSuggestion.message}`,
+														"Suggestion output:",
+														codeWithAppliedSuggestion,
+													].join("\n"),
 												);
-											}
 
-											assert.ok(
-												hasOwnProperty(
-													expectedSuggestion,
-													"output",
-												),
-												`${suggestionPrefix} The "output" property is required.`,
-											);
-											const codeWithAppliedSuggestion =
-												SourceCodeFixer.applyFixes(
-													item.code,
-													[actualSuggestion],
-												).output;
-
-											// Verify if suggestion fix makes a syntax error or not.
-											const errorMessageInSuggestion =
-												linter
-													.verify(
-														codeWithAppliedSuggestion,
-														result.configs,
-														result.filename,
-													)
-													.find(m => m.fatal);
-
-											assert(
-												!errorMessageInSuggestion,
-												[
-													"A fatal parsing error occurred in suggestion fix.",
-													`Error: ${errorMessageInSuggestion && errorMessageInSuggestion.message}`,
-													"Suggestion output:",
+												assert.strictEqual(
 													codeWithAppliedSuggestion,
-												].join("\n"),
-											);
-
-											assert.strictEqual(
-												codeWithAppliedSuggestion,
-												expectedSuggestion.output,
-												`Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`,
-											);
-											assert.notStrictEqual(
-												expectedSuggestion.output,
-												item.code,
-												`The output of a suggestion should differ from the original source code for suggestion at index: ${index} on error with message: "${message.message}"`,
-											);
-										},
-									);
-								} else {
-									assert.fail(
-										"Test error object property 'suggestions' should be an array or a number",
-									);
+													expectedSuggestion.output,
+													`Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`,
+												);
+												assert.notStrictEqual(
+													expectedSuggestion.output,
+													item.code,
+													`The output of a suggestion should differ from the original source code for suggestion at index: ${index} on error with message: "${message.message}"`,
+												);
+											},
+										);
+									} else {
+										assert.fail(
+											"Test error object property 'suggestions' should be an array or a number",
+										);
+									}
 								}
 							}
 						}
-					} else {
-						// Message was an unexpected type
-						assert.fail(
-							`Error should be a string, object, or RegExp, but found (${util.inspect(message)})`,
-						);
+					} catch (error) {
+						if (error instanceof Error) {
+							error.errorIndex = i;
+						}
+						throw error;
 					}
 				}
 			}
@@ -1599,7 +1900,7 @@ class RuleTester {
 			if (test.valid.length > 0) {
 				this.constructor.describe("valid", () => {
 					const seenTestCases = new Set();
-					test.valid.forEach(valid => {
+					test.valid.forEach((valid, index) => {
 						const item = normalizeTestCase(valid);
 						this.constructor[valid.only ? "itOnly" : "it"](
 							sanitize(item.name || item.code),
@@ -1608,6 +1909,21 @@ class RuleTester {
 									runHook(item, "before");
 									assertValidTestCase(item, seenTestCases);
 									testValidTemplate(item);
+								} catch (error) {
+									if (error instanceof Error) {
+										error.scenarioType = "valid";
+										error.scenarioIndex = index;
+										error.stack = error.stack.replace(
+											/^ +at /mu,
+											[
+												`    roughly at RuleTester.run.valid[${index}] (${estimateTestLocation(`valid[${index}]`)})`,
+												`    roughly at RuleTester.run.valid (${estimateTestLocation("valid")})`,
+												`    at RuleTester.run (${estimateTestLocation("root")})`,
+												"    at ",
+											].join("\n"),
+										);
+									}
+									throw error;
 								} finally {
 									runHook(item, "after");
 								}
@@ -1620,7 +1936,7 @@ class RuleTester {
 			if (test.invalid.length > 0) {
 				this.constructor.describe("invalid", () => {
 					const seenTestCases = new Set();
-					test.invalid.forEach(invalid => {
+					test.invalid.forEach((invalid, index) => {
 						const item = normalizeTestCase(invalid);
 						this.constructor[item.only ? "itOnly" : "it"](
 							sanitize(item.name || item.code),
@@ -1631,8 +1947,31 @@ class RuleTester {
 										item,
 										seenTestCases,
 										ruleName,
+										test.assertionOptions,
 									);
 									testInvalidTemplate(item);
+								} catch (error) {
+									if (error instanceof Error) {
+										error.scenarioType = "invalid";
+										error.scenarioIndex = index;
+										const errorIndex = error.errorIndex;
+										error.stack = error.stack.replace(
+											/^ +at /mu,
+											[
+												...(typeof errorIndex ===
+												"number"
+													? [
+															`    roughly at RuleTester.run.invalid[${index}].error[${errorIndex}] (${estimateTestLocation(`invalid[${index}].errors[${errorIndex}]`)})`,
+														]
+													: []),
+												`    roughly at RuleTester.run.invalid[${index}] (${estimateTestLocation(`invalid[${index}]`)})`,
+												`    roughly at RuleTester.run.invalid (${estimateTestLocation("invalid")})`,
+												`    at RuleTester.run (${estimateTestLocation("root")})`,
+												"    at ",
+											].join("\n"),
+										);
+									}
+									throw error;
 								} finally {
 									runHook(item, "after");
 								}