@alwaysmeticulous/cli 2.38.0 → 2.40.0

package/dist/commands/bootstrap/bootstrap.command.js

@@ -22,7 +22,7 @@ const handler = async () => {
     logger.info(`Setting up ${chalk_1.default.green("test:meticulous")} script...`);
     await (0, npm_set_script_utils_1.npmSetScript)({
         script: "test:meticulous",
-        command: "meticulous run-all-tests --headless --parallelize --deflake",
+        command: "meticulous run-all-tests --headless --parallelize",
     });
 };
 exports.bootstrapCommand = (0, command_builder_1.buildCommand)("bootstrap")

package/dist/commands/create-test/create-test.command.d.ts

@@ -78,10 +78,6 @@ export declare const createTestCommand: import("yargs").CommandModule<unknown, i
         boolean: true;
         description: string;
     };
-    screenshotSelector: {
-        string: true;
-        description: string;
-    };
     moveBeforeClick: {
         readonly boolean: true;
         readonly description: "Simulate mouse movement before clicking";

package/dist/commands/create-test/create-test.command.js

@@ -59,7 +59,7 @@ apiToken, commitSha, devTools, bypassCSP,
 // Record options
 width, height, uploadIntervalMs, incognito, trace, 
 // Replay options
-headless, screenshotSelector, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, }) => {
+headless, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     logger.info("Creating a new Meticulous test");
     logger.info("Step 1: record a new test");
@@ -95,7 +95,6 @@ headless, screenshotSelector, shiftTime, networkStubbing, moveBeforeClick, cooki
         devTools,
         bypassCSP,
         screenshot: true,
-        screenshotSelector,
         shiftTime,
         disableRemoteFonts,
         noSandbox,
@@ -148,10 +147,6 @@ exports.createTestCommand = (0, command_builder_1.buildCommand)("create-test")
         description: "Enable verbose logging",
     },
     // Replay options
-    screenshotSelector: {
-        string: true,
-        description: "Query selector to screenshot a specific DOM element instead of the whole page",
-    },
     moveBeforeClick: common_options_1.OPTIONS.moveBeforeClick,
     cookiesFile: {
         string: true,

package/dist/commands/replay/replay.command.d.ts

@@ -23,7 +23,6 @@ export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, O
     screenshot: boolean;
     appUrl: string | null | undefined;
     simulationIdForAssets: string | null | undefined;
-    screenshotSelector: string | null | undefined;
     maxDurationMs: number | null | undefined;
     maxEventCount: number | null | undefined;
     storyboard: boolean;
@@ -36,7 +35,7 @@ interface AdditionalReplayOptions {
     cookiesFile: string | null | undefined;
     debugger: boolean;
 }
-export declare const rawReplayCommandHandler: ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }: RawReplayCommandHandlerOptions) => Promise<Replay>;
+export declare const rawReplayCommandHandler: ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }: RawReplayCommandHandlerOptions) => Promise<Replay>;
 export declare const getReplayTarget: ({ appUrl, simulationIdForAssets, }: {
     appUrl: string | null;
     simulationIdForAssets: string | null;
@@ -134,10 +133,6 @@ export declare const replayCommand: import("yargs").CommandModule<unknown, impor
         description: string;
         default: boolean;
     };
-    screenshotSelector: {
-        string: true;
-        description: string;
-    };
     debugger: {
         boolean: true;
         description: string;

package/dist/commands/replay/replay.command.js

@@ -232,7 +232,7 @@ const serveOrGetAppUrl = async (client, replayTarget) => {
 const unknownReplayTargetType = (replayTarget) => {
     throw new Error(`Unknown type of replay target: ${JSON.stringify(replayTarget)}`);
 };
-const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }) => {
+const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -254,7 +254,6 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
     const screenshottingOptions = screenshot
         ? {
             enabled: true,
-            screenshotSelector: screenshotSelector !== null && screenshotSelector !== void 0 ? screenshotSelector : null,
             diffOptions: { diffPixelThreshold, diffThreshold },
             storyboardOptions,
         }
@@ -317,10 +316,6 @@ exports.replayCommand = (0, command_builder_1.buildCommand)("simulate")
         description: "Take a screenshot at the end of simulation",
         default: true,
     },
-    screenshotSelector: {
-        string: true,
-        description: "Query selector to screenshot a specific DOM element instead of the whole page",
-    },
     debugger: {
         boolean: true,
         description: "Opens a step through debugger to advance through the replay event by event",

package/dist/commands/run-all-tests/run-all-tests.command.d.ts

@@ -97,16 +97,16 @@ export declare const runAllTestsCommand: import("yargs").CommandModule<unknown,
         readonly description: "Number of tasks to run in parallel (defaults to two per CPU)";
         readonly coerce: (value: number | null | undefined) => number | null | undefined;
     };
-    readonly deflake: {
-        readonly boolean: true;
-        readonly description: "Attempt to deflake failing tests";
-        readonly default: false;
-    };
     readonly maxRetriesOnFailure: {
         readonly number: true;
         readonly description: "If set to a value greater than 0 then will re-run any replays that give a screenshot diff and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.";
         readonly default: 0;
     };
+    readonly rerunTestsNTimes: {
+        readonly number: true;
+        readonly description: "If set to a value greater than 0 then will re-run all replays the specified number of times and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.";
+        readonly default: 0;
+    };
     readonly useCache: {
         readonly boolean: true;
         readonly description: "Use result cache";

package/dist/commands/run-all-tests/run-all-tests.command.js

@@ -7,7 +7,7 @@ const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const run_all_tests_1 = require("../../parallel-tests/run-all-tests");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
-const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, maxRetriesOnFailure, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, baseTestRunId, }) => {
+const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, maxRetriesOnFailure, rerunTestsNTimes, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, baseTestRunId, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -27,7 +27,6 @@ const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl,
         : { enabled: false };
     const screenshottingOptions = {
         enabled: true,
-        screenshotSelector: null,
         diffOptions: { diffPixelThreshold, diffThreshold },
         storyboardOptions,
     };
@@ -47,8 +46,8 @@ const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl,
         baseTestRunId: baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : null,
         appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
         parallelTasks: parrelelTasks !== null && parrelelTasks !== void 0 ? parrelelTasks : null,
-        deflake,
         maxRetriesOnFailure,
+        rerunTestsNTimes,
         cachedTestRunResults,
         githubSummary,
     });
@@ -89,16 +88,16 @@ exports.runAllTestsCommand = (0, command_builder_1.buildCommand)("run-all-tests"
             return value;
         },
     },
-    deflake: {
-        boolean: true,
-        description: "Attempt to deflake failing tests",
-        default: false,
-    },
     maxRetriesOnFailure: {
         number: true,
         description: "If set to a value greater than 0 then will re-run any replays that give a screenshot diff and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.",
         default: 0,
     },
+    rerunTestsNTimes: {
+        number: true,
+        description: "If set to a value greater than 0 then will re-run all replays the specified number of times and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.",
+        default: 0,
+    },
     useCache: {
         boolean: true,
         description: "Use result cache",

package/dist/commands/screenshot-diff/utils/__tests__/get-screenshot-identifier.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/screenshot-diff/utils/__tests__/get-screenshot-identifier.spec.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const get_screenshot_identifier_1 = require("../get-screenshot-identifier");
+describe("get-screenshot-identifier", () => {
+    it("can parse identifiers without version numbers", () => {
+        expect((0, get_screenshot_identifier_1.getScreenshotIdentifier)("final-state.png")).toEqual({
+            type: "end-state",
+        });
+        expect((0, get_screenshot_identifier_1.getScreenshotIdentifier)("screenshot-after-event-7.png")).toEqual({
+            type: "after-event",
+            eventNumber: 7,
+        });
+    });
+    it("can parse identifiers with version numbers", () => {
+        expect((0, get_screenshot_identifier_1.getScreenshotIdentifier)("final-state-v2.png")).toEqual({
+            type: "end-state",
+            logicVersion: 2,
+        });
+        expect((0, get_screenshot_identifier_1.getScreenshotIdentifier)("screenshot-after-event-7-v2.png")).toEqual({
+            type: "after-event",
+            eventNumber: 7,
+            logicVersion: 2,
+        });
+    });
+});

package/dist/commands/screenshot-diff/utils/get-screenshot-filename.js

@@ -5,11 +5,15 @@ exports.getScreenshotFilename = void 0;
 // in replay-node, and in https://github.com/alwaysmeticulous/meticulous-sdk/blob/395af4394dc51d9b51ba1136fc26b23fcbba5604/packages/replayer/src/screenshot.utils.ts#L42
 const getScreenshotFilename = (identifier) => {
     if (identifier.type === "end-state") {
-        return "final-state.png";
+        return identifier.logicVersion == null
+            ? "final-state.png"
+            : `final-state-v${identifier.logicVersion}.png`;
     }
     else if (identifier.type === "after-event") {
         const eventIndexStr = identifier.eventNumber.toString().padStart(5, "0");
-        return `screenshot-after-event-${eventIndexStr}.png`;
+        return identifier.logicVersion == null
+            ? `screenshot-after-event-${eventIndexStr}.png`
+            : `screenshot-after-event-${eventIndexStr}-v${identifier.logicVersion}.png`;
     }
     else {
         throw new Error("Unexpected screenshot identifier: " + JSON.stringify(identifier));

package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.js

@@ -3,19 +3,34 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getScreenshotIdentifier = void 0;
 const path_1 = require("path");
 const getScreenshotIdentifier = (filename) => {
+    var _a;
     const name = (0, path_1.basename)(filename);
     if (name === "final-state.png") {
         return {
             type: "end-state",
         };
     }
+    if (name.startsWith("final-state-v")) {
+        const match = name.match(/^final-state-v(\d+)[.]png$/);
+        const logicVersionNumber = match ? parseInt(match[1], 10) : undefined;
+        if (match && logicVersionNumber != null && !isNaN(logicVersionNumber)) {
+            return {
+                type: "end-state",
+                logicVersion: logicVersionNumber,
+            };
+        }
+    }
     if (name.startsWith("screenshot-after-event")) {
-        const match = name.match(/^(?:.*)-(\d+)[.]png$/);
+        const match = (_a = name.match(/^(?:.*)-(\d+)-v(\d+)?[.]png$/)) !== null && _a !== void 0 ? _a : name.match(/^(?:.*)-(\d+)[.]png$/);
         const eventNumber = match ? parseInt(match[1], 10) : undefined;
+        const logicVersionNumber = match ? parseInt(match[2], 10) : undefined;
         if (match && eventNumber != null && !isNaN(eventNumber)) {
             return {
                 type: "after-event",
                 eventNumber,
+                ...(logicVersionNumber != null && !isNaN(logicVersionNumber)
+                    ? { logicVersion: logicVersionNumber }
+                    : {}),
             };
         }
     }

package/dist/config/config.js

@@ -22,10 +22,9 @@ const getConfigFilePath = async () => {
     return configFilePath;
 };
 const validateReplayOptions = (prevOptions) => {
-    const { appUrl, screenshotSelector, diffThreshold, diffPixelThreshold, moveBeforeClick, simulationIdForAssets, } = prevOptions;
+    const { appUrl, diffThreshold, diffPixelThreshold, moveBeforeClick, simulationIdForAssets, } = prevOptions;
     return {
         ...(appUrl != null ? { appUrl } : {}),
-        ...(screenshotSelector != null ? { screenshotSelector } : {}),
         ...(diffThreshold != null ? { diffThreshold } : {}),
         ...(diffPixelThreshold != null ? { diffPixelThreshold } : {}),
         ...(moveBeforeClick != null ? { moveBeforeClick } : {}),

package/dist/parallel-tests/__tests__/merge-test-results.spec.js

@@ -12,14 +12,16 @@ describe("mergeResults", () => {
         });
         expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.noDiff)(1)]));
     });
-    it("ignores diffs to screenshots which originally passed", () => {
+    it("doesn't ignore diffs to screenshots which originally passed", () => {
         const currentResult = (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)]);
         const comparisonToHeadReplay = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1)]);
         const mergedResult = (0, merge_test_results_1.mergeResults)({
             currentResult,
             comparisonToHeadReplay,
         });
-        expect(mergedResult).toEqual((0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)]));
+        expect(mergedResult).toEqual(
+        // All screenshots are either "no-diff" or "flake" thus the overall result is "flake"
+        (0, mock_test_results_1.testResult)("flake", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.noDiff)(), [(0, mock_test_results_1.diff)()])]));
     });
     it("marks screenshots as flakes if the screenshot comparison originally failed, but the second retry gives a different screenshot again", () => {
         const currentResult = (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.diff)(1), (0, mock_test_results_1.diff)(2)]);

package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.js

@@ -17,6 +17,7 @@ describe("runAllTestsInParallel", () => {
             testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
             parallelTasks: 2,
             maxRetriesOnFailure: 0,
+            rerunTestsNTimes: 0,
             executeTest: async (testCase) => {
                 var _a;
                 testsStarted++;
@@ -77,6 +78,7 @@ describe("runAllTestsInParallel", () => {
             testsToRun: [testCase(0), testCase(1), testCase(2), testCase(3)],
             parallelTasks: 2,
             maxRetriesOnFailure: 0,
+            rerunTestsNTimes: 0,
             executeTest: async (testCase) => {
                 var _a;
                 await testRunPromises[parseInt((_a = testCase.title) !== null && _a !== void 0 ? _a : "")].promise;
@@ -109,6 +111,7 @@ describe("runAllTestsInParallel", () => {
             testsToRun: [testCase(0)],
             parallelTasks: 2,
             maxRetriesOnFailure: 0,
+            rerunTestsNTimes: 0,
             executeTest: async (testCase) => {
                 return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0)], testCase);
             },
@@ -121,6 +124,7 @@ describe("runAllTestsInParallel", () => {
             testsToRun: [testCase(0)],
             parallelTasks: 2,
             maxRetriesOnFailure: 10,
+            rerunTestsNTimes: 0,
             executeTest: async (testCase, isRetry) => {
                 if (!isRetry) {
                     return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
@@ -145,6 +149,7 @@ describe("runAllTestsInParallel", () => {
             testsToRun: [testCase(0)],
             parallelTasks: 2,
             maxRetriesOnFailure: 3,
+            rerunTestsNTimes: 0,
             executeTest: async (testCase, isRetry) => {
                 if (!isRetry) {
                     return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.diff)(1)], testCase);
@@ -156,6 +161,31 @@ describe("runAllTestsInParallel", () => {
             (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)(), (0, mock_test_results_1.diff)()])], testCase(0)),
         ]);
     });
+    it("retries N times if rerunTestsNTimes > 0", async () => {
+        let rerunNum = 0;
+        const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+            testsToRun: [testCase(0)],
+            parallelTasks: 2,
+            maxRetriesOnFailure: 0,
+            rerunTestsNTimes: 3,
+            executeTest: async (testCase, isRetry) => {
+                if (!isRetry) {
+                    return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.diff)(0), (0, mock_test_results_1.noDiff)(1)], testCase);
+                }
+                if (rerunNum === 2) {
+                    return (0, mock_test_results_1.testResult)("fail", [(0, mock_test_results_1.missingHead)(0), (0, mock_test_results_1.diff)(1)], testCase);
+                }
+                rerunNum++;
+                return (0, mock_test_results_1.testResult)("pass", [(0, mock_test_results_1.noDiff)(0), (0, mock_test_results_1.noDiff)(1)], testCase);
+            },
+        });
+        expect(rerunNum).toEqual(2);
+        expect(results).toEqual([
+            (0, mock_test_results_1.testResult)(
+            // Flake outcome is returned iff all screenshots are flaky.
+            "flake", [(0, mock_test_results_1.flake)(0, (0, mock_test_results_1.diff)(), [(0, mock_test_results_1.missingHead)()]), (0, mock_test_results_1.flake)(1, (0, mock_test_results_1.noDiff)(), [(0, mock_test_results_1.diff)()])], testCase(0)),
+        ]);
+    });
 });
 const testCase = (num) => ({
     sessionId: "mock-session-id",

package/dist/parallel-tests/merge-test-results.d.ts

@@ -1,4 +1,5 @@
-import { DetailedTestCaseResult } from "../config/config.types";
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
+import { DetailedTestCaseResult, TestCaseResult } from "../config/config.types";
 export interface ResultsToMerge {
     currentResult: DetailedTestCaseResult;
     /**
@@ -8,3 +9,4 @@ export interface ResultsToMerge {
     comparisonToHeadReplay: DetailedTestCaseResult;
 }
 export declare const mergeResults: ({ currentResult, comparisonToHeadReplay, }: ResultsToMerge) => DetailedTestCaseResult;
+export declare const testRunOutcomeFromDiffResults: (currentResult: TestCaseResult["result"], newScreenshotDiffResults: ScreenshotDiffResult[]) => TestCaseResult["result"];

package/dist/parallel-tests/merge-test-results.js

@@ -3,9 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mergeResults = void 0;
-const utils_1 = require("@sentry/utils");
+exports.testRunOutcomeFromDiffResults = exports.mergeResults = void 0;
 const fast_json_stable_stringify_1 = __importDefault(require("fast-json-stable-stringify"));
+const has_notable_differences_1 = require("../commands/screenshot-diff/utils/has-notable-differences");
 const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
     // If any of the screenshots diffs in comparisonToHeadReplay show a diff against one
@@ -20,9 +20,6 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
         retryDiffById.set(hash, result);
     });
     const newScreenshotDiffResults = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(currentResult).map((currentDiff) => {
-        if (currentDiff.outcome === "no-diff") {
-            return currentDiff;
-        }
         const hash = hashScreenshotIdentifier(currentDiff.identifier);
         const diffWhenRetrying = retryDiffById.get(hash);
         // diffWhenRetrying is null in the case that there is a base screenshot for the base replay,
@@ -75,12 +72,9 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
             };
         }
     });
-    const noLongerHasFailures = newScreenshotDiffResults.every(({ outcome }) => outcome === "flake" || outcome === "no-diff");
     return {
         ...currentResult,
-        result: currentResult.result === "fail" && noLongerHasFailures
-            ? "flake"
-            : currentResult.result,
+        result: (0, exports.testRunOutcomeFromDiffResults)(currentResult.result, newScreenshotDiffResults),
         screenshotDiffResultsByBaseReplayId: (0, screenshot_diff_results_utils_1.groupScreenshotDiffResults)(newScreenshotDiffResults),
     };
 };
@@ -91,20 +85,22 @@ identifier: _identifier,
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 baseReplayId: _baseReplayId, ...rest }) => rest;
 const hashScreenshotIdentifier = (identifier) => {
-    if (identifier.type === "end-state") {
-        return "end-state";
-    }
-    else if (identifier.type === "after-event") {
-        return `after-event-${identifier.eventNumber}`;
+    return (0, fast_json_stable_stringify_1.default)(identifier);
+};
+const testRunOutcomeFromDiffResults = (currentResult, newScreenshotDiffResults) => {
+    // If a test run is already flakey, we don't want to overwrite that with a 'fail' result.
+    if (currentResult === "flake") {
+        return "flake";
     }
-    else {
-        unknownScreenshotIdentifierType(identifier);
-        // The identifier is probably from a newer version of the bundle script
-        // and we're on an old version of the CLI. Our best bet is to stringify it
-        // and use that as a hash.
-        return (0, fast_json_stable_stringify_1.default)(identifier);
+    const hasOnlyFlakes = newScreenshotDiffResults.every(({ outcome }) => outcome === "flake" || outcome === "no-diff");
+    if (hasOnlyFlakes) {
+        return "flake";
     }
+    // If we've had a test run has already failed, we don't want to overwrite that with a 'pass' result.
+    // Otherwise, if there are any notable differences, we want to fail the test run.
+    return currentResult === "fail" ||
+        (0, has_notable_differences_1.hasNotableDifferences)(newScreenshotDiffResults)
+        ? "fail"
+        : "pass";
 };
-const unknownScreenshotIdentifierType = (identifier) => {
-    utils_1.logger.error(`Unknown type of screenshot identifier: ${JSON.stringify(identifier)}`);
-};
+exports.testRunOutcomeFromDiffResults = testRunOutcomeFromDiffResults;

package/dist/parallel-tests/messages.types.d.ts

@@ -1,12 +1,12 @@
 import log from "loglevel";
 import { DetailedTestCaseResult } from "../config/config.types";
-import { DeflakeReplayCommandHandlerOptions } from "../deflake-tests/deflake-tests.handler";
+import { ParallelTestsReplayOptions } from "./parallel-replay.types";
 export interface InitMessage {
     kind: "init";
     data: {
         logLevel: log.LogLevel[keyof log.LogLevel];
         dataDir: string;
-        replayOptions: DeflakeReplayCommandHandlerOptions;
+        replayOptions: ParallelTestsReplayOptions;
     };
 }
 export interface ResultMessage {

package/dist/parallel-tests/parallel-replay.handler.d.ts

@@ -0,0 +1,3 @@
+import { DetailedTestCaseResult } from "../config/config.types";
+import { ParallelTestsReplayOptions } from "./parallel-replay.types";
+export declare const handleReplay: ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, baseTestRunId, }: ParallelTestsReplayOptions) => Promise<DetailedTestCaseResult>;

package/dist/{deflake-tests/deflake-tests.handler.js → parallel-tests/parallel-replay.handler.js}

@@ -1,11 +1,6 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deflakeReplayCommandHandler = void 0;
-const common_1 = require("@alwaysmeticulous/common");
-const loglevel_1 = __importDefault(require("loglevel"));
+exports.handleReplay = void 0;
 const replay_command_1 = require("../commands/replay/replay.command");
 const has_notable_differences_1 = require("../commands/screenshot-diff/utils/has-notable-differences");
 const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, baseTestRunId, }) => {
@@ -35,21 +30,7 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
         screenshotDiffResultsByBaseReplayId,
     };
 };
-const deflakeReplayCommandHandler = async ({ deflake, ...otherOptions }) => {
-    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const firstResult = await handleReplay(otherOptions);
-    if (firstResult.result === "pass" || !deflake) {
-        return firstResult;
-    }
-    const secondResult = await handleReplay(otherOptions);
-    if (secondResult.result === "fail") {
-        return secondResult;
-    }
-    const thirdResult = await handleReplay(otherOptions);
-    logger.info(`FLAKE: ${thirdResult.title} => ${thirdResult.result}`);
-    return thirdResult;
-};
-exports.deflakeReplayCommandHandler = deflakeReplayCommandHandler;
+exports.handleReplay = handleReplay;
 const applyTestCaseExecutionOptionOverrides = (executionOptionsFromCliFlags, overridesFromTestCase) => {
     var _a;
     // Options specified in the test case override those passed as CLI flags
@@ -60,7 +41,7 @@ const applyTestCaseExecutionOptionOverrides = (executionOptionsFromCliFlags, ove
     };
 };
 const applyTestCaseScreenshottingOptionsOverrides = (screenshottingOptionsFromCliFlags, overridesFromTestCase) => {
-    var _a, _b, _c;
+    var _a, _b;
     // Options specified in the test case override those passed as CLI flags
     // (CLI flags set the defaults)
     const diffOptions = {
@@ -69,7 +50,6 @@ const applyTestCaseScreenshottingOptionsOverrides = (screenshottingOptionsFromCl
     };
     return {
         enabled: true,
-        screenshotSelector: (_c = overridesFromTestCase.screenshotSelector) !== null && _c !== void 0 ? _c : screenshottingOptionsFromCliFlags.screenshotSelector,
         diffOptions,
         storyboardOptions: screenshottingOptionsFromCliFlags.storyboardOptions,
     };

package/dist/{deflake-tests/deflake-tests.handler.d.ts → parallel-tests/parallel-replay.types.d.ts}

@@ -1,11 +1,7 @@
 import { TestCase } from "@alwaysmeticulous/api";
 import { GeneratedBy, ReplayEventsDependencies, ReplayExecutionOptions, ReplayTarget } from "@alwaysmeticulous/common";
 import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
-import { DetailedTestCaseResult } from "../config/config.types";
-export interface DeflakeReplayCommandHandlerOptions extends HandleReplayOptions {
-    deflake: boolean;
-}
-interface HandleReplayOptions {
+export interface ParallelTestsReplayOptions {
     replayTarget: ReplayTarget;
     executionOptions: ReplayExecutionOptions;
     screenshottingOptions: ScreenshotAssertionsEnabledOptions;
@@ -18,5 +14,3 @@ interface HandleReplayOptions {
     replayEventsDependencies: ReplayEventsDependencies;
     suppressScreenshotDiffLogging: boolean;
 }
-export declare const deflakeReplayCommandHandler: ({ deflake, ...otherOptions }: DeflakeReplayCommandHandlerOptions) => Promise<DetailedTestCaseResult>;
-export {};

package/dist/parallel-tests/parallel-replay.types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/parallel-tests/parallel-tests.handler.d.ts

@@ -5,6 +5,7 @@ export interface RunAllTestsInParallelOptions {
     testsToRun: TestCase[];
     parallelTasks: number | null;
     maxRetriesOnFailure: number;
+    rerunTestsNTimes: number;
     executeTest: (testCase: TestCase, isRetry: boolean) => Promise<DetailedTestCaseResult>;
     onTestFinished?: (progress: TestRunProgress, resultsSoFar: DetailedTestCaseResult[]) => Promise<void>;
     onTestFailedToRun?: (progress: TestRunProgress) => Promise<void>;

package/dist/parallel-tests/parallel-tests.handler.js

@@ -10,7 +10,10 @@ const loglevel_1 = __importDefault(require("loglevel"));
 const merge_test_results_1 = require("./merge-test-results");
 const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, executeTest, onTestFinished, onTestFailedToRun, }) => {
+const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, executeTest, onTestFinished, onTestFailedToRun, }) => {
+    if (maxRetriesOnFailure && rerunTestsNTimes) {
+        throw new Error("maxRetriesOnFailure and rerunTestsNTimes are mutually exclusive.");
+    }
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     let nextId = 0;
     let queue = testsToRun.map((test) => ({
@@ -47,8 +50,16 @@ const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFa
             .then(async (result) => {
             var _a;
             const resultsForTestCase = resultsByTestId.get(id);
+            // Re-run the test if needed, comparing to the base replay.
+            if (!isRetry) {
+                queue.push(...Array.from(new Array(rerunTestsNTimes)).map(() => ({
+                    ...testCase,
+                    id,
+                    baseReplayId: result === null || result === void 0 ? void 0 : result.headReplayId,
+                })));
+            }
             if (resultsForTestCase != null && result != null) {
-                logRetrySummary(testName({ id, ...testCase }), result);
+                logRetrySummary(testName({ id, ...testCase }), result, resultsForTestCase.currentResult);
             }
             if (resultsForTestCase != null &&
                 resultsForTestCase.currentResult.result === "flake") {
@@ -81,8 +92,15 @@ const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFa
             // Our work is done for this test case if the first result was a pass,
             // we've performed all the retries, or one of the retries already proved
             // the result as flakey
-            const isFinalResult = mergedResult.result !== "fail" ||
-                numberOfRetriesExecuted === maxRetriesOnFailure;
+            let isFinalResult;
+            if (rerunTestsNTimes > 0) {
+                isFinalResult = numberOfRetriesExecuted >= rerunTestsNTimes;
+            }
+            else {
+                isFinalResult =
+                    mergedResult.result !== "fail" ||
+                        numberOfRetriesExecuted >= maxRetriesOnFailure;
+            }
             if (isFinalResult) {
                 // Cancel any replays that are still scheduled
                 queue = queue.filter((otherReplay) => otherReplay.id !== id);
@@ -119,7 +137,12 @@ const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFa
         }
         ++inProgress;
         if (resultsByTestId.has(testCase.id)) {
-            logger.info(`Test ${testName(testCase)} failed. Retrying to check for flakes...`);
+            if (maxRetriesOnFailure > 0) {
+                logger.info(`Test ${testName(testCase)} failed. Retrying to check for flakes...`);
+            }
+            else if (rerunTestsNTimes > 0) {
+                logger.info(`Re-running ${testName(testCase)} to check for flakes...`);
+            }
         }
         startTask(testCase);
         process.nextTick(checkNextTask);
@@ -129,10 +152,10 @@ const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFa
     return finalResults;
 };
 exports.runAllTestsInParallel = runAllTestsInParallel;
-const logRetrySummary = (nameOfTest, retryResult) => {
+const logRetrySummary = (nameOfTest, retryResult, resultSoFar) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     if (retryResult.result === "pass") {
-        logger.info(`Retried taking screenshots for failed test ${nameOfTest}, but got the same results`);
+        logger.info(`Retried taking screenshots for${resultSoFar.result == "fail" ? " failed" : ""} test ${nameOfTest}, but got the same results`);
     }
     else {
         const numDifferingScreenshots = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(retryResult).filter((result) => result.outcome !== "no-diff").length;

package/dist/parallel-tests/run-all-tests.d.ts

@@ -25,13 +25,22 @@ export interface Options {
      * Set to 1 to disable parralelism.
      */
     parallelTasks: number | null;
-    deflake: boolean;
     /**
      * If set to a value greater than 1 then will re-run any replays that give a screenshot diff
      * and mark them as a flake if the screenshot generated on one of the retryed replays differs from that
      * in the first replay.
      */
     maxRetriesOnFailure: number;
+    /**
+     * If set to a value greater than 0 then will re-run all replays the specified number of times
+     * and mark them as a flake if the screenshot generated on one of the retryed replays differs from that
+     * in the first replay.
+     *
+     * This is useful for checking flake rates.
+     *
+     * This option is mutually exclusive with maxRetriesOnFailure.
+     */
+    rerunTestsNTimes: number;
     githubSummary: boolean;
     /**
      * If provided it will incorportate the cachedTestRunResults in any calls to store
@@ -61,4 +70,4 @@ export interface RunAllTestsResult {
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;
+export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;

package/dist/parallel-tests/run-all-tests.js

@@ -12,21 +12,18 @@ const replay_diff_api_1 = require("../api/replay-diff.api");
 const test_run_api_1 = require("../api/test-run.api");
 const config_1 = require("../config/config");
 const replay_assets_1 = require("../local-data/replay-assets");
-const parallel_tests_handler_1 = require("../parallel-tests/parallel-tests.handler");
 const config_utils_1 = require("../utils/config.utils");
 const github_summary_utils_1 = require("../utils/github-summary.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 const test_run_environment_utils_1 = require("../utils/test-run-environment.utils");
 const version_utils_1 = require("../utils/version.utils");
 const execute_test_in_child_process_1 = require("./execute-test-in-child-process");
+const parallel_tests_handler_1 = require("./parallel-tests.handler");
 /**
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
-    if (deflake && maxRetriesOnFailure > 1) {
-        throw new Error("Arguments deflake and maxRetriesOnFailure are mutually exclusive");
-    }
+const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, maxRetriesOnFailure, rerunTestsNTimes, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const client = (0, client_1.createClient)({ apiToken });
     const project = await (0, project_api_1.getProject)(client);
@@ -60,7 +57,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                 baseCommitSha,
                 appUrl,
                 parallelTasks,
-                deflake,
                 githubSummary,
             },
             environment: (0, test_run_environment_utils_1.getEnvironment)(environment),
@@ -145,6 +141,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         testsToRun,
         parallelTasks,
         maxRetriesOnFailure,
+        rerunTestsNTimes,
         executeTest: (testCase, isRetry) => {
             var _a;
             const initMessage = {
@@ -156,7 +153,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                         apiToken,
                         commitSha,
                         testCase,
-                        deflake,
                         replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
                             appUrl,
                             testCase,

package/dist/parallel-tests/task.handler.js

@@ -30,9 +30,9 @@ const common_1 = require("@alwaysmeticulous/common");
 const Sentry = __importStar(require("@sentry/node"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const luxon_1 = require("luxon");
-const deflake_tests_handler_1 = require("../deflake-tests/deflake-tests.handler");
 const logger_utils_1 = require("../utils/logger.utils");
 const sentry_utils_1 = require("../utils/sentry.utils");
+const parallel_replay_handler_1 = require("./parallel-replay.handler");
 const INIT_TIMEOUT = luxon_1.Duration.fromObject({ second: 1 });
 const waitForInitMessage = () => {
     return Promise.race([
@@ -67,7 +67,7 @@ const main = async () => {
     const { logLevel, dataDir, replayOptions } = initMessage.data;
     logger.setLevel(logLevel);
     (0, common_1.setMeticulousLocalDataDir)(dataDir);
-    const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)(replayOptions);
+    const result = await (0, parallel_replay_handler_1.handleReplay)(replayOptions);
     const resultMessage = {
         kind: "result",
         data: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alwaysmeticulous/cli",
-  "version": "2.38.0",
+  "version": "2.40.0",
   "description": "The Meticulous CLI",
   "license": "ISC",
   "main": "dist/index.js",
@@ -26,7 +26,7 @@
     "test": "jest"
   },
   "dependencies": {
-    "@alwaysmeticulous/common": "^2.38.0",
+    "@alwaysmeticulous/common": "^2.40.0",
     "@sentry/node": "^7.36.0",
     "@sentry/tracing": "^7.36.0",
     "adm-zip": "^0.5.9",
@@ -45,7 +45,7 @@
     "yargs": "^17.5.1"
   },
   "devDependencies": {
-    "@alwaysmeticulous/api": "^2.37.0",
+    "@alwaysmeticulous/api": "^2.40.0",
     "@types/express": "^4.17.14",
     "@types/proper-lockfile": "^4.1.2"
   },
@@ -95,5 +95,5 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   },
-  "gitHead": "2c4fbc9225d5e0dc09f14a1e90a67008a5e3f376"
+  "gitHead": "46c8a774b634b1e43ca8e792a6ed573adba296ef"
 }