@alwaysmeticulous/cli 2.19.3 → 2.20.0

package/dist/parallel-tests/parallel-tests.handler.js

@@ -11,13 +11,28 @@ const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const config_utils_1 = require("../utils/config.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
+const merge_test_results_1 = require("./merge-test-results");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiToken, commitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, replayEventsDependencies, onTestFinished, }) => {
+const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, commitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, replayEventsDependencies, onTestFinished, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const results = [];
+    let nextId = 0;
+    let queue = testsToRun.map((test) => ({
+        ...test,
+        id: ++nextId,
+    }));
+    /**
+     * The current results, which may still be being updated if we're re-running a test
+     * to check for flakes.
+     */
+    const resultsByTestId = new Map();
+    /**
+     * Results that have been fully checked for flakes. At most one per test case.
+     */
+    const finalResults = [];
     const progress = {
         runningTestCases: queue.length,
         failedTestCases: 0,
+        flakedTestCases: 0,
         passedTestCases: 0,
     };
     const allTasksDone = (0, common_1.defer)();
@@ -26,7 +41,8 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiTo
     logger.debug(`Running with ${maxTasks} maximum tasks in parallel`);
     const taskHandler = (0, path_1.join)(__dirname, "task.handler.js");
     // Starts running a test case in a child process
-    const startTask = (testCase) => {
+    const startTask = (rerunnableTestCase) => {
+        const { id, ...testCase } = rerunnableTestCase;
         const deferredResult = (0, common_1.defer)();
         const child = (0, child_process_1.fork)(taskHandler, [], { stdio: "inherit" });
         const messageHandler = (message) => {
@@ -53,6 +69,7 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiTo
         });
         child.on("message", messageHandler);
         // Send test case and arguments to child process
+        const isRetry = resultsByTestId.has(id);
         const initMessage = {
             kind: "init",
             data: {
@@ -73,35 +90,62 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiTo
                     generatedBy: { type: "testRun", runId: testRun.id },
                     testRunId: testRun.id,
                     replayEventsDependencies,
+                    suppressScreenshotDiffLogging: isRetry,
                 },
             },
         };
         child.send(initMessage);
         // Handle task completion
         deferredResult.promise
-            .catch(() => {
-            // If it threw an error then it's something fatal, rather than just a failed diff
-            // (it resolves successfully on a failed diff)
-            const result = {
-                ...testCase,
-                headReplayId: "",
-                result: "fail",
-                screenshotDiffResults: [],
-            };
-            return result;
-        })
+            .catch(() => null)
             .then(async (result) => {
+            var _a;
+            const resultsForTestCase = resultsByTestId.get(id);
+            if (resultsForTestCase != null && result != null) {
+                logRetrySummary(testName({ id, ...testCase }), result);
+            }
+            if (resultsForTestCase != null &&
+                resultsForTestCase.currentResult.result === "flake") {
+                // This test has already been declared as flakey, and we can ignore this result. This result
+                // was from an already executing test that we weren't able to cancel.
+                process.nextTick(checkNextTask);
+                return;
+            }
             --inProgress;
-            results.push(result);
-            progress.failedTestCases += result.result === "fail" ? 1 : 0;
-            progress.passedTestCases += result.result === "pass" ? 1 : 0;
-            --progress.runningTestCases;
-            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, results).then(() => {
-                var _a;
-                if (results.length === (((_a = config.testCases) === null || _a === void 0 ? void 0 : _a.length) || 0)) {
-                    allTasksDone.resolve();
-                }
+            if ((result === null || result === void 0 ? void 0 : result.result) === "fail" && resultsForTestCase == null) {
+                queue.push(...Array.from(new Array(maxRetriesOnFailure)).map(() => ({
+                    ...testCase,
+                    id,
+                    baseReplayId: result.headReplayId,
+                })));
+            }
+            const mergedResult = getNewMergedResult(testCase, (_a = resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) !== null && _a !== void 0 ? _a : null, result);
+            const numberOfRetriesExecuted = resultsForTestCase == null
+                ? 0
+                : resultsForTestCase.numberOfRetriesExecuted + 1;
+            // Our work is done for this test case if the first result was a pass,
+            // we've performed all the retries, or one of the retries already proved
+            // the result as flakey
+            const isFinalResult = mergedResult.result !== "fail" ||
+                numberOfRetriesExecuted === maxRetriesOnFailure;
+            resultsByTestId.set(id, {
+                currentResult: mergedResult,
+                numberOfRetriesExecuted,
             });
+            if (isFinalResult) {
+                // Cancel any replays that are still scheduled
+                queue = queue.filter(({ id }) => id !== id);
+                finalResults.push(mergedResult);
+                --progress.runningTestCases;
+                progress.failedTestCases += mergedResult.result === "fail" ? 1 : 0;
+                progress.flakedTestCases += mergedResult.result === "flake" ? 1 : 0;
+                progress.passedTestCases += mergedResult.result === "pass" ? 1 : 0;
+                await (onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, finalResults).then(() => {
+                    if (queue.length === 0 && inProgress === 0) {
+                        allTasksDone.resolve();
+                    }
+                }));
+            }
             process.nextTick(checkNextTask);
         });
     };
@@ -115,11 +159,56 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiTo
             return;
         }
         ++inProgress;
+        if (resultsByTestId.has(testCase.id)) {
+            logger.info(`Test ${testName(testCase)} failed. Retrying to check for flakes...`);
+        }
         startTask(testCase);
         process.nextTick(checkNextTask);
     };
     process.nextTick(checkNextTask);
     await allTasksDone.promise;
-    return (0, run_all_tests_utils_1.sortResults)({ results, testCases: config.testCases || [] });
+    return (0, run_all_tests_utils_1.sortResults)({
+        results: finalResults,
+        testCases: config.testCases || [],
+    });
 };
 exports.runAllTestsInParallel = runAllTestsInParallel;
+const logRetrySummary = (nameOfTest, retryResult) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    if (retryResult.result === "pass") {
+        logger.info(`Retried taking screenshots for failed test ${nameOfTest}, but got the same results`);
+    }
+    else {
+        const numDifferingScreenshots = retryResult.screenshotDiffResults.filter((result) => result.outcome !== "no-diff").length;
+        logger.info(`Retried taking screenshots for failed test ${nameOfTest}, and ${numDifferingScreenshots} screenshots came out different. Results for these screenshots are assumed to be flakes, and so will be ignored.`);
+    }
+};
+const testName = (testCase) => testCase.title != null ? `'${testCase.title}'` : `#${testCase.id + 1}`;
+const getNewMergedResult = (testCase, currentMergedResult, newResult) => {
+    // If currentMergedResult is null then this is our first try, our original head replay
+    if (currentMergedResult == null) {
+        if (newResult == null) {
+            // This means our original head replay failed fatally (not just a failed diff, but failed to even run)
+            // In this case we just return it as result = fail, with no screenshot diffs
+            return {
+                ...testCase,
+                headReplayId: "",
+                result: "fail",
+                screenshotDiffResults: [],
+            };
+        }
+        // In this case the newResult is our first head replay, our first result,
+        // so lets initialize the mergedResult to this
+        return newResult;
+    }
+    // If currentMergedResult is not null then newResult is a retry, containing comparison screenshots to our original head replay
+    if (newResult == null) {
+        // In this case newResult is a retry of the head replay, but it failed fatally (not just a failed diff, but failed to even run)
+        // So we just ignore this retry
+        return currentMergedResult;
+    }
+    return (0, merge_test_results_1.mergeResults)({
+        currentResult: currentMergedResult,
+        comparisonToHeadReplay: newResult,
+    });
+};

package/dist/parallel-tests/run-all-tests.d.ts

@@ -23,6 +23,12 @@ export interface Options {
      */
     parallelTasks: number | null;
     deflake: boolean;
+    /**
+     * If set to a value greater than 1 then will re-run any replays that give a screenshot diff
+     * and mark them as a flake if the screenshot generated on one of the retryed replays differs from that
+     * in the first replay.
+     */
+    maxRetriesOnFailure: number;
     githubSummary: boolean;
     /**
      * If provided it will incorportate the cachedTestRunResults in any calls to store
@@ -57,4 +63,4 @@ export interface TestRun {
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;
+export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;

package/dist/parallel-tests/run-all-tests.js

@@ -7,12 +7,11 @@ exports.runAllTests = void 0;
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const client_1 = require("../api/client");
+const replay_diff_api_1 = require("../api/replay-diff.api");
 const test_run_api_1 = require("../api/test-run.api");
 const config_1 = require("../config/config");
-const deflake_tests_handler_1 = require("../deflake-tests/deflake-tests.handler");
 const replay_assets_1 = require("../local-data/replay-assets");
 const parallel_tests_handler_1 = require("../parallel-tests/parallel-tests.handler");
-const config_utils_1 = require("../utils/config.utils");
 const github_summary_utils_1 = require("../utils/github-summary.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 const test_run_environment_utils_1 = require("../utils/test-run-environment.utils");
@@ -21,10 +20,13 @@ const version_utils_1 = require("../utils/version.utils");
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
+const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
     if (appUrl != null && useAssetsSnapshottedInBaseSimulation) {
         throw new Error("Arguments useAssetsSnapshottedInBaseSimulation and appUrl are mutually exclusive");
     }
+    if (deflake && maxRetriesOnFailure > 1) {
+        throw new Error("Arguments deflake and maxRetriesOnFailure are mutually exclusive");
+    }
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const client = (0, client_1.createClient)({ apiToken });
     const cachedTestRunResults = cachedTestRunResults_ !== null && cachedTestRunResults_ !== void 0 ? cachedTestRunResults_ : [];
@@ -66,6 +68,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         status: "Running",
         progress: {
             failedTestCases: 0,
+            flakedTestCases: 0,
             passedTestCases: cachedTestRunResults.length,
             runningTestCases: testCases.length,
         },
@@ -109,59 +112,37 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                 passedTestCases: progress.passedTestCases + cachedTestRunResults.length,
             },
         });
-        await storeTestRunResults("Running", resultsSoFar);
-    };
-    const getResults = async () => {
-        if (parallelTasks == null || parallelTasks > 1) {
-            const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
-                config,
-                testRun,
-                testsToRun,
-                executionOptions,
-                screenshottingOptions,
-                apiToken,
-                commitSha,
-                appUrl,
-                useAssetsSnapshottedInBaseSimulation,
-                parallelTasks,
-                deflake,
-                replayEventsDependencies,
-                onTestFinished,
-            });
-            return results;
-        }
-        const results = [];
-        const progress = {
-            runningTestCases: testsToRun.length,
-            failedTestCases: 0,
-            passedTestCases: 0,
-        };
-        for (const testCase of testsToRun) {
-            const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)({
-                replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
-                    useAssetsSnapshottedInBaseSimulation,
-                    appUrl,
-                    testCase,
-                }),
-                executionOptions,
-                screenshottingOptions,
-                testCase,
-                apiToken,
-                commitSha,
-                deflake,
-                generatedBy: { type: "testRun", runId: testRun.id },
+        const newResult = resultsSoFar.at(-1);
+        if ((newResult === null || newResult === void 0 ? void 0 : newResult.baseReplayId) != null) {
+            await (0, replay_diff_api_1.createReplayDiff)({
+                client,
+                headReplayId: newResult.headReplayId,
+                baseReplayId: newResult.baseReplayId,
                 testRunId: testRun.id,
-                replayEventsDependencies,
+                data: {
+                    screenshotAssertionsOptions: screenshottingOptions,
+                    screenshotDiffResults: newResult.screenshotDiffResults,
+                },
             });
-            results.push(result);
-            progress.failedTestCases += result.result === "fail" ? 1 : 0;
-            progress.passedTestCases += result.result === "pass" ? 1 : 0;
-            --progress.runningTestCases;
-            await onTestFinished(progress, results);
         }
-        return (0, run_all_tests_utils_1.sortResults)({ results, testCases });
+        await storeTestRunResults("Running", resultsSoFar);
     };
-    const results = await getResults();
+    const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+        config,
+        testRun,
+        testsToRun,
+        executionOptions,
+        screenshottingOptions,
+        apiToken,
+        commitSha,
+        appUrl,
+        useAssetsSnapshottedInBaseSimulation,
+        parallelTasks,
+        deflake,
+        replayEventsDependencies,
+        onTestFinished,
+        maxRetriesOnFailure,
+    });
     const runAllFailure = results.find(({ result }) => result === "fail");
     const overallStatus = runAllFailure ? "Failure" : "Success";
     await storeTestRunResults(overallStatus, results);
@@ -182,6 +163,8 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
             id: testRun.id,
             status: overallStatus,
             progress: {
+                flakedTestCases: results.filter(({ result }) => result === "flake")
+                    .length,
                 passedTestCases: results.filter(({ result }) => result === "pass")
                     .length,
                 failedTestCases: results.filter(({ result }) => result === "fail")

package/dist/parallel-tests/run-all-tests.types.d.ts

@@ -1,5 +1,6 @@
 export interface TestRunProgress {
     failedTestCases: number;
+    flakedTestCases: number;
     passedTestCases: number;
     runningTestCases: number;
 }