@alwaysmeticulous/cli 2.21.1 → 2.21.3-alpha.1

package/dist/parallel-tests/parallel-tests.handler.js

@@ -4,16 +4,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runAllTestsInParallel = void 0;
-const child_process_1 = require("child_process");
 const os_1 = require("os");
-const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
-const config_utils_1 = require("../utils/config.utils");
-const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 const merge_test_results_1 = require("./merge-test-results");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, commitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, replayEventsDependencies, onTestFinished, }) => {
+const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, executeTest, onTestFinished, onTestFailedToRun, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     let nextId = 0;
     let queue = testsToRun.map((test) => ({
@@ -29,7 +25,7 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, co
      * Results that have been fully checked for flakes. At most one per test case.
      */
     const finalResults = [];
-    const progress = {
+    let progress = {
         runningTestCases: queue.length,
         failedTestCases: 0,
         flakedTestCases: 0,
@@ -39,68 +35,15 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, co
     let inProgress = 0;
     const maxTasks = parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : Math.max((0, os_1.cpus)().length, 1);
     logger.debug(`Running with ${maxTasks} maximum tasks in parallel`);
-    const taskHandler = (0, path_1.join)(__dirname, "task.handler.js");
     // Starts running a test case in a child process
     const startTask = (rerunnableTestCase) => {
         const { id, ...testCase } = rerunnableTestCase;
-        const deferredResult = (0, common_1.defer)();
-        const child = (0, child_process_1.fork)(taskHandler, [], { stdio: "inherit" });
-        const messageHandler = (message) => {
-            if (message &&
-                typeof message === "object" &&
-                message["kind"] === "result") {
-                const resultMessage = message;
-                deferredResult.resolve(resultMessage.data.result);
-                child.off("message", messageHandler);
-            }
-        };
-        child.on("error", (error) => {
-            if (deferredResult.getState() === "pending") {
-                deferredResult.reject(error);
-            }
-        });
-        child.on("exit", (code) => {
-            if (code) {
-                logger.debug(`child exited with code: ${code}`);
-            }
-            if (deferredResult.getState() === "pending") {
-                deferredResult.reject(new Error("No result"));
-            }
-        });
-        child.on("message", messageHandler);
-        // Send test case and arguments to child process
         const isRetry = resultsByTestId.has(id);
-        const initMessage = {
-            kind: "init",
-            data: {
-                logLevel: logger.getLevel(),
-                dataDir: (0, common_1.getMeticulousLocalDataDir)(),
-                replayOptions: {
-                    apiToken,
-                    commitSha,
-                    testCase,
-                    deflake,
-                    replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
-                        useAssetsSnapshottedInBaseSimulation,
-                        appUrl,
-                        testCase,
-                    }),
-                    executionOptions,
-                    screenshottingOptions,
-                    generatedBy: { type: "testRun", runId: testRun.id },
-                    testRunId: testRun.id,
-                    replayEventsDependencies,
-                    suppressScreenshotDiffLogging: isRetry,
-                },
-            },
-        };
-        child.send(initMessage);
         // Handle task completion
-        deferredResult.promise
+        executeTest(testCase, isRetry)
             .catch(() => null)
             .then(async (result) => {
             var _a;
-            --inProgress;
             const resultsForTestCase = resultsByTestId.get(id);
             if (resultsForTestCase != null && result != null) {
                 logRetrySummary(testName({ id, ...testCase }), result);
@@ -109,50 +52,58 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, co
                 resultsForTestCase.currentResult.result === "flake") {
                 // This test has already been declared as flakey, and we can ignore this result. This result
                 // was from an already executing test that we weren't able to cancel.
-                process.nextTick(checkNextTask);
                 return;
             }
             if ((result === null || result === void 0 ? void 0 : result.result) === "fail" && resultsForTestCase == null) {
+                // Let's auto-retry to see if this failure persists
                 queue.push(...Array.from(new Array(maxRetriesOnFailure)).map(() => ({
                     ...testCase,
                     id,
                     baseReplayId: result.headReplayId,
                 })));
             }
-            const mergedResult = getNewMergedResult(testCase, (_a = resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) !== null && _a !== void 0 ? _a : null, result);
+            if (result == null && (resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) == null) {
+                // This means our original head replay failed fatally (not just a failed diff, but failed to even run)
+                progress = updateProgress(progress, "fail");
+                await (onTestFailedToRun === null || onTestFailedToRun === void 0 ? void 0 : onTestFailedToRun(progress));
+                return;
+            }
+            const mergedResult = getNewMergedResult((_a = resultsForTestCase === null || resultsForTestCase === void 0 ? void 0 : resultsForTestCase.currentResult) !== null && _a !== void 0 ? _a : null, result);
             const numberOfRetriesExecuted = resultsForTestCase == null
                 ? 0
                 : resultsForTestCase.numberOfRetriesExecuted + 1;
+            resultsByTestId.set(id, {
+                currentResult: mergedResult,
+                numberOfRetriesExecuted,
+            });
             // Our work is done for this test case if the first result was a pass,
             // we've performed all the retries, or one of the retries already proved
             // the result as flakey
             const isFinalResult = mergedResult.result !== "fail" ||
                 numberOfRetriesExecuted === maxRetriesOnFailure;
-            resultsByTestId.set(id, {
-                currentResult: mergedResult,
-                numberOfRetriesExecuted,
-            });
             if (isFinalResult) {
                 // Cancel any replays that are still scheduled
-                queue = queue.filter(({ id }) => id !== id);
+                queue = queue.filter((otherReplay) => otherReplay.id !== id);
                 finalResults.push(mergedResult);
-                --progress.runningTestCases;
-                progress.failedTestCases += mergedResult.result === "fail" ? 1 : 0;
-                progress.flakedTestCases += mergedResult.result === "flake" ? 1 : 0;
-                progress.passedTestCases += mergedResult.result === "pass" ? 1 : 0;
-                await (onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, finalResults).then(() => {
-                    if (queue.length === 0 && inProgress === 0) {
-                        allTasksDone.resolve();
-                    }
-                }));
+                progress = updateProgress(progress, mergedResult.result);
+                await (onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, finalResults));
             }
-            process.nextTick(checkNextTask);
         })
             .catch((err) => {
             logger.error(`Error processing result of completed task for test '${testName({
                 id,
                 ...testCase,
             })}'`, err);
+        })
+            .finally(() => {
+            // We only decrement inProgress at the very end,
+            // otherwise another promise may call allTasksDone.resolve() while
+            // we're still saving results
+            --inProgress;
+            if (queue.length === 0 && inProgress === 0) {
+                allTasksDone.resolve();
+            }
+            process.nextTick(checkNextTask);
         });
     };
     // Checks if we can start a new child process
@@ -173,10 +124,7 @@ const runAllTestsInParallel = async ({ config, testRun, testsToRun, apiToken, co
     };
     process.nextTick(checkNextTask);
     await allTasksDone.promise;
-    return (0, run_all_tests_utils_1.sortResults)({
-        results: finalResults,
-        testCases: config.testCases || [],
-    });
+    return finalResults;
 };
 exports.runAllTestsInParallel = runAllTestsInParallel;
 const logRetrySummary = (nameOfTest, retryResult) => {
@@ -190,18 +138,11 @@ const logRetrySummary = (nameOfTest, retryResult) => {
     }
 };
 const testName = (testCase) => testCase.title != null ? `'${testCase.title}'` : `#${testCase.id + 1}`;
-const getNewMergedResult = (testCase, currentMergedResult, newResult) => {
+const getNewMergedResult = (currentMergedResult, newResult) => {
     // If currentMergedResult is null then this is our first try, our original head replay
     if (currentMergedResult == null) {
         if (newResult == null) {
-            // This means our original head replay failed fatally (not just a failed diff, but failed to even run)
-            // In this case we just return it as result = fail, with no screenshot diffs
-            return {
-                ...testCase,
-                headReplayId: "",
-                result: "fail",
-                screenshotDiffResults: [],
-            };
+            throw new Error("Expected either newResult to be non-null or currentMergedResult to be non-null, but both were null. This case should be handled before getNewMergedResult is called");
         }
         // In this case the newResult is our first head replay, our first result,
         // so lets initialize the mergedResult to this
@@ -218,3 +159,11 @@ const getNewMergedResult = (testCase, currentMergedResult, newResult) => {
         comparisonToHeadReplay: newResult,
     });
 };
+const updateProgress = (progress, newResult) => {
+    return {
+        runningTestCases: progress.runningTestCases - 1,
+        failedTestCases: progress.failedTestCases + (newResult === "fail" ? 1 : 0),
+        flakedTestCases: progress.flakedTestCases + (newResult === "flake" ? 1 : 0),
+        passedTestCases: progress.passedTestCases + (newResult === "pass" ? 1 : 0),
+    };
+};

package/dist/parallel-tests/run-all-tests.js

@@ -12,10 +12,12 @@ const test_run_api_1 = require("../api/test-run.api");
 const config_1 = require("../config/config");
 const replay_assets_1 = require("../local-data/replay-assets");
 const parallel_tests_handler_1 = require("../parallel-tests/parallel-tests.handler");
+const config_utils_1 = require("../utils/config.utils");
 const github_summary_utils_1 = require("../utils/github-summary.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 const test_run_environment_utils_1 = require("../utils/test-run-environment.utils");
 const version_utils_1 = require("../utils/version.utils");
+const execute_test_in_child_process_1 = require("./execute-test-in-child-process");
 /**
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
@@ -102,7 +104,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
             logger.error(`Error while pushing partial results: ${error}`);
         }
     };
-    const onTestFinished = async (progress, resultsSoFar) => {
+    const onProgressUpdated = async (progress) => {
         onTestFinished_ === null || onTestFinished_ === void 0 ? void 0 : onTestFinished_({
             id: testRun.id,
             url: testRunUrl,
@@ -112,6 +114,9 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                 passedTestCases: progress.passedTestCases + cachedTestRunResults.length,
             },
         });
+    };
+    const onTestFinished = async (progress, resultsSoFar) => {
+        onProgressUpdated(progress);
         const newResult = resultsSoFar.at(-1);
         if ((newResult === null || newResult === void 0 ? void 0 : newResult.baseReplayId) != null) {
             await (0, replay_diff_api_1.createReplayDiff)({
@@ -128,30 +133,52 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         await storeTestRunResults("Running", resultsSoFar);
     };
     const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
-        config,
-        testRun,
         testsToRun,
-        executionOptions,
-        screenshottingOptions,
-        apiToken,
-        commitSha,
-        appUrl,
-        useAssetsSnapshottedInBaseSimulation,
         parallelTasks,
-        deflake,
-        replayEventsDependencies,
-        onTestFinished,
         maxRetriesOnFailure,
+        executeTest: (testCase, isRetry) => {
+            const initMessage = {
+                kind: "init",
+                data: {
+                    logLevel: logger.getLevel(),
+                    dataDir: (0, common_1.getMeticulousLocalDataDir)(),
+                    replayOptions: {
+                        apiToken,
+                        commitSha,
+                        testCase,
+                        deflake,
+                        replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
+                            useAssetsSnapshottedInBaseSimulation,
+                            appUrl,
+                            testCase,
+                        }),
+                        executionOptions,
+                        screenshottingOptions,
+                        generatedBy: { type: "testRun", runId: testRun.id },
+                        testRunId: testRun.id,
+                        replayEventsDependencies,
+                        suppressScreenshotDiffLogging: isRetry,
+                    },
+                },
+            };
+            return (0, execute_test_in_child_process_1.executeTestInChildProcess)(initMessage);
+        },
+        onTestFinished,
+        onTestFailedToRun: onProgressUpdated,
     });
-    const runAllFailure = results.find(({ result }) => result === "fail");
+    const sortedResults = (0, run_all_tests_utils_1.sortResults)({
+        results: results,
+        testCases: config.testCases || [],
+    });
+    const runAllFailure = sortedResults.find(({ result }) => result === "fail");
     const overallStatus = runAllFailure ? "Failure" : "Success";
-    await storeTestRunResults(overallStatus, results);
+    await storeTestRunResults(overallStatus, sortedResults);
     logger.info("");
     logger.info("Results");
     logger.info("=======");
     logger.info(`URL: ${testRunUrl}`);
     logger.info("=======");
-    results.forEach(({ title, result }) => {
+    sortedResults.forEach(({ title, result }) => {
         logger.info(`${title} => ${result}`);
     });
     if (githubSummary) {
@@ -163,16 +190,15 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
             id: testRun.id,
             status: overallStatus,
             progress: {
-                flakedTestCases: results.filter(({ result }) => result === "flake")
-                    .length,
-                passedTestCases: results.filter(({ result }) => result === "pass")
+                flakedTestCases: sortedResults.filter(({ result }) => result === "flake").length,
+                passedTestCases: sortedResults.filter(({ result }) => result === "pass")
                     .length,
-                failedTestCases: results.filter(({ result }) => result === "fail")
+                failedTestCases: sortedResults.filter(({ result }) => result === "fail")
                     .length,
                 runningTestCases: 0,
             },
         },
-        testCaseResults: results,
+        testCaseResults: sortedResults,
     };
 };
 exports.runAllTests = runAllTests;