@alwaysmeticulous/cli 2.16.1 → 2.18.0

package/dist/parallel-tests/parallel-tests.handler.js

@@ -9,19 +9,17 @@ const os_1 = require("os");
 const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
-const test_run_api_1 = require("../api/test-run.api");
 const config_utils_1 = require("../utils/config.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults, replayEventsDependencies, }) => {
+const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiToken, commitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, replayEventsDependencies, onTestFinished, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const results = [...cachedTestRunResults];
-    const queue = await (0, run_all_tests_utils_1.getTestsToRun)({
-        client,
-        testCases: config.testCases || [],
-        cachedTestRunResults,
-        baseCommitSha,
-    });
+    const results = [];
+    const progress = {
+        runningTestCases: queue.length,
+        failedTestCases: 0,
+        passedTestCases: 0,
+    };
     const allTasksDone = (0, common_1.defer)();
     let inProgress = 0;
     const maxTasks = parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : Math.max((0, os_1.cpus)().length, 1);
@@ -82,26 +80,23 @@ const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commit
         // Handle task completion
         deferredResult.promise
             .catch(() => {
+            // If it threw an error then it's something fatal, rather than just a failed diff
+            // (it resolves successfully on a failed diff)
             const result = {
                 ...testCase,
                 headReplayId: "",
                 result: "fail",
+                screenshotDiffResults: [],
             };
             return result;
         })
             .then(async (result) => {
             --inProgress;
             results.push(result);
-            (0, test_run_api_1.putTestRunResults)({
-                client,
-                testRunId: testRun.id,
-                status: "Running",
-                resultData: { results },
-            })
-                .catch((error) => {
-                logger.error(`Error while pushing partial results: ${error}`);
-            })
-                .then(() => {
+            progress.failedTestCases += result.result === "fail" ? 1 : 0;
+            progress.passedTestCases += result.result === "pass" ? 1 : 0;
+            --progress.runningTestCases;
+            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, results).then(() => {
                 var _a;
                 if (results.length === (((_a = config.testCases) === null || _a === void 0 ? void 0 : _a.length) || 0)) {
                     allTasksDone.resolve();

package/dist/parallel-tests/run-all-tests.d.ts

@@ -0,0 +1,55 @@
+import { ReplayExecutionOptions } from "@alwaysmeticulous/common";
+import { TestRunStatus } from "../api/test-run.api";
+import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
+import { DetailedTestCaseResult, TestCaseResult } from "../config/config.types";
+import { TestRunProgress } from "./run-all-tests.types";
+export interface Options {
+    testsFile: string | null;
+    executionOptions: ReplayExecutionOptions;
+    screenshottingOptions: ScreenshotAssertionsEnabledOptions;
+    apiToken: string | null;
+    commitSha: string;
+    /**
+     * The base commit to compare test results against for test cases that don't have a baseReplayId specified.
+     */
+    baseCommitSha: string | null;
+    appUrl: string | null;
+    useAssetsSnapshottedInBaseSimulation: boolean;
+    /**
+     * If null runs in parralel with a sensible number of parrelel tasks for the given machine.
+     *
+     * Set to 1 to disable parralelism.
+     */
+    parallelTasks: number | null;
+    deflake: boolean;
+    githubSummary: boolean;
+    /**
+     * If provided it will incorportate the cachedTestRunResults in any calls to store
+     * test run results in the BE, but won't include the cachedTestRunResults in the returned
+     * RunAllTestsResult.
+     */
+    cachedTestRunResults?: TestCaseResult[];
+    onTestRunCreated?: (testRun: TestRun & {
+        status: "Running";
+    }) => void;
+    onTestFinished?: (testRun: TestRun & {
+        status: "Running";
+    }) => void;
+}
+export interface RunAllTestsResult {
+    testRun: TestRun & {
+        status: "Success" | "Failure";
+    };
+    testCaseResults: DetailedTestCaseResult[];
+}
+export interface TestRun {
+    id: string;
+    url: string;
+    status: TestRunStatus;
+    progress: TestRunProgress;
+}
+/**
+ * Runs all the test cases in the provided file.
+ * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
+ */
+export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults: cachedTestRunResults_, githubSummary, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;

package/dist/parallel-tests/run-all-tests.js

@@ -0,0 +1,180 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAllTests = void 0;
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const client_1 = require("../api/client");
+const test_run_api_1 = require("../api/test-run.api");
+const config_1 = require("../config/config");
+const deflake_tests_handler_1 = require("../deflake-tests/deflake-tests.handler");
+const replay_assets_1 = require("../local-data/replay-assets");
+const parallel_tests_handler_1 = require("../parallel-tests/parallel-tests.handler");
+const config_utils_1 = require("../utils/config.utils");
+const github_summary_utils_1 = require("../utils/github-summary.utils");
+const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
+const version_utils_1 = require("../utils/version.utils");
+/**
+ * Runs all the test cases in the provided file.
+ * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
+ */
+const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults: cachedTestRunResults_, githubSummary, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
+    if (appUrl != null && useAssetsSnapshottedInBaseSimulation) {
+        throw new Error("Arguments useAssetsSnapshottedInBaseSimulation and appUrl are mutually exclusive");
+    }
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const client = (0, client_1.createClient)({ apiToken });
+    const cachedTestRunResults = cachedTestRunResults_ !== null && cachedTestRunResults_ !== void 0 ? cachedTestRunResults_ : [];
+    const config = await (0, config_1.readConfig)(testsFile || undefined);
+    const allTestCases = config.testCases || [];
+    if (!allTestCases.length) {
+        throw new Error("Error! No test case defined");
+    }
+    // Only run the uncached test cases
+    const testCases = allTestCases.filter(({ sessionId, baseReplayId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
+        cached.baseReplayId === baseReplayId &&
+        cached.title === title));
+    const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
+    const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
+    const testRun = await (0, test_run_api_1.createTestRun)({
+        client,
+        commitSha,
+        meticulousSha,
+        configData: config,
+    });
+    const testRunUrl = (0, test_run_api_1.getTestRunUrl)(testRun);
+    onTestRunCreated === null || onTestRunCreated === void 0 ? void 0 : onTestRunCreated({
+        id: testRun.id,
+        url: testRunUrl,
+        status: "Running",
+        progress: {
+            failedTestCases: 0,
+            passedTestCases: cachedTestRunResults.length,
+            runningTestCases: testCases.length,
+        },
+    });
+    logger.info("");
+    logger.info(`Test run URL: ${testRunUrl}`);
+    logger.info("");
+    const testsToRun = await (0, run_all_tests_utils_1.getTestsToRun)({
+        testCases,
+        client,
+        baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
+    });
+    const storeTestRunResults = async (status, resultsSoFar) => {
+        const resultsToSendToBE = [
+            ...cachedTestRunResults,
+            ...resultsSoFar.map(
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            ({ screenshotDiffResults, ...result }) => result),
+        ];
+        try {
+            await (0, test_run_api_1.putTestRunResults)({
+                client,
+                testRunId: testRun.id,
+                status,
+                resultData: {
+                    results: resultsToSendToBE,
+                },
+            });
+        }
+        catch (error) {
+            logger.error(`Error while pushing partial results: ${error}`);
+        }
+    };
+    const onTestFinished = async (progress, resultsSoFar) => {
+        onTestFinished_ === null || onTestFinished_ === void 0 ? void 0 : onTestFinished_({
+            id: testRun.id,
+            url: testRunUrl,
+            status: "Running",
+            progress: {
+                ...progress,
+                passedTestCases: progress.passedTestCases + cachedTestRunResults.length,
+            },
+        });
+        await storeTestRunResults("Running", resultsSoFar);
+    };
+    const getResults = async () => {
+        if (parallelTasks == null || parallelTasks > 1) {
+            const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+                config,
+                testRun,
+                testsToRun,
+                executionOptions,
+                screenshottingOptions,
+                apiToken,
+                commitSha,
+                appUrl,
+                useAssetsSnapshottedInBaseSimulation,
+                parallelTasks,
+                deflake,
+                replayEventsDependencies,
+                onTestFinished,
+            });
+            return results;
+        }
+        const results = [];
+        const progress = {
+            runningTestCases: testsToRun.length,
+            failedTestCases: 0,
+            passedTestCases: 0,
+        };
+        for (const testCase of testsToRun) {
+            const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)({
+                replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
+                    useAssetsSnapshottedInBaseSimulation,
+                    appUrl,
+                    testCase,
+                }),
+                executionOptions,
+                screenshottingOptions,
+                testCase,
+                apiToken,
+                commitSha,
+                deflake,
+                generatedBy: { type: "testRun", runId: testRun.id },
+                testRunId: testRun.id,
+                replayEventsDependencies,
+            });
+            results.push(result);
+            progress.failedTestCases += result.result === "fail" ? 1 : 0;
+            progress.passedTestCases += result.result === "pass" ? 1 : 0;
+            --progress.runningTestCases;
+            await onTestFinished(progress, results);
+        }
+        return (0, run_all_tests_utils_1.sortResults)({ results, testCases });
+    };
+    const results = await getResults();
+    const runAllFailure = results.find(({ result }) => result === "fail");
+    const overallStatus = runAllFailure ? "Failure" : "Success";
+    await storeTestRunResults(overallStatus, results);
+    logger.info("");
+    logger.info("Results");
+    logger.info("=======");
+    logger.info(`URL: ${testRunUrl}`);
+    logger.info("=======");
+    results.forEach(({ title, result }) => {
+        logger.info(`${title} => ${result}`);
+    });
+    if (githubSummary) {
+        await (0, github_summary_utils_1.writeGitHubSummary)({ testRunUrl, results });
+    }
+    return {
+        testRun: {
+            url: testRunUrl,
+            id: testRun.id,
+            status: overallStatus,
+            progress: {
+                passedTestCases: results.filter(({ result }) => result === "pass")
+                    .length,
+                failedTestCases: results.filter(({ result }) => result === "fail")
+                    .length,
+                runningTestCases: 0,
+            },
+        },
+        testCaseResults: results,
+    };
+};
+exports.runAllTests = runAllTests;

package/dist/parallel-tests/run-all-tests.types.d.ts

@@ -0,0 +1,5 @@
+export interface TestRunProgress {
+    failedTestCases: number;
+    passedTestCases: number;
+    runningTestCases: number;
+}

package/dist/parallel-tests/run-all-tests.types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });