@alwaysmeticulous/cli 2.16.0 → 2.17.0

package/dist/parallel-tests/parallel-tests.handler.d.ts

@@ -3,6 +3,7 @@ import { AxiosInstance } from "axios";
 import { TestRun } from "../api/test-run.api";
 import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
 import { MeticulousCliConfig, TestCaseResult } from "../config/config.types";
+import { TestRunProgress } from "./run-all-tests.types";
 export interface RunAllTestsInParallelOptions {
     config: MeticulousCliConfig;
     client: AxiosInstance;
@@ -21,6 +22,7 @@ export interface RunAllTestsInParallelOptions {
     deflake: boolean;
     cachedTestRunResults: TestCaseResult[];
     replayEventsDependencies: ReplayEventsDependencies;
+    onTestFinished?: (progress: TestRunProgress) => void;
 }
 /** Handler for running Meticulous tests in parallel using child processes */
 export declare const runAllTestsInParallel: (options: RunAllTestsInParallelOptions) => Promise<TestCaseResult[]>;

package/dist/parallel-tests/parallel-tests.handler.js

@@ -13,7 +13,7 @@ const test_run_api_1 = require("../api/test-run.api");
 const config_utils_1 = require("../utils/config.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults, replayEventsDependencies, }) => {
+const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults, replayEventsDependencies, onTestFinished, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const results = [...cachedTestRunResults];
     const queue = await (0, run_all_tests_utils_1.getTestsToRun)({
@@ -22,6 +22,11 @@ const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commit
         cachedTestRunResults,
         baseCommitSha,
     });
+    const progress = {
+        runningTestCases: queue.length,
+        failedTestCases: 0,
+        passedTestCases: 0,
+    };
     const allTasksDone = (0, common_1.defer)();
     let inProgress = 0;
     const maxTasks = parallelTasks !== null && parallelTasks !== void 0 ? parallelTasks : Math.max((0, os_1.cpus)().length, 1);
@@ -92,6 +97,10 @@ const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commit
             .then(async (result) => {
             --inProgress;
             results.push(result);
+            progress.failedTestCases += result.result === "fail" ? 1 : 0;
+            progress.passedTestCases += result.result === "pass" ? 1 : 0;
+            --progress.runningTestCases;
+            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress);
             (0, test_run_api_1.putTestRunResults)({
                 client,
                 testRunId: testRun.id,

package/dist/parallel-tests/run-all-tests.d.ts

@@ -0,0 +1,46 @@
+import { ReplayExecutionOptions } from "@alwaysmeticulous/common";
+import { TestRunStatus } from "../api/test-run.api";
+import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
+import { TestCaseResult } from "../config/config.types";
+import { TestRunProgress } from "./run-all-tests.types";
+export interface Options {
+    testsFile: string | null;
+    executionOptions: ReplayExecutionOptions;
+    screenshottingOptions: ScreenshotAssertionsEnabledOptions;
+    apiToken: string | null;
+    commitSha: string | null;
+    /**
+     * The base commit to compare test results against for test cases that don't have a baseReplayId specified.
+     */
+    baseCommitSha: string | null;
+    appUrl: string | null;
+    useAssetsSnapshottedInBaseSimulation: boolean;
+    /**
+     * If null runs in parralel with a sensible number of parrelel tasks for the given machine.
+     *
+     * Set to 1 to disable parralelism.
+     */
+    parallelTasks: number | null;
+    deflake: boolean;
+    useCache: boolean;
+    githubSummary: boolean;
+    onTestRunCreated?: (testRun: TestRun & {
+        status: "Running";
+    }) => void;
+    onTestFinished?: (testRun: TestRun & {
+        status: "Running";
+    }) => void;
+}
+export interface RunAllTestsResult {
+    testRun: TestRun & {
+        status: "Success" | "Failure";
+    };
+    testCaseResults: TestCaseResult[];
+}
+export interface TestRun {
+    id: string;
+    url: string;
+    status: TestRunStatus;
+    progress: TestRunProgress;
+}
+export declare const runAllTests: ({ testsFile, apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, useCache, githubSummary, onTestRunCreated, onTestFinished, }: Options) => Promise<RunAllTestsResult>;

package/dist/parallel-tests/run-all-tests.js

@@ -0,0 +1,169 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAllTests = void 0;
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const client_1 = require("../api/client");
+const test_run_api_1 = require("../api/test-run.api");
+const config_1 = require("../config/config");
+const deflake_tests_handler_1 = require("../deflake-tests/deflake-tests.handler");
+const replay_assets_1 = require("../local-data/replay-assets");
+const parallel_tests_handler_1 = require("../parallel-tests/parallel-tests.handler");
+const commit_sha_utils_1 = require("../utils/commit-sha.utils");
+const config_utils_1 = require("../utils/config.utils");
+const github_summary_utils_1 = require("../utils/github-summary.utils");
+const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
+const version_utils_1 = require("../utils/version.utils");
+const runAllTests = async ({ testsFile, apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, useCache, githubSummary, onTestRunCreated, onTestFinished, }) => {
+    if (appUrl != null && useAssetsSnapshottedInBaseSimulation) {
+        throw new Error("Arguments useAssetsSnapshottedInBaseSimulation and appUrl are mutually exclusive");
+    }
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const client = (0, client_1.createClient)({ apiToken });
+    const config = await (0, config_1.readConfig)(testsFile || undefined);
+    const testCases = config.testCases || [];
+    if (!testCases.length) {
+        throw new Error("Error! No test case defined");
+    }
+    const commitSha = (await (0, commit_sha_utils_1.getCommitSha)(commitSha_)) || "unknown";
+    const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
+    const cachedTestRunResults = useCache
+        ? await (0, test_run_api_1.getCachedTestRunResults)({ client, commitSha })
+        : [];
+    const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
+    const testRun = await (0, test_run_api_1.createTestRun)({
+        client,
+        commitSha,
+        meticulousSha,
+        configData: config,
+    });
+    const testRunUrl = (0, test_run_api_1.getTestRunUrl)(testRun);
+    onTestRunCreated === null || onTestRunCreated === void 0 ? void 0 : onTestRunCreated({
+        id: testRun.id,
+        url: testRunUrl,
+        status: "Running",
+        progress: {
+            failedTestCases: 0,
+            passedTestCases: 0,
+            runningTestCases: testCases.length,
+        },
+    });
+    logger.info("");
+    logger.info(`Test run URL: ${testRunUrl}`);
+    logger.info("");
+    const getResults = async () => {
+        if (parallelTasks == null || parallelTasks > 1) {
+            const results = await (0, parallel_tests_handler_1.runAllTestsInParallel)({
+                config,
+                client,
+                testRun,
+                executionOptions,
+                screenshottingOptions,
+                apiToken,
+                commitSha,
+                appUrl,
+                useAssetsSnapshottedInBaseSimulation,
+                parallelTasks,
+                deflake,
+                cachedTestRunResults,
+                replayEventsDependencies,
+                baseCommitSha,
+                onTestFinished: (progress) => {
+                    onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished({
+                        id: testRun.id,
+                        url: testRunUrl,
+                        status: "Running",
+                        progress,
+                    });
+                },
+            });
+            return results;
+        }
+        const results = [...cachedTestRunResults];
+        const testsToRun = await (0, run_all_tests_utils_1.getTestsToRun)({
+            testCases,
+            cachedTestRunResults,
+            client,
+            baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
+        });
+        const progress = {
+            runningTestCases: testRun.length,
+            failedTestCases: 0,
+            passedTestCases: 0,
+        };
+        for (const testCase of testsToRun) {
+            const result = await (0, deflake_tests_handler_1.deflakeReplayCommandHandler)({
+                replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
+                    useAssetsSnapshottedInBaseSimulation,
+                    appUrl,
+                    testCase,
+                }),
+                executionOptions,
+                screenshottingOptions,
+                testCase,
+                apiToken,
+                commitSha,
+                deflake,
+                generatedBy: { type: "testRun", runId: testRun.id },
+                testRunId: testRun.id,
+                replayEventsDependencies,
+            });
+            results.push(result);
+            progress.failedTestCases += result.result === "fail" ? 1 : 0;
+            progress.passedTestCases += result.result === "pass" ? 1 : 0;
+            --progress.runningTestCases;
+            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished({
+                id: testRun.id,
+                url: testRunUrl,
+                status: "Running",
+                progress,
+            });
+            await (0, test_run_api_1.putTestRunResults)({
+                client,
+                testRunId: testRun.id,
+                status: "Running",
+                resultData: { results },
+            });
+        }
+        return (0, run_all_tests_utils_1.sortResults)({ results, testCases });
+    };
+    const results = await getResults();
+    const runAllFailure = results.find(({ result }) => result === "fail");
+    const overallStatus = runAllFailure ? "Failure" : "Success";
+    await (0, test_run_api_1.putTestRunResults)({
+        client,
+        testRunId: testRun.id,
+        status: overallStatus,
+        resultData: { results },
+    });
+    logger.info("");
+    logger.info("Results");
+    logger.info("=======");
+    logger.info(`URL: ${testRunUrl}`);
+    logger.info("=======");
+    results.forEach(({ title, result }) => {
+        logger.info(`${title} => ${result}`);
+    });
+    if (githubSummary) {
+        await (0, github_summary_utils_1.writeGitHubSummary)({ testRunUrl, results });
+    }
+    return {
+        testRun: {
+            url: testRunUrl,
+            id: testRun.id,
+            status: overallStatus,
+            progress: {
+                passedTestCases: results.filter(({ result }) => result === "pass")
+                    .length,
+                failedTestCases: results.filter(({ result }) => result === "fail")
+                    .length,
+                runningTestCases: 0,
+            },
+        },
+        testCaseResults: results,
+    };
+};
+exports.runAllTests = runAllTests;

package/dist/parallel-tests/run-all-tests.types.d.ts

@@ -0,0 +1,5 @@
+export interface TestRunProgress {
+    failedTestCases: number;
+    passedTestCases: number;
+    runningTestCases: number;
+}

package/dist/parallel-tests/run-all-tests.types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });