@alwaysmeticulous/cli 2.35.0 → 2.37.0

package/dist/parallel-tests/run-all-tests.js

@@ -23,10 +23,7 @@ const execute_test_in_child_process_1 = require("./execute-test-in-child-process
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
-    if (appUrl != null && useAssetsSnapshottedInBaseSimulation) {
-        throw new Error("Arguments useAssetsSnapshottedInBaseSimulation and appUrl are mutually exclusive");
-    }
+const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
     if (deflake && maxRetriesOnFailure > 1) {
         throw new Error("Arguments deflake and maxRetriesOnFailure are mutually exclusive");
     }
@@ -44,8 +41,8 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         throw new Error("Error! No test case defined");
     }
     // Only run the uncached test cases
-    const testCases = allTestCases.filter(({ sessionId, baseReplayId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
-        cached.baseReplayId === baseReplayId &&
+    const testCases = allTestCases.filter(({ sessionId, baseTestRunId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
+        cached.baseTestRunId === baseTestRunId &&
         cached.title === title));
     const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
     const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
@@ -62,7 +59,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                 commitSha,
                 baseCommitSha,
                 appUrl,
-                useAssetsSnapshottedInBaseSimulation,
                 parallelTasks,
                 deflake,
                 githubSummary,
@@ -86,17 +82,19 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     logger.info("");
     logger.info(`Test run URL: ${testRunUrl}`);
     logger.info("");
-    const testsToRun = await (0, run_all_tests_utils_1.getTestsToRun)({
-        testCases,
+    const testsToRun = await getTestCasesWithBaseTestRunId({
+        baseCommitSha,
+        baseTestRunId: baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : null,
         client,
-        baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
+        logger,
+        testCases,
     });
     const storeTestRunResults = async (status, resultsSoFar) => {
         const resultsToSendToBE = [
             ...cachedTestRunResults,
             ...resultsSoFar.map(
             // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            ({ screenshotDiffResults, ...result }) => result),
+            ({ screenshotDiffResultsByBaseReplayId, ...result }) => result),
         ];
         try {
             await (0, test_run_api_1.putTestRunResults)({
@@ -127,17 +125,19 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     const onTestFinished = async (progress, resultsSoFar) => {
         onProgressUpdated(progress);
         const newResult = resultsSoFar.at(-1);
-        if ((newResult === null || newResult === void 0 ? void 0 : newResult.baseReplayId) != null) {
-            await (0, replay_diff_api_1.createReplayDiff)({
-                client,
-                headReplayId: newResult.headReplayId,
-                baseReplayId: newResult.baseReplayId,
-                testRunId: testRun.id,
-                data: {
-                    screenshotAssertionsOptions: screenshottingOptions,
-                    screenshotDiffResults: newResult.screenshotDiffResults,
-                },
-            });
+        if (newResult != null) {
+            for (const [baseReplayId, screenshotDiffResults] of Object.entries(newResult.screenshotDiffResultsByBaseReplayId)) {
+                await (0, replay_diff_api_1.createReplayDiff)({
+                    client,
+                    headReplayId: newResult.headReplayId,
+                    baseReplayId: baseReplayId,
+                    testRunId: testRun.id,
+                    data: {
+                        screenshotAssertionsOptions: screenshottingOptions,
+                        screenshotDiffResults,
+                    },
+                });
+            }
         }
         await storeTestRunResults("Running", resultsSoFar);
     };
@@ -146,6 +146,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         parallelTasks,
         maxRetriesOnFailure,
         executeTest: (testCase, isRetry) => {
+            var _a;
             const initMessage = {
                 kind: "init",
                 data: {
@@ -157,7 +158,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                         testCase,
                         deflake,
                         replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
-                            useAssetsSnapshottedInBaseSimulation,
                             appUrl,
                             testCase,
                         }),
@@ -165,6 +165,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                         screenshottingOptions,
                         generatedBy: { type: "testRun", runId: testRun.id },
                         testRunId: testRun.id,
+                        baseTestRunId: (_a = testCase.baseTestRunId) !== null && _a !== void 0 ? _a : null,
                         replayEventsDependencies,
                         suppressScreenshotDiffLogging: isRetry,
                     },
@@ -212,3 +213,29 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     };
 };
 exports.runAllTests = runAllTests;
+const getTestCasesWithBaseTestRunId = async ({ logger, client, baseCommitSha, baseTestRunId, testCases, }) => {
+    const defaultBaseTestRunId = baseCommitSha != null
+        ? await (0, test_run_api_1.getLatestTestRunId)({
+            client,
+            commitSha: baseCommitSha,
+        })
+        : null;
+    const testsToRun = testCases.map((test) => {
+        // We use the baseTestRunId specified in the test case if it exists, otherwise we use
+        // use the baseTestRunId specified from the CLI args if it exists, otherwise we use the
+        // baseTestRunId for the base commit if it exists, otherwise we use null (don't compare screenshots).
+        const fallbackTestRunId = baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : defaultBaseTestRunId;
+        if (test.baseTestRunId != null || fallbackTestRunId == null) {
+            return test;
+        }
+        return { ...test, baseTestRunId: fallbackTestRunId };
+    });
+    if (baseCommitSha != null) {
+        testsToRun
+            .filter((test) => test.baseTestRunId == null)
+            .forEach((test) => {
+            logger.warn(`Skipping comparisons for test "${test.title}" since no result to compare against stored for base commit ${baseCommitSha}`);
+        });
+    }
+    return testsToRun;
+};

package/dist/parallel-tests/screenshot-diff-results.utils.d.ts

@@ -0,0 +1,7 @@
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
+import { DetailedTestCaseResult } from "../config/config.types";
+export type ScreenshotDiffResultWithBaseReplayId = ScreenshotDiffResult & {
+    baseReplayId: string;
+};
+export declare const flattenScreenshotDiffResults: (testCaseResult: DetailedTestCaseResult) => ScreenshotDiffResultWithBaseReplayId[];
+export declare const groupScreenshotDiffResults: (results: ScreenshotDiffResultWithBaseReplayId[]) => Record<string, ScreenshotDiffResult[]>;

package/dist/parallel-tests/screenshot-diff-results.utils.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.groupScreenshotDiffResults = exports.flattenScreenshotDiffResults = void 0;
+const flattenScreenshotDiffResults = (testCaseResult) => {
+    return Object.entries(testCaseResult.screenshotDiffResultsByBaseReplayId).flatMap(([baseReplayId, diffs]) => {
+        return diffs.map((diff) => ({ ...diff, baseReplayId }));
+    });
+};
+exports.flattenScreenshotDiffResults = flattenScreenshotDiffResults;
+const groupScreenshotDiffResults = (results) => {
+    const groupedResults = {};
+    results.forEach(({ baseReplayId, ...result }) => {
+        var _a;
+        const resultsForBaseReplayId = (_a = groupedResults[baseReplayId]) !== null && _a !== void 0 ? _a : [];
+        resultsForBaseReplayId.push(result);
+        groupedResults[baseReplayId] = resultsForBaseReplayId;
+    });
+    return groupedResults;
+};
+exports.groupScreenshotDiffResults = groupScreenshotDiffResults;

package/dist/utils/config.utils.d.ts

@@ -1,8 +1,7 @@
 import { TestCase } from "@alwaysmeticulous/api";
 import { ReplayTarget } from "@alwaysmeticulous/common/dist/types/replay.types";
 export declare const addTestCase: (testCase: TestCase) => Promise<void>;
-export declare const getReplayTargetForTestCase: ({ useAssetsSnapshottedInBaseSimulation, appUrl, testCase, }: {
-    useAssetsSnapshottedInBaseSimulation: boolean;
+export declare const getReplayTargetForTestCase: ({ appUrl, testCase, }: {
     appUrl: string | null;
     testCase: TestCase;
 }) => ReplayTarget;

package/dist/utils/config.utils.js

@@ -11,7 +11,7 @@ const addTestCase = async (testCase) => {
     await (0, config_1.saveConfig)(newConfig);
 };
 exports.addTestCase = addTestCase;
-const getReplayTargetForTestCase = ({ useAssetsSnapshottedInBaseSimulation, appUrl, testCase, }) => {
+const getReplayTargetForTestCase = ({ appUrl, testCase, }) => {
     var _a, _b, _c;
     if (((_a = testCase.options) === null || _a === void 0 ? void 0 : _a.simulationIdForAssets) != null) {
         return {
@@ -19,15 +19,6 @@ const getReplayTargetForTestCase = ({ useAssetsSnapshottedInBaseSimulation, appU
             simulationIdForAssets: (_b = testCase.options) === null || _b === void 0 ? void 0 : _b.simulationIdForAssets,
         };
     }
-    if (useAssetsSnapshottedInBaseSimulation) {
-        if (testCase.baseReplayId == null) {
-            throw new Error(`--useAssetsSnapshottedInBaseSimulation flag set, but test case "${testCase.title}" does not have a baseReplayId.`);
-        }
-        return {
-            type: "snapshotted-assets",
-            simulationIdForAssets: testCase.baseReplayId,
-        };
-    }
     if ((_c = testCase.options) === null || _c === void 0 ? void 0 : _c.appUrl) {
         if (appUrl) {
             throw new Error(`Test cases "${testCase.title}" has an "appUrl" option but --appUrl is also provided.`);

package/dist/utils/run-all-tests.utils.d.ts

@@ -14,4 +14,3 @@ export interface GetTestsToRunOptions {
      */
     baseCommitSha: string | null;
 }
-export declare const getTestsToRun: ({ testCases, client, baseCommitSha, }: GetTestsToRunOptions) => Promise<TestCase[]>;

package/dist/utils/run-all-tests.utils.js

@@ -1,12 +1,6 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTestsToRun = exports.sortResults = exports.mergeTestCases = void 0;
-const common_1 = require("@alwaysmeticulous/common");
-const loglevel_1 = __importDefault(require("loglevel"));
-const test_run_api_1 = require("../api/test-run.api");
+exports.sortResults = exports.mergeTestCases = void 0;
 const mergeTestCases = (...testSuites) => {
     const seenSessionIds = new Set();
     return testSuites.flatMap((testSuite) => {
@@ -22,9 +16,9 @@ exports.mergeTestCases = mergeTestCases;
 const sortResults = ({ results: unsorted_, testCases }) => {
     const unsorted = [...unsorted_];
     const results = [];
-    testCases.forEach(({ title, baseReplayId, sessionId }) => {
+    testCases.forEach(({ title, baseTestRunId, sessionId }) => {
         const idx = unsorted.findIndex((result) => result.title === title &&
-            result.baseReplayId === baseReplayId &&
+            result.baseTestRunId === baseTestRunId &&
             result.sessionId === sessionId);
         if (idx == -1) {
             return;
@@ -36,44 +30,3 @@ const sortResults = ({ results: unsorted_, testCases }) => {
     return results;
 };
 exports.sortResults = sortResults;
-const getTestsToRun = async ({ testCases, client, baseCommitSha, }) => {
-    const testCasesMissingBaseReplayId = testCases.filter((testCase) => testCase.baseReplayId == null);
-    const testCasesWithBaseReplayId = testCases.flatMap((testCase) => testCase.baseReplayId == null ? [] : [testCase]);
-    if (testCasesMissingBaseReplayId.length === 0) {
-        return testCasesWithBaseReplayId;
-    }
-    const baseReplayIdBySessionId = await getBaseReplayIdsBySessionId({
-        client,
-        baseCommitSha,
-    });
-    return testCases.flatMap((test) => {
-        if (test.baseReplayId != null) {
-            return [test];
-        }
-        const baseReplayId = baseReplayIdBySessionId[test.sessionId];
-        if (baseReplayId == null) {
-            const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-            logger.warn(`Skipping comparisons for test "${test.title}" since no result to compare against stored for base commit ${baseCommitSha}`);
-            return [test];
-        }
-        return [{ ...test, baseReplayId }];
-    });
-};
-exports.getTestsToRun = getTestsToRun;
-const getBaseReplayIdsBySessionId = async ({ client, baseCommitSha, }) => {
-    var _a, _b;
-    if (!baseCommitSha) {
-        return {};
-    }
-    const baseTestRun = await (0, test_run_api_1.getLatestTestRunResults)({
-        client,
-        commitSha: baseCommitSha,
-    });
-    const baseReplays = (_b = (_a = baseTestRun === null || baseTestRun === void 0 ? void 0 : baseTestRun.resultData) === null || _a === void 0 ? void 0 : _a.results) !== null && _b !== void 0 ? _b : [];
-    const baseReplayIdBySessionId = {};
-    // If there are multiple replays for a given session we take the last in the list
-    baseReplays.forEach((replay) => {
-        baseReplayIdBySessionId[replay.sessionId] = replay.headReplayId;
-    });
-    return baseReplayIdBySessionId;
-};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alwaysmeticulous/cli",
-  "version": "2.35.0",
+  "version": "2.37.0",
   "description": "The Meticulous CLI",
   "license": "ISC",
   "main": "dist/index.js",
@@ -22,10 +22,11 @@
     "lint:fix": "eslint src --ext=ts,tsx,js --cache --fix",
     "cli": "node dist/main.js",
     "cli:dev": "ts-node src/main.ts",
+    "cli:dev-localhost": "METICULOUS_API_URL=http://localhost:3001/api/ ts-node src/main.ts",
     "test": "jest"
   },
   "dependencies": {
-    "@alwaysmeticulous/common": "^2.35.0",
+    "@alwaysmeticulous/common": "^2.37.0",
     "@sentry/node": "^7.36.0",
     "@sentry/tracing": "^7.36.0",
     "adm-zip": "^0.5.9",
@@ -34,6 +35,7 @@
     "chalk": "^4.1.2",
     "cosmiconfig": "^8.0.0",
     "express": "^4.18.1",
+    "fast-json-stable-stringify": "^2.1.0",
     "find-free-port": "^2.0.0",
     "inquirer": "^8.2.4",
     "luxon": "^3.2.1",
@@ -43,7 +45,7 @@
     "yargs": "^17.5.1"
   },
   "devDependencies": {
-    "@alwaysmeticulous/api": "^2.34.0",
+    "@alwaysmeticulous/api": "^2.37.0",
     "@types/express": "^4.17.14",
     "@types/proper-lockfile": "^4.1.2"
   },
@@ -93,5 +95,5 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   },
-  "gitHead": "2d87c57d10bb368950a1bf7d12d2da85ab701f00"
+  "gitHead": "4ad32ef54767510ad29b08254603b774e4001f4b"
 }