@alwaysmeticulous/cli 2.36.0 → 2.38.0

package/dist/parallel-tests/merge-test-results.js

@@ -1,20 +1,25 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mergeResults = void 0;
 const utils_1 = require("@sentry/utils");
+const fast_json_stable_stringify_1 = __importDefault(require("fast-json-stable-stringify"));
+const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
     // If any of the screenshots diffs in comparisonToHeadReplay show a diff against one
     // of the screenshots that orignally failed in currentResult then we have a flake
     // on our hands
     const retryDiffById = new Map();
-    comparisonToHeadReplay.screenshotDiffResults.forEach((result) => {
+    (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(comparisonToHeadReplay).forEach((result) => {
         const hash = hashScreenshotIdentifier(result.identifier);
         if (retryDiffById.has(hash)) {
             throw new Error(`Received two screenshots for the same identifier '${hash}'. Screenshots should be unique.`);
         }
         retryDiffById.set(hash, result);
     });
-    const newScreenshotDiffResults = currentResult.screenshotDiffResults.map((currentDiff) => {
+    const newScreenshotDiffResults = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(currentResult).map((currentDiff) => {
         if (currentDiff.outcome === "no-diff") {
             return currentDiff;
         }
@@ -53,7 +58,7 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
                 ...currentDiff,
                 diffsToHeadScreenshotOnRetries: [
                     ...currentDiff.diffsToHeadScreenshotOnRetries,
-                    withoutIdentifier(diffWhenRetrying),
+                    withoutIdentifiers(diffWhenRetrying),
                 ],
             };
         }
@@ -62,10 +67,11 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
         }
         else {
             return {
+                baseReplayId: currentDiff.baseReplayId,
                 identifier: currentDiff.identifier,
                 outcome: "flake",
-                diffToBaseScreenshot: withoutIdentifier(currentDiff),
-                diffsToHeadScreenshotOnRetries: [withoutIdentifier(diffWhenRetrying)],
+                diffToBaseScreenshot: withoutIdentifiers(currentDiff),
+                diffsToHeadScreenshotOnRetries: [withoutIdentifiers(diffWhenRetrying)],
             };
         }
     });
@@ -75,12 +81,15 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
         result: currentResult.result === "fail" && noLongerHasFailures
             ? "flake"
             : currentResult.result,
-        screenshotDiffResults: newScreenshotDiffResults,
+        screenshotDiffResultsByBaseReplayId: (0, screenshot_diff_results_utils_1.groupScreenshotDiffResults)(newScreenshotDiffResults),
     };
 };
 exports.mergeResults = mergeResults;
-const withoutIdentifier = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
-...rest }) => rest;
+const withoutIdentifiers = ({ 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+identifier: _identifier, 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+baseReplayId: _baseReplayId, ...rest }) => rest;
 const hashScreenshotIdentifier = (identifier) => {
     if (identifier.type === "end-state") {
         return "end-state";
@@ -93,7 +102,7 @@ const hashScreenshotIdentifier = (identifier) => {
         // The identifier is probably from a newer version of the bundle script
         // and we're on an old version of the CLI. Our best bet is to stringify it
         // and use that as a hash.
-        return JSON.stringify(identifier);
+        return (0, fast_json_stable_stringify_1.default)(identifier);
     }
 };
 const unknownScreenshotIdentifierType = (identifier) => {

package/dist/parallel-tests/parallel-tests.handler.js

@@ -8,6 +8,7 @@ const os_1 = require("os");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const merge_test_results_1 = require("./merge-test-results");
+const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
 const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, executeTest, onTestFinished, onTestFailedToRun, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
@@ -134,7 +135,7 @@ const logRetrySummary = (nameOfTest, retryResult) => {
         logger.info(`Retried taking screenshots for failed test ${nameOfTest}, but got the same results`);
     }
     else {
-        const numDifferingScreenshots = retryResult.screenshotDiffResults.filter((result) => result.outcome !== "no-diff").length;
+        const numDifferingScreenshots = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(retryResult).filter((result) => result.outcome !== "no-diff").length;
         logger.info(`Retried taking screenshots for failed test ${nameOfTest}, and ${numDifferingScreenshots} screenshots came out different. Results for these screenshots are assumed to be flakes, and so will be ignored.`);
     }
 };

package/dist/parallel-tests/run-all-tests.d.ts

@@ -19,7 +19,6 @@ export interface Options {
      */
     baseCommitSha: string | null;
     appUrl: string | null;
-    useAssetsSnapshottedInBaseSimulation: boolean;
     /**
      * If null runs in parralel with a sensible number of parrelel tasks for the given machine.
      *
@@ -44,6 +43,7 @@ export interface Options {
      * Captured environment for this run
      */
     environment?: TestRunEnvironment;
+    baseTestRunId: string | null;
     onTestRunCreated?: (testRun: RunAllTestsTestRun & {
         status: "Running";
     }) => void;
@@ -61,4 +61,4 @@ export interface RunAllTestsResult {
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;
+export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;

package/dist/parallel-tests/run-all-tests.js

@@ -23,10 +23,7 @@ const execute_test_in_child_process_1 = require("./execute-test-in-child-process
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
-    if (appUrl != null && useAssetsSnapshottedInBaseSimulation) {
-        throw new Error("Arguments useAssetsSnapshottedInBaseSimulation and appUrl are mutually exclusive");
-    }
+const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }) => {
     if (deflake && maxRetriesOnFailure > 1) {
         throw new Error("Arguments deflake and maxRetriesOnFailure are mutually exclusive");
     }
@@ -44,8 +41,8 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         throw new Error("Error! No test case defined");
     }
     // Only run the uncached test cases
-    const testCases = allTestCases.filter(({ sessionId, baseReplayId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
-        cached.baseReplayId === baseReplayId &&
+    const testCases = allTestCases.filter(({ sessionId, baseTestRunId, title }) => !cachedTestRunResults.find((cached) => cached.sessionId === sessionId &&
+        cached.baseTestRunId === baseTestRunId &&
         cached.title === title));
     const meticulousSha = await (0, version_utils_1.getMeticulousVersion)();
     const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
@@ -62,7 +59,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                 commitSha,
                 baseCommitSha,
                 appUrl,
-                useAssetsSnapshottedInBaseSimulation,
                 parallelTasks,
                 deflake,
                 githubSummary,
@@ -86,17 +82,19 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     logger.info("");
     logger.info(`Test run URL: ${testRunUrl}`);
     logger.info("");
-    const testsToRun = await (0, run_all_tests_utils_1.getTestsToRun)({
-        testCases,
+    const testsToRun = await getTestCasesWithBaseTestRunId({
+        baseCommitSha,
+        baseTestRunId: baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : null,
         client,
-        baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
+        logger,
+        testCases,
     });
     const storeTestRunResults = async (status, resultsSoFar) => {
         const resultsToSendToBE = [
             ...cachedTestRunResults,
             ...resultsSoFar.map(
             // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            ({ screenshotDiffResults, ...result }) => result),
+            ({ screenshotDiffResultsByBaseReplayId, ...result }) => result),
         ];
         try {
             await (0, test_run_api_1.putTestRunResults)({
@@ -127,17 +125,19 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     const onTestFinished = async (progress, resultsSoFar) => {
         onProgressUpdated(progress);
         const newResult = resultsSoFar.at(-1);
-        if ((newResult === null || newResult === void 0 ? void 0 : newResult.baseReplayId) != null) {
-            await (0, replay_diff_api_1.createReplayDiff)({
-                client,
-                headReplayId: newResult.headReplayId,
-                baseReplayId: newResult.baseReplayId,
-                testRunId: testRun.id,
-                data: {
-                    screenshotAssertionsOptions: screenshottingOptions,
-                    screenshotDiffResults: newResult.screenshotDiffResults,
-                },
-            });
+        if (newResult != null) {
+            for (const [baseReplayId, screenshotDiffResults] of Object.entries(newResult.screenshotDiffResultsByBaseReplayId)) {
+                await (0, replay_diff_api_1.createReplayDiff)({
+                    client,
+                    headReplayId: newResult.headReplayId,
+                    baseReplayId: baseReplayId,
+                    testRunId: testRun.id,
+                    data: {
+                        screenshotAssertionsOptions: screenshottingOptions,
+                        screenshotDiffResults,
+                    },
+                });
+            }
         }
         await storeTestRunResults("Running", resultsSoFar);
     };
@@ -146,6 +146,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
         parallelTasks,
         maxRetriesOnFailure,
         executeTest: (testCase, isRetry) => {
+            var _a;
             const initMessage = {
                 kind: "init",
                 data: {
@@ -157,7 +158,6 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                         testCase,
                         deflake,
                         replayTarget: (0, config_utils_1.getReplayTargetForTestCase)({
-                            useAssetsSnapshottedInBaseSimulation,
                             appUrl,
                             testCase,
                         }),
@@ -165,6 +165,7 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
                         screenshottingOptions,
                         generatedBy: { type: "testRun", runId: testRun.id },
                         testRunId: testRun.id,
+                        baseTestRunId: (_a = testCase.baseTestRunId) !== null && _a !== void 0 ? _a : null,
                         replayEventsDependencies,
                         suppressScreenshotDiffLogging: isRetry,
                     },
@@ -212,3 +213,29 @@ const runAllTests = async ({ testsFile, apiToken, commitSha, baseCommitSha, appU
     };
 };
 exports.runAllTests = runAllTests;
+const getTestCasesWithBaseTestRunId = async ({ logger, client, baseCommitSha, baseTestRunId, testCases, }) => {
+    const defaultBaseTestRunId = baseCommitSha != null
+        ? await (0, test_run_api_1.getLatestTestRunId)({
+            client,
+            commitSha: baseCommitSha,
+        })
+        : null;
+    const testsToRun = testCases.map((test) => {
+        // We use the baseTestRunId specified in the test case if it exists, otherwise we use
+        // use the baseTestRunId specified from the CLI args if it exists, otherwise we use the
+        // baseTestRunId for the base commit if it exists, otherwise we use null (don't compare screenshots).
+        const fallbackTestRunId = baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : defaultBaseTestRunId;
+        if (test.baseTestRunId != null || fallbackTestRunId == null) {
+            return test;
+        }
+        return { ...test, baseTestRunId: fallbackTestRunId };
+    });
+    if (baseCommitSha != null) {
+        testsToRun
+            .filter((test) => test.baseTestRunId == null)
+            .forEach((test) => {
+            logger.warn(`Skipping comparisons for test "${test.title}" since no result to compare against stored for base commit ${baseCommitSha}`);
+        });
+    }
+    return testsToRun;
+};

package/dist/parallel-tests/screenshot-diff-results.utils.d.ts

@@ -0,0 +1,7 @@
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
+import { DetailedTestCaseResult } from "../config/config.types";
+export type ScreenshotDiffResultWithBaseReplayId = ScreenshotDiffResult & {
+    baseReplayId: string;
+};
+export declare const flattenScreenshotDiffResults: (testCaseResult: DetailedTestCaseResult) => ScreenshotDiffResultWithBaseReplayId[];
+export declare const groupScreenshotDiffResults: (results: ScreenshotDiffResultWithBaseReplayId[]) => Record<string, ScreenshotDiffResult[]>;

package/dist/parallel-tests/screenshot-diff-results.utils.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.groupScreenshotDiffResults = exports.flattenScreenshotDiffResults = void 0;
+const flattenScreenshotDiffResults = (testCaseResult) => {
+    return Object.entries(testCaseResult.screenshotDiffResultsByBaseReplayId).flatMap(([baseReplayId, diffs]) => {
+        return diffs.map((diff) => ({ ...diff, baseReplayId }));
+    });
+};
+exports.flattenScreenshotDiffResults = flattenScreenshotDiffResults;
+const groupScreenshotDiffResults = (results) => {
+    const groupedResults = {};
+    results.forEach(({ baseReplayId, ...result }) => {
+        var _a;
+        const resultsForBaseReplayId = (_a = groupedResults[baseReplayId]) !== null && _a !== void 0 ? _a : [];
+        resultsForBaseReplayId.push(result);
+        groupedResults[baseReplayId] = resultsForBaseReplayId;
+    });
+    return groupedResults;
+};
+exports.groupScreenshotDiffResults = groupScreenshotDiffResults;

package/dist/utils/config.utils.d.ts

@@ -1,8 +1,7 @@
 import { TestCase } from "@alwaysmeticulous/api";
 import { ReplayTarget } from "@alwaysmeticulous/common/dist/types/replay.types";
 export declare const addTestCase: (testCase: TestCase) => Promise<void>;
-export declare const getReplayTargetForTestCase: ({ useAssetsSnapshottedInBaseSimulation, appUrl, testCase, }: {
-    useAssetsSnapshottedInBaseSimulation: boolean;
+export declare const getReplayTargetForTestCase: ({ appUrl, testCase, }: {
     appUrl: string | null;
     testCase: TestCase;
 }) => ReplayTarget;

package/dist/utils/config.utils.js

@@ -11,7 +11,7 @@ const addTestCase = async (testCase) => {
     await (0, config_1.saveConfig)(newConfig);
 };
 exports.addTestCase = addTestCase;
-const getReplayTargetForTestCase = ({ useAssetsSnapshottedInBaseSimulation, appUrl, testCase, }) => {
+const getReplayTargetForTestCase = ({ appUrl, testCase, }) => {
     var _a, _b, _c;
     if (((_a = testCase.options) === null || _a === void 0 ? void 0 : _a.simulationIdForAssets) != null) {
         return {
@@ -19,15 +19,6 @@ const getReplayTargetForTestCase = ({ useAssetsSnapshottedInBaseSimulation, appU
             simulationIdForAssets: (_b = testCase.options) === null || _b === void 0 ? void 0 : _b.simulationIdForAssets,
         };
     }
-    if (useAssetsSnapshottedInBaseSimulation) {
-        if (testCase.baseReplayId == null) {
-            throw new Error(`--useAssetsSnapshottedInBaseSimulation flag set, but test case "${testCase.title}" does not have a baseReplayId.`);
-        }
-        return {
-            type: "snapshotted-assets",
-            simulationIdForAssets: testCase.baseReplayId,
-        };
-    }
     if ((_c = testCase.options) === null || _c === void 0 ? void 0 : _c.appUrl) {
         if (appUrl) {
             throw new Error(`Test cases "${testCase.title}" has an "appUrl" option but --appUrl is also provided.`);

package/dist/utils/run-all-tests.utils.d.ts

@@ -14,4 +14,3 @@ export interface GetTestsToRunOptions {
      */
     baseCommitSha: string | null;
 }
-export declare const getTestsToRun: ({ testCases, client, baseCommitSha, }: GetTestsToRunOptions) => Promise<TestCase[]>;

package/dist/utils/run-all-tests.utils.js

@@ -1,12 +1,6 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTestsToRun = exports.sortResults = exports.mergeTestCases = void 0;
-const common_1 = require("@alwaysmeticulous/common");
-const loglevel_1 = __importDefault(require("loglevel"));
-const test_run_api_1 = require("../api/test-run.api");
+exports.sortResults = exports.mergeTestCases = void 0;
 const mergeTestCases = (...testSuites) => {
     const seenSessionIds = new Set();
     return testSuites.flatMap((testSuite) => {
@@ -22,9 +16,9 @@ exports.mergeTestCases = mergeTestCases;
 const sortResults = ({ results: unsorted_, testCases }) => {
     const unsorted = [...unsorted_];
     const results = [];
-    testCases.forEach(({ title, baseReplayId, sessionId }) => {
+    testCases.forEach(({ title, baseTestRunId, sessionId }) => {
         const idx = unsorted.findIndex((result) => result.title === title &&
-            result.baseReplayId === baseReplayId &&
+            result.baseTestRunId === baseTestRunId &&
             result.sessionId === sessionId);
         if (idx == -1) {
             return;
@@ -36,44 +30,3 @@ const sortResults = ({ results: unsorted_, testCases }) => {
     return results;
 };
 exports.sortResults = sortResults;
-const getTestsToRun = async ({ testCases, client, baseCommitSha, }) => {
-    const testCasesMissingBaseReplayId = testCases.filter((testCase) => testCase.baseReplayId == null);
-    const testCasesWithBaseReplayId = testCases.flatMap((testCase) => testCase.baseReplayId == null ? [] : [testCase]);
-    if (testCasesMissingBaseReplayId.length === 0) {
-        return testCasesWithBaseReplayId;
-    }
-    const baseReplayIdBySessionId = await getBaseReplayIdsBySessionId({
-        client,
-        baseCommitSha,
-    });
-    return testCases.flatMap((test) => {
-        if (test.baseReplayId != null) {
-            return [test];
-        }
-        const baseReplayId = baseReplayIdBySessionId[test.sessionId];
-        if (baseReplayId == null) {
-            const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-            logger.warn(`Skipping comparisons for test "${test.title}" since no result to compare against stored for base commit ${baseCommitSha}`);
-            return [test];
-        }
-        return [{ ...test, baseReplayId }];
-    });
-};
-exports.getTestsToRun = getTestsToRun;
-const getBaseReplayIdsBySessionId = async ({ client, baseCommitSha, }) => {
-    var _a, _b;
-    if (!baseCommitSha) {
-        return {};
-    }
-    const baseTestRun = await (0, test_run_api_1.getLatestTestRunResults)({
-        client,
-        commitSha: baseCommitSha,
-    });
-    const baseReplays = (_b = (_a = baseTestRun === null || baseTestRun === void 0 ? void 0 : baseTestRun.resultData) === null || _a === void 0 ? void 0 : _a.results) !== null && _b !== void 0 ? _b : [];
-    const baseReplayIdBySessionId = {};
-    // If there are multiple replays for a given session we take the last in the list
-    baseReplays.forEach((replay) => {
-        baseReplayIdBySessionId[replay.sessionId] = replay.headReplayId;
-    });
-    return baseReplayIdBySessionId;
-};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alwaysmeticulous/cli",
-  "version": "2.36.0",
+  "version": "2.38.0",
   "description": "The Meticulous CLI",
   "license": "ISC",
   "main": "dist/index.js",
@@ -22,10 +22,11 @@
     "lint:fix": "eslint src --ext=ts,tsx,js --cache --fix",
     "cli": "node dist/main.js",
     "cli:dev": "ts-node src/main.ts",
+    "cli:dev-localhost": "METICULOUS_API_URL=http://localhost:3001/api/ ts-node src/main.ts",
     "test": "jest"
   },
   "dependencies": {
-    "@alwaysmeticulous/common": "^2.36.0",
+    "@alwaysmeticulous/common": "^2.38.0",
     "@sentry/node": "^7.36.0",
     "@sentry/tracing": "^7.36.0",
     "adm-zip": "^0.5.9",
@@ -34,6 +35,7 @@
     "chalk": "^4.1.2",
     "cosmiconfig": "^8.0.0",
     "express": "^4.18.1",
+    "fast-json-stable-stringify": "^2.1.0",
     "find-free-port": "^2.0.0",
     "inquirer": "^8.2.4",
     "luxon": "^3.2.1",
@@ -43,7 +45,7 @@
     "yargs": "^17.5.1"
   },
   "devDependencies": {
-    "@alwaysmeticulous/api": "^2.36.0",
+    "@alwaysmeticulous/api": "^2.37.0",
     "@types/express": "^4.17.14",
     "@types/proper-lockfile": "^4.1.2"
   },
@@ -93,5 +95,5 @@
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
   },
-  "gitHead": "fda3a87f90ef42bab4f7e51620494083e95c4b5e"
+  "gitHead": "2c4fbc9225d5e0dc09f14a1e90a67008a5e3f376"
 }