@alwaysmeticulous/cli 2.36.0 → 2.37.0

package/dist/commands/screenshot-diff/screenshot-diff.command.js

@@ -3,114 +3,106 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.screenshotDiffCommand = exports.summarizeDifferences = exports.diffScreenshots = void 0;
+exports.screenshotDiffCommand = exports.logDifferences = exports.diffScreenshots = void 0;
 const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
+const fast_json_stable_stringify_1 = __importDefault(require("fast-json-stable-stringify"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const client_1 = require("../../api/client");
-const replay_api_1 = require("../../api/replay.api");
 const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const diff_utils_1 = require("../../image/diff.utils");
 const io_utils_1 = require("../../image/io.utils");
 const replays_1 = require("../../local-data/replays");
 const screenshot_diffs_1 = require("../../local-data/screenshot-diffs");
-const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, logger, }) => {
+const has_notable_differences_1 = require("./utils/has-notable-differences");
+const diffScreenshots = async ({ headReplayId, baseReplayId, baseScreenshotsDir, headScreenshotsDir, baseReplayScreenshots, headReplayScreenshots, diffOptions, }) => {
     const { diffThreshold, diffPixelThreshold } = diffOptions;
-    const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
-    const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
-    const missingHeadImages = new Set(baseReplayScreenshots.filter((file) => !headReplayScreenshots.includes(file)));
-    const missingHeadImagesResults = Array.from(missingHeadImages).flatMap((screenshotFileName) => {
-        const identifier = getScreenshotIdentifier(screenshotFileName);
-        if (identifier == null) {
-            return [];
-        }
+    const headReplayScreenshotsByIdentifier = new Map(headReplayScreenshots.map((screenshot) => [
+        (0, fast_json_stable_stringify_1.default)(screenshot.identifier),
+        screenshot,
+    ]));
+    const baseReplayScreenshotsByIdentifier = new Map(baseReplayScreenshots.map((screenshot) => [
+        (0, fast_json_stable_stringify_1.default)(screenshot.identifier),
+        screenshot,
+    ]));
+    const missingHeadImages = new Set(baseReplayScreenshots.filter((file) => !headReplayScreenshotsByIdentifier.has((0, fast_json_stable_stringify_1.default)(file.identifier))));
+    const missingHeadImagesResults = Array.from(missingHeadImages).flatMap(({ identifier, fileName }) => {
         return [
             {
                 identifier,
                 outcome: "missing-head",
-                baseScreenshotFile: `screenshots/${screenshotFileName}`,
+                baseScreenshotFile: `screenshots/${fileName}`,
             },
         ];
     });
-    const diffAgainstBase = async (screenshotFileName) => {
-        const identifier = getScreenshotIdentifier(screenshotFileName);
-        if (identifier == null) {
-            return [];
-        }
-        if (!baseReplayScreenshots.includes(screenshotFileName)) {
+    const diffAgainstBase = async (headScreenshot) => {
+        const baseScreenshot = baseReplayScreenshotsByIdentifier.get((0, fast_json_stable_stringify_1.default)(headScreenshot.identifier));
+        if (baseScreenshot == null) {
             return [
                 {
-                    identifier,
+                    identifier: headScreenshot.identifier,
                     outcome: "missing-base",
-                    headScreenshotFile: `screenshots/${screenshotFileName}`,
+                    headScreenshotFile: `screenshots/${headScreenshot.fileName}`,
                 },
             ];
         }
-        const baseScreenshotFile = (0, path_1.join)(baseScreenshotsDir, screenshotFileName);
-        const headScreenshotFile = (0, path_1.join)(headScreenshotsDir, screenshotFileName);
-        const baseScreenshot = await (0, io_utils_1.readPng)(baseScreenshotFile);
-        const headScreenshot = await (0, io_utils_1.readPng)(headScreenshotFile);
-        if (baseScreenshot.width !== headScreenshot.width ||
-            baseScreenshot.height !== headScreenshot.height) {
+        const baseScreenshotFile = (0, path_1.join)(baseScreenshotsDir, baseScreenshot.fileName);
+        const headScreenshotFile = (0, path_1.join)(headScreenshotsDir, headScreenshot.fileName);
+        const baseScreenshotContents = await (0, io_utils_1.readPng)(baseScreenshotFile);
+        const headScreenshotContents = await (0, io_utils_1.readPng)(headScreenshotFile);
+        if (baseScreenshotContents.width !== headScreenshotContents.width ||
+            baseScreenshotContents.height !== headScreenshotContents.height) {
             return [
                 {
-                    identifier,
+                    identifier: headScreenshot.identifier,
                     outcome: "different-size",
-                    headScreenshotFile: `screenshots/${screenshotFileName}`,
-                    baseScreenshotFile: `screenshots/${screenshotFileName}`,
-                    baseWidth: baseScreenshot.width,
-                    baseHeight: baseScreenshot.height,
-                    headWidth: headScreenshot.width,
-                    headHeight: headScreenshot.height,
+                    baseScreenshotFile: `screenshots/${baseScreenshot.fileName}`,
+                    headScreenshotFile: `screenshots/${headScreenshot.fileName}`,
+                    baseWidth: baseScreenshotContents.width,
+                    baseHeight: baseScreenshotContents.height,
+                    headWidth: headScreenshotContents.width,
+                    headHeight: headScreenshotContents.height,
                 },
             ];
         }
         const comparisonResult = (0, diff_utils_1.compareImages)({
-            base: baseScreenshot,
-            head: headScreenshot,
+            base: baseScreenshotContents,
+            head: headScreenshotContents,
             pixelThreshold: diffPixelThreshold,
         });
         await (0, screenshot_diffs_1.writeScreenshotDiff)({
             baseReplayId,
             headReplayId,
-            screenshotFileName,
+            screenshotFileName: headScreenshot.fileName,
             diff: comparisonResult.diff,
         });
         return [
             {
-                identifier,
+                identifier: headScreenshot.identifier,
                 outcome: comparisonResult.mismatchFraction > diffThreshold
                     ? "diff"
                     : "no-diff",
-                headScreenshotFile: `screenshots/${screenshotFileName}`,
-                baseScreenshotFile: `screenshots/${screenshotFileName}`,
-                width: baseScreenshot.width,
-                height: baseScreenshot.height,
+                baseScreenshotFile: `screenshots/${baseScreenshot.fileName}`,
+                headScreenshotFile: `screenshots/${headScreenshot.fileName}`,
+                width: baseScreenshotContents.width,
+                height: baseScreenshotContents.height,
                 mismatchPixels: comparisonResult.mismatchPixels,
                 mismatchFraction: comparisonResult.mismatchFraction,
             },
         ];
     };
     const headDiffResults = (await Promise.all(headReplayScreenshots.map(diffAgainstBase))).flat();
-    const diffUrl = await (0, replay_api_1.getDiffUrl)(client, baseReplayId, headReplayId);
-    logger.info(`View screenshot diff at ${diffUrl}`);
     return [...missingHeadImagesResults, ...headDiffResults];
 };
 exports.diffScreenshots = diffScreenshots;
-const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions, logger, }) => {
+const logDifferences = ({ results, diffOptions, logger, }) => {
     const missingHeadImagesResults = results.flatMap((result) => result.outcome === "missing-head" ? [result] : []);
     if (missingHeadImagesResults.length) {
         const message = `Head replay is missing screenshots: ${missingHeadImagesResults
             .map(({ baseScreenshotFile }) => (0, path_1.basename)(baseScreenshotFile))
             .sort()} => FAIL!`;
         logger.info(message);
-        return {
-            hasDiffs: true,
-            summaryMessage: message,
-            baseReplayId,
-            headReplayId,
-        };
     }
     const missingBaseImagesResults = results.flatMap((result) => result.outcome === "missing-base" ? [result] : []);
     if (missingHeadImagesResults.length) {
@@ -119,60 +111,21 @@ const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions
             .sort()}`;
         logger.info(message);
     }
-    const diffs = results.flatMap((result) => {
+    results.forEach((result) => {
         const { outcome } = result;
         if (outcome === "different-size") {
             const message = `Screenshots ${(0, path_1.basename)(result.headScreenshotFile)} have different sizes => FAIL!`;
             logger.info(message);
-            return [
-                {
-                    hasDiffs: true,
-                    summaryMessage: message,
-                    baseReplayId,
-                    headReplayId,
-                },
-            ];
         }
         if (outcome === "diff" || outcome === "no-diff") {
             const mismatch = (result.mismatchFraction * 100).toFixed(3);
             const threshold = (diffOptions.diffThreshold * 100).toFixed(3);
             const message = `${mismatch}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${threshold}%) => ${outcome === "no-diff" ? "PASS" : "FAIL!"}`;
             logger.info(message);
-            if (outcome === "diff") {
-                return [
-                    {
-                        hasDiffs: true,
-                        summaryMessage: message,
-                        baseReplayId,
-                        headReplayId,
-                    },
-                ];
-            }
         }
-        return [];
     });
-    return diffs.length > 0 ? diffs[0] : { hasDiffs: false };
-};
-exports.summarizeDifferences = summarizeDifferences;
-const getScreenshotIdentifier = (filename) => {
-    const name = (0, path_1.basename)(filename);
-    if (name === "final-state.png") {
-        return {
-            type: "end-state",
-        };
-    }
-    if (name.startsWith("screenshot-after-event")) {
-        const match = name.match(/^(?:.*)-(\d+)[.]png$/);
-        const eventNumber = match ? parseInt(match[1], 10) : undefined;
-        if (match && eventNumber != null && !isNaN(eventNumber)) {
-            return {
-                type: "after-event",
-                eventNumber,
-            };
-        }
-    }
-    return undefined;
 };
+exports.logDifferences = logDifferences;
 const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulationId: headReplayId, threshold, pixelThreshold, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const client = (0, client_1.createClient)({ apiToken });
@@ -186,24 +139,24 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
         diffThreshold: threshold,
         diffPixelThreshold: pixelThreshold,
     };
+    const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
+    const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
     const results = await (0, exports.diffScreenshots)({
-        client,
         baseReplayId,
         headReplayId,
         baseScreenshotsDir,
         headScreenshotsDir,
+        baseReplayScreenshots,
+        headReplayScreenshots,
         diffOptions,
-        logger,
     });
     logger.debug(results);
-    const diffSummary = (0, exports.summarizeDifferences)({
-        baseReplayId,
-        headReplayId,
+    (0, exports.logDifferences)({
         results,
         diffOptions,
         logger,
     });
-    if (diffSummary.hasDiffs) {
+    if ((0, has_notable_differences_1.hasNotableDifferences)(results)) {
         process.exit(1);
     }
 };

package/dist/commands/screenshot-diff/utils/get-screenshot-filename.d.ts

@@ -0,0 +1,2 @@
+import { ScreenshotIdentifier } from "@alwaysmeticulous/api";
+export declare const getScreenshotFilename: (identifier: ScreenshotIdentifier) => string;

package/dist/commands/screenshot-diff/utils/get-screenshot-filename.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getScreenshotFilename = void 0;
+// Note: ideally this should match the filenames produced by the screenshotting code
+// in replay-node, and in https://github.com/alwaysmeticulous/meticulous-sdk/blob/395af4394dc51d9b51ba1136fc26b23fcbba5604/packages/replayer/src/screenshot.utils.ts#L42
+const getScreenshotFilename = (identifier) => {
+    if (identifier.type === "end-state") {
+        return "final-state.png";
+    }
+    else if (identifier.type === "after-event") {
+        const eventIndexStr = identifier.eventNumber.toString().padStart(5, "0");
+        return `screenshot-after-event-${eventIndexStr}.png`;
+    }
+    else {
+        throw new Error("Unexpected screenshot identifier: " + JSON.stringify(identifier));
+    }
+};
+exports.getScreenshotFilename = getScreenshotFilename;

package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.d.ts

@@ -0,0 +1,2 @@
+import { ScreenshotIdentifier } from "@alwaysmeticulous/api";
+export declare const getScreenshotIdentifier: (filename: string) => ScreenshotIdentifier | undefined;

package/dist/commands/screenshot-diff/utils/get-screenshot-identifier.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getScreenshotIdentifier = void 0;
+const path_1 = require("path");
+const getScreenshotIdentifier = (filename) => {
+    const name = (0, path_1.basename)(filename);
+    if (name === "final-state.png") {
+        return {
+            type: "end-state",
+        };
+    }
+    if (name.startsWith("screenshot-after-event")) {
+        const match = name.match(/^(?:.*)-(\d+)[.]png$/);
+        const eventNumber = match ? parseInt(match[1], 10) : undefined;
+        if (match && eventNumber != null && !isNaN(eventNumber)) {
+            return {
+                type: "after-event",
+                eventNumber,
+            };
+        }
+    }
+    return undefined;
+};
+exports.getScreenshotIdentifier = getScreenshotIdentifier;

package/dist/commands/screenshot-diff/utils/has-notable-differences.d.ts

@@ -0,0 +1,2 @@
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
+export declare const hasNotableDifferences: (screenshotDiffResults: ScreenshotDiffResult[]) => boolean;

package/dist/commands/screenshot-diff/utils/has-notable-differences.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasNotableDifferences = void 0;
+const hasNotableDifferences = (screenshotDiffResults) => {
+    // Note: we ignore flakes & missing-bases here
+    return screenshotDiffResults.some((diff) => diff.outcome === "diff" ||
+        diff.outcome === "missing-head" ||
+        diff.outcome === "different-size");
+};
+exports.hasNotableDifferences = hasNotableDifferences;

package/dist/config/config.js

@@ -35,14 +35,13 @@ const validateReplayOptions = (prevOptions) => {
 const validateConfig = (prevConfig) => {
     const { testCases, ...rest } = prevConfig;
     const nextTestCases = (testCases || [])
-        .map(({ title, sessionId, baseReplayId, options }) => ({
+        .map(({ title, sessionId, options }) => ({
         title: typeof title === "string" ? title : "",
         sessionId: typeof sessionId === "string" ? sessionId : "",
-        ...(typeof baseReplayId === "string" ? { baseReplayId } : {}),
         ...(options ? { options: validateReplayOptions(options) } : {}),
     }))
         .map(({ title, ...rest }) => ({
-        title: title || `${rest.sessionId} | ${rest.baseReplayId}`,
+        title: title || `${rest.sessionId}`,
         ...rest,
     }))
         .filter(({ sessionId }) => sessionId);

package/dist/config/config.types.d.ts

@@ -12,5 +12,5 @@ export interface TestCaseResult extends TestCase {
     result: "pass" | "fail" | "flake";
 }
 export interface DetailedTestCaseResult extends TestCaseResult {
-    screenshotDiffResults: ScreenshotDiffResult[];
+    screenshotDiffResultsByBaseReplayId: Record<string, ScreenshotDiffResult[]>;
 }

package/dist/deflake-tests/deflake-tests.handler.d.ts

@@ -14,6 +14,7 @@ interface HandleReplayOptions {
     commitSha: string;
     generatedBy: GeneratedBy;
     testRunId: string | null;
+    baseTestRunId: string | null;
     replayEventsDependencies: ReplayEventsDependencies;
     suppressScreenshotDiffLogging: boolean;
 }

package/dist/deflake-tests/deflake-tests.handler.js

@@ -7,16 +7,17 @@ exports.deflakeReplayCommandHandler = void 0;
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const replay_command_1 = require("../commands/replay/replay.command");
-const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, }) => {
+const has_notable_differences_1 = require("../commands/screenshot-diff/utils/has-notable-differences");
+const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, baseTestRunId, }) => {
     var _a, _b;
-    const { replay, screenshotDiffResults, screenshotDiffsSummary } = await (0, replay_command_1.replayCommandHandler)({
+    const { replay, screenshotDiffResultsByBaseReplayId } = await (0, replay_command_1.replayCommandHandler)({
         replayTarget,
         executionOptions: applyTestCaseExecutionOptionOverrides(executionOptions, (_a = testCase.options) !== null && _a !== void 0 ? _a : {}),
         screenshottingOptions: applyTestCaseScreenshottingOptionsOverrides(screenshottingOptions, (_b = testCase.options) !== null && _b !== void 0 ? _b : {}),
         apiToken,
         commitSha,
         sessionId: testCase.sessionId,
-        baseSimulationId: testCase.baseReplayId,
+        baseTestRunId,
         cookiesFile: null,
         generatedBy,
         testRunId,
@@ -24,11 +25,14 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
         suppressScreenshotDiffLogging,
         debugger: false,
     });
+    const result = (0, has_notable_differences_1.hasNotableDifferences)(Object.values(screenshotDiffResultsByBaseReplayId).flat())
+        ? "fail"
+        : "pass";
     return {
         ...testCase,
         headReplayId: replay.id,
-        result: screenshotDiffsSummary.hasDiffs ? "fail" : "pass",
-        screenshotDiffResults,
+        result,
+        screenshotDiffResultsByBaseReplayId,
     };
 };
 const deflakeReplayCommandHandler = async ({ deflake, ...otherOptions }) => {

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { getLatestTestRunResults, GetLatestTestRunResultsOptions, } from "./api/test-run.api";
+export { GetLatestTestRunResultsOptions } from "./api/test-run.api";
 export { bootstrapCommand } from "./commands/bootstrap/bootstrap.command";
 export { createTestCommand } from "./commands/create-test/create-test.command";
 export { downloadReplayCommand } from "./commands/download-replay/download-replay.command";

package/dist/index.js

@@ -1,8 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initSentry = exports.setLogLevel = exports.initLogger = exports.runAllTests = exports.updateTestsCommand = exports.showProjectCommand = exports.screenshotDiffCommand = exports.runAllTestsCommand = exports.replayCommand = exports.recordCommand = exports.downloadSessionCommand = exports.downloadReplayCommand = exports.createTestCommand = exports.bootstrapCommand = exports.getLatestTestRunResults = void 0;
-var test_run_api_1 = require("./api/test-run.api");
-Object.defineProperty(exports, "getLatestTestRunResults", { enumerable: true, get: function () { return test_run_api_1.getLatestTestRunResults; } });
+exports.initSentry = exports.setLogLevel = exports.initLogger = exports.runAllTests = exports.updateTestsCommand = exports.showProjectCommand = exports.screenshotDiffCommand = exports.runAllTestsCommand = exports.replayCommand = exports.recordCommand = exports.downloadSessionCommand = exports.downloadReplayCommand = exports.createTestCommand = exports.bootstrapCommand = void 0;
 var bootstrap_command_1 = require("./commands/bootstrap/bootstrap.command");
 Object.defineProperty(exports, "bootstrapCommand", { enumerable: true, get: function () { return bootstrap_command_1.bootstrapCommand; } });
 var create_test_command_1 = require("./commands/create-test/create-test.command");

package/dist/local-data/replays.d.ts

@@ -1,3 +1,4 @@
+import { ScreenshotIdentifier } from "@alwaysmeticulous/api";
 import { Replay } from "@alwaysmeticulous/common";
 import { AxiosInstance } from "axios";
 export declare const getOrFetchReplay: (client: AxiosInstance, replayId: string) => Promise<{
@@ -7,7 +8,11 @@ export declare const getOrFetchReplay: (client: AxiosInstance, replayId: string)
 export declare const getOrFetchReplayArchive: (client: AxiosInstance, replayId: string) => Promise<{
     fileName: string;
 }>;
-export declare const getScreenshotFiles: (screenshotsDirPath: string) => Promise<string[]>;
+export interface IdentifiedScreenshotFile {
+    identifier: ScreenshotIdentifier;
+    fileName: string;
+}
+export declare const getScreenshotFiles: (screenshotsDirPath: string) => Promise<IdentifiedScreenshotFile[]>;
 export declare const getSnapshottedAssetsDir: (replayId: string) => string;
 export declare const getScreenshotsDir: (replayDir: string) => string;
 export declare const getReplayDir: (replayId: string) => string;

package/dist/local-data/replays.js

@@ -11,6 +11,7 @@ const adm_zip_1 = __importDefault(require("adm-zip"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const download_1 = require("../api/download");
 const replay_api_1 = require("../api/replay.api");
+const get_screenshot_identifier_1 = require("../commands/screenshot-diff/utils/get-screenshot-identifier");
 const local_data_utils_1 = require("./local-data.utils");
 const getOrFetchReplay = async (client, replayId) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
@@ -62,13 +63,20 @@ exports.getOrFetchReplayArchive = getOrFetchReplayArchive;
 const getScreenshotFiles = async (screenshotsDirPath) => {
     const screenshotFiles = [];
     const screenshotsDir = await (0, promises_1.opendir)(screenshotsDirPath);
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     for await (const dirEntry of screenshotsDir) {
-        if (dirEntry.isFile() && dirEntry.name.endsWith(".png")) {
-            screenshotFiles.push(dirEntry.name);
+        if (!dirEntry.isFile() || !dirEntry.name.endsWith(".png")) {
+            continue;
+        }
+        const identifier = (0, get_screenshot_identifier_1.getScreenshotIdentifier)(dirEntry.name);
+        if (identifier == null) {
+            logger.error(`Ignoring screenshot file with unrecognized name pattern: ${dirEntry.name}`);
+            continue;
         }
+        screenshotFiles.push({ identifier, fileName: dirEntry.name });
     }
     // Sort files alphabetically to help when reading results.
-    return screenshotFiles.sort();
+    return screenshotFiles.sort((a, b) => a.fileName.localeCompare(b.fileName));
 };
 exports.getScreenshotFiles = getScreenshotFiles;
 const getSnapshottedAssetsDir = (replayId) => (0, path_1.join)((0, exports.getReplayDir)(replayId), "snapshotted-assets");

package/dist/local-data/screenshot-diffs.js

@@ -13,7 +13,7 @@ const writeScreenshotDiff = async ({ baseReplayId, headReplayId, screenshotFileN
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const diffDir = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "screenshot-diffs");
     await (0, promises_1.mkdir)(diffDir, { recursive: true });
-    const diffFile = (0, path_1.join)(diffDir, `${baseReplayId}+${headReplayId}+${screenshotFileName}.png`);
+    const diffFile = (0, path_1.join)(diffDir, `${baseReplayId}-vs-${headReplayId}-${screenshotFileName}.png`);
     await (0, io_utils_1.writePng)(diff, diffFile);
     logger.debug(`Screenshot diff written to ${diffFile}`);
 };

package/dist/local-data/serve-assets-from-simulation.js

@@ -20,7 +20,7 @@ async function serveAssetsFromSimulation(client, simulationId) {
     const snapshottedAssetsDir = (0, replays_1.getSnapshottedAssetsDir)(simulationId);
     if (!(0, fs_1.existsSync)(snapshottedAssetsDir)) {
         logger.error(`No snapshotted assets found for simulation '${simulationId}'.` +
-            " Please re-run without the --simulationIdForAssets or --useAssetsSnapshottedInBaseSimulation flag." +
+            " Please re-run without the --simulationIdForAssets flag." +
             " You can optionally specify an --appUrl to run the simulation against.");
         process.exit(1);
     }

package/dist/parallel-tests/__tests__/mock-test-results.js

@@ -13,7 +13,9 @@ const testResult = (result, screenshotDiffResults, testCase) => {
         sessionId: (_a = testCase === null || testCase === void 0 ? void 0 : testCase.sessionId) !== null && _a !== void 0 ? _a : "mock-session-id",
         headReplayId: "mock-head-replay-id",
         result,
-        screenshotDiffResults,
+        screenshotDiffResultsByBaseReplayId: {
+            "mock-base-replay-id": screenshotDiffResults,
+        },
     };
 };
 exports.testResult = testResult;

package/dist/parallel-tests/__tests__/parrallel-tests.handler.spec.js

@@ -160,7 +160,7 @@ describe("runAllTestsInParallel", () => {
 const testCase = (num) => ({
     sessionId: "mock-session-id",
     title: `${num}`,
-    baseReplayId: "mock-base-replay-id",
+    baseTestRunId: "mock-base-test-run-id",
 });
 const expectPromiseToNotHaveResolved = async (promise) => {
     let resolved = false;

package/dist/parallel-tests/merge-test-results.js

@@ -1,20 +1,25 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mergeResults = void 0;
 const utils_1 = require("@sentry/utils");
+const fast_json_stable_stringify_1 = __importDefault(require("fast-json-stable-stringify"));
+const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
     // If any of the screenshots diffs in comparisonToHeadReplay show a diff against one
     // of the screenshots that orignally failed in currentResult then we have a flake
     // on our hands
     const retryDiffById = new Map();
-    comparisonToHeadReplay.screenshotDiffResults.forEach((result) => {
+    (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(comparisonToHeadReplay).forEach((result) => {
         const hash = hashScreenshotIdentifier(result.identifier);
         if (retryDiffById.has(hash)) {
             throw new Error(`Received two screenshots for the same identifier '${hash}'. Screenshots should be unique.`);
         }
         retryDiffById.set(hash, result);
     });
-    const newScreenshotDiffResults = currentResult.screenshotDiffResults.map((currentDiff) => {
+    const newScreenshotDiffResults = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(currentResult).map((currentDiff) => {
         if (currentDiff.outcome === "no-diff") {
             return currentDiff;
         }
@@ -53,7 +58,7 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
                 ...currentDiff,
                 diffsToHeadScreenshotOnRetries: [
                     ...currentDiff.diffsToHeadScreenshotOnRetries,
-                    withoutIdentifier(diffWhenRetrying),
+                    withoutIdentifiers(diffWhenRetrying),
                 ],
             };
         }
@@ -62,10 +67,11 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
         }
         else {
             return {
+                baseReplayId: currentDiff.baseReplayId,
                 identifier: currentDiff.identifier,
                 outcome: "flake",
-                diffToBaseScreenshot: withoutIdentifier(currentDiff),
-                diffsToHeadScreenshotOnRetries: [withoutIdentifier(diffWhenRetrying)],
+                diffToBaseScreenshot: withoutIdentifiers(currentDiff),
+                diffsToHeadScreenshotOnRetries: [withoutIdentifiers(diffWhenRetrying)],
             };
         }
     });
@@ -75,12 +81,15 @@ const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
         result: currentResult.result === "fail" && noLongerHasFailures
             ? "flake"
             : currentResult.result,
-        screenshotDiffResults: newScreenshotDiffResults,
+        screenshotDiffResultsByBaseReplayId: (0, screenshot_diff_results_utils_1.groupScreenshotDiffResults)(newScreenshotDiffResults),
     };
 };
 exports.mergeResults = mergeResults;
-const withoutIdentifier = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
-...rest }) => rest;
+const withoutIdentifiers = ({ 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+identifier: _identifier, 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+baseReplayId: _baseReplayId, ...rest }) => rest;
 const hashScreenshotIdentifier = (identifier) => {
     if (identifier.type === "end-state") {
         return "end-state";
@@ -93,7 +102,7 @@ const hashScreenshotIdentifier = (identifier) => {
         // The identifier is probably from a newer version of the bundle script
         // and we're on an old version of the CLI. Our best bet is to stringify it
         // and use that as a hash.
-        return JSON.stringify(identifier);
+        return (0, fast_json_stable_stringify_1.default)(identifier);
     }
 };
 const unknownScreenshotIdentifierType = (identifier) => {

package/dist/parallel-tests/parallel-tests.handler.js

@@ -8,6 +8,7 @@ const os_1 = require("os");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const merge_test_results_1 = require("./merge-test-results");
+const screenshot_diff_results_utils_1 = require("./screenshot-diff-results.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
 const runAllTestsInParallel = async ({ testsToRun, parallelTasks, maxRetriesOnFailure, executeTest, onTestFinished, onTestFailedToRun, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
@@ -134,7 +135,7 @@ const logRetrySummary = (nameOfTest, retryResult) => {
         logger.info(`Retried taking screenshots for failed test ${nameOfTest}, but got the same results`);
     }
     else {
-        const numDifferingScreenshots = retryResult.screenshotDiffResults.filter((result) => result.outcome !== "no-diff").length;
+        const numDifferingScreenshots = (0, screenshot_diff_results_utils_1.flattenScreenshotDiffResults)(retryResult).filter((result) => result.outcome !== "no-diff").length;
         logger.info(`Retried taking screenshots for failed test ${nameOfTest}, and ${numDifferingScreenshots} screenshots came out different. Results for these screenshots are assumed to be flakes, and so will be ignored.`);
     }
 };

package/dist/parallel-tests/run-all-tests.d.ts

@@ -19,7 +19,6 @@ export interface Options {
      */
     baseCommitSha: string | null;
     appUrl: string | null;
-    useAssetsSnapshottedInBaseSimulation: boolean;
     /**
      * If null runs in parralel with a sensible number of parrelel tasks for the given machine.
      *
@@ -44,6 +43,7 @@ export interface Options {
      * Captured environment for this run
      */
     environment?: TestRunEnvironment;
+    baseTestRunId: string | null;
     onTestRunCreated?: (testRun: RunAllTestsTestRun & {
         status: "Running";
     }) => void;
@@ -61,4 +61,4 @@ export interface RunAllTestsResult {
  * Runs all the test cases in the provided file.
  * @returns The results of the tests that were executed (note that this does not include results from any cachedTestRunResults passed in)
  */
-export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;
+export declare const runAllTests: ({ testsFile, apiToken, commitSha, baseCommitSha, appUrl, executionOptions, screenshottingOptions, parallelTasks, deflake, maxRetriesOnFailure, cachedTestRunResults: cachedTestRunResults_, githubSummary, environment, baseTestRunId, onTestRunCreated, onTestFinished: onTestFinished_, }: Options) => Promise<RunAllTestsResult>;