@alwaysmeticulous/cli 2.17.0 → 2.18.0

package/dist/commands/replay/replay.command.d.ts

@@ -1,16 +1,22 @@
 /// <reference types="yargs" />
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { GeneratedBy, Replay, ReplayEventsDependencies, ReplayExecutionOptions, ReplayTarget } from "@alwaysmeticulous/common";
 import { ScreenshotAssertionsOptions, ScreenshotDiffOptions } from "../../command-utils/common-types";
+import { ScreenshotDiffsSummary } from "../screenshot-diff/screenshot-diff.command";
 export interface ReplayOptions extends AdditionalReplayOptions {
     replayTarget: ReplayTarget;
     executionOptions: ReplayExecutionOptions;
     screenshottingOptions: ScreenshotAssertionsOptions;
-    exitOnMismatch: boolean;
     generatedBy: GeneratedBy;
     testRunId: string | null;
     replayEventsDependencies: ReplayEventsDependencies;
 }
-export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }: ReplayOptions) => Promise<Replay>;
+export interface ReplayResult {
+    replay: Replay;
+    screenshotDiffResults: ScreenshotDiffResult[] | null;
+    screenshotDiffsSummary: ScreenshotDiffsSummary;
+}
+export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }: ReplayOptions) => Promise<ReplayResult>;
 export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, Omit<ReplayExecutionOptions, "maxDurationMs" | "maxEventCount">, AdditionalReplayOptions {
     screenshot: boolean;
     appUrl: string | null | undefined;

package/dist/commands/replay/replay.command.js

@@ -10,7 +10,6 @@ const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const luxon_1 = require("luxon");
 const client_1 = require("../../api/client");
-const replay_diff_api_1 = require("../../api/replay-diff.api");
 const replay_api_1 = require("../../api/replay.api");
 const upload_1 = require("../../api/upload");
 const archive_1 = require("../../archive/archive");
@@ -18,14 +17,13 @@ const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const local_data_utils_1 = require("../../local-data/local-data.utils");
 const replay_assets_1 = require("../../local-data/replay-assets");
-const replays_1 = require("../../local-data/replays");
 const serve_assets_from_simulation_1 = require("../../local-data/serve-assets-from-simulation");
 const sessions_1 = require("../../local-data/sessions");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
 const config_utils_1 = require("../../utils/config.utils");
 const version_utils_1 = require("../../utils/version.utils");
-const screenshot_diff_command_1 = require("../screenshot-diff/screenshot-diff.command");
-const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, exitOnMismatch, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }) => {
+const compute_and_save_diff_1 = require("./utils/compute-and-save-diff");
+const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const client = (0, client_1.createClient)({ apiToken });
     // 1. Check session files
@@ -82,93 +80,69 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
     logger.info("Sending simulation results to Meticulous");
     // 8. Create a Zip archive containing the replay files
     const archivePath = await (0, archive_1.createReplayArchive)(tempDir);
-    // 9. Get upload URL
-    const replay = await (0, replay_api_1.createReplay)({
-        client,
-        commitSha,
-        sessionId,
-        meticulousSha,
-        version: "v2",
-        metadata: { generatedBy },
-    });
-    const uploadUrlData = await (0, replay_api_1.getReplayPushUrl)(client, replay.id);
-    if (!uploadUrlData) {
-        logger.error("Error: Could not get a push URL from the Meticulous API");
-        process.exit(1);
-    }
-    const uploadUrl = uploadUrlData.pushUrl;
-    // 10. Send archive to S3
     try {
-        await (0, upload_1.uploadArchive)(uploadUrl, archivePath);
-    }
-    catch (error) {
-        await (0, replay_api_1.putReplayPushedStatus)(client, replay.id, "failure", replayCommandId).catch(logger.error);
-        logger.error(error);
-        process.exit(1);
-    }
-    // 11. Report successful upload to Meticulous
-    const updatedProjectBuild = await (0, replay_api_1.putReplayPushedStatus)(client, replay.id, "success", replayCommandId);
-    logger.info("Simulation artifacts successfully sent to Meticulous");
-    logger.debug(updatedProjectBuild);
-    const replayUrl = (0, replay_api_1.getReplayUrl)(replay);
-    logger.info("=======");
-    logger.info(`View simulation at: ${replayUrl}`);
-    logger.info("=======");
-    // 12. Diff against base replay screenshot if one is provided
-    const baseReplayId = baseReplayId_ || "";
-    if (screenshottingOptions.enabled && baseReplayId) {
-        logger.info(`Diffing screenshots against replay ${baseReplayId}`);
-        await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
-        await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
-        const baseReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(baseReplayId));
-        const headReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)(tempDir);
-        const screenshotDiffResults = await (0, screenshot_diff_command_1.diffScreenshots)({
+        // 9. Get upload URL
+        const replay = await (0, replay_api_1.createReplay)({
             client,
-            baseReplayId,
-            headReplayId: replay.id,
-            baseScreenshotsDir: baseReplayScreenshotsDir,
-            headScreenshotsDir: headReplayScreenshotsDir,
-            diffOptions: screenshottingOptions.diffOptions,
-        });
-        const replayDiff = await (0, replay_diff_api_1.createReplayDiff)({
-            client,
-            headReplayId: replay.id,
-            baseReplayId,
-            testRunId,
-            data: {
-                screenshotAssertionsOptions: screenshottingOptions,
-                screenshotDiffResults,
-            },
+            commitSha,
+            sessionId,
+            meticulousSha,
+            version: "v2",
+            metadata: { generatedBy },
         });
-        logger.debug(replayDiff);
+        const uploadUrlData = await (0, replay_api_1.getReplayPushUrl)(client, replay.id);
+        if (!uploadUrlData) {
+            logger.error("Error: Could not get a push URL from the Meticulous API");
+            process.exit(1);
+        }
+        const uploadUrl = uploadUrlData.pushUrl;
+        // 10. Send archive to S3
         try {
-            (0, screenshot_diff_command_1.checkScreenshotDiffResult)({
-                baseReplayId,
-                headReplayId: replay.id,
-                results: screenshotDiffResults,
-                diffOptions: screenshottingOptions.diffOptions,
-            });
+            await (0, upload_1.uploadArchive)(uploadUrl, archivePath);
         }
         catch (error) {
-            if (exitOnMismatch) {
-                process.exit(1);
+            await (0, replay_api_1.putReplayPushedStatus)(client, replay.id, "failure", replayCommandId).catch(logger.error);
+            logger.error(error);
+            process.exit(1);
+        }
+        // 11. Report successful upload to Meticulous
+        const updatedProjectBuild = await (0, replay_api_1.putReplayPushedStatus)(client, replay.id, "success", replayCommandId);
+        logger.info("Simulation artifacts successfully sent to Meticulous");
+        logger.debug(updatedProjectBuild);
+        const replayUrl = (0, replay_api_1.getReplayUrl)(replay);
+        logger.info("=======");
+        logger.info(`View simulation at: ${replayUrl}`);
+        logger.info("=======");
+        // 12. Diff against base replay screenshot if one is provided
+        const { screenshotDiffResults, screenshotDiffsSummary } = screenshottingOptions.enabled && baseReplayId_
+            ? await (0, compute_and_save_diff_1.computeAndSaveDiff)({
+                client,
+                baseReplayId: baseReplayId_ !== null && baseReplayId_ !== void 0 ? baseReplayId_ : "",
+                headReplayId: replay.id,
+                tempDir,
+                screenshottingOptions,
+                testRunId,
+            })
+            : {
+                screenshotDiffResults: null,
+                screenshotDiffsSummary: { hasDiffs: false },
+            };
+        // 13. Add test case to meticulous.json if --save option is passed
+        if (save) {
+            if (!screenshottingOptions.enabled) {
+                logger.error("Warning: saving a new test case without screenshot enabled.");
             }
-            throw error;
+            await (0, config_utils_1.addTestCase)({
+                title: `${sessionId} | ${replay.id}`,
+                sessionId,
+                baseReplayId: replay.id,
+            });
         }
+        return { replay, screenshotDiffResults, screenshotDiffsSummary };
     }
-    // 13. Add test case to meticulous.json if --save option is passed
-    if (save) {
-        if (!screenshottingOptions.enabled) {
-            logger.error("Warning: saving a new test case without screenshot enabled.");
-        }
-        await (0, config_utils_1.addTestCase)({
-            title: `${sessionId} | ${replay.id}`,
-            sessionId,
-            baseReplayId: replay.id,
-        });
+    finally {
+        await (0, archive_1.deleteArchive)(archivePath);
     }
-    await (0, archive_1.deleteArchive)(archivePath);
-    return replay;
 };
 exports.replayCommandHandler = replayCommandHandler;
 const serveOrGetAppUrl = async (client, replayTarget) => {
@@ -218,7 +192,7 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         }
         : { enabled: false };
     const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
-    return (0, exports.replayCommandHandler)({
+    const { replay, screenshotDiffsSummary } = await (0, exports.replayCommandHandler)({
         replayTarget: (0, exports.getReplayTarget)({
             appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
             simulationIdForAssets: simulationIdForAssets !== null && simulationIdForAssets !== void 0 ? simulationIdForAssets : null,
@@ -231,11 +205,14 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         sessionId,
         baseSimulationId,
         save,
-        exitOnMismatch: true,
         generatedBy: generatedByOption,
         testRunId: null,
         replayEventsDependencies,
     });
+    if (screenshotDiffsSummary.hasDiffs) {
+        process.exit(1);
+    }
+    return replay;
 };
 exports.rawReplayCommandHandler = rawReplayCommandHandler;
 const getReplayTarget = ({ appUrl, simulationIdForAssets, }) => {

package/dist/commands/replay/utils/compute-and-save-diff.d.ts

@@ -0,0 +1,15 @@
+import { ScreenshotAssertionsEnabledOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
+import { AxiosInstance } from "axios";
+import { ScreenshotDiffsSummary } from "../../screenshot-diff/screenshot-diff.command";
+export interface ComputeAndSaveDiffOptions {
+    client: AxiosInstance;
+    testRunId: string | null;
+    baseReplayId: string;
+    headReplayId: string;
+    tempDir: string;
+    screenshottingOptions: ScreenshotAssertionsEnabledOptions;
+}
+export declare const computeAndSaveDiff: ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, testRunId, }: ComputeAndSaveDiffOptions) => Promise<{
+    screenshotDiffResults: ScreenshotDiffResult[];
+    screenshotDiffsSummary: ScreenshotDiffsSummary;
+}>;

package/dist/commands/replay/utils/compute-and-save-diff.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeAndSaveDiff = void 0;
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const replay_diff_api_1 = require("../../../api/replay-diff.api");
+const replays_1 = require("../../../local-data/replays");
+const screenshot_diff_command_1 = require("../../screenshot-diff/screenshot-diff.command");
+const computeAndSaveDiff = async ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, testRunId, }) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    logger.info(`Diffing screenshots against replay ${baseReplayId}`);
+    await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
+    await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
+    const baseReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(baseReplayId));
+    const headReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)(tempDir);
+    const screenshotDiffResults = await (0, screenshot_diff_command_1.diffScreenshots)({
+        client,
+        baseReplayId,
+        headReplayId,
+        baseScreenshotsDir: baseReplayScreenshotsDir,
+        headScreenshotsDir: headReplayScreenshotsDir,
+        diffOptions: screenshottingOptions.diffOptions,
+    });
+    const replayDiff = await (0, replay_diff_api_1.createReplayDiff)({
+        client,
+        headReplayId,
+        baseReplayId,
+        testRunId,
+        data: {
+            screenshotAssertionsOptions: screenshottingOptions,
+            screenshotDiffResults,
+        },
+    });
+    logger.debug(replayDiff);
+    const screenshotDiffsSummary = (0, screenshot_diff_command_1.summarizeDifferences)({
+        baseReplayId,
+        headReplayId,
+        results: screenshotDiffResults,
+        diffOptions: screenshottingOptions.diffOptions,
+    });
+    return { screenshotDiffResults, screenshotDiffsSummary };
+};
+exports.computeAndSaveDiff = computeAndSaveDiff;

package/dist/commands/run-all-tests/run-all-tests.command.js

@@ -1,10 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runAllTestsCommand = void 0;
+const client_1 = require("../../api/client");
+const test_run_api_1 = require("../../api/test-run.api");
 const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const run_all_tests_1 = require("../../parallel-tests/run-all-tests");
-const handler = async ({ apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, }) => {
+const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
+const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -29,21 +32,26 @@ const handler = async ({ apiToken, commitSha, baseCommitSha, appUrl, useAssetsSn
         storyboardOptions,
     };
     const parrelelTasks = parallelize ? parrelelTasks_ : 1;
-    const result = await (0, run_all_tests_1.runAllTests)({
+    const commitSha = (await (0, commit_sha_utils_1.getCommitSha)(commitSha_)) || "unknown";
+    const client = (0, client_1.createClient)({ apiToken });
+    const cachedTestRunResults = useCache
+        ? await (0, test_run_api_1.getCachedTestRunResults)({ client, commitSha })
+        : [];
+    const { testRun } = await (0, run_all_tests_1.runAllTests)({
         testsFile: testsFile !== null && testsFile !== void 0 ? testsFile : null,
         executionOptions,
         screenshottingOptions,
         apiToken: apiToken !== null && apiToken !== void 0 ? apiToken : null,
-        commitSha: commitSha !== null && commitSha !== void 0 ? commitSha : null,
+        commitSha,
         baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
         appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
         useAssetsSnapshottedInBaseSimulation,
         parallelTasks: parrelelTasks !== null && parrelelTasks !== void 0 ? parrelelTasks : null,
         deflake,
-        useCache,
+        cachedTestRunResults,
         githubSummary,
     });
-    if (result.testRun.status === "Failure") {
+    if (testRun.status === "Failure") {
         process.exit(1);
     }
 };

package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts

@@ -9,21 +9,21 @@ export declare const diffScreenshots: ({ client, headReplayId, baseReplayId, hea
     headScreenshotsDir: string;
     diffOptions: ScreenshotDiffOptions;
 }) => Promise<ScreenshotDiffResult[]>;
-export declare const checkScreenshotDiffResult: ({ baseReplayId, headReplayId, results, diffOptions, }: {
+export declare const summarizeDifferences: ({ baseReplayId, headReplayId, results, diffOptions, }: {
     baseReplayId: string;
     headReplayId: string;
     results: ScreenshotDiffResult[];
     diffOptions: ScreenshotDiffOptions;
-}) => void;
-export declare class ScreenshotDiffError extends Error {
-    readonly extras?: {
-        baseReplayId: string;
-        headReplayId: string;
-    } | undefined;
-    constructor(message: string, extras?: {
-        baseReplayId: string;
-        headReplayId: string;
-    } | undefined);
+}) => ScreenshotDiffsSummary;
+export declare type ScreenshotDiffsSummary = HasDiffsScreenshotDiffsResult | NoDiffsScreenshotDiffsResult;
+export interface HasDiffsScreenshotDiffsResult {
+    hasDiffs: true;
+    summaryMessage: string;
+    baseReplayId: string;
+    headReplayId: string;
+}
+export interface NoDiffsScreenshotDiffsResult {
+    hasDiffs: false;
 }
 export declare const screenshotDiffCommand: import("yargs").CommandModule<unknown, import("yargs").InferredOptionTypes<{
     apiToken: {

package/dist/commands/screenshot-diff/screenshot-diff.command.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.screenshotDiffCommand = exports.ScreenshotDiffError = exports.checkScreenshotDiffResult = exports.diffScreenshots = void 0;
+exports.screenshotDiffCommand = exports.summarizeDifferences = exports.diffScreenshots = void 0;
 const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
@@ -99,7 +99,7 @@ const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreens
     return [...missingHeadImagesResults, ...headDiffResults];
 };
 exports.diffScreenshots = diffScreenshots;
-const checkScreenshotDiffResult = ({ baseReplayId, headReplayId, results, diffOptions, }) => {
+const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const missingHeadImagesResults = results.flatMap((result) => result.outcome === "missing-head" ? [result] : []);
     if (missingHeadImagesResults.length) {
@@ -107,10 +107,12 @@ const checkScreenshotDiffResult = ({ baseReplayId, headReplayId, results, diffOp
             .map(({ baseScreenshotFile }) => (0, path_1.basename)(baseScreenshotFile))
             .sort()} => FAIL!`;
         logger.info(message);
-        throw new ScreenshotDiffError(message, {
+        return {
+            hasDiffs: true,
+            summaryMessage: message,
             baseReplayId,
             headReplayId,
-        });
+        };
     }
     const missingBaseImagesResults = results.flatMap((result) => result.outcome === "missing-base" ? [result] : []);
     if (missingHeadImagesResults.length) {
@@ -124,10 +126,12 @@ const checkScreenshotDiffResult = ({ baseReplayId, headReplayId, results, diffOp
         if (outcome === "different-size") {
             const message = `Screenshots ${(0, path_1.basename)(result.headScreenshotFile)} have different sizes => FAIL!`;
             logger.info(message);
-            throw new ScreenshotDiffError(message, {
+            return {
+                hasDiffs: true,
+                summaryMessage: message,
                 baseReplayId,
                 headReplayId,
-            });
+            };
         }
         if (outcome === "diff" || outcome === "no-diff") {
             const mismatch = (result.mismatchFraction * 100).toFixed(3);
@@ -135,22 +139,18 @@ const checkScreenshotDiffResult = ({ baseReplayId, headReplayId, results, diffOp
             const message = `${mismatch}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${threshold}%) => ${outcome === "no-diff" ? "PASS" : "FAIL!"}`;
             logger.info(message);
             if (outcome === "diff") {
-                throw new ScreenshotDiffError(message, {
+                return {
+                    hasDiffs: true,
+                    summaryMessage: message,
                     baseReplayId,
                     headReplayId,
-                });
+                };
             }
         }
     });
+    return { hasDiffs: false };
 };
-exports.checkScreenshotDiffResult = checkScreenshotDiffResult;
-class ScreenshotDiffError extends Error {
-    constructor(message, extras) {
-        super(message);
-        this.extras = extras;
-    }
-}
-exports.ScreenshotDiffError = ScreenshotDiffError;
+exports.summarizeDifferences = summarizeDifferences;
 const getScreenshotIdentifier = (filename) => {
     const name = (0, path_1.basename)(filename);
     if (name === "final-state.png") {
@@ -192,12 +192,15 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
         diffOptions,
     });
     logger.debug(results);
-    (0, exports.checkScreenshotDiffResult)({
+    const diffSummary = (0, exports.summarizeDifferences)({
         baseReplayId,
         headReplayId,
         results,
         diffOptions,
     });
+    if (diffSummary.hasDiffs) {
+        process.exit(1);
+    }
 };
 exports.screenshotDiffCommand = (0, command_builder_1.buildCommand)("screenshot-diff")
     .details({ describe: "Diff two replay screenshots" })

package/dist/config/config.types.d.ts

@@ -1,3 +1,4 @@
+import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { ScreenshotDiffOptions } from "../command-utils/common-types";
 export interface TestCaseReplayOptions extends Partial<ScreenshotDiffOptions> {
     appUrl?: string | null | undefined;
@@ -21,3 +22,6 @@ export interface TestCaseResult extends TestCase {
     headReplayId: string;
     result: "pass" | "fail";
 }
+export interface DetailedTestCaseResult extends TestCaseResult {
+    screenshotDiffResults: ScreenshotDiffResult[];
+}

package/dist/deflake-tests/deflake-tests.handler.d.ts

@@ -1,6 +1,6 @@
 import { GeneratedBy, ReplayEventsDependencies, ReplayExecutionOptions, ReplayTarget } from "@alwaysmeticulous/common";
 import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
-import { TestCase, TestCaseResult } from "../config/config.types";
+import { DetailedTestCaseResult, TestCase } from "../config/config.types";
 export interface DeflakeReplayCommandHandlerOptions extends HandleReplayOptions {
     deflake: boolean;
 }
@@ -15,5 +15,5 @@ interface HandleReplayOptions {
     testRunId: string | null;
     replayEventsDependencies: ReplayEventsDependencies;
 }
-export declare const deflakeReplayCommandHandler: (options: DeflakeReplayCommandHandlerOptions) => Promise<TestCaseResult>;
+export declare const deflakeReplayCommandHandler: ({ deflake, ...otherOptions }: DeflakeReplayCommandHandlerOptions) => Promise<DetailedTestCaseResult>;
 export {};

package/dist/deflake-tests/deflake-tests.handler.js

@@ -7,11 +7,9 @@ exports.deflakeReplayCommandHandler = void 0;
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const replay_command_1 = require("../commands/replay/replay.command");
-const screenshot_diff_command_1 = require("../commands/screenshot-diff/screenshot-diff.command");
 const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, }) => {
     var _a, _b;
-    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const replayPromise = (0, replay_command_1.replayCommandHandler)({
+    const { replay, screenshotDiffResults, screenshotDiffsSummary } = await (0, replay_command_1.replayCommandHandler)({
         replayTarget,
         executionOptions: applyTestCaseExecutionOptionOverrides(executionOptions, (_a = testCase.options) !== null && _a !== void 0 ? _a : {}),
         screenshottingOptions: applyTestCaseScreenshottingOptionsOverrides(screenshottingOptions, (_b = testCase.options) !== null && _b !== void 0 ? _b : {}),
@@ -20,30 +18,20 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
         sessionId: testCase.sessionId,
         baseSimulationId: testCase.baseReplayId,
         save: false,
-        exitOnMismatch: false,
         cookiesFile: null,
         generatedBy,
         testRunId,
         replayEventsDependencies,
     });
-    const result = await replayPromise
-        .then((replay) => ({
+    if (screenshotDiffResults == null) {
+        throw new Error(`replayCommandHandler returned a null screenshotDiffResults, but was called with screenshottingOptions.enabled = true`);
+    }
+    return {
         ...testCase,
         headReplayId: replay.id,
-        result: "pass",
-    }))
-        .catch((error) => {
-        if (error instanceof screenshot_diff_command_1.ScreenshotDiffError && error.extras) {
-            return {
-                ...testCase,
-                headReplayId: error.extras.headReplayId,
-                result: "fail",
-            };
-        }
-        logger.error(error);
-        return { ...testCase, headReplayId: "", result: "fail" };
-    });
-    return result;
+        result: screenshotDiffsSummary.hasDiffs ? "pass" : "fail",
+        screenshotDiffResults,
+    };
 };
 const deflakeReplayCommandHandler = async ({ deflake, ...otherOptions }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);

package/dist/parallel-tests/messages.types.d.ts

@@ -1,5 +1,5 @@
 import log from "loglevel";
-import { TestCaseResult } from "../config/config.types";
+import { DetailedTestCaseResult } from "../config/config.types";
 import { DeflakeReplayCommandHandlerOptions } from "../deflake-tests/deflake-tests.handler";
 export interface InitMessage {
     kind: "init";
@@ -12,6 +12,6 @@ export interface InitMessage {
 export interface ResultMessage {
     kind: "result";
     data: {
-        result: TestCaseResult;
+        result: DetailedTestCaseResult;
     };
 }

package/dist/parallel-tests/parallel-tests.handler.d.ts

@@ -1,28 +1,22 @@
 import { ReplayEventsDependencies, ReplayExecutionOptions } from "@alwaysmeticulous/common";
-import { AxiosInstance } from "axios";
 import { TestRun } from "../api/test-run.api";
 import { ScreenshotAssertionsEnabledOptions } from "../command-utils/common-types";
-import { MeticulousCliConfig, TestCaseResult } from "../config/config.types";
+import { DetailedTestCaseResult, MeticulousCliConfig, TestCase } from "../config/config.types";
 import { TestRunProgress } from "./run-all-tests.types";
 export interface RunAllTestsInParallelOptions {
     config: MeticulousCliConfig;
-    client: AxiosInstance;
     testRun: TestRun;
+    testsToRun: TestCase[];
     executionOptions: ReplayExecutionOptions;
     screenshottingOptions: ScreenshotAssertionsEnabledOptions;
     apiToken: string | null;
     commitSha: string;
-    /**
-     * The base commit to compare test results against for test cases that don't have a baseReplayId specified.
-     */
-    baseCommitSha: string | null;
     appUrl: string | null;
     useAssetsSnapshottedInBaseSimulation: boolean;
     parallelTasks: number | null;
     deflake: boolean;
-    cachedTestRunResults: TestCaseResult[];
     replayEventsDependencies: ReplayEventsDependencies;
-    onTestFinished?: (progress: TestRunProgress) => void;
+    onTestFinished?: (progress: TestRunProgress, resultsSoFar: DetailedTestCaseResult[]) => Promise<void>;
 }
 /** Handler for running Meticulous tests in parallel using child processes */
-export declare const runAllTestsInParallel: (options: RunAllTestsInParallelOptions) => Promise<TestCaseResult[]>;
+export declare const runAllTestsInParallel: (options: RunAllTestsInParallelOptions) => Promise<DetailedTestCaseResult[]>;

package/dist/parallel-tests/parallel-tests.handler.js

@@ -9,19 +9,12 @@ const os_1 = require("os");
 const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
-const test_run_api_1 = require("../api/test-run.api");
 const config_utils_1 = require("../utils/config.utils");
 const run_all_tests_utils_1 = require("../utils/run-all-tests.utils");
 /** Handler for running Meticulous tests in parallel using child processes */
-const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commitSha, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, cachedTestRunResults, replayEventsDependencies, onTestFinished, }) => {
+const runAllTestsInParallel = async ({ config, testRun, testsToRun: queue, apiToken, commitSha, appUrl, useAssetsSnapshottedInBaseSimulation, executionOptions, screenshottingOptions, parallelTasks, deflake, replayEventsDependencies, onTestFinished, }) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const results = [...cachedTestRunResults];
-    const queue = await (0, run_all_tests_utils_1.getTestsToRun)({
-        client,
-        testCases: config.testCases || [],
-        cachedTestRunResults,
-        baseCommitSha,
-    });
+    const results = [];
     const progress = {
         runningTestCases: queue.length,
         failedTestCases: 0,
@@ -87,10 +80,13 @@ const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commit
         // Handle task completion
         deferredResult.promise
             .catch(() => {
+            // If it threw an error then it's something fatal, rather than just a failed diff
+            // (it resolves successfully on a failed diff)
             const result = {
                 ...testCase,
                 headReplayId: "",
                 result: "fail",
+                screenshotDiffResults: [],
             };
             return result;
         })
@@ -100,17 +96,7 @@ const runAllTestsInParallel = async ({ config, client, testRun, apiToken, commit
             progress.failedTestCases += result.result === "fail" ? 1 : 0;
             progress.passedTestCases += result.result === "pass" ? 1 : 0;
             --progress.runningTestCases;
-            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress);
-            (0, test_run_api_1.putTestRunResults)({
-                client,
-                testRunId: testRun.id,
-                status: "Running",
-                resultData: { results },
-            })
-                .catch((error) => {
-                logger.error(`Error while pushing partial results: ${error}`);
-            })
-                .then(() => {
+            onTestFinished === null || onTestFinished === void 0 ? void 0 : onTestFinished(progress, results).then(() => {
                 var _a;
                 if (results.length === (((_a = config.testCases) === null || _a === void 0 ? void 0 : _a.length) || 0)) {
                     allTasksDone.resolve();