@alwaysmeticulous/cli 2.35.0 → 2.37.0

package/dist/api/replay.api.d.ts

@@ -34,4 +34,3 @@ export interface ScreenshotDiffStats {
 }
 export declare const postScreenshotDiffStats: (client: AxiosInstance, options: ScreenshotDiffStats) => Promise<void>;
 export declare const getReplayUrl: (replay: any) => string;
-export declare const getDiffUrl: (client: AxiosInstance, baseReplayId: string, headReplayId: string) => Promise<string>;

package/dist/api/replay.api.js

@@ -3,9 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDiffUrl = exports.getReplayUrl = exports.postScreenshotDiffStats = exports.getReplayCommandId = exports.putReplayPushedStatus = exports.getReplayDownloadUrl = exports.getReplayPushUrl = exports.createReplay = exports.getReplay = void 0;
+exports.getReplayUrl = exports.postScreenshotDiffStats = exports.getReplayCommandId = exports.putReplayPushedStatus = exports.getReplayDownloadUrl = exports.getReplayPushUrl = exports.createReplay = exports.getReplay = void 0;
 const axios_1 = __importDefault(require("axios"));
-const project_api_1 = require("./project.api");
 const getReplay = async (client, replayId) => {
     const { data } = await client.get(`replays/${replayId}`).catch((error) => {
         var _a;
@@ -88,14 +87,3 @@ const getReplayUrl = (replay) => {
     return replayUrl;
 };
 exports.getReplayUrl = getReplayUrl;
-const getDiffUrl = async (client, baseReplayId, headReplayId) => {
-    const project = await (0, project_api_1.getProject)(client);
-    if (!project) {
-        throw new Error(`Unexpected error: could not retrieve project data`);
-    }
-    const organizationName = encodeURIComponent(project.organization.name);
-    const projectName = encodeURIComponent(project.name);
-    const diffUrl = `https://app.meticulous.ai/projects/${organizationName}/${projectName}/simulations/${headReplayId}/diff/${baseReplayId}`;
-    return diffUrl;
-};
-exports.getDiffUrl = getDiffUrl;

package/dist/api/test-run.api.d.ts

@@ -1,7 +1,7 @@
 import { TestRunConfigData } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
 import { TestCaseResult } from "../config/config.types";
-import { TestRun, TestRunStatus } from "./types";
+import { ScreenshotLocator, TestRun, TestRunStatus } from "./types";
 export declare const getTestRun: (options: {
     client: AxiosInstance;
     testRunId: string;
@@ -26,4 +26,10 @@ export interface GetLatestTestRunResultsOptions {
     commitSha: string;
 }
 export declare const getCachedTestRunResults: ({ client, commitSha, }: GetLatestTestRunResultsOptions) => Promise<TestCaseResult[]>;
-export declare const getLatestTestRunResults: ({ client, commitSha, }: GetLatestTestRunResultsOptions) => Promise<TestRun | null>;
+export declare const getLatestTestRunId: (opts: GetLatestTestRunResultsOptions) => Promise<string | null>;
+export interface GetBaseScreenshotLocatorsOptions {
+    client: AxiosInstance;
+    testRunId: string;
+    sessionId: string;
+}
+export declare const getBaseScreenshots: ({ client, testRunId, sessionId, }: GetBaseScreenshotLocatorsOptions) => Promise<ScreenshotLocator[]>;

package/dist/api/test-run.api.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLatestTestRunResults = exports.getCachedTestRunResults = exports.getTestRunUrl = exports.putTestRunResults = exports.createTestRun = exports.getTestRun = void 0;
+exports.getBaseScreenshots = exports.getLatestTestRunId = exports.getCachedTestRunResults = exports.getTestRunUrl = exports.putTestRunResults = exports.createTestRun = exports.getTestRun = void 0;
 const common_1 = require("@alwaysmeticulous/common");
 const axios_1 = __importStar(require("axios"));
 const loglevel_1 = __importDefault(require("loglevel"));
@@ -75,7 +75,7 @@ const getCachedTestRunResults = async ({ client, commitSha, }) => {
         logger.warn("Test run cache not supported: no commit hash");
         return [];
     }
-    const results = (_c = (_b = (_a = (await (0, exports.getLatestTestRunResults)({ client, commitSha }))) === null || _a === void 0 ? void 0 : _a.resultData) === null || _b === void 0 ? void 0 : _b.results) !== null && _c !== void 0 ? _c : [];
+    const results = (_c = (_b = (_a = (await getLatestTestRunResults({ client, commitSha }))) === null || _a === void 0 ? void 0 : _a.resultData) === null || _b === void 0 ? void 0 : _b.results) !== null && _c !== void 0 ? _c : [];
     // Only return passing tests
     return results.filter(({ result }) => result === "pass");
 };
@@ -93,4 +93,22 @@ const getLatestTestRunResults = async ({ client, commitSha, }) => {
     });
     return (_a = data) !== null && _a !== void 0 ? _a : null;
 };
-exports.getLatestTestRunResults = getLatestTestRunResults;
+const getLatestTestRunId = async (opts) => {
+    var _a, _b;
+    return (_b = (_a = (await getLatestTestRunResults(opts))) === null || _a === void 0 ? void 0 : _a.id) !== null && _b !== void 0 ? _b : null;
+};
+exports.getLatestTestRunId = getLatestTestRunId;
+const getBaseScreenshots = async ({ client, testRunId, sessionId, }) => {
+    var _a;
+    const { data } = await client
+        .get(`test-runs/${encodeURIComponent(testRunId)}/base-screenshots?sessionId=${encodeURIComponent(sessionId)}`)
+        .catch((error) => {
+        var _a;
+        if (error instanceof axios_1.AxiosError && ((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 404) {
+            return { data: null };
+        }
+        throw error;
+    });
+    return (_a = data) !== null && _a !== void 0 ? _a : [];
+};
+exports.getBaseScreenshots = getBaseScreenshots;

package/dist/api/types.d.ts

@@ -1,3 +1,4 @@
+import { ScreenshotIdentifier } from "@alwaysmeticulous/api";
 import { TestCaseResult } from "../config/config.types";
 interface Organization {
     id: string;
@@ -27,4 +28,12 @@ export interface TestRun {
     };
     [key: string]: any;
 }
+/**
+ * All the information required to locate & download a screenshot of a replay.
+ */
+export interface ScreenshotLocator {
+    replayId: string;
+    screenshotIdentifier: ScreenshotIdentifier;
+    screenshotUrl: string;
+}
 export {};

package/dist/commands/create-test/create-test.command.js

@@ -48,7 +48,6 @@ const handleTestCreation = async (replay, sessionId) => {
     await (0, config_utils_1.addTestCase)({
         title: testNameResponse.name,
         sessionId,
-        baseReplayId: replay.id,
     });
     logger.info(chalk_1.default.bold.white(`Test saved to ${chalk_1.default.green("meticulous.json")}.`));
 };
@@ -105,12 +104,10 @@ headless, screenshotSelector, padTime, shiftTime, networkStubbing, moveBeforeCli
         moveBeforeClick,
         cookiesFile,
         skipPauses,
-        save: false,
         // We replay against the original recorded URL
         appUrl: null,
         simulationIdForAssets: null,
         // We don't try comparing to the original screenshot, so just set these to their defaults
-        baseSimulationId: null,
         diffThreshold: common_options_1.OPTIONS.diffThreshold.default,
         diffPixelThreshold: common_options_1.OPTIONS.diffPixelThreshold.default,
         // We don't expose these options

package/dist/commands/replay/replay.command.d.ts

@@ -2,7 +2,6 @@
 import { ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { GeneratedBy, Replay, ReplayEventsDependencies, ReplayExecutionOptions, ReplayTarget } from "@alwaysmeticulous/common";
 import { ScreenshotAssertionsOptions, ScreenshotDiffOptions } from "../../command-utils/common-types";
-import { ScreenshotDiffsSummary } from "../screenshot-diff/screenshot-diff.command";
 export interface ReplayOptions extends AdditionalReplayOptions {
     replayTarget: ReplayTarget;
     executionOptions: ReplayExecutionOptions;
@@ -17,14 +16,10 @@ export interface ReplayResult {
     /**
      * Empty if screenshottingOptions.enabled was false.
      */
-    screenshotDiffResults: ScreenshotDiffResult[];
-    /**
-     * Returned as { hasDiffs: false } if screenshottingOptions.enabled was false.
-     */
-    screenshotDiffsSummary: ScreenshotDiffsSummary;
+    screenshotDiffResultsByBaseReplayId: Record<string, ScreenshotDiffResult[]>;
 }
-export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseSimulationId: baseReplayId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, debugger: enableStepThroughDebugger, suppressScreenshotDiffLogging, }: ReplayOptions) => Promise<ReplayResult>;
-export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, Omit<ReplayExecutionOptions, "maxDurationMs" | "maxEventCount">, AdditionalReplayOptions {
+export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseTestRunId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, debugger: enableStepThroughDebugger, suppressScreenshotDiffLogging, }: ReplayOptions) => Promise<ReplayResult>;
+export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, Omit<ReplayExecutionOptions, "maxDurationMs" | "maxEventCount">, Omit<AdditionalReplayOptions, "baseTestRunId"> {
     screenshot: boolean;
     appUrl: string | null | undefined;
     simulationIdForAssets: string | null | undefined;
@@ -32,17 +27,16 @@ export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, O
     maxDurationMs: number | null | undefined;
     maxEventCount: number | null | undefined;
     storyboard: boolean;
-    save: boolean | null | undefined;
 }
 interface AdditionalReplayOptions {
     apiToken: string | null | undefined;
     commitSha: string | null | undefined;
     sessionId: string;
-    baseSimulationId: string | null | undefined;
+    baseTestRunId: string | null | undefined;
     cookiesFile: string | null | undefined;
     debugger: boolean;
 }
-export declare const rawReplayCommandHandler: ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId, diffThreshold, diffPixelThreshold, save, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }: RawReplayCommandHandlerOptions) => Promise<Replay>;
+export declare const rawReplayCommandHandler: ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }: RawReplayCommandHandlerOptions) => Promise<Replay>;
 export declare const getReplayTarget: ({ appUrl, simulationIdForAssets, }: {
     appUrl: string | null;
     simulationIdForAssets: string | null;
@@ -149,15 +143,6 @@ export declare const replayCommand: import("yargs").CommandModule<unknown, impor
         string: true;
         description: string;
     };
-    baseSimulationId: {
-        string: true;
-        description: string;
-        alias: string;
-    };
-    save: {
-        boolean: true;
-        description: string;
-    };
     debugger: {
         boolean: true;
         description: string;

package/dist/commands/replay/replay.command.js

@@ -34,7 +34,6 @@ const Sentry = __importStar(require("@sentry/node"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const luxon_1 = require("luxon");
 const client_1 = require("../../api/client");
-const replay_diff_api_1 = require("../../api/replay-diff.api");
 const replay_api_1 = require("../../api/replay.api");
 const upload_1 = require("../../api/upload");
 const archive_1 = require("../../archive/archive");
@@ -45,11 +44,10 @@ const replay_assets_1 = require("../../local-data/replay-assets");
 const serve_assets_from_simulation_1 = require("../../local-data/serve-assets-from-simulation");
 const sessions_1 = require("../../local-data/sessions");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
-const config_utils_1 = require("../../utils/config.utils");
 const version_utils_1 = require("../../utils/version.utils");
 const compute_diff_1 = require("./utils/compute-diff");
 const exit_early_if_skip_upload_env_var_set_1 = require("./utils/exit-early-if-skip-upload-env-var-set");
-const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseSimulationId: baseReplayId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, debugger: enableStepThroughDebugger, suppressScreenshotDiffLogging, }) => {
+const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseTestRunId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, debugger: enableStepThroughDebugger, suppressScreenshotDiffLogging, }) => {
     var _a;
     if (executionOptions.headless === true &&
         enableStepThroughDebugger === true) {
@@ -137,7 +135,7 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
     closeServer === null || closeServer === void 0 ? void 0 : closeServer();
     const endTime = luxon_1.DateTime.utc();
     logger.info(`Simulation time: ${endTime.diff(startTime).as("seconds")} seconds`);
-    (0, exit_early_if_skip_upload_env_var_set_1.exitEarlyIfSkipUploadEnvVarSet)(baseReplayId);
+    (0, exit_early_if_skip_upload_env_var_set_1.exitEarlyIfSkipUploadEnvVarSet)(baseTestRunId);
     logger.info("Sending simulation results to Meticulous");
     // 8. Create a Zip archive containing the replay files
     const createReplayArchiveSpan = transaction.startChild({
@@ -195,21 +193,19 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
         else {
             computeDiffsLogger.setLevel(logger.getLevel(), false);
         }
-        const { screenshotDiffResults, screenshotDiffsSummary } = screenshottingOptions.enabled && baseReplayId
+        const screenshotDiffResultsByBaseReplayId = screenshottingOptions.enabled && baseTestRunId
             ? await (0, compute_diff_1.computeDiff)({
                 client,
-                baseReplayId,
+                baseTestRunId,
+                sessionId,
                 headReplayId: replay.id,
                 tempDir,
                 screenshottingOptions,
                 logger: computeDiffsLogger,
             })
-            : {
-                screenshotDiffResults: [],
-                screenshotDiffsSummary: { hasDiffs: false },
-            };
+            : {};
         computeDiffSpan.finish();
-        return { replay, screenshotDiffResults, screenshotDiffsSummary };
+        return { replay, screenshotDiffResultsByBaseReplayId };
     }
     finally {
         await (0, archive_1.deleteArchive)(archivePath);
@@ -236,7 +232,7 @@ const serveOrGetAppUrl = async (client, replayTarget) => {
 const unknownReplayTargetType = (replayTarget) => {
     throw new Error(`Unknown type of replay target: ${JSON.stringify(replayTarget)}`);
 };
-const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, baseSimulationId, diffThreshold, diffPixelThreshold, save, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }) => {
+const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl, simulationIdForAssets, headless, devTools, bypassCSP, screenshot, screenshotSelector, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, moveBeforeClick, cookiesFile, disableRemoteFonts, noSandbox, skipPauses, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, debugger: enableStepThroughDebugger, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -265,7 +261,7 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         }
         : { enabled: false };
     const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
-    const { replay, screenshotDiffsSummary, screenshotDiffResults } = await (0, exports.replayCommandHandler)({
+    const { replay } = await (0, exports.replayCommandHandler)({
         replayTarget: (0, exports.getReplayTarget)({
             appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
             simulationIdForAssets: simulationIdForAssets !== null && simulationIdForAssets !== void 0 ? simulationIdForAssets : null,
@@ -276,43 +272,13 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         commitSha,
         cookiesFile,
         sessionId,
-        baseSimulationId,
         generatedBy: generatedByOption,
+        baseTestRunId: null,
         testRunId: null,
         replayEventsDependencies,
         debugger: enableStepThroughDebugger,
         suppressScreenshotDiffLogging: false,
     });
-    // Add test case to meticulous.json if --save option is passed
-    if (save) {
-        if (!screenshottingOptions.enabled) {
-            const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-            logger.error("Warning: saving a new test case without screenshot enabled.");
-        }
-        await (0, config_utils_1.addTestCase)({
-            title: `${sessionId} | ${replay.id}`,
-            sessionId,
-            baseReplayId: replay.id,
-        });
-    }
-    if (screenshotDiffsSummary.hasDiffs) {
-        const client = (0, client_1.createClient)({ apiToken });
-        if (baseSimulationId == null) {
-            throw new Error("baseSimulationId must have been defined if there are diffs, but was null-ish");
-        }
-        // Store the diff
-        await (0, replay_diff_api_1.createReplayDiff)({
-            client,
-            headReplayId: replay.id,
-            baseReplayId: baseSimulationId,
-            testRunId: null,
-            data: {
-                screenshotAssertionsOptions: screenshottingOptions,
-                screenshotDiffResults,
-            },
-        });
-        process.exit(1);
-    }
     return replay;
 };
 exports.rawReplayCommandHandler = rawReplayCommandHandler;
@@ -356,15 +322,6 @@ exports.replayCommand = (0, command_builder_1.buildCommand)("simulate")
         string: true,
         description: "Query selector to screenshot a specific DOM element instead of the whole page",
     },
-    baseSimulationId: {
-        string: true,
-        description: "Base simulation id to diff the final state screenshot against",
-        alias: "baseReplayId",
-    },
-    save: {
-        boolean: true,
-        description: "Adds the simulation to the list of test cases in meticulous.json",
-    },
     debugger: {
         boolean: true,
         description: "Opens a step through debugger to advance through the replay event by event",

package/dist/commands/replay/utils/compute-diff.d.ts

@@ -1,16 +1,13 @@
 import { ScreenshotAssertionsEnabledOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
 import log from "loglevel";
-import { ScreenshotDiffsSummary } from "../../screenshot-diff/screenshot-diff.command";
 export interface ComputeAndSaveDiffOptions {
     client: AxiosInstance;
-    baseReplayId: string;
+    baseTestRunId: string;
+    sessionId: string;
     headReplayId: string;
     tempDir: string;
     screenshottingOptions: ScreenshotAssertionsEnabledOptions;
     logger: log.Logger;
 }
-export declare const computeDiff: ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, logger, }: ComputeAndSaveDiffOptions) => Promise<{
-    screenshotDiffResults: ScreenshotDiffResult[];
-    screenshotDiffsSummary: ScreenshotDiffsSummary;
-}>;
+export declare const computeDiff: ({ client, baseTestRunId, sessionId, tempDir, headReplayId, screenshottingOptions, logger, }: ComputeAndSaveDiffOptions) => Promise<Record<string, ScreenshotDiffResult[]>>;

package/dist/commands/replay/utils/compute-diff.js

@@ -1,30 +1,95 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.computeDiff = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const download_1 = require("../../../api/download");
+const test_run_api_1 = require("../../../api/test-run.api");
 const replays_1 = require("../../../local-data/replays");
 const screenshot_diff_command_1 = require("../../screenshot-diff/screenshot-diff.command");
-const computeDiff = async ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, logger, }) => {
-    logger.info(`Diffing screenshots against replay ${baseReplayId}`);
-    await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
-    await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
-    const baseReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)((0, replays_1.getReplayDir)(baseReplayId));
-    const headReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)(tempDir);
-    const screenshotDiffResults = await (0, screenshot_diff_command_1.diffScreenshots)({
+const get_screenshot_filename_1 = require("../../screenshot-diff/utils/get-screenshot-filename");
+const computeDiff = async ({ client, baseTestRunId, sessionId, tempDir, headReplayId, screenshottingOptions, logger, }) => {
+    var _a;
+    logger.info(`Diffing screenshots against replays of session ${sessionId} in test run ${baseTestRunId}`);
+    const baseScreenshotsDir = (0, path_1.join)((0, replays_1.getScreenshotsDir)(tempDir), "base-screenshots");
+    await (0, promises_1.mkdir)(baseScreenshotsDir, { recursive: true });
+    const baseScreenshots = await (0, test_run_api_1.getBaseScreenshots)({
         client,
-        baseReplayId,
-        headReplayId,
-        baseScreenshotsDir: baseReplayScreenshotsDir,
-        headScreenshotsDir: headReplayScreenshotsDir,
-        diffOptions: screenshottingOptions.diffOptions,
-        logger,
+        testRunId: baseTestRunId,
+        sessionId,
     });
-    const screenshotDiffsSummary = (0, screenshot_diff_command_1.summarizeDifferences)({
-        baseReplayId,
-        headReplayId,
-        results: screenshotDiffResults,
+    const screenshotIdentifiersByBaseReplayId = groupByBaseReplayId(baseScreenshots);
+    await downloadScreenshots(baseScreenshots, baseScreenshotsDir);
+    logger.debug(`Downloaded ${baseScreenshots.length} base screenshots to ${baseScreenshotsDir}}`);
+    const headReplayScreenshotsDir = (0, replays_1.getScreenshotsDir)(tempDir);
+    const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headReplayScreenshotsDir);
+    const screenshotDiffResultsByBaseReplayId = {};
+    // Diff the screenshots for each base replay
+    for (const baseReplayId of screenshotIdentifiersByBaseReplayId.keys()) {
+        const baseScreenshotIdentifiers = (_a = screenshotIdentifiersByBaseReplayId.get(baseReplayId)) !== null && _a !== void 0 ? _a : [];
+        const baseReplayScreenshots = baseScreenshotIdentifiers.map((identifier) => ({
+            identifier,
+            fileName: (0, get_screenshot_filename_1.getScreenshotFilename)(identifier),
+        }));
+        const resultsForBaseReplay = await (0, screenshot_diff_command_1.diffScreenshots)({
+            baseReplayId,
+            headReplayId,
+            baseScreenshotsDir: baseScreenshotsDir,
+            headScreenshotsDir: headReplayScreenshotsDir,
+            headReplayScreenshots,
+            baseReplayScreenshots,
+            diffOptions: screenshottingOptions.diffOptions,
+        });
+        screenshotDiffResultsByBaseReplayId[baseReplayId] = resultsForBaseReplay;
+    }
+    (0, screenshot_diff_command_1.logDifferences)({
+        results: Object.values(screenshotDiffResultsByBaseReplayId).flat(),
         diffOptions: screenshottingOptions.diffOptions,
         logger,
     });
-    return { screenshotDiffResults, screenshotDiffsSummary };
+    // Delete base screenshots directory and all files inside it
+    // (we don't want to upload it to the backend)
+    await deleteScreenshotsDirectory(baseScreenshotsDir);
+    return screenshotDiffResultsByBaseReplayId;
 };
 exports.computeDiff = computeDiff;
+const deleteScreenshotsDirectory = async (directory) => {
+    const files = await (0, promises_1.opendir)(directory);
+    for await (const file of files) {
+        if (file.isFile()) {
+            await (0, promises_1.rm)((0, path_1.join)(directory, file.name));
+        }
+        else {
+            throw new Error(`Expected screenshots directory to only contain screenshot files, but it contained: ${file.name}`);
+        }
+    }
+    await (0, promises_1.rmdir)(directory);
+};
+const groupByBaseReplayId = (screenshots) => {
+    const screenshotIdentifiersByBaseReplayId = new Map();
+    screenshots.forEach(({ replayId, screenshotIdentifier }) => {
+        var _a;
+        const screenshotIdentifiers = (_a = screenshotIdentifiersByBaseReplayId.get(replayId)) !== null && _a !== void 0 ? _a : [];
+        screenshotIdentifiers.push(screenshotIdentifier);
+        screenshotIdentifiersByBaseReplayId.set(replayId, screenshotIdentifiers);
+    });
+    return screenshotIdentifiersByBaseReplayId;
+};
+const downloadScreenshots = async (screenshots, directory) => {
+    // Download in batches of 20
+    const screenshotBatches = chunk(screenshots, 20);
+    for (const screenshotBatch of screenshotBatches) {
+        const screenshotDownloads = screenshotBatch.map((screenshot) => {
+            const filePath = (0, path_1.join)(directory, (0, get_screenshot_filename_1.getScreenshotFilename)(screenshot.screenshotIdentifier));
+            return (0, download_1.downloadFile)(screenshot.screenshotUrl, filePath);
+        });
+        await Promise.all(screenshotDownloads);
+    }
+};
+const chunk = (array, size) => {
+    const chunkedArray = [];
+    for (let i = 0; i < array.length; i += size) {
+        chunkedArray.push(array.slice(i, i + size));
+    }
+    return chunkedArray;
+};

package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.d.ts

@@ -1 +1 @@
-export declare const exitEarlyIfSkipUploadEnvVarSet: (baseReplayId: string | null | undefined) => void;
+export declare const exitEarlyIfSkipUploadEnvVarSet: (baseTestRunId: string | null | undefined) => void;

package/dist/commands/replay/utils/exit-early-if-skip-upload-env-var-set.js

@@ -9,12 +9,12 @@ const loglevel_1 = __importDefault(require("loglevel"));
 const METICULOUS_SKIP_UPLOAD_ENV_VAR = "METICULOUS_SKIP_UPLOAD";
 // Uploading the zip can take a long time, so we expose a env variable to support skipping it
 // We don't expose this as a first-class CLI option because we don't want to pollute the list of CLI options
-const exitEarlyIfSkipUploadEnvVarSet = (baseReplayId) => {
+const exitEarlyIfSkipUploadEnvVarSet = (baseTestRunId) => {
     if (!shouldSkipUpload()) {
         return;
     }
-    if (baseReplayId != null) {
-        throw new Error(`Cannot specify baseReplayId and compare to base results when ${METICULOUS_SKIP_UPLOAD_ENV_VAR} is set to true`);
+    if (baseTestRunId != null) {
+        throw new Error(`Cannot specify baseTestRunId and compare to base results when ${METICULOUS_SKIP_UPLOAD_ENV_VAR} is set to true`);
     }
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     logger.info(`Skipping upload / exiting early since ${METICULOUS_SKIP_UPLOAD_ENV_VAR} is set to true`);

package/dist/commands/run-all-tests/run-all-tests.command.d.ts

@@ -87,11 +87,6 @@ export declare const runAllTestsCommand: import("yargs").CommandModule<unknown,
         readonly string: true;
         readonly description: "The URL to execute the tests against. If left absent here and in the test cases file, then will use the URL the test was originally recorded against.";
     };
-    readonly useAssetsSnapshottedInBaseSimulation: {
-        readonly boolean: true;
-        readonly description: string;
-        readonly default: false;
-    };
     readonly githubSummary: {
         readonly boolean: true;
         readonly description: "Outputs a summary page for GitHub actions";
@@ -126,6 +121,10 @@ export declare const runAllTestsCommand: import("yargs").CommandModule<unknown,
         readonly string: true;
         readonly description: string;
     };
+    readonly baseTestRunId: {
+        readonly string: true;
+        readonly description: "The id of a test run to compare screenshots against.";
+    };
     readonly moveBeforeClick: {
         readonly boolean: true;
         readonly description: "Simulate mouse movement before clicking";

package/dist/commands/run-all-tests/run-all-tests.command.js

@@ -7,7 +7,7 @@ const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const run_all_tests_1 = require("../../parallel-tests/run-all-tests");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
-const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, maxRetriesOnFailure, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, }) => {
+const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, maxRetriesOnFailure, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, essentialFeaturesOnly, baseTestRunId, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -45,8 +45,8 @@ const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl,
         apiToken: apiToken !== null && apiToken !== void 0 ? apiToken : null,
         commitSha,
         baseCommitSha: baseCommitSha !== null && baseCommitSha !== void 0 ? baseCommitSha : null,
+        baseTestRunId: baseTestRunId !== null && baseTestRunId !== void 0 ? baseTestRunId : null,
         appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
-        useAssetsSnapshottedInBaseSimulation,
         parallelTasks: parrelelTasks !== null && parrelelTasks !== void 0 ? parrelelTasks : null,
         deflake,
         maxRetriesOnFailure,
@@ -70,13 +70,6 @@ exports.runAllTestsCommand = (0, command_builder_1.buildCommand)("run-all-tests"
         string: true,
         description: "The URL to execute the tests against. If left absent here and in the test cases file, then will use the URL the test was originally recorded against.",
     },
-    useAssetsSnapshottedInBaseSimulation: {
-        boolean: true,
-        description: "If present will run each session against a local server serving up previously snapshotted assets (HTML, JS, CSS etc.)" +
-            " from the base simulation/replay the test is comparing against. The sessions will then be replayed against those local urls." +
-            " This is an alternative to specifying an appUrl.",
-        default: false,
-    },
     githubSummary: {
         boolean: true,
         description: "Outputs a summary page for GitHub actions",
@@ -117,6 +110,10 @@ exports.runAllTestsCommand = (0, command_builder_1.buildCommand)("run-all-tests"
         description: "The path to the meticulous.json file containing the list of tests you want to run." +
             " If not set a search will be performed to find a meticulous.json file in the current directory or the nearest parent directory.",
     },
+    baseTestRunId: {
+        string: true,
+        description: "The id of a test run to compare screenshots against.",
+    },
     moveBeforeClick: common_options_1.OPTIONS.moveBeforeClick,
     ...common_options_1.COMMON_REPLAY_OPTIONS,
     ...common_options_1.SCREENSHOT_DIFF_OPTIONS,

package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts

@@ -1,23 +1,21 @@
 /// <reference types="yargs" />
 import { ScreenshotDiffOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
-import { AxiosInstance } from "axios";
 import log from "loglevel";
-export declare const diffScreenshots: ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, logger, }: {
-    client: AxiosInstance;
+import { IdentifiedScreenshotFile } from "../../local-data/replays";
+export declare const diffScreenshots: ({ headReplayId, baseReplayId, baseScreenshotsDir, headScreenshotsDir, baseReplayScreenshots, headReplayScreenshots, diffOptions, }: {
     baseReplayId: string;
     headReplayId: string;
     baseScreenshotsDir: string;
     headScreenshotsDir: string;
+    baseReplayScreenshots: IdentifiedScreenshotFile[];
+    headReplayScreenshots: IdentifiedScreenshotFile[];
     diffOptions: ScreenshotDiffOptions;
-    logger: log.Logger;
 }) => Promise<ScreenshotDiffResult[]>;
-export declare const summarizeDifferences: ({ baseReplayId, headReplayId, results, diffOptions, logger, }: {
-    baseReplayId: string;
-    headReplayId: string;
+export declare const logDifferences: ({ results, diffOptions, logger, }: {
     results: ScreenshotDiffResult[];
     diffOptions: ScreenshotDiffOptions;
     logger: log.Logger;
-}) => ScreenshotDiffsSummary;
+}) => void;
 export type ScreenshotDiffsSummary = HasDiffsScreenshotDiffsResult | NoDiffsScreenshotDiffsResult;
 export interface HasDiffsScreenshotDiffsResult {
     hasDiffs: true;