@alwaysmeticulous/cli 2.19.3 → 2.20.1

package/dist/local-data/replays.js

@@ -11,25 +11,19 @@ const adm_zip_1 = __importDefault(require("adm-zip"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const download_1 = require("../api/download");
 const replay_api_1 = require("../api/replay.api");
+const local_data_utils_1 = require("./local-data.utils");
 const getOrFetchReplay = async (client, replayId) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const replayDir = (0, exports.getReplayDir)(replayId);
-    await (0, promises_1.mkdir)(replayDir, { recursive: true });
-    const replayFile = (0, path_1.join)(replayDir, `${replayId}.json`);
-    const existingReplay = await (0, promises_1.readFile)(replayFile)
-        .then((data) => JSON.parse(data.toString("utf-8")))
-        .catch(() => null);
-    if (existingReplay) {
-        logger.debug(`Reading replay from local copy in ${replayFile}`);
-        return existingReplay;
-    }
-    const replay = await (0, replay_api_1.getReplay)(client, replayId);
+    const replayFile = (0, path_1.join)((0, exports.getReplayDir)(replayId), `${replayId}.json`);
+    const replay = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: replayFile,
+        dataDescription: "replay",
+        downloadJson: () => (0, replay_api_1.getReplay)(client, replayId),
+    });
     if (!replay) {
         logger.error(`Error: Could not retrieve replay with id "${replayId}". Is the API token correct?`);
         process.exit(1);
     }
-    await (0, promises_1.writeFile)(replayFile, JSON.stringify(replay, null, 2));
-    logger.debug(`Wrote replay to ${replayFile}`);
     return replay;
 };
 exports.getOrFetchReplay = getOrFetchReplay;
@@ -37,27 +31,31 @@ const getOrFetchReplayArchive = async (client, replayId) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     const replayDir = (0, exports.getReplayDir)(replayId);
     await (0, promises_1.mkdir)(replayDir, { recursive: true });
-    const replayArchiveFile = (0, path_1.join)(replayDir, `${replayId}.zip`);
-    const paramsFile = (0, path_1.join)(replayDir, "replayEventsParams.json");
-    // Check if "replayEventsParams.json" exists. If yes, we assume the replay
-    // zip archive has been downloaded and extracted.
-    const paramsFileExists = await (0, promises_1.access)(paramsFile)
-        .then(() => true)
-        .catch(() => false);
-    if (paramsFileExists) {
-        logger.debug(`Replay archive already downloaded at ${replayDir}`);
-        return;
+    const releaseLock = await (0, local_data_utils_1.waitToAcquireLockOnDirectory)(replayDir);
+    try {
+        const replayArchiveFile = (0, path_1.join)(replayDir, `${replayId}.zip`);
+        const paramsFile = (0, path_1.join)(replayDir, "replayEventsParams.json");
+        // Check if "replayEventsParams.json" exists. If yes, we assume the replay
+        // zip archive has been downloaded and extracted.
+        if (await (0, local_data_utils_1.fileExists)(paramsFile)) {
+            logger.debug(`Replay archive already downloaded at ${replayDir}`);
+            return;
+        }
+        const downloadUrlData = await (0, replay_api_1.getReplayDownloadUrl)(client, replayId);
+        if (!downloadUrlData) {
+            logger.error("Error: Could not retrieve replay archive URL. This may be an invalid replay");
+            await releaseLock();
+            process.exit(1);
+        }
+        await (0, download_1.downloadFile)(downloadUrlData.dowloadUrl, replayArchiveFile);
+        const zipFile = new adm_zip_1.default(replayArchiveFile);
+        zipFile.extractAllTo(replayDir, /*overwrite=*/ true);
+        await (0, promises_1.rm)(replayArchiveFile);
+        logger.debug(`Extracted replay archive in ${replayDir}`);
     }
-    const downloadUrlData = await (0, replay_api_1.getReplayDownloadUrl)(client, replayId);
-    if (!downloadUrlData) {
-        logger.error("Error: Could not retrieve replay archive URL. This may be an invalid replay");
-        process.exit(1);
+    finally {
+        await releaseLock();
     }
-    await (0, download_1.downloadFile)(downloadUrlData.dowloadUrl, replayArchiveFile);
-    const zipFile = new adm_zip_1.default(replayArchiveFile);
-    zipFile.extractAllTo(replayDir, /*overwrite=*/ true);
-    await (0, promises_1.rm)(replayArchiveFile);
-    logger.debug(`Exrtracted replay archive in ${replayDir}`);
 };
 exports.getOrFetchReplayArchive = getOrFetchReplayArchive;
 const getScreenshotFiles = async (screenshotsDirPath) => {

package/dist/local-data/sessions.d.ts

@@ -1,3 +1,4 @@
+import { SessionData } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
 export declare const getOrFetchRecordedSession: (client: AxiosInstance, sessionId: string) => Promise<any>;
-export declare const getOrFetchRecordedSessionData: (client: AxiosInstance, sessionId: string) => Promise<any>;
+export declare const getOrFetchRecordedSessionData: (client: AxiosInstance, sessionId: string) => Promise<SessionData>;

package/dist/local-data/sessions.js

@@ -4,7 +4,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getOrFetchRecordedSessionData = exports.getOrFetchRecordedSession = void 0;
-const promises_1 = require("fs/promises");
 const path_1 = require("path");
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
@@ -12,45 +11,31 @@ const session_api_1 = require("../api/session.api");
 const local_data_utils_1 = require("./local-data.utils");
 const getOrFetchRecordedSession = async (client, sessionId) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const sessionsDir = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions");
-    await (0, promises_1.mkdir)(sessionsDir, { recursive: true });
-    const sessionFile = (0, path_1.join)(sessionsDir, `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}.json`);
-    const existingSession = await (0, promises_1.readFile)(sessionFile)
-        .then((data) => JSON.parse(data.toString("utf-8")))
-        .catch(() => null);
-    if (existingSession) {
-        logger.debug(`Reading session from local copy in ${sessionFile}`);
-        return existingSession;
-    }
-    const session = await (0, session_api_1.getRecordedSession)(client, sessionId);
+    const sessionFile = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions", `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}.json`);
+    const session = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: sessionFile,
+        dataDescription: "session",
+        downloadJson: () => (0, session_api_1.getRecordedSession)(client, sessionId),
+    });
     if (!session) {
         logger.error("Error: Could not retrieve session. Is the API token correct?");
         process.exit(1);
     }
-    await (0, promises_1.writeFile)(sessionFile, JSON.stringify(session, null, 2));
-    logger.debug(`Wrote session to ${sessionFile}`);
     return session;
 };
 exports.getOrFetchRecordedSession = getOrFetchRecordedSession;
 const getOrFetchRecordedSessionData = async (client, sessionId) => {
     const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
-    const sessionsDir = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions");
-    await (0, promises_1.mkdir)(sessionsDir, { recursive: true });
-    const sessionDataFile = (0, path_1.join)(sessionsDir, `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}_data.json`);
-    const existingSessionData = await (0, promises_1.readFile)(sessionDataFile)
-        .then((data) => JSON.parse(data.toString("utf-8")))
-        .catch(() => null);
-    if (existingSessionData) {
-        logger.debug(`Reading session data from local copy in ${sessionDataFile}`);
-        return existingSessionData;
-    }
-    const sessionData = await (0, session_api_1.getRecordedSessionData)(client, sessionId);
+    const sessionFile = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions", `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}_data.json`);
+    const sessionData = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: sessionFile,
+        dataDescription: "session data",
+        downloadJson: () => (0, session_api_1.getRecordedSessionData)(client, sessionId),
+    });
     if (!sessionData) {
         logger.error("Error: Could not retrieve session data. This may be an invalid session");
         process.exit(1);
     }
-    await (0, promises_1.writeFile)(sessionDataFile, JSON.stringify(sessionData, null, 2));
-    logger.debug(`Wrote session data to ${sessionDataFile}`);
     return sessionData;
 };
 exports.getOrFetchRecordedSessionData = getOrFetchRecordedSessionData;

package/dist/main.js

@@ -28,7 +28,7 @@ const main = async () => {
     (0, logger_utils_1.initLogger)();
     const meticulousVersion = await (0, version_utils_1.getMeticulousVersion)();
     (0, sentry_utils_1.initSentry)(meticulousVersion);
-    const promise = yargs_1.default
+    yargs_1.default
         .scriptName("meticulous")
         .usage(`$0 <command>
 
@@ -63,12 +63,6 @@ const main = async () => {
         (argv) => handleDataDir(argv.dataDir),
         (argv) => (0, sentry_utils_1.setOptions)(argv),
     ]).argv;
-    if (promise instanceof Promise) {
-        promise.catch((error) => {
-            console.error(error);
-            process.exit(1);
-        });
-    }
 };
 exports.main = main;
 (0, exports.main)();

package/dist/parallel-tests/__tests__/merge-test-results.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/parallel-tests/__tests__/merge-test-results.spec.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const merge_test_results_1 = require("../merge-test-results");
+describe("mergeResults", () => {
+    it("keeps the result as a failure when all retried screenshots are 'identical'", () => {
+        const currentResult = testResult("fail", [diff(0), noDiff(1)]);
+        const comparisonToHeadReplay = testResult("pass", [noDiff(0), noDiff(1)]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("fail", [diff(0), noDiff(1)]));
+    });
+    it("ignores diffs to screenshots which originally passed", () => {
+        const currentResult = testResult("pass", [noDiff(0), noDiff(1)]);
+        const comparisonToHeadReplay = testResult("fail", [noDiff(0), diff(1)]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("pass", [noDiff(0), noDiff(1)]));
+    });
+    it("marks screenshots as flakes if the screenshot comparison originally failed, but the second retry gives a different screenshot again", () => {
+        const currentResult = testResult("fail", [noDiff(0), diff(1), diff(2)]);
+        const comparisonToHeadReplay = testResult("fail", [
+            noDiff(0),
+            diff(1),
+            noDiff(2),
+        ]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("fail", [noDiff(0), flake(1, diff(), [diff()]), diff(2)]));
+    });
+    it("marks overall test as a flake if there are only flakey screenshots and no failed screenshots", () => {
+        const currentResult = testResult("fail", [noDiff(0), diff(1), diff(2)]);
+        const comparisonToHeadReplay = testResult("fail", [
+            noDiff(0),
+            diff(1),
+            diff(2),
+        ]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("flake", [
+            noDiff(0),
+            flake(1, diff(), [diff()]),
+            flake(2, diff(), [diff()]),
+        ]));
+    });
+    it("adds to diffsToHeadScreenshotOnRetries for existing flakes", () => {
+        const currentResult = testResult("fail", [
+            diff(0),
+            flake(1, diff(), [missingHead()]),
+            flake(2, differentSize(), [diff()]),
+            flake(3, missingBase(), [diff(), diff()]),
+        ]);
+        const comparisonToHeadReplay = testResult("fail", [
+            noDiff(0),
+            diff(1),
+            differentSize(2),
+            missingHead(3),
+        ]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("fail", [
+            diff(0),
+            flake(1, diff(), [missingHead(), diff()]),
+            flake(2, differentSize(), [diff(), differentSize()]),
+            flake(3, missingBase(), [diff(), diff(), missingHead()]),
+        ]));
+    });
+    it("keeps a missing-head as is, if there is no corresponding retry screenshot", () => {
+        const currentResult = testResult("fail", [missingHead(0)]);
+        const comparisonToHeadReplay = testResult("pass", []);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("fail", [missingHead(0)]));
+    });
+    it("adds to diffsToHeadScreenshotOnRetries for a flakey missing-head, if there is no corresponding retry screenshot", () => {
+        const currentResult = testResult("fail", [
+            flake(0, missingHead(), [missingBase()]),
+            diff(1),
+        ]);
+        const comparisonToHeadReplay = testResult("pass", [noDiff(1)]);
+        const mergedResult = (0, merge_test_results_1.mergeResults)({
+            currentResult,
+            comparisonToHeadReplay,
+        });
+        expect(mergedResult).toEqual(testResult("fail", [
+            flake(0, missingHead(), [missingBase(), missingBaseAndHead()]),
+            diff(1),
+        ]));
+    });
+});
+const id = (eventNumber = 0) => ({
+    type: "after-event",
+    eventNumber,
+});
+const testResult = (result, screenshotDiffResults) => {
+    return {
+        sessionId: "mock-session-id",
+        headReplayId: "mock-head-replay-id",
+        result,
+        screenshotDiffResults,
+    };
+};
+const diff = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "diff",
+        headScreenshotFile: "mock-head-file",
+        baseScreenshotFile: "mock-base-file",
+        width: 1920,
+        height: 1080,
+        mismatchFraction: 0.01,
+        mismatchPixels: 1000,
+    };
+};
+const noDiff = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "no-diff",
+        headScreenshotFile: "mock-head-file",
+        baseScreenshotFile: "mock-base-file",
+        width: 1920,
+        height: 1080,
+        mismatchFraction: 0.01,
+        mismatchPixels: 1000,
+    };
+};
+const flake = (eventNumber, diffToBaseScreenshot, diffsToHeadScreenshotOnRetries) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "flake",
+        diffToBaseScreenshot: asSingleTryDiff(diffToBaseScreenshot),
+        diffsToHeadScreenshotOnRetries: diffsToHeadScreenshotOnRetries.map(asRetryDiff),
+    };
+};
+const missingBase = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "missing-base",
+        headScreenshotFile: "mock-head-file",
+    };
+};
+const missingHead = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "missing-head",
+        baseScreenshotFile: "mock-base-file",
+    };
+};
+const missingBaseAndHead = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "missing-base-and-head",
+    };
+};
+const differentSize = (eventNumber) => {
+    return {
+        identifier: id(eventNumber),
+        outcome: "different-size",
+        baseScreenshotFile: "mock-base-file",
+        headScreenshotFile: "mock-head-file",
+        baseWidth: 1920,
+        baseHeight: 1080,
+        headWidth: 10,
+        headHeight: 5,
+    };
+};
+const asSingleTryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
+...rest }) => {
+    if (rest.outcome === "flake") {
+        throw new Error("Must not be a diff with a flake outcome");
+    }
+    return rest;
+};
+const asRetryDiff = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
+...rest }) => {
+    if (rest.outcome === "flake") {
+        throw new Error("Must not be a diff with a flake outcome");
+    }
+    return rest;
+};

package/dist/parallel-tests/merge-test-results.d.ts

@@ -0,0 +1,10 @@
+import { DetailedTestCaseResult } from "../config/config.types";
+export interface ResultsToMerge {
+    currentResult: DetailedTestCaseResult;
+    /**
+     * The result of replaying the session once more against the head commit and
+     * comparing the screenshots to those taken on the first replay against the head commit.
+     */
+    comparisonToHeadReplay: DetailedTestCaseResult;
+}
+export declare const mergeResults: ({ currentResult, comparisonToHeadReplay, }: ResultsToMerge) => DetailedTestCaseResult;

package/dist/parallel-tests/merge-test-results.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mergeResults = void 0;
+const utils_1 = require("@sentry/utils");
+const mergeResults = ({ currentResult, comparisonToHeadReplay, }) => {
+    // If any of the screenshots diffs in comparisonToHeadReplay show a diff against one
+    // of the screenshots that orignally failed in currentResult then we have a flake
+    // on our hands
+    const retryDiffById = new Map();
+    comparisonToHeadReplay.screenshotDiffResults.forEach((result) => {
+        const hash = hashScreenshotIdentifier(result.identifier);
+        if (retryDiffById.has(hash)) {
+            throw new Error(`Received two screenshots for the same identifier '${hash}'. Screenshots should be unique.`);
+        }
+        retryDiffById.set(hash, result);
+    });
+    const newScreenshotDiffResults = currentResult.screenshotDiffResults.map((currentDiff) => {
+        if (currentDiff.outcome === "no-diff") {
+            return currentDiff;
+        }
+        const hash = hashScreenshotIdentifier(currentDiff.identifier);
+        const diffWhenRetrying = retryDiffById.get(hash);
+        // diffWhenRetrying is null in the case that there is a base screenshot for the base replay,
+        // but the first replay on head did not generate a head screenshot (got 'missing-head'),
+        // and the replay of that did not generate a head screenshot either (if the first replay on head
+        // did generate a screenshot, then diffWhenRetrying.outcome would be 'missing-head' instead)
+        if (diffWhenRetrying == null) {
+            const diffToBaseScreenshotOutcome = currentDiff.outcome === "flake"
+                ? currentDiff.diffToBaseScreenshot.outcome
+                : currentDiff.outcome;
+            if (diffToBaseScreenshotOutcome !== "missing-head") {
+                throw new Error(`Expected to find a screenshot comparison for ${hash}, but none was found. The screenshot must
+            have existed in the orginal replay, since the orginal comparison outcome was not '${diffToBaseScreenshotOutcome}'.`);
+            }
+            if (currentDiff.outcome === "flake") {
+                // Original comparison had missing-head, so we re-ran to check if it was flakey. In this case
+                // we got the same result.
+                return {
+                    ...currentDiff,
+                    diffsToHeadScreenshotOnRetries: [
+                        ...currentDiff.diffsToHeadScreenshotOnRetries,
+                        { outcome: "missing-base-and-head" },
+                    ],
+                };
+            }
+            return currentDiff; // no difference, both screenshots were missing
+        }
+        if (diffWhenRetrying.outcome === "flake") {
+            throw new Error("Expected diffs when retrying and comparing to the original head screenshot to be first-try diffs, but got a flake.");
+        }
+        if (currentDiff.outcome === "flake") {
+            return {
+                ...currentDiff,
+                diffsToHeadScreenshotOnRetries: [
+                    ...currentDiff.diffsToHeadScreenshotOnRetries,
+                    withoutIdentifier(diffWhenRetrying),
+                ],
+            };
+        }
+        else if ((diffWhenRetrying === null || diffWhenRetrying === void 0 ? void 0 : diffWhenRetrying.outcome) === "no-diff") {
+            return currentDiff;
+        }
+        else {
+            return {
+                identifier: currentDiff.identifier,
+                outcome: "flake",
+                diffToBaseScreenshot: withoutIdentifier(currentDiff),
+                diffsToHeadScreenshotOnRetries: [withoutIdentifier(diffWhenRetrying)],
+            };
+        }
+    });
+    const noLongerHasFailures = newScreenshotDiffResults.every(({ outcome }) => outcome === "flake" || outcome === "no-diff");
+    return {
+        ...currentResult,
+        result: currentResult.result === "fail" && noLongerHasFailures
+            ? "flake"
+            : currentResult.result,
+        screenshotDiffResults: newScreenshotDiffResults,
+    };
+};
+exports.mergeResults = mergeResults;
+const withoutIdentifier = ({ identifier, // eslint-disable-line @typescript-eslint/no-unused-vars
+...rest }) => rest;
+const hashScreenshotIdentifier = (identifier) => {
+    if (identifier.type === "end-state") {
+        return "end-state";
+    }
+    else if (identifier.type === "after-event") {
+        return `after-event-${identifier.eventNumber}`;
+    }
+    else {
+        unknownScreenshotIdentifierType(identifier);
+        // The identifier is probably from a newer version of the bundle script
+        // and we're on an old version of the CLI. Our best bet is to stringify it
+        // and use that as a hash.
+        return JSON.stringify(identifier);
+    }
+};
+const unknownScreenshotIdentifierType = (identifier) => {
+    utils_1.logger.error(`Unknown type of screenshot identifier: ${JSON.stringify(identifier)}`);
+};

package/dist/parallel-tests/parallel-tests.handler.d.ts

@@ -17,6 +17,7 @@ export interface RunAllTestsInParallelOptions {
     parallelTasks: number | null;
     deflake: boolean;
     replayEventsDependencies: ReplayEventsDependencies;
+    maxRetriesOnFailure: number;
     onTestFinished?: (progress: TestRunProgress, resultsSoFar: DetailedTestCaseResult[]) => Promise<void>;
 }
 /** Handler for running Meticulous tests in parallel using child processes */