@alwaysmeticulous/cli 2.19.2 → 2.20.0

package/dist/api/replay.api.d.ts

@@ -1,6 +1,6 @@
 import { Replay } from "@alwaysmeticulous/common";
 import { AxiosInstance } from "axios";
-export declare const getReplay: (client: AxiosInstance, replayId: string) => Promise<any>;
+export declare const getReplay: (client: AxiosInstance, replayId: string) => Promise<Replay>;
 export declare const createReplay: (options: {
     client: AxiosInstance;
     commitSha: string;

package/dist/api/session.api.d.ts

@@ -1,5 +1,6 @@
+import { SessionData } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
 export declare const getRecordedSession: (client: AxiosInstance, sessionId: string) => Promise<any>;
-export declare const getRecordedSessionData: (client: AxiosInstance, sessionId: string) => Promise<any>;
+export declare const getRecordedSessionData: (client: AxiosInstance, sessionId: string) => Promise<SessionData>;
 export declare const getRecordingCommandId: (client: AxiosInstance) => Promise<string>;
 export declare const postSessionIdNotification: (client: AxiosInstance, sessionId: string, recordingCommandId: string) => Promise<void>;

package/dist/commands/replay/replay.command.d.ts

@@ -10,13 +10,20 @@ export interface ReplayOptions extends AdditionalReplayOptions {
     generatedBy: GeneratedBy;
     testRunId: string | null;
     replayEventsDependencies: ReplayEventsDependencies;
+    suppressScreenshotDiffLogging: boolean;
 }
 export interface ReplayResult {
     replay: Replay;
-    screenshotDiffResults: ScreenshotDiffResult[] | null;
+    /**
+     * Empty if screenshottingOptions.enabled was false.
+     */
+    screenshotDiffResults: ScreenshotDiffResult[];
+    /**
+     * Returned as { hasDiffs: false } if screenshottingOptions.enabled was false.
+     */
     screenshotDiffsSummary: ScreenshotDiffsSummary;
 }
-export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }: ReplayOptions) => Promise<ReplayResult>;
+export declare const replayCommandHandler: ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseSimulationId: baseReplayId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, }: ReplayOptions) => Promise<ReplayResult>;
 export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, Omit<ReplayExecutionOptions, "maxDurationMs" | "maxEventCount">, AdditionalReplayOptions {
     screenshot: boolean;
     appUrl: string | null | undefined;
@@ -25,12 +32,12 @@ export interface RawReplayCommandHandlerOptions extends ScreenshotDiffOptions, O
     maxDurationMs: number | null | undefined;
     maxEventCount: number | null | undefined;
     storyboard: boolean;
+    save: boolean | null | undefined;
 }
 interface AdditionalReplayOptions {
     apiToken: string | null | undefined;
     commitSha: string | null | undefined;
     sessionId: string;
-    save: boolean | null | undefined;
     baseSimulationId: string | null | undefined;
     cookiesFile: string | null | undefined;
 }

package/dist/commands/replay/replay.command.js

@@ -34,6 +34,7 @@ const Sentry = __importStar(require("@sentry/node"));
 const loglevel_1 = __importDefault(require("loglevel"));
 const luxon_1 = require("luxon");
 const client_1 = require("../../api/client");
+const replay_diff_api_1 = require("../../api/replay-diff.api");
 const replay_api_1 = require("../../api/replay.api");
 const upload_1 = require("../../api/upload");
 const archive_1 = require("../../archive/archive");
@@ -46,8 +47,8 @@ const sessions_1 = require("../../local-data/sessions");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
 const config_utils_1 = require("../../utils/config.utils");
 const version_utils_1 = require("../../utils/version.utils");
-const compute_and_save_diff_1 = require("./utils/compute-and-save-diff");
-const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, save, baseSimulationId: baseReplayId_, cookiesFile, generatedBy, testRunId, replayEventsDependencies, }) => {
+const compute_diff_1 = require("./utils/compute-diff");
+const replayCommandHandler = async ({ replayTarget, executionOptions, screenshottingOptions, apiToken, sessionId, commitSha: commitSha_, baseSimulationId: baseReplayId, cookiesFile, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, }) => {
     const transaction = Sentry.startTransaction({
         name: "replay.command_handler",
         description: "Handle the replay command",
@@ -163,32 +164,30 @@ const replayCommandHandler = async ({ replayTarget, executionOptions, screenshot
         logger.info(`View simulation at: ${replayUrl}`);
         logger.info("=======");
         // 12. Diff against base replay screenshot if one is provided
-        const computeAndSaveDiffSpan = transaction.startChild({ op: "computeAndSaveDiff" });
-        const { screenshotDiffResults, screenshotDiffsSummary } = screenshottingOptions.enabled && baseReplayId_
-            ? await (0, compute_and_save_diff_1.computeAndSaveDiff)({
+        const computeDiffSpan = transaction.startChild({
+            op: "computeDiff",
+        });
+        const computeDiffsLogger = loglevel_1.default.getLogger(`METICULOUS_LOGGER_NAME/compute-diffs`);
+        if (suppressScreenshotDiffLogging) {
+            computeDiffsLogger.setLevel("ERROR", false);
+        }
+        else {
+            computeDiffsLogger.setLevel(logger.getLevel(), false);
+        }
+        const { screenshotDiffResults, screenshotDiffsSummary } = screenshottingOptions.enabled && baseReplayId
+            ? await (0, compute_diff_1.computeDiff)({
                 client,
-                baseReplayId: baseReplayId_ !== null && baseReplayId_ !== void 0 ? baseReplayId_ : "",
+                baseReplayId,
                 headReplayId: replay.id,
                 tempDir,
                 screenshottingOptions,
-                testRunId,
+                logger: computeDiffsLogger,
             })
             : {
-                screenshotDiffResults: null,
+                screenshotDiffResults: [],
                 screenshotDiffsSummary: { hasDiffs: false },
             };
-        computeAndSaveDiffSpan.finish();
-        // 13. Add test case to meticulous.json if --save option is passed
-        if (save) {
-            if (!screenshottingOptions.enabled) {
-                logger.error("Warning: saving a new test case without screenshot enabled.");
-            }
-            await (0, config_utils_1.addTestCase)({
-                title: `${sessionId} | ${replay.id}`,
-                sessionId,
-                baseReplayId: replay.id,
-            });
-        }
+        computeDiffSpan.finish();
         return { replay, screenshotDiffResults, screenshotDiffsSummary };
     }
     finally {
@@ -244,7 +243,7 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         }
         : { enabled: false };
     const replayEventsDependencies = await (0, replay_assets_1.loadReplayEventsDependencies)();
-    const { replay, screenshotDiffsSummary } = await (0, exports.replayCommandHandler)({
+    const { replay, screenshotDiffsSummary, screenshotDiffResults } = await (0, exports.replayCommandHandler)({
         replayTarget: (0, exports.getReplayTarget)({
             appUrl: appUrl !== null && appUrl !== void 0 ? appUrl : null,
             simulationIdForAssets: simulationIdForAssets !== null && simulationIdForAssets !== void 0 ? simulationIdForAssets : null,
@@ -256,12 +255,39 @@ const rawReplayCommandHandler = async ({ apiToken, commitSha, sessionId, appUrl,
         cookiesFile,
         sessionId,
         baseSimulationId,
-        save,
         generatedBy: generatedByOption,
         testRunId: null,
         replayEventsDependencies,
+        suppressScreenshotDiffLogging: false,
     });
+    // Add test case to meticulous.json if --save option is passed
+    if (save) {
+        if (!screenshottingOptions.enabled) {
+            const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+            logger.error("Warning: saving a new test case without screenshot enabled.");
+        }
+        await (0, config_utils_1.addTestCase)({
+            title: `${sessionId} | ${replay.id}`,
+            sessionId,
+            baseReplayId: replay.id,
+        });
+    }
     if (screenshotDiffsSummary.hasDiffs) {
+        const client = (0, client_1.createClient)({ apiToken });
+        if (baseSimulationId == null) {
+            throw new Error("baseSimulationId must have been defined if there are diffs, but was null-ish");
+        }
+        // Store the diff
+        await (0, replay_diff_api_1.createReplayDiff)({
+            client,
+            headReplayId: replay.id,
+            baseReplayId: baseSimulationId,
+            testRunId: null,
+            data: {
+                screenshotAssertionsOptions: screenshottingOptions,
+                screenshotDiffResults,
+            },
+        });
         process.exit(1);
     }
     return replay;

package/dist/commands/replay/utils/{compute-and-save-diff.d.ts → compute-diff.d.ts}

@@ -1,15 +1,16 @@
 import { ScreenshotAssertionsEnabledOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
+import log from "loglevel";
 import { ScreenshotDiffsSummary } from "../../screenshot-diff/screenshot-diff.command";
 export interface ComputeAndSaveDiffOptions {
     client: AxiosInstance;
-    testRunId: string | null;
     baseReplayId: string;
     headReplayId: string;
     tempDir: string;
     screenshottingOptions: ScreenshotAssertionsEnabledOptions;
+    logger: log.Logger;
 }
-export declare const computeAndSaveDiff: ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, testRunId, }: ComputeAndSaveDiffOptions) => Promise<{
+export declare const computeDiff: ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, logger, }: ComputeAndSaveDiffOptions) => Promise<{
     screenshotDiffResults: ScreenshotDiffResult[];
     screenshotDiffsSummary: ScreenshotDiffsSummary;
 }>;

package/dist/commands/replay/utils/{compute-and-save-diff.js → compute-diff.js}

@@ -1,16 +1,9 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.computeAndSaveDiff = void 0;
-const common_1 = require("@alwaysmeticulous/common");
-const loglevel_1 = __importDefault(require("loglevel"));
-const replay_diff_api_1 = require("../../../api/replay-diff.api");
+exports.computeDiff = void 0;
 const replays_1 = require("../../../local-data/replays");
 const screenshot_diff_command_1 = require("../../screenshot-diff/screenshot-diff.command");
-const computeAndSaveDiff = async ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, testRunId, }) => {
-    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+const computeDiff = async ({ client, baseReplayId, tempDir, headReplayId, screenshottingOptions, logger, }) => {
     logger.info(`Diffing screenshots against replay ${baseReplayId}`);
     await (0, replays_1.getOrFetchReplay)(client, baseReplayId);
     await (0, replays_1.getOrFetchReplayArchive)(client, baseReplayId);
@@ -23,24 +16,15 @@ const computeAndSaveDiff = async ({ client, baseReplayId, tempDir, headReplayId,
         baseScreenshotsDir: baseReplayScreenshotsDir,
         headScreenshotsDir: headReplayScreenshotsDir,
         diffOptions: screenshottingOptions.diffOptions,
+        logger,
     });
-    const replayDiff = await (0, replay_diff_api_1.createReplayDiff)({
-        client,
-        headReplayId,
-        baseReplayId,
-        testRunId,
-        data: {
-            screenshotAssertionsOptions: screenshottingOptions,
-            screenshotDiffResults,
-        },
-    });
-    logger.debug(replayDiff);
     const screenshotDiffsSummary = (0, screenshot_diff_command_1.summarizeDifferences)({
         baseReplayId,
         headReplayId,
         results: screenshotDiffResults,
         diffOptions: screenshottingOptions.diffOptions,
+        logger,
     });
     return { screenshotDiffResults, screenshotDiffsSummary };
 };
-exports.computeAndSaveDiff = computeAndSaveDiff;
+exports.computeDiff = computeDiff;

package/dist/commands/run-all-tests/run-all-tests.command.d.ts

@@ -107,6 +107,11 @@ export declare const runAllTestsCommand: import("yargs").CommandModule<unknown,
         readonly description: "Attempt to deflake failing tests";
         readonly default: false;
     };
+    readonly maxRetriesOnFailure: {
+        readonly number: true;
+        readonly description: "If set to a value greater than 0 then will re-run any replays that give a screenshot diff and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.";
+        readonly default: 0;
+    };
     readonly useCache: {
         readonly boolean: true;
         readonly description: "Use result cache";

package/dist/commands/run-all-tests/run-all-tests.command.js

@@ -7,7 +7,7 @@ const command_builder_1 = require("../../command-utils/command-builder");
 const common_options_1 = require("../../command-utils/common-options");
 const run_all_tests_1 = require("../../parallel-tests/run-all-tests");
 const commit_sha_utils_1 = require("../../utils/commit-sha.utils");
-const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, }) => {
+const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl, useAssetsSnapshottedInBaseSimulation, headless, devTools, bypassCSP, diffThreshold, diffPixelThreshold, padTime, shiftTime, networkStubbing, githubSummary, parallelize, parallelTasks: parrelelTasks_, deflake, maxRetriesOnFailure, useCache, testsFile, disableRemoteFonts, noSandbox, skipPauses, moveBeforeClick, maxDurationMs, maxEventCount, storyboard, }) => {
     const executionOptions = {
         headless,
         devTools,
@@ -48,6 +48,7 @@ const handler = async ({ apiToken, commitSha: commitSha_, baseCommitSha, appUrl,
         useAssetsSnapshottedInBaseSimulation,
         parallelTasks: parrelelTasks !== null && parrelelTasks !== void 0 ? parrelelTasks : null,
         deflake,
+        maxRetriesOnFailure,
         cachedTestRunResults,
         githubSummary,
     });
@@ -100,6 +101,11 @@ exports.runAllTestsCommand = (0, command_builder_1.buildCommand)("run-all-tests"
         description: "Attempt to deflake failing tests",
         default: false,
     },
+    maxRetriesOnFailure: {
+        number: true,
+        description: "If set to a value greater than 0 then will re-run any replays that give a screenshot diff and mark them as a flake if the screenshot generated on one of the retryed replays differs from that in the first replay.",
+        default: 0,
+    },
     useCache: {
         boolean: true,
         description: "Use result cache",

package/dist/commands/screenshot-diff/screenshot-diff.command.d.ts

@@ -1,19 +1,22 @@
 /// <reference types="yargs" />
 import { ScreenshotDiffOptions, ScreenshotDiffResult } from "@alwaysmeticulous/api";
 import { AxiosInstance } from "axios";
-export declare const diffScreenshots: ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, }: {
+import log from "loglevel";
+export declare const diffScreenshots: ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, logger, }: {
     client: AxiosInstance;
     baseReplayId: string;
     headReplayId: string;
     baseScreenshotsDir: string;
     headScreenshotsDir: string;
     diffOptions: ScreenshotDiffOptions;
+    logger: log.Logger;
 }) => Promise<ScreenshotDiffResult[]>;
-export declare const summarizeDifferences: ({ baseReplayId, headReplayId, results, diffOptions, }: {
+export declare const summarizeDifferences: ({ baseReplayId, headReplayId, results, diffOptions, logger, }: {
     baseReplayId: string;
     headReplayId: string;
     results: ScreenshotDiffResult[];
     diffOptions: ScreenshotDiffOptions;
+    logger: log.Logger;
 }) => ScreenshotDiffsSummary;
 export declare type ScreenshotDiffsSummary = HasDiffsScreenshotDiffsResult | NoDiffsScreenshotDiffsResult;
 export interface HasDiffsScreenshotDiffsResult {

package/dist/commands/screenshot-diff/screenshot-diff.command.js

@@ -15,8 +15,7 @@ const diff_utils_1 = require("../../image/diff.utils");
 const io_utils_1 = require("../../image/io.utils");
 const replays_1 = require("../../local-data/replays");
 const screenshot_diffs_1 = require("../../local-data/screenshot-diffs");
-const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, }) => {
-    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreenshotsDir, baseScreenshotsDir, diffOptions, logger, }) => {
     const { diffThreshold, diffPixelThreshold } = diffOptions;
     const baseReplayScreenshots = await (0, replays_1.getScreenshotFiles)(baseScreenshotsDir);
     const headReplayScreenshots = await (0, replays_1.getScreenshotFiles)(headScreenshotsDir);
@@ -99,8 +98,7 @@ const diffScreenshots = async ({ client, headReplayId, baseReplayId, headScreens
     return [...missingHeadImagesResults, ...headDiffResults];
 };
 exports.diffScreenshots = diffScreenshots;
-const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions, }) => {
-    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions, logger, }) => {
     const missingHeadImagesResults = results.flatMap((result) => result.outcome === "missing-head" ? [result] : []);
     if (missingHeadImagesResults.length) {
         const message = `Head replay is missing screenshots: ${missingHeadImagesResults
@@ -121,17 +119,19 @@ const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions
             .sort()}`;
         logger.info(message);
     }
-    results.forEach((result) => {
+    const diffs = results.flatMap((result) => {
         const { outcome } = result;
         if (outcome === "different-size") {
             const message = `Screenshots ${(0, path_1.basename)(result.headScreenshotFile)} have different sizes => FAIL!`;
             logger.info(message);
-            return {
-                hasDiffs: true,
-                summaryMessage: message,
-                baseReplayId,
-                headReplayId,
-            };
+            return [
+                {
+                    hasDiffs: true,
+                    summaryMessage: message,
+                    baseReplayId,
+                    headReplayId,
+                },
+            ];
         }
         if (outcome === "diff" || outcome === "no-diff") {
             const mismatch = (result.mismatchFraction * 100).toFixed(3);
@@ -139,16 +139,19 @@ const summarizeDifferences = ({ baseReplayId, headReplayId, results, diffOptions
             const message = `${mismatch}% pixel mismatch for screenshot ${(0, path_1.basename)(result.headScreenshotFile)} (threshold is ${threshold}%) => ${outcome === "no-diff" ? "PASS" : "FAIL!"}`;
             logger.info(message);
             if (outcome === "diff") {
-                return {
-                    hasDiffs: true,
-                    summaryMessage: message,
-                    baseReplayId,
-                    headReplayId,
-                };
+                return [
+                    {
+                        hasDiffs: true,
+                        summaryMessage: message,
+                        baseReplayId,
+                        headReplayId,
+                    },
+                ];
             }
         }
+        return [];
     });
-    return { hasDiffs: false };
+    return diffs.length > 0 ? diffs[0] : { hasDiffs: false };
 };
 exports.summarizeDifferences = summarizeDifferences;
 const getScreenshotIdentifier = (filename) => {
@@ -190,6 +193,7 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
         baseScreenshotsDir,
         headScreenshotsDir,
         diffOptions,
+        logger,
     });
     logger.debug(results);
     const diffSummary = (0, exports.summarizeDifferences)({
@@ -197,6 +201,7 @@ const handler = async ({ apiToken, baseSimulationId: baseReplayId, headSimulatio
         headReplayId,
         results,
         diffOptions,
+        logger,
     });
     if (diffSummary.hasDiffs) {
         process.exit(1);

package/dist/config/config.types.d.ts

@@ -4,7 +4,12 @@ export interface MeticulousCliConfig {
 }
 export interface TestCaseResult extends TestCase {
     headReplayId: string;
-    result: "pass" | "fail";
+    /**
+     * A test case is marked as a flake if there were screenshot comparison failures,
+     * but for every one of those failures regenerating the screenshot on head sometimes gave
+     * a different screenshot to the original screenshot taken on head.
+     */
+    result: "pass" | "fail" | "flake";
 }
 export interface DetailedTestCaseResult extends TestCaseResult {
     screenshotDiffResults: ScreenshotDiffResult[];

package/dist/deflake-tests/deflake-tests.handler.d.ts

@@ -15,6 +15,7 @@ interface HandleReplayOptions {
     generatedBy: GeneratedBy;
     testRunId: string | null;
     replayEventsDependencies: ReplayEventsDependencies;
+    suppressScreenshotDiffLogging: boolean;
 }
 export declare const deflakeReplayCommandHandler: ({ deflake, ...otherOptions }: DeflakeReplayCommandHandlerOptions) => Promise<DetailedTestCaseResult>;
 export {};

package/dist/deflake-tests/deflake-tests.handler.js

@@ -7,7 +7,7 @@ exports.deflakeReplayCommandHandler = void 0;
 const common_1 = require("@alwaysmeticulous/common");
 const loglevel_1 = __importDefault(require("loglevel"));
 const replay_command_1 = require("../commands/replay/replay.command");
-const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, }) => {
+const handleReplay = async ({ testCase, replayTarget, executionOptions, screenshottingOptions, apiToken, commitSha, generatedBy, testRunId, replayEventsDependencies, suppressScreenshotDiffLogging, }) => {
     var _a, _b;
     const { replay, screenshotDiffResults, screenshotDiffsSummary } = await (0, replay_command_1.replayCommandHandler)({
         replayTarget,
@@ -17,15 +17,12 @@ const handleReplay = async ({ testCase, replayTarget, executionOptions, screensh
         commitSha,
         sessionId: testCase.sessionId,
         baseSimulationId: testCase.baseReplayId,
-        save: false,
         cookiesFile: null,
         generatedBy,
         testRunId,
         replayEventsDependencies,
+        suppressScreenshotDiffLogging,
     });
-    if (screenshotDiffResults == null) {
-        throw new Error(`replayCommandHandler returned a null screenshotDiffResults, but was called with screenshottingOptions.enabled = true`);
-    }
     return {
         ...testCase,
         headReplayId: replay.id,

package/dist/local-data/local-data.utils.d.ts

@@ -1 +1,20 @@
 export declare const sanitizeFilename: (filename: string) => string;
+declare type ReleaseLock = () => Promise<void>;
+export interface LoadOrDownloadJsonFileOptions<T> {
+    filePath: string;
+    downloadJson: () => Promise<T | null>;
+    /**
+     * For debug messages e.g. 'session' or 'session data'
+     */
+    dataDescription: string;
+}
+/**
+ * Returns the JSON.parse'd contents of the file at the given path. If the file
+ * doesn't exist yet then it downloads the object, writes it to the file, and returns it.
+ *
+ * Handles concurrent processes trying to download to the same file at the same time.
+ */
+export declare const getOrDownloadJsonFile: <T>({ filePath, downloadJson, dataDescription, }: LoadOrDownloadJsonFileOptions<T>) => Promise<T | null>;
+export declare const fileExists: (filePath: string) => Promise<boolean>;
+export declare const waitToAcquireLockOnDirectory: (directoryPath: string) => Promise<ReleaseLock>;
+export {};

package/dist/local-data/local-data.utils.js

@@ -1,7 +1,101 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeFilename = void 0;
+exports.waitToAcquireLockOnDirectory = exports.fileExists = exports.getOrDownloadJsonFile = exports.sanitizeFilename = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const luxon_1 = require("luxon");
+const proper_lockfile_1 = require("proper-lockfile");
 const sanitizeFilename = (filename) => {
     return filename.replace(/[^a-zA-Z0-9]/g, "_");
 };
 exports.sanitizeFilename = sanitizeFilename;
+/**
+ * Returns the JSON.parse'd contents of the file at the given path. If the file
+ * doesn't exist yet then it downloads the object, writes it to the file, and returns it.
+ *
+ * Handles concurrent processes trying to download to the same file at the same time.
+ */
+const getOrDownloadJsonFile = async ({ filePath, downloadJson, dataDescription, }) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    await (0, promises_1.mkdir)((0, path_1.dirname)(filePath), { recursive: true });
+    // We create a lock file so that if multiple processes try downloading at the same
+    // time they don't interfere with each other. The second process to run will
+    // wait for the first process to complete, and then return straight away because
+    // it'll notice the file already exists.
+    const releaseLock = await waitToAcquireLockOnFile(filePath);
+    try {
+        const existingData = await (0, promises_1.readFile)(filePath)
+            .then((data) => JSON.parse(data.toString("utf-8")))
+            .catch(() => null);
+        if (existingData) {
+            logger.debug(`Reading ${dataDescription} from local copy in ${filePath}`);
+            return existingData;
+        }
+        const downloadedData = await downloadJson();
+        if (downloadedData) {
+            await (0, promises_1.writeFile)(filePath, JSON.stringify(downloadedData, null, 2));
+            logger.debug(`Wrote ${dataDescription} to ${filePath}`);
+        }
+        return downloadedData;
+    }
+    finally {
+        await releaseLock();
+    }
+};
+exports.getOrDownloadJsonFile = getOrDownloadJsonFile;
+const waitToAcquireLockOnFile = async (filePath) => {
+    // In many cases the file doesn't exist yet, and can't exist yet (need to download the data, and creating an
+    // empty file beforehand is risky if the process crashes, and a second process tries reading the empty file).
+    // However proper-lockfile requires us to pass a file or directory as the first arg. This path is just used
+    // to detect if the same process tries taking out multiple locks on the same file. It just needs to be calculated
+    // as something that's unique to the file, and gives the same path for a given file everytime. So we create our
+    // own lock-target directory for this purpose (directory not file since mkdir is guaranteed to be synchronous).
+    // The path needs to actually exist, since proper-lockfile resolves symlinks on it.
+    const lockDirectory = `${filePath}.lock-target`;
+    await (0, promises_1.mkdir)(lockDirectory, { recursive: true });
+    try {
+        const releaseLock = await (0, proper_lockfile_1.lock)(lockDirectory, {
+            retries: LOCK_RETRY_OPTIONS,
+            lockfilePath: `${filePath}.lock`,
+        });
+        return async () => {
+            // Clean up our directory _before_ releasing the lock
+            // Note: it may have already been deleted: if we're just coming out of
+            // a lock released by someone else then they will have already deleted
+            // the directory
+            await (0, promises_1.rm)(lockDirectory, { recursive: true, force: true });
+            await releaseLock();
+        };
+    }
+    catch (err) {
+        await (0, promises_1.rm)(lockDirectory, { recursive: true, force: true });
+        throw err;
+    }
+};
+const fileExists = (filePath) => (0, promises_1.access)(filePath)
+    .then(() => true)
+    .catch(() => false);
+exports.fileExists = fileExists;
+const waitToAcquireLockOnDirectory = (directoryPath) => {
+    return (0, proper_lockfile_1.lock)(directoryPath, {
+        retries: LOCK_RETRY_OPTIONS,
+        lockfilePath: (0, path_1.join)(directoryPath, "dir.lock"),
+    });
+};
+exports.waitToAcquireLockOnDirectory = waitToAcquireLockOnDirectory;
+const LOCK_RETRY_OPTIONS = {
+    // We want to keep on retrying till we get the maxRetryTime, so we set retries, which is a maximum, to a high value
+    retries: 1000,
+    factor: 1.05,
+    minTimeout: 500,
+    maxTimeout: 2000,
+    // Wait a maximum of 120s for the other process to finish downloading and/or extracting
+    maxRetryTime: luxon_1.Duration.fromObject({ minutes: 2 }).as("milliseconds"),
+    // Randomize so processes are less likely to clash on their retries
+    randomize: true,
+};

package/dist/local-data/replays.d.ts

@@ -1,5 +1,6 @@
+import { Replay } from "@alwaysmeticulous/common";
 import { AxiosInstance } from "axios";
-export declare const getOrFetchReplay: (client: AxiosInstance, replayId: string) => Promise<any>;
+export declare const getOrFetchReplay: (client: AxiosInstance, replayId: string) => Promise<Replay>;
 export declare const getOrFetchReplayArchive: (client: AxiosInstance, replayId: string) => Promise<void>;
 export declare const getScreenshotFiles: (screenshotsDirPath: string) => Promise<string[]>;
 export declare const getSnapshottedAssetsDir: (replayId: string) => string;