@alwaysmeticulous/downloading-helpers 2.40.4

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2022, Meticulous Contributors
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,3 @@
+# Meticulous Download Helpers
+
+Helper utilities for downloading files & scripts required to execute replays

package/dist/config/snippets.d.ts

@@ -0,0 +1,4 @@
+export declare const getSnippetsBaseUrl: () => string;
+export declare class ConfigurationError extends Error {
+    constructor(message: string);
+}

package/dist/config/snippets.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConfigurationError = exports.getSnippetsBaseUrl = void 0;
+const common_1 = require("@alwaysmeticulous/common");
+const getSnippetsBaseUrl = () => {
+    const baseUrl = process.env["METICULOUS_SNIPPETS_BASE_URL"] || common_1.BASE_SNIPPETS_URL;
+    try {
+        return new URL(baseUrl).href;
+    }
+    catch (e) {
+        if (e instanceof TypeError) {
+            throw new ConfigurationError(`Invalid base snippets URL: ${baseUrl}`);
+        }
+        throw e;
+    }
+};
+exports.getSnippetsBaseUrl = getSnippetsBaseUrl;
+class ConfigurationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ConfigurationError";
+    }
+}
+exports.ConfigurationError = ConfigurationError;

package/dist/file-downloads/download-file.d.ts

@@ -0,0 +1 @@
+export declare const downloadFile: (fileUrl: string, path: string) => Promise<void>;

package/dist/file-downloads/download-file.js

@@ -0,0 +1,21 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadFile = void 0;
+const fs_1 = require("fs");
+const stream_1 = require("stream");
+const util_1 = require("util");
+const axios_1 = __importDefault(require("axios"));
+const promisifiedFinished = (0, util_1.promisify)(stream_1.finished);
+const downloadFile = async (fileUrl, path) => {
+    const writer = (0, fs_1.createWriteStream)(path);
+    return axios_1.default
+        .request({ method: "GET", url: fileUrl, responseType: "stream" })
+        .then(async (response) => {
+        response.data.pipe(writer);
+        return promisifiedFinished(writer);
+    });
+};
+exports.downloadFile = downloadFile;

package/dist/file-downloads/local-data.utils.d.ts

@@ -0,0 +1,20 @@
+export declare const sanitizeFilename: (filename: string) => string;
+type ReleaseLock = () => Promise<void>;
+export interface LoadOrDownloadJsonFileOptions<T> {
+    filePath: string;
+    downloadJson: () => Promise<T | null>;
+    /**
+     * For debug messages e.g. 'session' or 'session data'
+     */
+    dataDescription: string;
+}
+/**
+ * Returns the JSON.parse'd contents of the file at the given path. If the file
+ * doesn't exist yet then it downloads the object, writes it to the file, and returns it.
+ *
+ * Handles concurrent processes trying to download to the same file at the same time.
+ */
+export declare const getOrDownloadJsonFile: <T>({ filePath, downloadJson, dataDescription, }: LoadOrDownloadJsonFileOptions<T>) => Promise<T | null>;
+export declare const fileExists: (filePath: string) => Promise<boolean>;
+export declare const waitToAcquireLockOnDirectory: (directoryPath: string) => Promise<ReleaseLock>;
+export {};

package/dist/file-downloads/local-data.utils.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.waitToAcquireLockOnDirectory = exports.fileExists = exports.getOrDownloadJsonFile = exports.sanitizeFilename = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const luxon_1 = require("luxon");
+const proper_lockfile_1 = require("proper-lockfile");
+const sanitizeFilename = (filename) => {
+    return filename.replace(/[^a-zA-Z0-9]/g, "_");
+};
+exports.sanitizeFilename = sanitizeFilename;
+/**
+ * Returns the JSON.parse'd contents of the file at the given path. If the file
+ * doesn't exist yet then it downloads the object, writes it to the file, and returns it.
+ *
+ * Handles concurrent processes trying to download to the same file at the same time.
+ */
+const getOrDownloadJsonFile = async ({ filePath, downloadJson, dataDescription, }) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    await (0, promises_1.mkdir)((0, path_1.dirname)(filePath), { recursive: true });
+    // We create a lock file so that if multiple processes try downloading at the same
+    // time they don't interfere with each other. The second process to run will
+    // wait for the first process to complete, and then return straight away because
+    // it'll notice the file already exists.
+    const releaseLock = await waitToAcquireLockOnFile(filePath);
+    try {
+        const existingData = await (0, promises_1.readFile)(filePath)
+            .then((data) => JSON.parse(data.toString("utf-8")))
+            .catch(() => null);
+        if (existingData) {
+            logger.debug(`Reading ${dataDescription} from local copy in ${filePath}`);
+            return existingData;
+        }
+        const downloadedData = await downloadJson();
+        if (downloadedData) {
+            await (0, promises_1.writeFile)(filePath, JSON.stringify(downloadedData, null, 2));
+            logger.debug(`Wrote ${dataDescription} to ${filePath}`);
+        }
+        return downloadedData;
+    }
+    finally {
+        await releaseLock();
+    }
+};
+exports.getOrDownloadJsonFile = getOrDownloadJsonFile;
+const waitToAcquireLockOnFile = async (filePath) => {
+    // In many cases the file doesn't exist yet, and can't exist yet (need to download the data, and creating an
+    // empty file beforehand is risky if the process crashes, and a second process tries reading the empty file).
+    // However proper-lockfile requires us to pass a file or directory as the first arg. This path is just used
+    // to detect if the same process tries taking out multiple locks on the same file. It just needs to be calculated
+    // as something that's unique to the file, and gives the same path for a given file everytime. So we create our
+    // own lock-target directory for this purpose (directory not file since mkdir is guaranteed to be synchronous).
+    // The path needs to actually exist, since proper-lockfile resolves symlinks on it.
+    //
+    // Note: we don't delete the lock directory afterwards because doing so without creating race-conditions is tricky
+    const lockDirectory = `${filePath}.lock-target`;
+    await (0, promises_1.mkdir)(lockDirectory, { recursive: true });
+    const releaseLock = await (0, proper_lockfile_1.lock)(lockDirectory, {
+        retries: LOCK_RETRY_OPTIONS,
+        lockfilePath: `${filePath}.lock`,
+    });
+    return async () => {
+        await releaseLock();
+    };
+};
+const fileExists = (filePath) => (0, promises_1.access)(filePath)
+    .then(() => true)
+    .catch(() => false);
+exports.fileExists = fileExists;
+const waitToAcquireLockOnDirectory = (directoryPath) => {
+    return (0, proper_lockfile_1.lock)(directoryPath, {
+        retries: LOCK_RETRY_OPTIONS,
+        lockfilePath: (0, path_1.join)(directoryPath, "dir.lock"),
+    });
+};
+exports.waitToAcquireLockOnDirectory = waitToAcquireLockOnDirectory;
+const LOCK_RETRY_OPTIONS = {
+    // We want to keep on retrying till we get the maxRetryTime, so we set retries, which is a maximum, to a high value
+    retries: 1000,
+    factor: 1.05,
+    minTimeout: 500,
+    maxTimeout: 2000,
+    // Wait a maximum of 120s for the other process to finish downloading and/or extracting
+    maxRetryTime: luxon_1.Duration.fromObject({ minutes: 2 }).as("milliseconds"),
+    // Randomize so processes are less likely to clash on their retries
+    randomize: true,
+};

package/dist/file-downloads/sessions.d.ts

@@ -0,0 +1,10 @@
+import { SessionData } from "@alwaysmeticulous/api";
+import { AxiosInstance } from "axios";
+export declare const getOrFetchRecordedSession: (client: AxiosInstance, sessionId: string) => Promise<{
+    fileName: string;
+    data: any;
+}>;
+export declare const getOrFetchRecordedSessionData: (client: AxiosInstance, sessionId: string) => Promise<{
+    fileName: string;
+    data: SessionData;
+}>;

package/dist/file-downloads/sessions.js

@@ -0,0 +1,41 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getOrFetchRecordedSessionData = exports.getOrFetchRecordedSession = void 0;
+const path_1 = require("path");
+const client_1 = require("@alwaysmeticulous/client");
+const common_1 = require("@alwaysmeticulous/common");
+const loglevel_1 = __importDefault(require("loglevel"));
+const local_data_utils_1 = require("./local-data.utils");
+const getOrFetchRecordedSession = async (client, sessionId) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const sessionFile = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions", `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}.json`);
+    const session = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: sessionFile,
+        dataDescription: "session",
+        downloadJson: () => (0, client_1.getRecordedSession)(client, sessionId),
+    });
+    if (!session) {
+        logger.error("Error: Could not retrieve session. Is the API token correct?");
+        process.exit(1);
+    }
+    return { fileName: sessionFile, data: session };
+};
+exports.getOrFetchRecordedSession = getOrFetchRecordedSession;
+const getOrFetchRecordedSessionData = async (client, sessionId) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const sessionFile = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "sessions", `${(0, local_data_utils_1.sanitizeFilename)(sessionId)}_data.json`);
+    const sessionData = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: sessionFile,
+        dataDescription: "session data",
+        downloadJson: () => (0, client_1.getRecordedSessionData)(client, sessionId),
+    });
+    if (!sessionData) {
+        logger.error("Error: Could not retrieve session data. This may be an invalid session");
+        process.exit(1);
+    }
+    return { fileName: sessionFile, data: sessionData };
+};
+exports.getOrFetchRecordedSessionData = getOrFetchRecordedSessionData;

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { sanitizeFilename } from "./file-downloads/local-data.utils";
+export { getOrFetchReplay, getOrFetchReplayArchive } from "./scripts/replays";
+export { getOrFetchRecordedSession, getOrFetchRecordedSessionData, } from "./file-downloads/sessions";
+export { fetchAsset } from "./scripts/replay-assets";
+export { downloadFile } from "./file-downloads/download-file";
+export { getReplayDir } from "./scripts/replays";

package/dist/index.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getReplayDir = exports.downloadFile = exports.fetchAsset = exports.getOrFetchRecordedSessionData = exports.getOrFetchRecordedSession = exports.getOrFetchReplayArchive = exports.getOrFetchReplay = exports.sanitizeFilename = void 0;
+var local_data_utils_1 = require("./file-downloads/local-data.utils");
+Object.defineProperty(exports, "sanitizeFilename", { enumerable: true, get: function () { return local_data_utils_1.sanitizeFilename; } });
+var replays_1 = require("./scripts/replays");
+Object.defineProperty(exports, "getOrFetchReplay", { enumerable: true, get: function () { return replays_1.getOrFetchReplay; } });
+Object.defineProperty(exports, "getOrFetchReplayArchive", { enumerable: true, get: function () { return replays_1.getOrFetchReplayArchive; } });
+var sessions_1 = require("./file-downloads/sessions");
+Object.defineProperty(exports, "getOrFetchRecordedSession", { enumerable: true, get: function () { return sessions_1.getOrFetchRecordedSession; } });
+Object.defineProperty(exports, "getOrFetchRecordedSessionData", { enumerable: true, get: function () { return sessions_1.getOrFetchRecordedSessionData; } });
+var replay_assets_1 = require("./scripts/replay-assets");
+Object.defineProperty(exports, "fetchAsset", { enumerable: true, get: function () { return replay_assets_1.fetchAsset; } });
+var download_file_1 = require("./file-downloads/download-file");
+Object.defineProperty(exports, "downloadFile", { enumerable: true, get: function () { return download_file_1.downloadFile; } });
+var replays_2 = require("./scripts/replays");
+Object.defineProperty(exports, "getReplayDir", { enumerable: true, get: function () { return replays_2.getReplayDir; } });

package/dist/scripts/replay-assets.d.ts

@@ -0,0 +1 @@
+export declare const fetchAsset: (path: string) => Promise<string>;

package/dist/scripts/replay-assets.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchAsset = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const common_1 = require("@alwaysmeticulous/common");
+const axios_1 = __importDefault(require("axios"));
+const loglevel_1 = __importDefault(require("loglevel"));
+const snippets_1 = require("../config/snippets");
+const ASSETS_FOLDER_NAME = "assets";
+const ASSET_METADATA_FILE_NAME = "assets.json";
+const fetchAsset = async (path) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const fetchUrl = new URL(path, (0, snippets_1.getSnippetsBaseUrl)()).href;
+    const assetFileName = `${(0, path_1.basename)(new URL(fetchUrl).pathname, ".js")}.cjs`;
+    const assetMetadata = await loadAssetMetadata();
+    const etag = (await axios_1.default.head(fetchUrl)).headers["etag"] || "";
+    const entry = assetMetadata.assets.find((item) => item.fileName === assetFileName);
+    const filePath = (0, path_1.join)(await getOrCreateAssetsDir(), assetFileName);
+    if (entry && etag !== "" && etag === entry.etag) {
+        logger.debug(`${fetchUrl} already present`);
+        return filePath;
+    }
+    const contents = (await axios_1.default.get(fetchUrl)).data;
+    await (0, promises_1.writeFile)(filePath, contents);
+    if (entry) {
+        logger.debug(`${fetchUrl} updated`);
+        entry.etag = etag;
+    }
+    else {
+        logger.debug(`${fetchUrl} downloaded`);
+        assetMetadata.assets.push({ fileName: assetFileName, etag, fetchUrl });
+    }
+    await saveAssetMetadata(assetMetadata);
+    return filePath;
+};
+exports.fetchAsset = fetchAsset;
+const getOrCreateAssetsDir = async () => {
+    const assetsDir = (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), ASSETS_FOLDER_NAME);
+    await (0, promises_1.mkdir)(assetsDir, { recursive: true });
+    return assetsDir;
+};
+const loadAssetMetadata = async () => {
+    const assetsFile = (0, path_1.join)(await getOrCreateAssetsDir(), ASSET_METADATA_FILE_NAME);
+    const existingMetadata = await (0, promises_1.readFile)(assetsFile)
+        .then((data) => JSON.parse(data.toString("utf-8")))
+        .catch(() => null);
+    if (existingMetadata) {
+        return existingMetadata;
+    }
+    return { assets: [] };
+};
+const saveAssetMetadata = async (assetMetadata) => {
+    const assetsFile = (0, path_1.join)(await getOrCreateAssetsDir(), ASSET_METADATA_FILE_NAME);
+    await (0, promises_1.writeFile)(assetsFile, JSON.stringify(assetMetadata, null, 2));
+};

package/dist/scripts/replays.d.ts

@@ -0,0 +1,8 @@
+import { AxiosInstance } from "axios";
+export declare const getOrFetchReplay: (client: AxiosInstance, replayId: string) => Promise<{
+    fileName: string;
+}>;
+export declare const getOrFetchReplayArchive: (client: AxiosInstance, replayId: string) => Promise<{
+    fileName: string;
+}>;
+export declare const getReplayDir: (replayId: string) => string;

package/dist/scripts/replays.js

@@ -0,0 +1,63 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getReplayDir = exports.getOrFetchReplayArchive = exports.getOrFetchReplay = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const client_1 = require("@alwaysmeticulous/client");
+const common_1 = require("@alwaysmeticulous/common");
+const adm_zip_1 = __importDefault(require("adm-zip"));
+const loglevel_1 = __importDefault(require("loglevel"));
+const download_file_1 = require("../file-downloads/download-file");
+const local_data_utils_1 = require("../file-downloads/local-data.utils");
+const getOrFetchReplay = async (client, replayId) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const replayFile = (0, path_1.join)((0, exports.getReplayDir)(replayId), `${replayId}.json`);
+    const replay = await (0, local_data_utils_1.getOrDownloadJsonFile)({
+        filePath: replayFile,
+        dataDescription: "replay",
+        downloadJson: () => (0, client_1.getReplay)(client, replayId),
+    });
+    if (!replay) {
+        logger.error(`Error: Could not retrieve replay with id "${replayId}". Is the API token correct?`);
+        process.exit(1);
+    }
+    return { fileName: replayFile };
+};
+exports.getOrFetchReplay = getOrFetchReplay;
+const getOrFetchReplayArchive = async (client, replayId) => {
+    const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
+    const replayDir = (0, exports.getReplayDir)(replayId);
+    await (0, promises_1.mkdir)(replayDir, { recursive: true });
+    const releaseLock = await (0, local_data_utils_1.waitToAcquireLockOnDirectory)(replayDir);
+    try {
+        const replayArchiveFile = (0, path_1.join)(replayDir, `${replayId}.zip`);
+        const paramsFile = (0, path_1.join)(replayDir, "replayEventsParams.json");
+        // Check if "replayEventsParams.json" exists. If yes, we assume the replay
+        // zip archive has been downloaded and extracted.
+        if (await (0, local_data_utils_1.fileExists)(paramsFile)) {
+            logger.debug(`Replay archive already downloaded at ${replayDir}`);
+            return { fileName: replayDir };
+        }
+        const downloadUrlData = await (0, client_1.getReplayDownloadUrl)(client, replayId);
+        if (!downloadUrlData) {
+            logger.error("Error: Could not retrieve replay archive URL. This may be an invalid replay");
+            await releaseLock();
+            process.exit(1);
+        }
+        await (0, download_file_1.downloadFile)(downloadUrlData.dowloadUrl, replayArchiveFile);
+        const zipFile = new adm_zip_1.default(replayArchiveFile);
+        zipFile.extractAllTo(replayDir, /*overwrite=*/ true);
+        await (0, promises_1.rm)(replayArchiveFile);
+        logger.debug(`Extracted replay archive in ${replayDir}`);
+        return { fileName: replayDir };
+    }
+    finally {
+        await releaseLock();
+    }
+};
+exports.getOrFetchReplayArchive = getOrFetchReplayArchive;
+const getReplayDir = (replayId) => (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "replays", replayId);
+exports.getReplayDir = getReplayDir;

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@alwaysmeticulous/downloading-helpers",
+  "version": "2.40.4",
+  "description": "Helper utilities for downloading files & scripts required to execute replays",
+  "license": "ISC",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "clean": "rimraf dist tsconfig.tsbuildinfo",
+    "build": "tsc --build tsconfig.json",
+    "dev": "tsc --build tsconfig.json --watch",
+    "format": "prettier --write src",
+    "lint": "eslint src --ext=ts,tsx,js --cache",
+    "lint:commit": "eslint --cache $(git diff --relative --name-only --diff-filter=ACMRTUXB master | grep  -E \"(.js$|.ts$|.tsx$)\")",
+    "lint:fix": "eslint src --ext=ts,tsx,js --cache --fix",
+    "depcheck": "depcheck --ignore-patterns=dist"
+  },
+  "dependencies": {
+    "@alwaysmeticulous/api": "^2.40.4",
+    "@alwaysmeticulous/client": "^2.40.4",
+    "@alwaysmeticulous/common": "^2.40.4",
+    "adm-zip": "^0.5.9",
+    "axios": "^1.2.6",
+    "loglevel": "^1.8.0",
+    "luxon": "^3.2.1",
+    "proper-lockfile": "^4.1.2"
+  },
+  "devDependencies": {
+    "@types/adm-zip": "^0.5.0",
+    "@types/luxon": "^3.2.0",
+    "@types/proper-lockfile": "^4.1.2"
+  },
+  "author": {
+    "name": "The Meticulous Team",
+    "email": "eng@meticulous.ai",
+    "url": "https://meticulous.ai"
+  },
+  "engines": {
+    "node": ">= 12"
+  },
+  "homepage": "https://github.com/alwaysmeticulous/meticulous-sdk",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/alwaysmeticulous/meticulous-sdk.git",
+    "directory": "packages/cli"
+  },
+  "bugs": {
+    "url": "https://github.com/alwaysmeticulous/meticulous-sdk/issues"
+  },
+  "gitHead": "11acce96c03e23cbb556779a121710a8136d24db"
+}