@datatruck/cli 0.29.1 → 0.30.0

package/Action/BackupAction.js

@@ -153,7 +153,7 @@ class BackupAction {
                     .map((item) => [
                     (0, cli_1.renderResult)(item.error),
                     renderTitle(item, true),
-                    renderData(item, true),
+                    renderData(item, true, result),
                     (0, date_1.duration)(item.elapsed),
                     (0, cli_1.renderError)(item.error, options.verbose),
                 ]),

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @datatruck/cli
 
+## 0.30.0
+
+### Minor Changes
+
+- [`e28b12d`](https://github.com/swordev/datatruck/commit/e28b12d08c36844317e506552443825ab3333139) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `allowlist` option to the datatruck server
+
+### Patch Changes
+
+- [`2f63e67`](https://github.com/swordev/datatruck/commit/2f63e67ee532892fda3a9d06e46336a42834e5ed) Thanks [@juanrgm](https://github.com/juanrgm)! - Add download progress in datatruck repository
+
+- [`258e933`](https://github.com/swordev/datatruck/commit/258e93385d9b6f93a435a28a2b26e53ea1763755) Thanks [@juanrgm](https://github.com/juanrgm)! - Copy backups safely
+
+- [`82b4c67`](https://github.com/swordev/datatruck/commit/82b4c67c55eee4f40753b48eb91345e6d6789f72) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix exit event
+
 ## 0.29.1
 
 ### Patch Changes

package/Command/CleanCacheCommand.js

@@ -1,13 +1,10 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CleanCacheCommand = void 0;
 const CleanCacheAction_1 = require("../Action/CleanCacheAction");
 const DataFormat_1 = require("../utils/DataFormat");
+const bytes_1 = require("../utils/bytes");
 const CommandAbstract_1 = require("./CommandAbstract");
-const bytes_1 = __importDefault(require("bytes"));
 class CleanCacheCommand extends CommandAbstract_1.CommandAbstract {
     onOptions() {
         return this.returnsOptions({});
@@ -29,7 +26,7 @@ class CleanCacheCommand extends CommandAbstract_1.CommandAbstract {
                         value: "Freed disk space",
                     },
                 ],
-                rows: () => [[result.path, (0, bytes_1.default)(result.freedSize)]],
+                rows: () => [[result.path, (0, bytes_1.formatBytes)(result.freedSize)]],
             },
         });
         await cleanCache.exec();

package/Command/SnapshotsCommand.js

@@ -1,15 +1,12 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SnapshotsCommand = void 0;
 const ConfigAction_1 = require("../Action/ConfigAction");
 const SnapshotsAction_1 = require("../Action/SnapshotsAction");
 const DataFormat_1 = require("../utils/DataFormat");
+const bytes_1 = require("../utils/bytes");
 const string_1 = require("../utils/string");
 const CommandAbstract_1 = require("./CommandAbstract");
-const bytes_1 = __importDefault(require("bytes"));
 class SnapshotsCommand extends CommandAbstract_1.CommandAbstract {
     onOptions() {
         const groupByValues = [
@@ -139,7 +136,7 @@ class SnapshotsCommand extends CommandAbstract_1.CommandAbstract {
                     item.date.replace("T", " ").replace("Z", ""),
                     item.packageName,
                     item.packageTaskName || "",
-                    (0, bytes_1.default)(item.size),
+                    (0, bytes_1.formatBytes)(item.size),
                     item.repositoryName,
                     item.repositoryType,
                 ]),

package/Config/Config.js

@@ -39,6 +39,7 @@ exports.configDefinition = {
             additionalProperties: false,
             properties: {
                 path: { type: "string" },
+                log: { type: "boolean" },
                 users: {
                     type: "array",
                     items: {
@@ -58,6 +59,27 @@ exports.configDefinition = {
                         address: { type: "string" },
                     },
                 },
+                trustProxy: {
+                    anyOf: [
+                        { type: "boolean" },
+                        {
+                            type: "object",
+                            additionalProperties: false,
+                            required: ["remoteAddressHeader"],
+                            properties: {
+                                remoteAddressHeader: { type: "string" },
+                            },
+                        },
+                    ],
+                },
+                allowlist: {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        enabled: { type: "boolean" },
+                        remoteAddreses: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
             },
         },
         prunePolicy: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.prunePolicy),

package/Repository/DatatruckRepository.js

@@ -58,7 +58,9 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     static zipBasenameTpl = `.*.dd.tar.gz`;
     static buildSnapshotName(snapshot, pkg) {
         const date = snapshot.date.replace(/:/g, "-");
-        const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
+        const pkgName = encodeURIComponent(pkg.name)
+            .replace(/%40/g, "@")
+            .replace("_", "%5F");
         const snapshotShortId = snapshot.id.slice(0, 8);
         return `${date}_${pkgName}_${snapshotShortId}`;
     }
@@ -260,38 +262,62 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
         if (data.options.verbose)
             (0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
-        await targetFs.mkdir(snapshotName);
-        await targetFs.ensureEmptyDir(snapshotName);
+        const tmpSnapshotName = `${snapshotName}_tmp`;
+        if (await targetFs.existsDir(snapshotName))
+            await targetFs.ensureEmptyDir(snapshotName);
+        try {
+            await targetFs.rmAll(tmpSnapshotName);
+        }
+        catch (_) { }
+        await targetFs.mkdir(tmpSnapshotName);
+        await targetFs.ensureEmptyDir(tmpSnapshotName);
         const entries = await sourceFs.readdir(snapshotName);
         const total = entries.length;
         let current = 0;
         for (const entry of entries) {
-            data.onProgress({
-                absolute: {
-                    current,
-                    description: "Copying",
-                    payload: entry,
-                    total,
-                    percent: (0, math_1.progressPercent)(total, current),
-                },
-            });
+            const absolute = {
+                current,
+                description: "Copying",
+                payload: entry,
+                total,
+                percent: (0, math_1.progressPercent)(total, current),
+            };
+            data.onProgress({ absolute });
             current++;
             const sourceEntry = `${snapshotName}/${entry}`;
+            const targetEntry = `${tmpSnapshotName}/${entry}`;
             if (targetFs.isLocal()) {
-                await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
+                await sourceFs.download(sourceEntry, targetFs.resolvePath(targetEntry), {
+                    onProgress: (progress) => data.onProgress({
+                        absolute,
+                        relative: {
+                            description: "Downloading",
+                            ...progress,
+                        },
+                    }),
+                });
             }
             else {
                 const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
                 const tempFile = (0, path_1.join)(tempDir, entry);
                 try {
-                    await sourceFs.download(sourceEntry, tempFile);
-                    await targetFs.upload(tempFile, sourceEntry);
+                    await sourceFs.download(sourceEntry, tempFile, {
+                        onProgress: (progress) => data.onProgress({
+                            absolute,
+                            relative: {
+                                description: "Downloading",
+                                ...progress,
+                            },
+                        }),
+                    });
+                    await targetFs.upload(tempFile, targetEntry);
                 }
                 finally {
                     await (0, fs_1.tryRm)(tempFile);
                 }
             }
         }
+        await targetFs.rename(tmpSnapshotName, snapshotName);
     }
     async restore(data) {
         const fs = (0, client_1.createFs)(this.config.backend);

package/cli.js

@@ -9,8 +9,8 @@ const AppError_1 = require("./Error/AppError");
 const CommandFactory_1 = require("./Factory/CommandFactory");
 const globalData_1 = __importDefault(require("./globalData"));
 const cli_1 = require("./utils/cli");
+const exit_1 = require("./utils/exit");
 const fs_1 = require("./utils/fs");
-const process_1 = require("./utils/process");
 const string_1 = require("./utils/string");
 const temp_1 = require("./utils/temp");
 const chalk_1 = require("chalk");
@@ -112,7 +112,7 @@ exports.buildArgs = buildArgs;
 function parseArgs(args) {
     program.parse(args);
     const verbose = getGlobalOptions().verbose;
-    (0, process_1.onExit)((eventName, error) => {
+    (0, exit_1.onExit)((eventName, error) => {
         if (eventName !== "exit") {
             process.stdout.write(cli_1.showCursorCommand);
             console.info(`\nClosing... (reason: ${eventName})`);

package/config.schema.json

@@ -1051,6 +1051,9 @@
             "path": {
               "type": "string"
             },
+            "log": {
+              "type": "boolean"
+            },
             "users": {
               "type": "array",
               "items": {
@@ -1077,6 +1080,37 @@
                   "type": "string"
                 }
               }
+            },
+            "trustProxy": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "remoteAddressHeader"
+                  ],
+                  "properties": {
+                    "remoteAddressHeader": {
+                      "type": "string"
+                    }
+                  }
+                }
+              ]
+            },
+            "allowlist": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "remoteAddreses": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
             }
           }
         },

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.29.1",
+  "version": "0.30.0",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",
     "async": "^3.2.4",
-    "bytes": "^3.1.2",
     "chalk": "^4.1.2",
     "commander": "^11.0.0",
     "dayjs": "^1.11.10",

package/utils/bytes.d.ts

@@ -0,0 +1,2 @@
+export declare function formatBytes(bytes: number): string;
+export declare function parseSize(size: string): number;

package/utils/bytes.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSize = exports.formatBytes = void 0;
+const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+const sizeRegex = new RegExp(`^(\\d+(?:\\.\\d+)?)\\s*(${units.join("|")})$`, "i");
+function formatBytes(bytes) {
+    let u = 0;
+    let n = bytes;
+    if (bytes < 0n)
+        throw new Error(`Invalid bytes: ${bytes.toString()}`);
+    while (n >= 1024n && ++u)
+        n = n / 1024;
+    return Number(n).toFixed(n < 10 && u > 0 ? 1 : 0) + units[u];
+}
+exports.formatBytes = formatBytes;
+function parseSize(size) {
+    const matches = sizeRegex.exec(size);
+    if (!matches)
+        throw new Error(`Invalid size: ${size}`);
+    const [, value, unit] = matches;
+    const unitIndex = units.findIndex((v) => v === unit.toUpperCase());
+    if (unitIndex === -1)
+        throw new Error(`Unit not found: ${unit}`);
+    let result = Number(value);
+    for (let i = 0; i < unitIndex; ++i)
+        result *= 1024;
+    return result;
+}
+exports.parseSize = parseSize;

package/utils/datatruck/client.d.ts

@@ -1,4 +1,5 @@
 import { DiskStats } from "../fs";
+import { BasicProgress } from "../progress";
 import { AbstractFs, FsOptions } from "../virtual-fs";
 export declare class RemoteFs extends AbstractFs {
     readonly options: FsOptions;
@@ -9,6 +10,7 @@ export declare class RemoteFs extends AbstractFs {
     protected fetchJson(name: string, params: any[]): Promise<any>;
     protected post(name: string, params: any[], data: string): Promise<void>;
     existsDir(path: string): Promise<any>;
+    rename(source: string, target: string): Promise<any>;
     mkdir(path: string): Promise<any>;
     readFile(path: string): Promise<any>;
     readdir(path: string): Promise<any>;
@@ -18,7 +20,10 @@ export declare class RemoteFs extends AbstractFs {
     rmAll(path: string): Promise<void>;
     fetchDiskStats(path: string): Promise<DiskStats>;
     upload(source: string, target: string): Promise<void>;
-    download(source: string, target: string, timeout?: number): Promise<void>;
+    download(source: string, target: string, options?: {
+        timeout?: number;
+        onProgress?: (progress: BasicProgress) => void;
+    }): Promise<void>;
 }
 export declare function isRemoteBackend(backend: string): boolean;
 export declare function createFs(backend: string): AbstractFs;

package/utils/datatruck/client.js

@@ -44,6 +44,9 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
     async existsDir(path) {
         return await this.fetchJson("existsDir", [path]);
     }
+    async rename(source, target) {
+        return await this.fetchJson("rename", [source, target]);
+    }
     async mkdir(path) {
         return await this.fetchJson("mkdir", [path]);
     }
@@ -76,13 +79,11 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
             },
         });
     }
-    async download(source, target, timeout = 100000) {
+    async download(source, target, options = {}) {
         await (0, http_1.downloadFile)(`${this.url}/download`, target, {
-            timeout,
+            ...options,
             headers: this.headers,
-            query: {
-                params: JSON.stringify([source]),
-            },
+            query: { params: JSON.stringify([source]) },
         });
     }
 }

package/utils/datatruck/server.d.ts

@@ -12,6 +12,16 @@ export type DatatruckServerOptions = {
         address?: string;
     };
     users?: User[];
+    trustProxy?: true | {
+        remoteAddressHeader: string;
+    };
+    allowlist?: {
+        /**
+         * @default true
+         */
+        enabled?: boolean;
+        remoteAddreses?: string[];
+    };
 };
 export declare const headerKey: {
     user: string;

package/utils/datatruck/server.js

@@ -4,6 +4,7 @@ exports.createDatatruckServer = exports.headerKey = void 0;
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
 const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
 const http_2 = require("http");
 function parseUrl(inUrl) {
     const url = new URL(`http://127.0.0.1${inUrl}`);
@@ -23,14 +24,26 @@ exports.headerKey = {
     user: "x-dtt-user",
     password: "x-dtt-password",
 };
-function validateRequest(req, users) {
-    const name = req.headers[exports.headerKey.user];
-    const password = req.headers[exports.headerKey.password];
-    return users.some((user) => user.name.length &&
-        user.password.length &&
-        user.name === name &&
-        user.password === password);
+function validateRequest(req, options) {
+    const list = options.allowlist;
+    if (list && (list.enabled ?? true) && list.remoteAddreses) {
+        const remoteAddress = getRemoteAddress(req, options);
+        if (!remoteAddress || list.remoteAddreses.includes(remoteAddress))
+            return false;
+    }
+    const name = req.headers[exports.headerKey.user]?.toString().trim();
+    const password = req.headers[exports.headerKey.password]?.toString().trim();
+    if (!name?.length || !password?.length)
+        return;
+    return (options.users?.some((user) => user.name === name && user.password === password) || false);
 }
+const getRemoteAddress = (req, options) => {
+    return ((options.trustProxy
+        ? options.trustProxy === true
+            ? req.headers["x-real-ip"]?.toString()
+            : req.headers[options.trustProxy.remoteAddressHeader]?.toString()
+        : undefined) ?? req.socket.remoteAddress);
+};
 function createDatatruckServer(options) {
     const log = options.log ?? true;
     return (0, http_2.createServer)(async (req, res) => {
@@ -38,7 +51,7 @@ function createDatatruckServer(options) {
             if (req.url === "/" || req.url === "/favicon.ico") {
                 return res.end();
             }
-            else if (!validateRequest(req, options.users || [])) {
+            else if (!validateRequest(req, options)) {
                 res.statusCode = 401;
                 return res.end();
             }
@@ -50,7 +63,8 @@ function createDatatruckServer(options) {
             const { action, params } = parseUrl(req.url);
             if (action === "upload") {
                 const [target] = params;
-                const file = (0, fs_1.createWriteStream)(fs.resolvePath(target));
+                const path = fs.resolvePath(target);
+                const file = (0, fs_1.createWriteStream)(path);
                 req.pipe(file);
                 await new Promise((resolve, reject) => {
                     req.on("error", reject);
@@ -60,7 +74,10 @@ function createDatatruckServer(options) {
             }
             else if (action === "download") {
                 const [target] = params;
-                const file = (0, fs_1.createReadStream)(fs.resolvePath(target));
+                const path = fs.resolvePath(target);
+                const file = (0, fs_1.createReadStream)(path);
+                const fileStat = await (0, promises_1.stat)(path);
+                res.setHeader("Content-Length", fileStat.size);
                 file.pipe(res);
                 await new Promise((resolve, reject) => {
                     req.on("error", reject);
@@ -82,14 +99,16 @@ function createDatatruckServer(options) {
                 if (json !== undefined)
                     res.write(JSON.stringify(json));
             }
-            res.end();
             if (log)
                 console.info(`<${action}`);
+            res.end();
         }
         catch (error) {
             if (log)
                 console.error(`<${req.url}`, error);
             res.statusCode = 500;
+            res.statusMessage = error.message;
+            res.end();
         }
     });
 }

package/utils/exit.d.ts

@@ -0,0 +1,6 @@
+type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
+export declare function triggerExitEvent(eventName: EventNameType, ...args: any[]): void;
+export declare function enableExitEvents(): void;
+export declare function disableExitEvents(): void;
+export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void, priority?: number): () => boolean;
+export {};

package/utils/exit.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.onExit = exports.disableExitEvents = exports.enableExitEvents = exports.triggerExitEvent = void 0;
+const eventNames = [
+    `exit`,
+    `SIGINT`,
+    `SIGUSR1`,
+    `SIGUSR2`,
+    `uncaughtException`,
+    `SIGTERM`,
+];
+const listeners = new Set();
+const disableExitDisposes = new Set();
+function triggerExitEvent(eventName, ...args) {
+    if (!disableExitDisposes.size) {
+        process.emit(eventName, ...args);
+        return;
+    }
+    try {
+        disableExitEvents();
+        const items = [...listeners].sort((a, b) => b.priority - a.priority);
+        for (const { cb } of items) {
+            try {
+                cb(eventName, ...args);
+            }
+            catch (_) { }
+        }
+    }
+    catch (_) {
+        process.exit(5);
+    }
+}
+exports.triggerExitEvent = triggerExitEvent;
+function enableExitEvents() {
+    disableExitEvents();
+    for (const eventName of eventNames) {
+        const listener = (...args) => triggerExitEvent(eventName, ...args);
+        process.on(eventName, listener);
+        disableExitDisposes.add(() => process.off(eventName, listener));
+    }
+}
+exports.enableExitEvents = enableExitEvents;
+function disableExitEvents() {
+    for (const dispose of disableExitDisposes)
+        dispose();
+    disableExitDisposes.clear();
+}
+exports.disableExitEvents = disableExitEvents;
+function onExit(cb, priority) {
+    if (!disableExitDisposes.size)
+        enableExitEvents();
+    const listener = { cb, priority: priority ?? 0 };
+    listeners.add(listener);
+    return () => listeners.delete(listener);
+}
+exports.onExit = onExit;

package/utils/fs.d.ts

@@ -133,4 +133,7 @@ export declare function fetchDiskStats(path: string): Promise<DiskStats>;
 export declare function checkFreeDiskSpace(stat: DiskStats, inSize: string | number): Promise<void>;
 export declare function ensureFreeDiskSpace(input: string[] | DiskStats, inSize: number | string): Promise<void>;
 export declare function groupFiles(inFiles: string[], suffixes?: string[], gzSuffix?: string): [string[], Record<string, string>];
+export declare function asFile(input: string | {
+    path: string;
+}): Promise<[string, (() => Promise<void>) | undefined]>;
 export {};

package/utils/fs.js

@@ -3,12 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.asFile = exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
+const bytes_1 = require("./bytes");
 const math_1 = require("./math");
 const path_1 = require("./path");
 const string_1 = require("./string");
+const temp_1 = require("./temp");
 const async_1 = require("async");
-const bytes_1 = __importDefault(require("bytes"));
 const fast_folder_size_1 = __importDefault(require("fast-folder-size"));
 const fast_glob_1 = __importDefault(require("fast-glob"));
 const fs_1 = require("fs");
@@ -238,7 +239,7 @@ async function writeGitIgnoreList(options) {
 }
 exports.writeGitIgnoreList = writeGitIgnoreList;
 async function waitForClose(stream) {
-    return new Promise(async (resolve, reject) => {
+    return new Promise((resolve, reject) => {
         stream.on("close", resolve);
         stream.on("error", reject);
         return stream;
@@ -566,10 +567,10 @@ async function fetchDiskStats(path) {
 }
 exports.fetchDiskStats = fetchDiskStats;
 async function checkFreeDiskSpace(stat, inSize) {
-    const humanSize = typeof inSize === "number" ? (0, bytes_1.default)(inSize) : inSize;
-    const size = bytes_1.default.parse(inSize);
+    const humanSize = typeof inSize === "number" ? (0, bytes_1.formatBytes)(inSize) : inSize;
+    const size = typeof inSize === "number" ? inSize : (0, bytes_1.parseSize)(inSize);
     if (stat.free < size)
-        throw new Error(`Free disk space is less than ${humanSize}: ${(0, bytes_1.default)(stat.free)}/${(0, bytes_1.default)(stat.total)}`);
+        throw new Error(`Free disk space is less than ${humanSize}: ${(0, bytes_1.formatBytes)(stat.free)}/${(0, bytes_1.formatBytes)(stat.total)}`);
 }
 exports.checkFreeDiskSpace = checkFreeDiskSpace;
 async function ensureFreeDiskSpace(input, inSize) {
@@ -608,3 +609,20 @@ function groupFiles(inFiles, suffixes, gzSuffix = ".tar.gz") {
     return [Object.keys(grouped), compressed];
 }
 exports.groupFiles = groupFiles;
+async function asFile(input) {
+    if (typeof input === "string") {
+        const dir = await (0, temp_1.mkTmpDir)("text-as-file");
+        const path = (0, path_2.join)(dir, "contents.txt");
+        await (0, promises_1.writeFile)(path, input);
+        return [
+            path,
+            async () => {
+                await (0, promises_1.rm)(dir, { recursive: true });
+            },
+        ];
+    }
+    else {
+        return [input.path, undefined];
+    }
+}
+exports.asFile = asFile;

package/utils/http.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+import { BasicProgress } from "./progress";
 import { IncomingMessage, Server } from "http";
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
@@ -14,6 +15,7 @@ export declare function downloadFile(url: string, output: string, options?: {
     headers?: Record<string, string>;
     query?: Record<string, string>;
     timeout?: number;
+    onProgress?: (progress: BasicProgress) => void;
 }): Promise<void>;
 export declare function uploadFile(url: string, path: string, options?: {
     headers?: Record<string, string>;

package/utils/http.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
+const math_1 = require("./math");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const http_1 = require("http");
@@ -89,13 +90,28 @@ async function post(url, data, options = {}) {
 }
 exports.post = post;
 async function downloadFile(url, output, options = {}) {
-    const timeout = options.timeout ?? 3600 * 100;
+    const timeout = options.timeout ?? 3600 * 1000; // 60m
     const file = (0, fs_1.createWriteStream)(output);
     await new Promise((resolve, reject) => {
         const req = request(href(url, options.query), {
             headers: options.headers,
         }, (res) => {
+            const contentLength = res.headers["content-length"] ?? "";
+            if (!/^\d+$/.test(contentLength))
+                return reject(new Error(`Invalid 'content-length': ${contentLength}`));
+            const total = Number(contentLength);
+            let current = 0;
             if (res.statusCode === 200) {
+                if (options.onProgress) {
+                    res.on("data", (chunk) => {
+                        current += chunk.byteLength;
+                        options.onProgress({
+                            percent: (0, math_1.progressPercent)(total, current),
+                            current,
+                            total,
+                        });
+                    });
+                }
                 res
                     .on("error", async (error) => {
                     file.destroy();

package/utils/list.d.ts

@@ -46,6 +46,7 @@ export declare class Listr3<T extends Listr3Context> extends Listr<void, "defaul
     };
     readonly resultMap: Record<string, Listr3TaskResult<T>>;
     readonly resultList: Listr3TaskResult<T>[];
+    readonly logger: List3Logger;
     protected execTimer: Timer;
     constructor($options: {
         streams?: Streams;
@@ -59,6 +60,7 @@ export declare class Listr3<T extends Listr3Context> extends Listr<void, "defaul
     add(tasks: ListrTask<void, ListrGetRendererClassFromValue<"default">> | ListrTask<void, ListrGetRendererClassFromValue<"default">>[]): this;
     getSummaryResult(): SummaryResult;
     getResult(): (SummaryResult | Listr3TaskResult<T>)[];
+    protected release(): void;
     exec(): Promise<(Listr3TaskResult<T> | SummaryResult)[]>;
 }
 export {};

package/utils/list.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Listr3 = exports.List3Logger = void 0;
 const date_1 = require("./date");
+const exit_1 = require("./exit");
 const stream_1 = require("./stream");
 const listr2_1 = require("listr2");
 class List3Logger extends listr2_1.ListrLogger {
@@ -17,8 +18,10 @@ class Listr3 extends listr2_1.Listr {
     $options;
     resultMap = {};
     resultList = [];
+    logger;
     execTimer;
     constructor($options) {
+        const logger = new List3Logger();
         super([], {
             renderer: "default",
             collectErrors: "minimal",
@@ -27,12 +30,12 @@ class Listr3 extends listr2_1.Listr {
             }),
             fallbackRenderer: "simple",
             fallbackRendererOptions: {
-                logger: new List3Logger(),
+                logger: logger,
                 timestamp: listr2_1.PRESET_TIMESTAMP,
                 timer: listr2_1.PRESET_TIMER,
             },
             rendererOptions: {
-                logger: new List3Logger(),
+                logger: logger,
                 collapseSubtasks: false,
                 collapseErrors: false,
                 timer: listr2_1.PRESET_TIMER,
@@ -40,6 +43,7 @@ class Listr3 extends listr2_1.Listr {
         });
         this.$options = $options;
         this.execTimer = (0, date_1.createTimer)();
+        this.logger = logger;
     }
     serializeKeyIndex(keyIndex) {
         return typeof keyIndex !== "undefined"
@@ -127,7 +131,18 @@ class Listr3 extends listr2_1.Listr {
     getResult() {
         return [...this.resultList, this.getSummaryResult()];
     }
+    release() {
+        for (const task of this.tasks)
+            if (task.isPending())
+                task.state$ = listr2_1.ListrTaskState.FAILED;
+        this["renderer"].end(new Error("Interrupted."));
+    }
     async exec() {
+        const dispose = (0, exit_1.onExit)(() => {
+            this.$options.progressManager?.dispose();
+            this.execTimer.reset();
+            this.release();
+        }, 1);
         try {
             this.$options.progressManager?.start();
             this.execTimer.reset();
@@ -138,7 +153,7 @@ class Listr3 extends listr2_1.Listr {
             throw error;
         }
         finally {
-            this.$options.progressManager?.dispose();
+            dispose();
         }
     }
 }

package/utils/process.d.ts

@@ -30,7 +30,7 @@ export interface ExecSettingsInterface {
         stream: Readable;
     };
     log?: ExecLogSettingsType | boolean;
-    onSpawn?: (p: ChildProcess) => any;
+    onSpawn?: (p: ChildProcess) => void | undefined;
     stdout?: {
         save?: boolean;
         parseLines?: boolean | "skip-empty";
@@ -103,6 +103,4 @@ export declare function createProcess<O1 extends boolean, O2 extends boolean>(co
     stderr: string;
 } : {})>;
 export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
-type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
-export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void): void;
 export {};

package/utils/process.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.onExit = exports.exec = exports.createProcess = exports.logProcessExec = exports.waitForClose = exports.parseStreamData = exports.logExecStderr = exports.logExecStdout = void 0;
+exports.exec = exports.createProcess = exports.logProcessExec = exports.waitForClose = exports.parseStreamData = exports.logExecStderr = exports.logExecStdout = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
@@ -179,46 +179,45 @@ async function exec(command, argv = [], options = null, settings = {}) {
     else if (settings.log) {
         log = settings.log;
     }
-    return new Promise(async (resolve, reject) => {
-        if (log.exec) {
-            logProcessExec(command, argv, {
-                env: options?.env,
-                envNames: log.envNames,
-                pipe: pipe?.stream,
-                toStderr: log.allToStderr,
-            });
-        }
-        if (typeof options?.cwd === "string" && !(await (0, fs_1.existsDir)(options.cwd)))
-            return reject(new Error(`Current working directory does not exist: ${options.cwd}`));
-        if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
-            const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
-            const totalBytes = fileInfo.size;
-            let currentBytes = 0;
-            pipe.stream.on("data", (data) => {
-                currentBytes += data.length;
-                pipe.onReadProgress?.({
-                    totalBytes: totalBytes,
-                    currentBytes: currentBytes,
-                    progress: (0, math_1.progressPercent)(totalBytes, currentBytes),
-                });
+    if (log.exec)
+        logProcessExec(command, argv, {
+            env: options?.env,
+            envNames: log.envNames,
+            pipe: pipe?.stream,
+            toStderr: log.allToStderr,
+        });
+    if (typeof options?.cwd === "string" && !(await (0, fs_1.existsDir)(options.cwd)))
+        throw new Error(`Current working directory does not exist: ${options.cwd}`);
+    if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
+        const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
+        const totalBytes = fileInfo.size;
+        let currentBytes = 0;
+        pipe.stream.on("data", (data) => {
+            currentBytes += data.length;
+            pipe.onReadProgress?.({
+                totalBytes: totalBytes,
+                currentBytes: currentBytes,
+                progress: (0, math_1.progressPercent)(totalBytes, currentBytes),
             });
-        }
-        const p = (0, child_process_1.spawn)(command, argv, options ?? {});
-        await settings.onSpawn?.(p);
-        let spawnError;
-        const spawnData = {
-            stdout: "",
-            stderr: "",
-            exitCode: 0,
-        };
-        let finishListeners = 1;
-        if (pipe?.stream instanceof fs_2.WriteStream)
-            finishListeners++;
-        if (settings.stdout?.parseLines)
-            finishListeners++;
-        if (settings.stderr?.parseLines)
-            finishListeners++;
-        let streamError;
+        });
+    }
+    const p = (0, child_process_1.spawn)(command, argv, options ?? {});
+    settings.onSpawn?.(p);
+    let spawnError;
+    const spawnData = {
+        stdout: "",
+        stderr: "",
+        exitCode: 0,
+    };
+    let finishListeners = 1;
+    if (pipe?.stream instanceof fs_2.WriteStream)
+        finishListeners++;
+    if (settings.stdout?.parseLines)
+        finishListeners++;
+    if (settings.stderr?.parseLines)
+        finishListeners++;
+    let streamError;
+    return new Promise((resolve, reject) => {
         const tryFinish = () => {
             if (!--finishListeners)
                 finish();
@@ -336,17 +335,3 @@ async function exec(command, argv = [], options = null, settings = {}) {
     });
 }
 exports.exec = exec;
-const eventNames = [
-    `exit`,
-    `SIGINT`,
-    `SIGUSR1`,
-    `SIGUSR2`,
-    `uncaughtException`,
-    `SIGTERM`,
-];
-function onExit(cb) {
-    for (const eventName of eventNames) {
-        process.on(eventName, (...args) => cb(eventName, ...args));
-    }
-}
-exports.onExit = onExit;

package/utils/progress.d.ts

@@ -1,5 +1,10 @@
 /// <reference types="node" />
 import { Timer } from "./date";
+export type BasicProgress = {
+    percent: number;
+    current: number;
+    total: number;
+};
 export type ProgressStats = {
     percent?: number;
     total?: number;
@@ -24,7 +29,7 @@ export declare class ProgressManager {
     };
     protected timer: Timer;
     protected interval: Timer | undefined;
-    protected keydownListener: ((data: Buffer) => void) | undefined;
+    protected keydownListener: ((data: Buffer | undefined) => void) | undefined;
     readonly tty: boolean;
     readonly enabled: boolean | "interval";
     constructor(options: {

package/utils/progress.js

@@ -1,12 +1,10 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.renderProgressStats = exports.renderProgress = exports.ProgressManager = void 0;
+const bytes_1 = require("./bytes");
 const cli_1 = require("./cli");
 const date_1 = require("./date");
-const bytes_1 = __importDefault(require("bytes"));
+const exit_1 = require("./exit");
 const chalk_1 = require("chalk");
 const readline_1 = require("readline");
 class ProgressManager {
@@ -41,10 +39,10 @@ class ProgressManager {
         process.stdin?.resume();
         process.stdin?.setEncoding("utf8");
         process.stdin?.on("keypress", (this.keydownListener = (inKey) => {
-            const key = inKey.toString();
+            const key = inKey?.toString() || "";
             if (key === "\u0003") {
                 process.stdin.setRawMode?.(false);
-                process.emit("SIGINT");
+                (0, exit_1.triggerExitEvent)("SIGINT");
             }
             else if (/^(\r\n)|\r|\n$/.test(key)) {
                 this.interval = undefined;
@@ -89,7 +87,7 @@ function renderProgressStats(stats, progressBar) {
         text.push(`${stats.percent.toFixed(2).padStart(5, " ")}%`);
     }
     if (typeof stats.current === "number" || typeof stats.total === "number") {
-        const format = (value) => stats.format === "size" ? (0, bytes_1.default)(value) : value.toString();
+        const format = (value) => stats.format === "size" ? (0, bytes_1.formatBytes)(value) : value.toString();
         const pad = 8;
         let values = [];
         if (typeof stats.current === "number" && typeof stats.total === "number") {

package/utils/tar.d.ts

@@ -1,12 +1,8 @@
+import { BasicProgress } from "./progress";
 import type { JSONSchema7 } from "json-schema";
-export type Progress = {
-    percent: number;
-    current: number;
-    total: number;
-};
 export type TarEntry = {
     path: string;
-    progress: Progress;
+    progress: BasicProgress;
 };
 export type CoresOptions = number | {
     percent: number;

package/utils/virtual-fs.d.ts

@@ -1,4 +1,5 @@
 import { DiskStats } from "./fs";
+import { BasicProgress } from "./progress";
 export declare function resolvePath(path: string): string;
 export type FsOptions = {
     backend: string;
@@ -10,6 +11,7 @@ export declare abstract class AbstractFs {
     abstract isLocal(): boolean;
     isRemote(): boolean;
     abstract existsDir(path: string): Promise<boolean>;
+    abstract rename(source: string, target: string): Promise<void>;
     abstract mkdir(path: string): Promise<void>;
     abstract readFile(path: string): Promise<string>;
     abstract rmAll(path: string): Promise<void>;
@@ -18,12 +20,16 @@ export declare abstract class AbstractFs {
     abstract ensureEmptyDir(path: string): Promise<void>;
     abstract writeFile(path: string, contents: string): Promise<void>;
     abstract upload(source: string, target: string): Promise<void>;
-    abstract download(source: string, target: string): Promise<void>;
+    abstract download(source: string, target: string, options?: {
+        timeout?: number;
+        onProgress?: (progress: BasicProgress) => void;
+    }): Promise<void>;
     abstract fetchDiskStats(source: string): Promise<DiskStats>;
 }
 export declare class LocalFs extends AbstractFs {
     isLocal(): boolean;
     existsDir(path: string): Promise<boolean>;
+    rename(source: string, target: string): Promise<void>;
     mkdir(path: string): Promise<void>;
     ensureEmptyDir(path: string): Promise<void>;
     readFile(path: string): Promise<string>;

package/utils/virtual-fs.js

@@ -29,6 +29,9 @@ class LocalFs extends AbstractFs {
     async existsDir(path) {
         return (0, fs_1.existsDir)(this.resolvePath(path));
     }
+    async rename(source, target) {
+        await (0, promises_1.rename)(this.resolvePath(source), this.resolvePath(target));
+    }
     async mkdir(path) {
         await (0, fs_1.mkdirIfNotExists)(this.resolvePath(path));
     }