@datatruck/cli 0.29.1 → 0.30.1

package/Action/BackupAction.d.ts

@@ -4,7 +4,7 @@ import { RepositoryConfigType } from "../Config/RepositoryConfig";
 import { PreSnapshot } from "../Repository/RepositoryAbstract";
 import { DataFormat } from "../utils/DataFormat";
 import { Listr3TaskResultEnd } from "../utils/list";
-import { Progress, ProgressManager } from "../utils/progress";
+import { Progress, ProgressMode } from "../utils/progress";
 import { Streams } from "../utils/stream";
 import { IfRequireKeys } from "../utils/ts";
 export type BackupActionOptions = {
@@ -17,8 +17,7 @@ export type BackupActionOptions = {
     verbose?: boolean;
     date?: string;
     tty?: "auto" | boolean;
-    progress?: "auto" | "interval" | boolean;
-    progressInterval?: number;
+    progress?: ProgressMode;
     streams?: Streams;
     prune?: boolean;
 };
@@ -51,7 +50,6 @@ type Context = {
 export declare class BackupAction<TRequired extends boolean = true> {
     readonly config: ConfigType;
     readonly options: IfRequireKeys<TRequired, BackupActionOptions>;
-    protected pm: ProgressManager;
     constructor(config: ConfigType, options?: IfRequireKeys<TRequired, BackupActionOptions>);
     protected prepareSnapshot(): PreSnapshot;
     protected getPackages(snapshot: PreSnapshot): PackageConfigType[];

package/Action/BackupAction.js

@@ -22,16 +22,9 @@ const dayjs_1 = __importDefault(require("dayjs"));
 class BackupAction {
     config;
     options;
-    pm;
     constructor(config, options = {}) {
         this.config = config;
         this.options = options;
-        this.pm = new progress_1.ProgressManager({
-            verbose: options.verbose,
-            tty: options.tty,
-            enabled: options.progress,
-            interval: options.progressInterval,
-        });
     }
     prepareSnapshot() {
         const date = this.options.date ?? new Date().toISOString();
@@ -153,7 +146,7 @@ class BackupAction {
                     .map((item) => [
                     (0, cli_1.renderResult)(item.error),
                     renderTitle(item, true),
-                    renderData(item, true),
+                    renderData(item, true, result),
                     (0, date_1.duration)(item.elapsed),
                     (0, cli_1.renderError)(item.error, options.verbose),
                 ]),
@@ -165,8 +158,7 @@ class BackupAction {
         const pm = new progress_1.ProgressManager({
             verbose: options.verbose,
             tty: options.tty,
-            enabled: options.progress,
-            interval: options.progressInterval,
+            mode: options.progress,
         });
         const l = new list_1.Listr3({
             streams: this.options.streams,
@@ -246,7 +238,7 @@ class BackupAction {
                                         repositoryName,
                                         snapshot,
                                         snapshotPath: taskResult?.snapshotPath,
-                                        onProgress: (p) => this.pm.update(p, (t) => (task.output = t)),
+                                        onProgress: (p) => pm.update(p, (t) => (task.output = t)),
                                     });
                                 },
                             })), l.$task({

package/Action/CopyAction.d.ts

@@ -2,6 +2,7 @@ import type { ConfigType } from "../Config/Config";
 import { Snapshot } from "../Repository/RepositoryAbstract";
 import { DataFormat } from "../utils/DataFormat";
 import { Listr3TaskResultEnd } from "../utils/list";
+import { ProgressMode } from "../utils/progress";
 import { Streams } from "../utils/stream";
 import { IfRequireKeys } from "../utils/ts";
 export type CopyActionOptionsType = {
@@ -13,8 +14,7 @@ export type CopyActionOptionsType = {
     repositoryNames2?: string[];
     verbose?: boolean;
     tty?: "auto" | boolean;
-    progress?: "auto" | "interval" | boolean;
-    progressInterval?: number;
+    progress?: ProgressMode;
 };
 export type CopyActionResult = {
     errors: Error[];

package/Action/CopyAction.js

@@ -69,8 +69,7 @@ class CopyAction {
         const pm = new progress_1.ProgressManager({
             verbose: options.verbose,
             tty: options.tty,
-            enabled: options.progress,
-            interval: options.progressInterval,
+            mode: options.progress,
         });
         const l = new list_1.Listr3({ progressManager: pm });
         return l

package/Action/RestoreAction.d.ts

@@ -4,7 +4,7 @@ import { Snapshot } from "../Repository/RepositoryAbstract";
 import { TaskAbstract } from "../Task/TaskAbstract";
 import { DataFormat } from "../utils/DataFormat";
 import { Listr3TaskResultEnd } from "../utils/list";
-import { Progress } from "../utils/progress";
+import { Progress, ProgressMode } from "../utils/progress";
 import { Streams } from "../utils/stream";
 import { GargabeCollector } from "../utils/temp";
 import { IfRequireKeys } from "../utils/ts";
@@ -19,8 +19,7 @@ export type RestoreActionOptions = {
     verbose?: boolean;
     initial?: boolean;
     tty?: "auto" | boolean;
-    progress?: "auto" | "interval" | boolean;
-    progressInterval?: number;
+    progress?: ProgressMode;
     streams?: Streams;
 };
 type RestoreSnapshot = Snapshot & {

package/Action/RestoreAction.js

@@ -143,8 +143,7 @@ class RestoreAction {
         const pm = new progress_1.ProgressManager({
             verbose: options.verbose,
             tty: options.tty,
-            enabled: options.progress,
-            interval: options.progressInterval,
+            mode: options.progress,
         });
         const l = new list_1.Listr3({
             streams: options.streams,

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @datatruck/cli
 
+## 0.30.1
+
+### Patch Changes
+
+- [`68e991b`](https://github.com/swordev/datatruck/commit/68e991b862ee3793e6b31d1fd5d6cebdf59524a4) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix allowlist in datatruck server
+
+- [`16f982c`](https://github.com/swordev/datatruck/commit/16f982c7da0d44cbbb691d5552da77fb27366f82) Thanks [@juanrgm](https://github.com/juanrgm)! - Reduce progress interval to 300 ms in auto progress mode
+
+## 0.30.0
+
+### Minor Changes
+
+- [`e28b12d`](https://github.com/swordev/datatruck/commit/e28b12d08c36844317e506552443825ab3333139) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `allowlist` option to the datatruck server
+
+### Patch Changes
+
+- [`2f63e67`](https://github.com/swordev/datatruck/commit/2f63e67ee532892fda3a9d06e46336a42834e5ed) Thanks [@juanrgm](https://github.com/juanrgm)! - Add download progress in datatruck repository
+
+- [`258e933`](https://github.com/swordev/datatruck/commit/258e93385d9b6f93a435a28a2b26e53ea1763755) Thanks [@juanrgm](https://github.com/juanrgm)! - Copy backups safely
+
+- [`82b4c67`](https://github.com/swordev/datatruck/commit/82b4c67c55eee4f40753b48eb91345e6d6789f72) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix exit event
+
 ## 0.29.1
 
 ### Patch Changes

package/Command/BackupCommand.js

@@ -61,7 +61,6 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
             date: this.options.date,
             tty: this.globalOptions.tty,
             progress: this.globalOptions.progress,
-            progressInterval: this.globalOptions.progressInterval,
             streams: this.streams,
             prune: this.options.prune,
         });

package/Command/CleanCacheCommand.js

@@ -1,13 +1,10 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CleanCacheCommand = void 0;
 const CleanCacheAction_1 = require("../Action/CleanCacheAction");
 const DataFormat_1 = require("../utils/DataFormat");
+const bytes_1 = require("../utils/bytes");
 const CommandAbstract_1 = require("./CommandAbstract");
-const bytes_1 = __importDefault(require("bytes"));
 class CleanCacheCommand extends CommandAbstract_1.CommandAbstract {
     onOptions() {
         return this.returnsOptions({});
@@ -29,7 +26,7 @@ class CleanCacheCommand extends CommandAbstract_1.CommandAbstract {
                         value: "Freed disk space",
                     },
                 ],
-                rows: () => [[result.path, (0, bytes_1.default)(result.freedSize)]],
+                rows: () => [[result.path, (0, bytes_1.formatBytes)(result.freedSize)]],
             },
         });
         await cleanCache.exec();

package/Command/CommandAbstract.d.ts

@@ -1,6 +1,7 @@
 import { ConfigType } from "../Config/Config";
 import { FormatType } from "../utils/DataFormat";
 import { OptionsType } from "../utils/cli";
+import { ProgressMode } from "../utils/progress";
 import { Streams } from "../utils/stream";
 import { If, SimilarObject } from "../utils/ts";
 export type GlobalOptions<TResolved = false> = {
@@ -8,8 +9,7 @@ export type GlobalOptions<TResolved = false> = {
     outputFormat?: FormatType;
     verbose?: number;
     tty?: If<TResolved, "auto" | boolean, "auto" | "true" | "false">;
-    progress?: If<TResolved, "auto" | boolean, "auto" | "true" | "false" | "interval">;
-    progressInterval?: number;
+    progress?: If<TResolved, ProgressMode, Exclude<ProgressMode, boolean> | "true" | "false">;
 };
 export type CommandConstructor<TUnresolvedOptions, TOptions extends SimilarObject<TUnresolvedOptions>> = {
     new (globalOptions: GlobalOptions<true>, options: TOptions): CommandAbstract<TUnresolvedOptions, TOptions>;

package/Command/CopyCommand.js

@@ -53,7 +53,6 @@ class CopyCommand extends CommandAbstract_1.CommandAbstract {
             verbose: verbose > 0,
             tty: this.globalOptions.tty,
             progress: this.globalOptions.progress,
-            progressInterval: this.globalOptions.progressInterval,
         });
         const result = await copy.exec();
         if (this.globalOptions.outputFormat)

package/Command/RestoreCommand.js

@@ -63,7 +63,6 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
             initial: this.options.initial,
             tty: this.globalOptions.tty,
             progress: this.globalOptions.progress,
-            progressInterval: this.globalOptions.progressInterval,
             streams: this.streams,
         });
         const result = await restore.exec();

package/Command/SnapshotsCommand.js

@@ -1,15 +1,12 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SnapshotsCommand = void 0;
 const ConfigAction_1 = require("../Action/ConfigAction");
 const SnapshotsAction_1 = require("../Action/SnapshotsAction");
 const DataFormat_1 = require("../utils/DataFormat");
+const bytes_1 = require("../utils/bytes");
 const string_1 = require("../utils/string");
 const CommandAbstract_1 = require("./CommandAbstract");
-const bytes_1 = __importDefault(require("bytes"));
 class SnapshotsCommand extends CommandAbstract_1.CommandAbstract {
     onOptions() {
         const groupByValues = [
@@ -139,7 +136,7 @@ class SnapshotsCommand extends CommandAbstract_1.CommandAbstract {
                     item.date.replace("T", " ").replace("Z", ""),
                     item.packageName,
                     item.packageTaskName || "",
-                    (0, bytes_1.default)(item.size),
+                    (0, bytes_1.formatBytes)(item.size),
                     item.repositoryName,
                     item.repositoryType,
                 ]),

package/Config/Config.js

@@ -39,6 +39,7 @@ exports.configDefinition = {
             additionalProperties: false,
             properties: {
                 path: { type: "string" },
+                log: { type: "boolean" },
                 users: {
                     type: "array",
                     items: {
@@ -58,6 +59,27 @@ exports.configDefinition = {
                         address: { type: "string" },
                     },
                 },
+                trustProxy: {
+                    anyOf: [
+                        { type: "boolean" },
+                        {
+                            type: "object",
+                            additionalProperties: false,
+                            required: ["remoteAddressHeader"],
+                            properties: {
+                                remoteAddressHeader: { type: "string" },
+                            },
+                        },
+                    ],
+                },
+                allowlist: {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        enabled: { type: "boolean" },
+                        remoteAddresses: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
             },
         },
         prunePolicy: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.prunePolicy),

package/Repository/DatatruckRepository.js

@@ -58,7 +58,9 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     static zipBasenameTpl = `.*.dd.tar.gz`;
     static buildSnapshotName(snapshot, pkg) {
         const date = snapshot.date.replace(/:/g, "-");
-        const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
+        const pkgName = encodeURIComponent(pkg.name)
+            .replace(/%40/g, "@")
+            .replace("_", "%5F");
         const snapshotShortId = snapshot.id.slice(0, 8);
         return `${date}_${pkgName}_${snapshotShortId}`;
     }
@@ -260,38 +262,64 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
         if (data.options.verbose)
             (0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
-        await targetFs.mkdir(snapshotName);
-        await targetFs.ensureEmptyDir(snapshotName);
+        const tmpSnapshotName = `${snapshotName}_tmp`;
+        if (await targetFs.existsDir(snapshotName))
+            await targetFs.ensureEmptyDir(snapshotName);
+        try {
+            await targetFs.rmAll(tmpSnapshotName);
+        }
+        catch (_) { }
+        await targetFs.mkdir(tmpSnapshotName);
+        await targetFs.ensureEmptyDir(tmpSnapshotName);
         const entries = await sourceFs.readdir(snapshotName);
         const total = entries.length;
         let current = 0;
         for (const entry of entries) {
-            data.onProgress({
-                absolute: {
-                    current,
-                    description: "Copying",
-                    payload: entry,
-                    total,
-                    percent: (0, math_1.progressPercent)(total, current),
-                },
-            });
+            const absolute = {
+                current,
+                description: "Copying",
+                payload: entry,
+                total,
+                percent: (0, math_1.progressPercent)(total, current),
+            };
+            data.onProgress({ absolute });
             current++;
             const sourceEntry = `${snapshotName}/${entry}`;
+            const targetEntry = `${tmpSnapshotName}/${entry}`;
             if (targetFs.isLocal()) {
-                await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
+                await sourceFs.download(sourceEntry, targetFs.resolvePath(targetEntry), {
+                    onProgress: (progress) => data.onProgress({
+                        absolute,
+                        relative: {
+                            description: "Downloading",
+                            format: "size",
+                            ...progress,
+                        },
+                    }),
+                });
             }
             else {
                 const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
                 const tempFile = (0, path_1.join)(tempDir, entry);
                 try {
-                    await sourceFs.download(sourceEntry, tempFile);
-                    await targetFs.upload(tempFile, sourceEntry);
+                    await sourceFs.download(sourceEntry, tempFile, {
+                        onProgress: (progress) => data.onProgress({
+                            absolute,
+                            relative: {
+                                description: "Downloading",
+                                format: "size",
+                                ...progress,
+                            },
+                        }),
+                    });
+                    await targetFs.upload(tempFile, targetEntry);
                 }
                 finally {
                     await (0, fs_1.tryRm)(tempFile);
                 }
             }
         }
+        await targetFs.rename(tmpSnapshotName, snapshotName);
     }
     async restore(data) {
         const fs = (0, client_1.createFs)(this.config.backend);

package/cli.js

@@ -9,8 +9,8 @@ const AppError_1 = require("./Error/AppError");
 const CommandFactory_1 = require("./Factory/CommandFactory");
 const globalData_1 = __importDefault(require("./globalData"));
 const cli_1 = require("./utils/cli");
+const exit_1 = require("./utils/exit");
 const fs_1 = require("./utils/fs");
-const process_1 = require("./utils/process");
 const string_1 = require("./utils/string");
 const temp_1 = require("./utils/temp");
 const chalk_1 = require("chalk");
@@ -19,12 +19,11 @@ const fs_2 = require("fs");
 const path_1 = require("path");
 function getGlobalOptions() {
     const result = program.opts();
+    const parseBool = (v) => v === "true" ? true : v === "false" ? false : v;
     return {
         ...result,
-        tty: result.tty === "auto" ? "auto" : result.tty === "true",
-        progress: result.progress === "auto" || result.progress === "interval"
-            ? result.progress
-            : result.progress === "true",
+        tty: parseBool(result.tty),
+        progress: parseBool(result.progress),
     };
 }
 function makeCommand(command) {
@@ -89,8 +88,7 @@ program.usage("dtt");
 program.option("-v,--verbose", "Verbose", (_, previous) => previous + 1, 0);
 program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFIG"] ?? (cwd.endsWith(path_1.sep) ? cwd : `${cwd}${path_1.sep}`));
 program.option("--tty <value>", "TTY mode (auto, true, false)", "auto");
-program.option("--progress <value>", "Progress type (auto, true, false, interval)", "auto");
-program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
+program.option("--progress <value>", "Progress type (auto, true, false, interval, interval:[ms])", "auto");
 program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
 makeCommand(CommandFactory_1.CommandEnum.startServer).alias("start");
 makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
@@ -112,7 +110,7 @@ exports.buildArgs = buildArgs;
 function parseArgs(args) {
     program.parse(args);
     const verbose = getGlobalOptions().verbose;
-    (0, process_1.onExit)((eventName, error) => {
+    (0, exit_1.onExit)((eventName, error) => {
         if (eventName !== "exit") {
             process.stdout.write(cli_1.showCursorCommand);
             console.info(`\nClosing... (reason: ${eventName})`);

package/config.schema.json

@@ -1051,6 +1051,9 @@
             "path": {
               "type": "string"
             },
+            "log": {
+              "type": "boolean"
+            },
             "users": {
               "type": "array",
               "items": {
@@ -1077,6 +1080,37 @@
                   "type": "string"
                 }
               }
+            },
+            "trustProxy": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "remoteAddressHeader"
+                  ],
+                  "properties": {
+                    "remoteAddressHeader": {
+                      "type": "string"
+                    }
+                  }
+                }
+              ]
+            },
+            "allowlist": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "enabled": {
+                  "type": "boolean"
+                },
+                "remoteAddresses": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
             }
           }
         },

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.29.1",
+  "version": "0.30.1",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",
     "async": "^3.2.4",
-    "bytes": "^3.1.2",
     "chalk": "^4.1.2",
     "commander": "^11.0.0",
     "dayjs": "^1.11.10",

package/utils/bytes.d.ts

@@ -0,0 +1,2 @@
+export declare function formatBytes(bytes: number): string;
+export declare function parseSize(size: string): number;

package/utils/bytes.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSize = exports.formatBytes = void 0;
+const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+const sizeRegex = new RegExp(`^(\\d+(?:\\.\\d+)?)\\s*(${units.join("|")})$`, "i");
+function formatBytes(bytes) {
+    let u = 0;
+    let n = bytes;
+    if (bytes < 0n)
+        throw new Error(`Invalid bytes: ${bytes.toString()}`);
+    while (n >= 1024n && ++u)
+        n = n / 1024;
+    return Number(n).toFixed(n < 10 && u > 0 ? 1 : 0) + units[u];
+}
+exports.formatBytes = formatBytes;
+function parseSize(size) {
+    const matches = sizeRegex.exec(size);
+    if (!matches)
+        throw new Error(`Invalid size: ${size}`);
+    const [, value, unit] = matches;
+    const unitIndex = units.findIndex((v) => v === unit.toUpperCase());
+    if (unitIndex === -1)
+        throw new Error(`Unit not found: ${unit}`);
+    let result = Number(value);
+    for (let i = 0; i < unitIndex; ++i)
+        result *= 1024;
+    return result;
+}
+exports.parseSize = parseSize;

package/utils/datatruck/client.d.ts

@@ -1,4 +1,5 @@
 import { DiskStats } from "../fs";
+import { BasicProgress } from "../progress";
 import { AbstractFs, FsOptions } from "../virtual-fs";
 export declare class RemoteFs extends AbstractFs {
     readonly options: FsOptions;
@@ -9,6 +10,7 @@ export declare class RemoteFs extends AbstractFs {
     protected fetchJson(name: string, params: any[]): Promise<any>;
     protected post(name: string, params: any[], data: string): Promise<void>;
     existsDir(path: string): Promise<any>;
+    rename(source: string, target: string): Promise<any>;
     mkdir(path: string): Promise<any>;
     readFile(path: string): Promise<any>;
     readdir(path: string): Promise<any>;
@@ -18,7 +20,10 @@ export declare class RemoteFs extends AbstractFs {
     rmAll(path: string): Promise<void>;
     fetchDiskStats(path: string): Promise<DiskStats>;
     upload(source: string, target: string): Promise<void>;
-    download(source: string, target: string, timeout?: number): Promise<void>;
+    download(source: string, target: string, options?: {
+        timeout?: number;
+        onProgress?: (progress: BasicProgress) => void;
+    }): Promise<void>;
 }
 export declare function isRemoteBackend(backend: string): boolean;
 export declare function createFs(backend: string): AbstractFs;

package/utils/datatruck/client.js

@@ -44,6 +44,9 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
     async existsDir(path) {
         return await this.fetchJson("existsDir", [path]);
     }
+    async rename(source, target) {
+        return await this.fetchJson("rename", [source, target]);
+    }
     async mkdir(path) {
         return await this.fetchJson("mkdir", [path]);
     }
@@ -76,13 +79,11 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
             },
         });
     }
-    async download(source, target, timeout = 100000) {
+    async download(source, target, options = {}) {
         await (0, http_1.downloadFile)(`${this.url}/download`, target, {
-            timeout,
+            ...options,
             headers: this.headers,
-            query: {
-                params: JSON.stringify([source]),
-            },
+            query: { params: JSON.stringify([source]) },
         });
     }
 }

package/utils/datatruck/server.d.ts

@@ -12,6 +12,16 @@ export type DatatruckServerOptions = {
         address?: string;
     };
     users?: User[];
+    trustProxy?: true | {
+        remoteAddressHeader: string;
+    };
+    allowlist?: {
+        /**
+         * @default true
+         */
+        enabled?: boolean;
+        remoteAddresses?: string[];
+    };
 };
 export declare const headerKey: {
     user: string;