@datatruck/cli 0.5.0 → 0.7.0

package/Action/BackupAction.d.ts

@@ -10,6 +10,7 @@ export declare type BackupActionOptionsType = {
     repositoryNames?: string[];
     repositoryTypes?: string[];
     packageNames?: string[];
+    packageTaskNames?: string[];
     tags?: string[];
     dryRun?: boolean;
     verbose?: boolean;

package/Action/BackupAction.js

@@ -21,6 +21,7 @@ class BackupAction {
         await session.initDrivers();
         let packages = (0, config_util_1.filterPackages)(this.config, {
             packageNames: this.options.packageNames,
+            packageTaskNames: this.options.packageTaskNames,
             repositoryNames: this.options.repositoryNames,
             repositoryTypes: this.options.repositoryTypes,
             sourceAction: "backup",
@@ -193,7 +194,9 @@ class BackupAction {
                 error: error?.message,
             });
         }
-        await session.endDrivers();
+        await session.endDrivers({
+            snapshotId: snapshot.id.slice(0, 8),
+        });
         return {
             total: total,
             errors: errors,

package/Action/RestoreAction.d.ts

@@ -10,6 +10,7 @@ export declare type RestoreActionOptionsType = {
     snapshotId: string;
     tags?: string[];
     packageNames?: string[];
+    packageTaskNames?: string[];
     repositoryNames?: string[];
     repositoryTypes?: string[];
     verbose?: boolean;

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # @datatruck/cli
 
+## 0.7.0
+
+### Minor Changes
+
+- [`3b8d6da`](https://github.com/swordev/datatruck/commit/3b8d6da01495799aceb848a63b35b8c46a7d1b0e) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `--package-task` cli option
+
+* [`69b34a0`](https://github.com/swordev/datatruck/commit/69b34a02b9cade48df2b071a92a8f79d5cfec23e) Thanks [@juanrgm](https://github.com/juanrgm)! - Allow restore multiple backups over the same database
+
+- [`69caf26`](https://github.com/swordev/datatruck/commit/69caf26881272331bd4c8d7d345b3b85d33e33ac) Thanks [@juanrgm](https://github.com/juanrgm)! - Add cli short option to `--tag`
+
+* [`377f0de`](https://github.com/swordev/datatruck/commit/377f0de345c9c8f45c772ac47e4ded81e91725d7) Thanks [@juanrgm](https://github.com/juanrgm)! - Rename cli short option to `-rt`
+
+### Patch Changes
+
+- [`c03200a`](https://github.com/swordev/datatruck/commit/c03200a6347d1e9f9fdad86dcb22df30bbefcab4) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix `sql-dump` tasks
+
+* [`f56a4bc`](https://github.com/swordev/datatruck/commit/f56a4bcb429a674c13f32de73985cd67eb1acc23) Thanks [@juanrgm](https://github.com/juanrgm)! - Show full error message
+
+- [`4324422`](https://github.com/swordev/datatruck/commit/4324422550474619811a8d455af55bc6e3b08aeb) Thanks [@juanrgm](https://github.com/juanrgm)! - Use connection port in `mysql-dump` task
+
+## 0.6.1
+
+### Patch Changes
+
+- [`0ba6229`](https://github.com/swordev/datatruck/commit/0ba6229348c109a59783e72242ab7c0e61f25e36) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix progress bar in restic repository
+
+## 0.6.0
+
+### Minor Changes
+
+- [`0c6877d`](https://github.com/swordev/datatruck/commit/0c6877d189761e75dd434b0a8d72b71621d024de) Thanks [@juanrgm](https://github.com/juanrgm)! - Show more progress stats
+
+* [`751e1f6`](https://github.com/swordev/datatruck/commit/751e1f6d6b33d3fa96eb40d998fdd140ce0e3875) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `fileCopyConcurrency` option
+
+- [`05487e6`](https://github.com/swordev/datatruck/commit/05487e6a33f875a3afb7ff0815b16da6f2a41301) Thanks [@juanrgm](https://github.com/juanrgm)! - Parse InnoDB error in `MariadbTask` to avoid infinite wait
+
+### Patch Changes
+
+- [`b62a6f8`](https://github.com/swordev/datatruck/commit/b62a6f8a82409339afd65d4f96476eb57bbfb5a2) Thanks [@juanrgm](https://github.com/juanrgm)! - Resolve target/restore path in local repository
+
 ## 0.5.0
 
 ### Minor Changes

package/Command/BackupCommand.d.ts

@@ -3,6 +3,7 @@ import { If } from "../util/ts-util";
 import { CommandAbstract } from "./CommandAbstract";
 export declare type BackupCommandOptionsType<TResolved = false> = {
     package?: If<TResolved, string[]>;
+    packageTask?: If<TResolved, string[]>;
     repository?: If<TResolved, string[]>;
     repositoryType?: If<TResolved, RepositoryConfigType["type"][]>;
     tag?: If<TResolved, string[]>;

package/Command/BackupCommand.js

@@ -21,6 +21,11 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
                 option: "-p,--package <values>",
                 parser: string_util_1.parseStringList,
             },
+            packageTask: {
+                description: "Package task names",
+                option: "-pt,--package-task <values>",
+                parser: string_util_1.parseStringList,
+            },
             repository: {
                 description: "Repository names",
                 option: "-r,--repository <values>",
@@ -28,12 +33,12 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
             },
             repositoryType: {
                 description: "Repository types",
-                option: "-t,--repository-type <values>",
+                option: "-rt,--repository-type <values>",
                 parser: (v) => (0, string_util_1.parseStringList)(v),
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
             date: {
@@ -47,6 +52,7 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
         const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
         const backup = new BackupAction_1.BackupAction(config, {
             packageNames: this.options.package,
+            packageTaskNames: this.options.packageTask,
             repositoryNames: this.options.repository,
             repositoryTypes: this.options.repositoryType,
             tags: this.options.tag,

package/Command/BackupSessionsCommand.js

@@ -24,7 +24,7 @@ class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
             limit: {

package/Command/ConfigCommand.d.ts

@@ -3,6 +3,7 @@ import { If } from "../util/ts-util";
 import { CommandAbstract } from "./CommandAbstract";
 export declare type ConfigCommandOptionsType<TResolved = false> = {
     package?: If<TResolved, string[]>;
+    packageTask?: If<TResolved, string[]>;
     repository?: If<TResolved, string[]>;
     repositoryType?: If<TResolved, RepositoryConfigType["type"][]>;
 };

package/Command/ConfigCommand.js

@@ -14,6 +14,11 @@ class ConfigCommand extends CommandAbstract_1.CommandAbstract {
                 option: "-p,--package <values>",
                 parser: string_util_1.parseStringList,
             },
+            packageTask: {
+                description: "Package task names",
+                option: "-pt,--package-task <values>",
+                parser: string_util_1.parseStringList,
+            },
             repository: {
                 description: "Repository names",
                 option: "-r,--repository <values>",
@@ -21,7 +26,7 @@ class ConfigCommand extends CommandAbstract_1.CommandAbstract {
             },
             repositoryType: {
                 description: "Repository types",
-                option: "-t,--repository-type <values>",
+                option: "-rt,--repository-type <values>",
                 parser: (v) => (0, string_util_1.parseStringList)(v),
             },
         });
@@ -30,6 +35,7 @@ class ConfigCommand extends CommandAbstract_1.CommandAbstract {
         const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
         const packages = (0, config_util_1.filterPackages)(config, {
             packageNames: this.options.package,
+            packageTaskNames: this.options.packageTask,
             repositoryNames: this.options.repository,
             repositoryTypes: this.options.repositoryType,
         });

package/Command/InitCommand.js

@@ -18,7 +18,7 @@ class InitCommand extends CommandAbstract_1.CommandAbstract {
             },
             repositoryType: {
                 description: "Repository types",
-                option: "-t,--repository-type <values>",
+                option: "-rt,--repository-type <values>",
                 parser: (v) => (0, string_util_1.parseStringList)(v),
             },
         });

package/Command/PruneCommand.js

@@ -81,12 +81,12 @@ class PruneCommand extends CommandAbstract_1.CommandAbstract {
             },
             repositoryType: {
                 description: "Repository types",
-                option: "-t,--repositoryType <values>",
+                option: "-rt,--repository-type <values>",
                 parser: (v) => (0, string_util_1.parseStringList)(v),
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
             showAll: {

package/Command/RestoreCommand.d.ts

@@ -4,6 +4,7 @@ import { CommandAbstract } from "./CommandAbstract";
 export declare type RestoreCommandOptionsType<TResolved = false> = {
     id: string;
     package?: If<TResolved, string[]>;
+    packageTask?: If<TResolved, string[]>;
     repository?: If<TResolved, string[]>;
     repositoryType?: If<TResolved, RepositoryConfigType["type"][]>;
     tag?: If<TResolved, string[]>;

package/Command/RestoreCommand.js

@@ -21,6 +21,11 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
                 option: "-p,--package <values>",
                 parser: string_util_1.parseStringList,
             },
+            packageTask: {
+                description: "Package task names",
+                option: "-pt,--package-task <values>",
+                parser: string_util_1.parseStringList,
+            },
             repository: {
                 description: "Repository names",
                 option: "-r,--repository <values>",
@@ -28,12 +33,12 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
             },
             repositoryType: {
                 description: "Repository types",
-                option: "-t,--repository-type <values>",
+                option: "-rt,--repository-type <values>",
                 parser: (v) => (0, string_util_1.parseStringList)(v),
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
         });
@@ -44,6 +49,7 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
         const restore = new RestoreAction_1.RestoreAction(config, {
             snapshotId: this.options.id,
             packageNames: this.options.package,
+            packageTaskNames: this.options.packageTask,
             repositoryNames: this.options.repository,
             repositoryTypes: this.options.repositoryType,
             tags: this.options.tag,

package/Command/RestoreSessionsCommand.js

@@ -24,7 +24,7 @@ class RestoreSessionsCommand extends CommandAbstract_1.CommandAbstract {
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
             limit: {

package/Command/SnapshotsCommand.js

@@ -76,7 +76,7 @@ class SnapshotsCommand extends CommandAbstract_1.CommandAbstract {
             },
             tag: {
                 description: "Tags",
-                option: "--tag <values>",
+                option: "-t,--tag <values>",
                 parser: string_util_1.parseStringList,
             },
         });

package/Repository/GitRepository.js

@@ -13,7 +13,6 @@ const string_util_1 = require("../util/string-util");
 const RepositoryAbstract_1 = require("./RepositoryAbstract");
 const assert_1 = require("assert");
 const fast_glob_1 = __importDefault(require("fast-glob"));
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
@@ -62,7 +61,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
             log: data.options.verbose,
         });
         if (await git.canBeInit(this.config.repo)) {
-            await (0, fs_extra_1.mkdir)(git.options.dir);
+            await (0, promises_1.mkdir)(git.options.dir);
             await git.exec(["init", "--bare", this.config.repo]);
         }
         const branchName = this.config.branch ?? "master";
@@ -182,7 +181,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
             const target = (0, path_1.join)(tmpPath, entry);
             const dir = (0, path_1.dirname)(target);
             if (!createdPaths.includes(dir)) {
-                await (0, fs_extra_1.mkdir)(dir, {
+                await (0, promises_1.mkdir)(dir, {
                     recursive: true,
                 });
                 createdPaths.push(dir);

package/Repository/LocalRepository.d.ts

@@ -10,6 +10,10 @@ export declare type MetaDataType = {
 export declare type LocalRepositoryConfigType = {
     outPath: string;
     compress?: boolean;
+    /**
+     * @default 1
+     */
+    fileCopyConcurrency?: number;
 };
 declare type CompressObjectType = {
     packs?: {

package/Repository/LocalRepository.js

@@ -16,11 +16,9 @@ const RepositoryAbstract_1 = require("./RepositoryAbstract");
 const assert_1 = require("assert");
 const fast_glob_1 = __importDefault(require("fast-glob"));
 const fs_1 = require("fs");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
-const path_2 = require("path");
 const readline_1 = require("readline");
 exports.localRepositoryName = "local";
 exports.localRepositoryDefinition = {
@@ -62,6 +60,10 @@ exports.localPackageRepositoryDefinition = {
                 },
             ],
         },
+        fileCopyConcurrency: {
+            type: "integer",
+            minimum: 1,
+        },
     },
 };
 class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
@@ -111,11 +113,11 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Deleting ${snapshotPath}`);
         if (await (0, fs_util_1.checkDir)(snapshotPath))
-            await (0, fs_extra_1.rm)(snapshotPath, {
+            await (0, promises_1.rm)(snapshotPath, {
                 recursive: true,
             });
         if (await (0, fs_util_1.checkFile)(metaPath))
-            await (0, fs_extra_1.rm)(metaPath);
+            await (0, promises_1.rm)(metaPath);
     }
     async onSnapshots(data) {
         if (!(await (0, fs_util_1.checkDir)(this.config.outPath)))
@@ -173,8 +175,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
             snapshotDate: data.snapshot.date,
             packageName: data.package.name,
         });
-        const outPath = (0, path_1.join)(this.config.outPath, snapshotName);
-        const createdPaths = [];
+        const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
         const pkg = data.package;
         await (0, promises_1.mkdir)(outPath, {
             recursive: true,
@@ -260,35 +261,26 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
         }
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
-        const reader = (0, readline_1.createInterface)({
-            input: (0, fs_1.createReadStream)(pathLists.path),
-        });
-        for await (const entry of reader) {
-            const source = (0, path_1.join)(sourcePath, entry);
-            const target = (0, path_1.join)(outPath, entry);
-            if (entry.endsWith("/")) {
-                await (0, promises_1.mkdir)(target, {
-                    recursive: true,
-                });
-            }
-            else {
+        await (0, fs_util_1.cpy)({
+            input: {
+                type: "pathList",
+                path: pathLists.path,
+                basePath: sourcePath,
+            },
+            targetPath: outPath,
+            concurrency: this.config.fileCopyConcurrency,
+            async onPath({ isDir, entryPath }) {
+                if (isDir)
+                    return;
                 currentFiles++;
                 await data.onProgress({
                     total: pathLists.total.all,
                     current: currentFiles,
                     percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
-                    step: entry,
+                    step: entryPath,
                 });
-                const dir = (0, path_2.dirname)(target);
-                if (!createdPaths.includes(dir)) {
-                    await (0, promises_1.mkdir)(dir, {
-                        recursive: true,
-                    });
-                    createdPaths.push(dir);
-                }
-                await (0, promises_1.copyFile)(source, target);
-            }
-        }
+            },
+        });
         const metaPath = `${outPath}.json`;
         const nodePkg = (0, fs_util_1.parsePackageFile)();
         const meta = {
@@ -303,8 +295,9 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
         await (0, promises_1.writeFile)(metaPath, LocalRepository.stringifyMetaData(meta));
     }
     async onRestore(data) {
-        const restorePath = data.targetPath ?? data.package.restorePath;
-        (0, assert_1.ok)(restorePath);
+        const relRestorePath = data.targetPath ?? data.package.restorePath;
+        (0, assert_1.ok)(relRestorePath);
+        const restorePath = (0, path_1.resolve)(relRestorePath);
         const [snapshot] = await this.onSnapshots({
             options: {
                 ids: [data.options.snapshotId],
@@ -325,19 +318,27 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
         }, true);
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
-        await (0, fs_extra_1.copy)(sourcePath, restorePath, {
-            filter: async (path) => {
-                const [firstFolder] = (0, path_1.relative)(sourcePath, path).split(path_1.sep);
-                const isZipFile = firstFolder.startsWith(".") && firstFolder.endsWith(".dd.zip");
+        await (0, fs_util_1.cpy)({
+            input: {
+                type: "glob",
+                sourcePath,
+            },
+            targetPath: restorePath,
+            concurrency: this.config.fileCopyConcurrency,
+            onPath: async ({ entryPath, entrySourcePath }) => {
+                const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
+                const isZipFile = isRootFile &&
+                    entryPath.startsWith(".") &&
+                    entryPath.endsWith(".dd.zip");
                 await data.onProgress({
                     total: totalFiles,
                     current: Math.max(currentFiles, 0),
                     percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
-                    step: (0, path_1.relative)(sourcePath, path),
+                    step: entryPath,
                 });
                 if (isZipFile) {
                     await (0, zip_util_1.unzip)({
-                        input: path,
+                        input: entrySourcePath,
                         output: restorePath,
                         verbose: data.options.verbose,
                         onStream: async (stream) => await data.onProgress({

package/Repository/RepositoryAbstract.d.ts

@@ -14,10 +14,10 @@ export declare type SnapshotResultType = SnapshotType & {
     tags: string[];
 };
 export declare type ProgressDataType = {
-    total: number;
-    current: number;
-    percent: number;
-    step: string;
+    total?: number;
+    current?: number;
+    percent?: number;
+    step?: string;
     stepPercent?: number | null;
 };
 export declare type InitDataType = {

package/Repository/ResticRepository.js

@@ -190,6 +190,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             });
             if (data.options.verbose)
                 (0, cli_util_1.logExec)(`Writing paths lists`);
+            await data.onProgress({
+                step: "Writing excluded paths list...",
+            });
             gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
                 paths: stream,
             });
@@ -197,12 +200,21 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
+        await data.onProgress({
+            step: "Fetching last snapshot...",
+        });
         const [lastSnapshot] = await restic.snapshots({
             json: true,
             tags: [packageTag],
             latest: 1,
         });
         const nodePkg = (0, fs_util_1.parsePackageFile)();
+        let lastProgress;
+        let totalFilesChanges = 0;
+        const totalFilesChangesLimit = 10;
+        await data.onProgress({
+            step: "Executing backup action...",
+        });
         await restic.backup({
             cwd: sourcePath,
             paths: ["."],
@@ -223,22 +235,31 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             ],
             onStream: async (streamData) => {
                 if (streamData.message_type === "status") {
-                    await data.onProgress({
-                        total: Number((streamData.total_bytes / 1024 / 1024).toFixed(2)),
-                        current: streamData.bytes_done
-                            ? Number((streamData.bytes_done / 1024 / 1024).toFixed(2))
+                    let showProgressBar = false;
+                    if (totalFilesChanges > totalFilesChangesLimit) {
+                        showProgressBar = true;
+                    }
+                    else if (lastProgress?.total !== streamData.total_files) {
+                        totalFilesChanges = 0;
+                    }
+                    else {
+                        totalFilesChanges++;
+                    }
+                    await data.onProgress((lastProgress = {
+                        total: Math.max(lastProgress?.total || 0, streamData.total_files),
+                        current: Math.max(lastProgress?.current || 0, streamData.files_done ?? 0),
+                        percent: showProgressBar
+                            ? Number((streamData.percent_done * 100).toFixed(2))
                             : 0,
-                        percent: Number((streamData.percent_done * 100).toFixed(2)),
                         step: streamData.current_files?.join(", ") ?? "-",
-                    });
+                    }));
                 }
             },
         });
         await data.onProgress({
-            current: 100,
+            total: lastProgress?.total || 0,
+            current: lastProgress?.total || 0,
             percent: 100,
-            step: "",
-            total: 100,
         });
     }
     async onRestore(data) {

package/SessionDriver/ConsoleSessionDriver.d.ts

@@ -26,10 +26,14 @@ export declare class ConsoleSessionDriver extends SessionDriverAbstract {
     protected rendering?: boolean;
     protected lastColumns?: number;
     protected startTime: number;
+    protected chron: {
+        start: () => number;
+        elapsed: (formatted?: boolean | undefined) => string | number;
+    };
     onInit(): Promise<void>;
-    onEnd(): Promise<void>;
+    onEnd(data?: Record<string, any>): Promise<void>;
     onRead(): Promise<ReadResultType[]>;
-    protected printMessage(message: MessageType): void;
+    protected printMessage(message: MessageType, endMessage: boolean): void;
     protected renderSpinner(text: string): string;
     protected renderMessage(message: MessageType): string;
     onWrite(data: WriteDataType): Promise<void>;