@datatruck/cli 0.3.2 → 0.6.0

package/Repository/ResticRepository.js

@@ -18,10 +18,22 @@ const path_1 = require("path");
 exports.resticRepositoryName = "restic";
 exports.resticRepositoryDefinition = {
     type: "object",
-    required: ["passwordFile", "repository"],
+    required: ["password", "repository"],
     additionalProperties: false,
     properties: {
-        passwordFile: { type: "string" },
+        password: {
+            anyOf: [
+                { type: "string" },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    required: ["path"],
+                    properties: {
+                        path: { type: "string" },
+                    },
+                },
+            ],
+        },
         repository: {
             type: "object",
             additionalProperties: false,
@@ -59,7 +71,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (this.env)
             return this.env;
         return (this.env = {
-            RESTIC_PASSWORD_FILE: (0, path_1.resolve)(this.config.passwordFile),
+            ...(typeof this.config.password === "string"
+                ? { RESTIC_PASSWORD: this.config.password }
+                : { RESTIC_PASSWORD_FILE: (0, path_1.resolve)(this.config.password.path) }),
             RESTIC_REPOSITORY: await ResticUtil_1.ResticUtil.formatRepository(this.config.repository),
         });
     }
@@ -176,6 +190,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             });
             if (data.options.verbose)
                 (0, cli_util_1.logExec)(`Writing paths lists`);
+            await data.onProgress({
+                step: "Writing excluded paths list...",
+            });
             gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
                 paths: stream,
             });
@@ -183,12 +200,19 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
+        await data.onProgress({
+            step: "Fetching last snapshot...",
+        });
         const [lastSnapshot] = await restic.snapshots({
             json: true,
             tags: [packageTag],
             latest: 1,
         });
         const nodePkg = (0, fs_util_1.parsePackageFile)();
+        let lastProgress;
+        await data.onProgress({
+            step: "Executing backup action...",
+        });
         await restic.backup({
             cwd: sourcePath,
             paths: ["."],
@@ -209,22 +233,19 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             ],
             onStream: async (streamData) => {
                 if (streamData.message_type === "status") {
-                    await data.onProgress({
-                        total: Number((streamData.total_bytes / 1024 / 1024).toFixed(2)),
-                        current: streamData.bytes_done
-                            ? Number((streamData.bytes_done / 1024 / 1024).toFixed(2))
-                            : 0,
+                    await data.onProgress((lastProgress = {
+                        total: streamData.total_files,
+                        current: streamData.files_done ?? 0,
                         percent: Number((streamData.percent_done * 100).toFixed(2)),
                         step: streamData.current_files?.join(", ") ?? "-",
-                    });
+                    }));
                 }
             },
         });
         await data.onProgress({
-            current: 100,
+            total: lastProgress?.total || 0,
+            current: lastProgress?.total || 0,
             percent: 100,
-            step: "",
-            total: 100,
         });
     }
     async onRestore(data) {

package/SessionDriver/ConsoleSessionDriver.js

@@ -82,18 +82,19 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
         const parts = [
             `${padding}${message.textPrefix} [${(0, chalk_1.grey)(sessionId)}] ${message.text}`,
             badges,
-            ...(haveProgressBar
-                ? [
-                    (0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressPercent ?? 0, 10)),
-                    `${message.progressPercent?.toFixed(2)}%`,
-                    `${message.progressCurrent}/${message.progressTotal}`,
-                    message.progressStep,
-                    `${message.progressStepPercent
-                        ? (0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressStepPercent ?? 0, 10))
-                        : ""}`,
-                ].filter((v) => !!v?.length)
-                : []),
         ];
+        if (typeof message.progressPercent === "number") {
+            parts.push((0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressPercent ?? 0, 10)), `${message.progressPercent?.toFixed(2)}%`);
+        }
+        if (typeof message.progressCurrent === "number" ||
+            typeof message.progressTotal === "number") {
+            parts.push(`${message.progressCurrent ?? "?"}/${message.progressTotal ?? "?"}`);
+        }
+        if (typeof message.progressStep === "string")
+            parts.push(message.progressStep);
+        if (typeof message.progressStepPercent === "number") {
+            parts.push((0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressStepPercent ?? 0, 10)));
+        }
         return parts.join(` ${sep} `);
     }
     async onWrite(data) {

package/Task/GitTask.d.ts

@@ -18,6 +18,10 @@ export declare type GitTaskConfigType = {
      * @default true
      */
     includeConfig?: boolean;
+    /**
+     * @default 1
+     */
+    fileCopyConcurrency?: number;
 };
 export declare const gitTaskName = "git";
 export declare const gitTaskDefinition: JSONSchema7;

package/Task/GitTask.js

@@ -9,7 +9,6 @@ const process_util_1 = require("../util/process-util");
 const TaskAbstract_1 = require("./TaskAbstract");
 const assert_1 = require("assert");
 const fs_1 = require("fs");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
@@ -49,6 +48,10 @@ exports.gitTaskDefinition = {
         includeConfig: {
             type: "boolean",
         },
+        fileCopyConcurrency: {
+            type: "integer",
+            minimum: 1,
+        },
     },
 };
 class GitTask extends TaskAbstract_1.TaskAbstract {
@@ -69,6 +72,9 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
         (0, assert_1.ok)(typeof targetPath === "string");
         // Bundle
         const bundlePath = (0, path_1.join)(targetPath, "repo.bundle");
+        await data.onProgress({
+            step: "Creating bundle...",
+        });
         await (0, process_util_1.exec)(this.command, ["bundle", "create", bundlePath, "--all"], {
             cwd: path,
         }, {
@@ -152,40 +158,28 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
         for (const option of lsFilesConfig) {
             if (!option.include)
                 continue;
-            const createdPaths = [];
             const outPath = (0, path_1.join)(targetPath, `repo.${option.name}`);
             await (0, fs_util_1.mkdirIfNotExists)(outPath);
             if (data.options.verbose)
                 (0, cli_util_1.logExec)(`Copying ${option.name} files to ${outPath}`);
-            const reader = (0, readline_1.createInterface)({
-                input: (0, fs_1.createReadStream)(option.pathsPath),
-            });
-            for await (const entry of reader) {
-                const source = (0, path_1.join)(path, entry);
-                const target = (0, path_1.join)(outPath, entry);
-                if (entry.endsWith("/")) {
-                    await (0, promises_1.mkdir)(target, {
-                        recursive: true,
-                    });
-                }
-                else {
+            await (0, fs_util_1.cpy)({
+                input: {
+                    type: "pathList",
+                    path: option.pathsPath,
+                    basePath: path,
+                },
+                targetPath: outPath,
+                concurrency: this.config.fileCopyConcurrency,
+                onPath: async ({ entryPath }) => {
                     currentFiles++;
                     await data.onProgress({
                         total,
                         current: currentFiles,
                         percent: (0, math_util_1.progressPercent)(total, currentFiles),
-                        step: entry,
+                        step: entryPath,
                     });
-                    const dir = (0, path_1.dirname)(target);
-                    if (!createdPaths.includes(dir)) {
-                        await (0, promises_1.mkdir)(dir, {
-                            recursive: true,
-                        });
-                        createdPaths.push(dir);
-                    }
-                    await (0, promises_1.copyFile)(source, target);
-                }
-            }
+                },
+            });
             await (0, promises_1.rm)(option.pathsPath);
         }
     }
@@ -235,10 +229,15 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
             if (await (0, fs_util_1.checkDir)(sourcePath)) {
                 if (data.options.verbose)
                     (0, cli_util_1.logExec)(`Copying ${name} files to ${restorePath}`);
-                await (0, fs_extra_1.copy)(sourcePath, restorePath, {
-                    filter: async (path) => {
-                        await incrementProgress((0, path_1.relative)(sourcePath, path));
-                        return true;
+                await (0, fs_util_1.cpy)({
+                    input: {
+                        type: "glob",
+                        sourcePath,
+                    },
+                    targetPath: restorePath,
+                    concurrency: this.config.fileCopyConcurrency,
+                    onPath: async ({ entryPath }) => {
+                        await incrementProgress(entryPath);
                     },
                 });
             }

package/Task/MariadbTask.js

@@ -83,21 +83,30 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         await (0, fs_util_1.forEachFile)(sourcePath, () => {
             total++;
         });
-        const onData = async (lines) => {
-            const regex = /\[\d{1,}\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} Copying (.+) to/;
-            let path = lines
-                .split(/\r?\n/)
-                .reduce((result, line) => {
-                const matches = regex.exec(line);
-                if (matches) {
-                    current++;
-                    result.push(matches[1]);
+        let childProcess;
+        const onData = async (strLines) => {
+            const paths = [];
+            const pathRegex = /\[\d{1,}\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} Copying (.+) to/;
+            const lines = strLines.split(/\r?\n/);
+            let fatalError = false;
+            for (const line of lines) {
+                if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
+                    line.includes("Error: cannot read redo log header")) {
+                    fatalError = true;
                 }
-                return result;
-            }, [])
-                .pop();
-            if (path) {
-                path = (0, posix_1.normalize)(path);
+                else {
+                    const matches = pathRegex.exec(line);
+                    if (matches) {
+                        current++;
+                        paths.push(matches[1]);
+                    }
+                }
+            }
+            if (fatalError) {
+                childProcess.kill();
+            }
+            else if (paths.length) {
+                const path = (0, posix_1.normalize)(paths[0]);
                 await data.onProgress({
                     current,
                     percent: (0, math_util_1.progressPercent)(total, current),
@@ -108,6 +117,9 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         };
         await (0, process_util_1.exec)(command, args, undefined, {
             log: this.verbose,
+            onSpawn: (p) => {
+                childProcess = p;
+            },
             stdout: {
                 onData,
             },

package/Task/MssqlTask.js

@@ -8,7 +8,6 @@ const fs_util_1 = require("../util/fs-util");
 const process_util_1 = require("../util/process-util");
 const TaskAbstract_1 = require("./TaskAbstract");
 const assert_1 = require("assert");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
@@ -85,7 +84,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
         const restorePath = data.package.restorePath;
         (0, assert_1.ok)(typeof restorePath === "string");
         await (0, fs_util_1.mkdirIfNotExists)(restorePath);
-        const files = await (0, fs_extra_1.readdir)(restorePath);
+        const files = await (0, promises_1.readdir)(restorePath);
         for (const file of files) {
             if (!file.endsWith(MssqlTask.SUFFIX))
                 continue;

package/Task/TaskAbstract.d.ts

@@ -3,10 +3,10 @@ import { RestoreActionOptionsType } from "../Action/RestoreAction";
 import { PackageConfigType } from "../Config/PackageConfig";
 import { SnapshotType } from "../Repository/RepositoryAbstract";
 export declare type ProgressDataType = {
-    total: number;
-    current: number;
-    percent: number;
-    step: string;
+    total?: number;
+    current?: number;
+    percent?: number;
+    step?: string;
     stepPercent?: number;
 };
 export declare type BackupDataType = {

package/cli.js

@@ -1,8 +1,13 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.exec = exports.parseArgs = exports.buildArgs = void 0;
+const ConfigAction_1 = require("./Action/ConfigAction");
 const AppError_1 = require("./Error/AppError");
 const CommandFactory_1 = require("./Factory/CommandFactory");
+const globalData_1 = __importDefault(require("./globalData"));
 const cli_util_1 = require("./util/cli-util");
 const fs_util_1 = require("./util/fs-util");
 const process_util_1 = require("./util/process-util");
@@ -35,7 +40,19 @@ function makeCommandAction(command) {
         let exitCode = 1;
         const globalOptions = getGlobalOptions();
         try {
-            exitCode = await (0, CommandFactory_1.CommandFactory)(command, getGlobalOptions(), options).onExec();
+            const configAction = new ConfigAction_1.ConfigAction({
+                path: globalOptions.config,
+                verbose: !!globalOptions.verbose,
+            });
+            const config = await configAction.exec();
+            if (config.data.tempDir)
+                globalData_1.default.tempDir = (0, path_1.isAbsolute)(config.data.tempDir)
+                    ? config.data.tempDir
+                    : (0, path_1.join)((0, path_1.dirname)(config.path), config.data.tempDir);
+            exitCode = await (0, CommandFactory_1.CommandFactory)(command, {
+                ...globalOptions,
+                config: config.data,
+            }, options).onExec();
         }
         catch (e) {
             const error = e;

package/config.schema.json

@@ -20,6 +20,37 @@
         "name": {
           "type": "string"
         },
+        "enabled": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "defaults": {
+                  "type": "boolean"
+                },
+                "backup": {
+                  "type": "boolean"
+                },
+                "init": {
+                  "type": "boolean"
+                },
+                "prune": {
+                  "type": "boolean"
+                },
+                "restore": {
+                  "type": "boolean"
+                },
+                "snapshots": {
+                  "type": "boolean"
+                }
+              }
+            }
+          ]
+        },
         "config": {}
       },
       "anyOf": [
@@ -456,19 +487,39 @@
               }
             }
           ]
+        },
+        "fileCopyConcurrency": {
+          "type": "integer",
+          "minimum": 1
         }
       }
     },
     "restic-repository": {
       "type": "object",
       "required": [
-        "passwordFile",
+        "password",
         "repository"
       ],
       "additionalProperties": false,
       "properties": {
-        "passwordFile": {
-          "type": "string"
+        "password": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "path"
+              ],
+              "properties": {
+                "path": {
+                  "type": "string"
+                }
+              }
+            }
+          ]
         },
         "repository": {
           "type": "object",
@@ -568,6 +619,10 @@
         },
         "includeConfig": {
           "type": "boolean"
+        },
+        "fileCopyConcurrency": {
+          "type": "integer",
+          "minimum": 1
         }
       }
     },
@@ -769,6 +824,9 @@
         "$schema": {
           "type": "string"
         },
+        "tempDir": {
+          "type": "string"
+        },
         "repositories": {
           "type": "array",
           "items": {

package/globalData.d.ts

@@ -0,0 +1,5 @@
+declare const globalData: {
+    configDir?: string;
+    tempDir: string;
+};
+export default globalData;

package/globalData.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const os_1 = require("os");
+const globalData = {
+    tempDir: (0, os_1.tmpdir)(),
+};
+exports.default = globalData;

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.3.2",
+  "version": "0.6.0",
   "dependencies": {
     "ajv": "^8.11.0",
+    "async": "^3.2.4",
     "chalk": "^4.1.2",
     "cli-table3": "^0.6.2",
     "commander": "^9.2.0",
     "dayjs": "^1.11.2",
     "fast-glob": "^3.2.11",
-    "fs-extra": "^10.1.0",
     "micromatch": "^4.0.5",
     "sqlite": "^4.1.1",
     "sqlite3": "^5.0.8"

package/util/GitUtil.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.GitUtil = void 0;
 const fs_util_1 = require("./fs-util");
 const process_util_1 = require("./process-util");
-const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
 class GitUtil {
     constructor(options) {
         this.options = options;
@@ -16,7 +16,7 @@ class GitUtil {
     }
     async canBeInit(repo) {
         return ((0, fs_util_1.isLocalDir)(repo) &&
-            (!(await (0, fs_util_1.checkDir)(repo)) || !(await (0, fs_extra_1.readdir)(repo)).length));
+            (!(await (0, fs_util_1.checkDir)(repo)) || !(await (0, promises_1.readdir)(repo)).length));
     }
     async clone(options) {
         return await this.exec([

package/util/ResticUtil.js

@@ -4,7 +4,6 @@ exports.ResticUtil = void 0;
 const fs_util_1 = require("./fs-util");
 const process_util_1 = require("./process-util");
 const string_util_1 = require("./string-util");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class ResticUtil {
@@ -20,7 +19,7 @@ class ResticUtil {
         if (input.passwordFile)
             input = {
                 ...input,
-                password: (await (0, fs_extra_1.readFile)(input.passwordFile)).toString(),
+                password: (await (0, promises_1.readFile)(input.passwordFile)).toString(),
             };
         return `${input.backend}:${(0, string_util_1.formatUri)(input, hidePassword)}`;
     }

package/util/datatruck/config-util.d.ts

@@ -1,10 +1,13 @@
 import { ConfigType } from "../../Config/Config";
 import type { PackageConfigType } from "../../Config/PackageConfig";
-export declare function findRepositoryOrFail(config: ConfigType, repositoryName: string): import("../../Config/RepositoryConfig").RepositoryConfigType;
+import { RepositoryConfigEnabledActionType, RepositoryConfigType } from "../../Config/RepositoryConfig";
+export declare function findRepositoryOrFail(config: ConfigType, repositoryName: string): RepositoryConfigType;
+export declare function filterRepository(repository: RepositoryConfigType, action?: RepositoryConfigEnabledActionType): boolean;
 export declare function filterPackages(config: ConfigType, options: {
     packageNames?: string[];
     repositoryNames?: string[];
     repositoryTypes?: string[];
+    sourceAction?: RepositoryConfigEnabledActionType;
 }): PackageConfigType[];
 declare type ResolvePackagePathParamsType = ResolvePackageParamsType & {
     packageName: string;

package/util/datatruck/config-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgExcludeParams = exports.pkgIncludeParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.findRepositoryOrFail = void 0;
+exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgExcludeParams = exports.pkgIncludeParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.filterRepository = exports.findRepositoryOrFail = void 0;
 const AppError_1 = require("../../Error/AppError");
 const fs_util_1 = require("../fs-util");
 const string_util_1 = require("../string-util");
@@ -12,6 +12,14 @@ function findRepositoryOrFail(config, repositoryName) {
     return repo;
 }
 exports.findRepositoryOrFail = findRepositoryOrFail;
+function filterRepository(repository, action) {
+    const enabled = repository.enabled ?? true;
+    if (typeof enabled === "boolean")
+        return enabled;
+    const defaults = enabled["defaults"] ?? true;
+    return action ? enabled[action] ?? defaults : defaults;
+}
+exports.filterRepository = filterRepository;
 function filterPackages(config, options) {
     const packagePatterns = (0, string_util_1.makePathPatterns)(options.packageNames);
     return config.packages
@@ -19,6 +27,8 @@ function filterPackages(config, options) {
         pkg = Object.assign({}, pkg);
         pkg.repositoryNames = (pkg.repositoryNames ?? []).filter((name) => {
             const repo = findRepositoryOrFail(config, name);
+            if (!filterRepository(repo, options?.sourceAction))
+                return false;
             return ((!options.repositoryNames ||
                 options.repositoryNames.includes(name)) &&
                 (!options.repositoryTypes ||

package/util/fs-util.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+import { Interface } from "readline";
 export declare function isLocalDir(path: string): boolean;
 export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
@@ -44,3 +45,35 @@ export declare function writePathLists(options: {
         multipleStats: Record<string, number>;
     };
 }>;
+export declare function cpy(options: {
+    input: {
+        type: "glob";
+        sourcePath: string;
+        include?: string[];
+        exclude?: string[];
+    } | {
+        type: "stream";
+        basePath: string;
+        value: Interface;
+    } | {
+        type: "pathList";
+        path: string;
+        basePath: string;
+    };
+    targetPath: string;
+    /**
+     * @default 1
+     */
+    concurrency?: number;
+    onPath?: (data: {
+        isDir: boolean;
+        entryPath: string;
+        entrySourcePath: string;
+        entryTargetPath: string;
+        stats: {
+            paths: number;
+            files: number;
+            dirs: number;
+        };
+    }) => Promise<boolean | void>;
+}): Promise<void>;