@datatruck/cli 0.17.2 → 0.19.0

package/Task/MariadbTask.d.ts

@@ -11,6 +11,17 @@ export type MariadbTaskConfigType = {
     excludeTables?: string[];
     includeDatabases?: string[];
     excludeDatabases?: string[];
+    /**
+     * @default "auto"
+     */
+    parallel?: number | "auto";
+    compress?: {
+        command: string;
+        args?: string[];
+    } | {
+        type: "gzip" | "pigz";
+        args?: string[];
+    };
 };
 export declare const mariadbTaskName = "mariadb";
 export declare const mariadbTaskDefinition: JSONSchema7;

package/Task/MariadbTask.js

@@ -6,11 +6,13 @@ const cli_1 = require("../utils/cli");
 const fs_1 = require("../utils/fs");
 const math_1 = require("../utils/math");
 const process_1 = require("../utils/process");
+const tar_1 = require("../utils/tar");
 const TaskAbstract_1 = require("./TaskAbstract");
 const assert_1 = require("assert");
+const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
+const os_1 = require("os");
 const path_1 = require("path");
-const posix_1 = require("path/posix");
 exports.mariadbTaskName = "mariadb";
 exports.mariadbTaskDefinition = {
     type: "object",
@@ -39,9 +41,67 @@ exports.mariadbTaskDefinition = {
         excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         includeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         excludeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        parallel: {
+            anyOf: [{ type: "integer", minimum: 1 }, { enum: ["auto"] }],
+        },
+        compress: {
+            anyOf: [
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        command: { type: "string" },
+                        args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        type: { enum: ["gzip", "pigz"] },
+                        args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
+            ],
+        },
     },
 };
+function normalizeConfig(input) {
+    let parallel = input.parallel ?? "auto";
+    let cores = (0, os_1.cpus)().length;
+    if (parallel === "auto") {
+        parallel = input.compress ? Math.round(cores / 2) : cores;
+        cores = Math.max(1, cores - parallel);
+    }
+    let compress;
+    if (input.compress && "type" in input.compress) {
+        if (input.compress.type === "pigz") {
+            compress = { command: "pigz", args: [] };
+            if (!input.compress.args?.includes("-p")) {
+                compress.args.push("-p", cores.toString());
+            }
+        }
+        else if (input.compress.type === "gzip") {
+            compress = { command: "gzip" };
+        }
+    }
+    else {
+        compress = input.compress;
+    }
+    return { parallel, compress };
+}
+const parseLineRegex = /^(?:\[(?<step>\d+)\] )?(?<dateTime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})?\s+(?<text>.+)\s*$/;
+function parseLine(line) {
+    const result = parseLineRegex.exec(line);
+    if (result) {
+        return result.groups;
+    }
+    else {
+        return { text: line };
+    }
+}
 class MariadbTask extends TaskAbstract_1.TaskAbstract {
+    verbose;
     get command() {
         return this.config.command ?? "mariabackup";
     }
@@ -58,18 +118,25 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         const targetPath = data.targetPath;
         (0, assert_1.ok)(typeof sourcePath === "string");
         (0, assert_1.ok)(typeof targetPath === "string");
+        const { parallel, compress } = normalizeConfig(config);
         const args = [
             `--backup`,
             `--datadir=${sourcePath}`,
-            `--target-dir=${targetPath}`,
             `--host=${config.hostname}`,
             `--user=${config.username}`,
+            `--parallel=${parallel}`,
             `--password=${typeof config.password === "string"
                 ? config.password
                 : config.password
                     ? (await (0, promises_1.readFile)(config.password.path)).toString()
                     : ""}`,
         ];
+        if (compress) {
+            args.push(`--stream=xbstream`);
+        }
+        else {
+            args.push(`--target-dir=${targetPath}`);
+        }
         if (config.includeDatabases)
             args.push(`--databases=${config.includeDatabases.join(" ")}`);
         if (config.excludeDatabases)
@@ -83,34 +150,23 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         await (0, fs_1.forEachFile)(sourcePath, () => {
             total++;
         });
-        let childProcess;
-        const onData = async (strLines) => {
-            const paths = [];
-            const pathRegex = /\[\d{1,}\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} Copying (.+) to/;
-            const lines = strLines.split(/\r?\n/);
-            let fatalError = false;
-            for (const line of lines) {
-                if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
-                    line.includes("Error: cannot read redo log header")) {
-                    fatalError = true;
-                }
-                else {
-                    const matches = pathRegex.exec(line);
-                    if (matches) {
-                        current++;
-                        paths.push(matches[1]);
-                    }
-                }
-            }
-            if (fatalError) {
-                childProcess.kill();
+        let p1;
+        let lastLineText;
+        const pathRegex = /((Copying|Streaming) .+) to/;
+        const onData = async (line) => {
+            const { text } = parseLine(line);
+            lastLineText = text;
+            if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
+                line.includes("Error: cannot read redo log header")) {
+                p1.kill();
             }
-            else if (paths.length) {
-                const path = (0, posix_1.normalize)(paths[0]);
+            else {
+                const matches = pathRegex.exec(text);
+                if (matches)
+                    current++;
                 await data.onProgress({
                     relative: {
-                        description: "Copying file",
-                        payload: path,
+                        payload: matches ? matches[1] : text,
                     },
                     absolute: {
                         current,
@@ -120,36 +176,163 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
                 });
             }
         };
-        await (0, process_1.exec)(command, args, undefined, {
-            log: this.verbose,
-            onSpawn: (p) => {
-                childProcess = p;
-            },
-            stdout: {
-                onData,
-            },
-            stderr: {
-                onData,
-            },
-        });
-        await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
-            log: this.verbose,
-            stderr: { onData: () => { } },
-        });
+        const stats = { xbFiles: total };
+        await (0, promises_1.writeFile)((0, path_1.join)(targetPath, "stats.dtt.json"), JSON.stringify(stats));
+        if (compress) {
+            const p0 = (0, fs_2.createWriteStream)((0, path_1.join)(targetPath, "db.xb.gz"));
+            p1 = (0, process_1.createProcess)(command, args, {
+                $log: {
+                    exec: this.verbose,
+                    stderr: this.verbose,
+                },
+                $stderr: {
+                    parseLines: true,
+                    onData,
+                },
+                $onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
+            });
+            const p2 = (0, process_1.createProcess)(compress.command, compress.args);
+            p1.stdout.pipe(p2.stdin, { end: true });
+            p2.stdout.pipe(p0, { end: true });
+            await Promise.all([p1, p2, (0, fs_1.waitForClose)(p0)]);
+        }
+        else {
+            p1 = (0, process_1.createProcess)(command, args, {
+                $log: this.verbose,
+                $stdout: {
+                    parseLines: true,
+                    onData,
+                },
+                $stderr: {
+                    parseLines: true,
+                    onData,
+                },
+                $onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
+            });
+            await p1;
+            await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
+                log: this.verbose,
+                stderr: { onData: () => { } },
+            });
+        }
     }
     async onRestore(data) {
         this.verbose = data.options.verbose;
         const restorePath = data.package.restorePath;
         (0, assert_1.ok)(typeof restorePath === "string");
         await (0, fs_1.mkdirIfNotExists)(restorePath);
-        const files = await (0, fs_1.readDir)(restorePath);
-        for (const file of files) {
-            if (file.startsWith("ib_logfile")) {
-                const filePath = (0, path_1.join)(restorePath, file);
-                if (this.verbose)
-                    (0, cli_1.logExec)("rm", [filePath]);
-                await (0, promises_1.rm)(filePath);
-            }
+        const removeFiles = [];
+        let files = [];
+        const reloadFiles = async (data = {}) => {
+            if (data.removeFile)
+                removeFiles.push(data.removeFile);
+            return (files = (await (0, fs_1.readDir)(restorePath)).filter((v) => !removeFiles.includes(v)));
+        };
+        await reloadFiles();
+        const absolute = {
+            format: "amount",
+            total: 3,
+            current: 0,
+            description: undefined,
+            payload: undefined,
+            percent: undefined,
+        };
+        // Stats
+        const statsFile = files.find((file) => file === "stats.dtt.json");
+        let stats;
+        if (statsFile) {
+            const statsFilePath = (0, path_1.join)(restorePath, statsFile);
+            const statsBuffer = await (0, promises_1.readFile)(statsFilePath);
+            stats = JSON.parse(statsBuffer.toString());
+            await reloadFiles({ removeFile: statsFile });
+        }
+        const zipFile = files.find((file) => file.endsWith(".gz"));
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        // Extract
+        if (files.length === 1 && zipFile) {
+            absolute.description = "Extracting";
+            absolute.payload = zipFile;
+            await data.onProgress({
+                absolute,
+            });
+            await (0, tar_1.extractTar)({
+                input: (0, path_1.join)(restorePath, zipFile),
+                output: restorePath,
+                verbose: this.verbose,
+                async onEntry(item) {
+                    await data.onProgress({
+                        absolute,
+                        relative: {
+                            payload: item.path,
+                            format: "amount",
+                            percent: item.progress.percent,
+                        },
+                    });
+                },
+            });
+            await reloadFiles({ removeFile: zipFile });
+        }
+        // Extract stream
+        const xbFile = files.find((file) => file.endsWith(".xb"));
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        if (files.length === 1 && xbFile) {
+            const xbFilePath = (0, path_1.join)(restorePath, xbFile);
+            const xbStream = (0, fs_2.createReadStream)(xbFilePath);
+            removeFiles.push(xbFile);
+            absolute.description = "Extracting stream";
+            absolute.payload = xbFile;
+            await data.onProgress({
+                absolute,
+            });
+            let currentXbFiles = 0;
+            const { parallel } = normalizeConfig({ parallel: this.config.parallel });
+            const p1 = (0, process_1.createProcess)("mbstream", ["-x", "-C", restorePath, "-v", "-p", parallel], {
+                $log: this.verbose,
+                $stderr: {
+                    parseLines: true,
+                    async onData(line) {
+                        const { text: path } = parseLine(line);
+                        await data.onProgress({
+                            absolute,
+                            relative: {
+                                payload: path,
+                                format: "amount",
+                                current: ++currentXbFiles,
+                                total: stats?.xbFiles,
+                                percent: stats?.xbFiles
+                                    ? (0, math_1.progressPercent)(stats.xbFiles, currentXbFiles)
+                                    : undefined,
+                            },
+                        });
+                    },
+                },
+            });
+            xbStream.pipe(p1.stdin, { end: true });
+            await Promise.all([(0, fs_1.waitForClose)(xbStream), p1]);
+        }
+        // Prepare
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        if (files.length === 1 && xbFile) {
+            absolute.description = "Preparing";
+            await data.onProgress({
+                absolute,
+            });
+            await (0, process_1.exec)(this.command, [`--prepare`, `--target-dir=${restorePath}`], undefined, {
+                log: this.verbose,
+                stderr: { onData: () => { } },
+            });
+        }
+        await reloadFiles();
+        removeFiles.push(...files.filter((file) => file.startsWith("ib_logfile")));
+        // Remove files
+        for (const file of removeFiles) {
+            const filePath = (0, path_1.join)(restorePath, file);
+            if (this.verbose)
+                (0, cli_1.logExec)("rm", [filePath]);
+            await (0, promises_1.rm)(filePath);
         }
     }
 }

package/Task/MssqlTask.js

@@ -25,6 +25,8 @@ exports.mssqlTaskDefinition = {
     },
 };
 class MssqlTask extends TaskAbstract_1.TaskAbstract {
+    static SUFFIX = ".BAK";
+    verbose;
     get command() {
         return this.config.command ?? "sqlcmd";
     }
@@ -106,4 +108,3 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
     }
 }
 exports.MssqlTask = MssqlTask;
-MssqlTask.SUFFIX = ".BAK";

package/Task/ScriptTask.js

@@ -92,6 +92,7 @@ exports.scriptTaskDefinition = {
     },
 };
 class ScriptTask extends TaskAbstract_1.TaskAbstract {
+    verbose;
     async onBeforeBackup() {
         return {
             targetPath: await this.mkTmpDir(ScriptTask.name),

package/Task/SqlDumpTaskAbstract.js

@@ -85,6 +85,7 @@ function parseSqlFile(fileName) {
     }
 }
 class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
+    verbose;
     async fetchPassword() {
         if (typeof this.config.password === "string")
             return this.config.password;

package/Task/TaskAbstract.js

@@ -3,9 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TaskAbstract = void 0;
 const fs_1 = require("../utils/fs");
 class TaskAbstract {
+    config;
+    tmpDirs = [];
     constructor(config) {
         this.config = config;
-        this.tmpDirs = [];
     }
     async mkTmpDir(prefix, id) {
         const dir = await (0, fs_1.mkTmpDir)(prefix, id);

package/config.schema.json

@@ -458,45 +458,31 @@
       "additionalProperties": false,
       "properties": {
         "compress": {
-          "anyOf": [
-            {
-              "type": "boolean"
-            },
-            {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "packs": {
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                      "include"
-                    ],
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "include": {
-                        "$ref": "#/definitions/stringlist-util"
-                      },
-                      "exclude": {
-                        "$ref": "#/definitions/stringlist-util"
-                      },
-                      "onePackByResult": {
-                        "type": "boolean"
-                      }
-                    }
-                  }
-                }
+          "type": "boolean"
+        },
+        "packs": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "include"
+            ],
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "include": {
+                "$ref": "#/definitions/stringlist-util"
+              },
+              "exclude": {
+                "$ref": "#/definitions/stringlist-util"
+              },
+              "onePackByResult": {
+                "type": "boolean"
               }
             }
-          ]
-        },
-        "fileCopyConcurrency": {
-          "type": "integer",
-          "minimum": 1
+          }
         }
       }
     },
@@ -798,6 +784,50 @@
         },
         "excludeDatabases": {
           "$ref": "#/definitions/stringlist-util"
+        },
+        "parallel": {
+          "anyOf": [
+            {
+              "type": "integer",
+              "minimum": 1
+            },
+            {
+              "enum": [
+                "auto"
+              ]
+            }
+          ]
+        },
+        "compress": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "command": {
+                  "type": "string"
+                },
+                "args": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "type": {
+                  "enum": [
+                    "gzip",
+                    "pigz"
+                  ]
+                },
+                "args": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
+            }
+          ]
         }
       }
     },

package/index.d.ts

@@ -0,0 +1 @@
+export {};

package/index.js

@@ -1 +1,2 @@
 "use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,23 +1,24 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.17.2",
+  "version": "0.19.0",
   "dependencies": {
-    "ajv": "^8.11.2",
+    "ajv": "^8.12.0",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
     "cli-table3": "^0.6.3",
-    "commander": "^9.4.1",
-    "dayjs": "^1.11.7",
-    "fast-folder-size": "^1.7.1",
-    "fast-glob": "^3.2.12",
+    "commander": "^11.0.0",
+    "dayjs": "^1.11.10",
+    "fast-folder-size": "^2.2.0",
+    "fast-glob": "^3.3.1",
     "micromatch": "^4.0.5",
     "pretty-bytes": "^5.6.0",
-    "sqlite": "^4.1.2",
-    "sqlite3": "^5.1.4"
+    "sqlite": "^5.0.1",
+    "sqlite3": "^5.1.6",
+    "tar": "^6.2.0"
   },
   "optionalDependencies": {
     "ts-node": "^10.9.1",
-    "yaml": "^2.2.0"
+    "yaml": "^2.3.2"
   },
   "engine": {
     "node": ">=16.0.0"

package/utils/DataFormat.js

@@ -10,6 +10,7 @@ const util_1 = require("util");
 const customPrefix = "custom=";
 const tplPrefix = "tpl=";
 class DataFormat {
+    options;
     constructor(options) {
         this.options = options;
     }

package/utils/Git.js

@@ -4,6 +4,7 @@ exports.Git = void 0;
 const fs_1 = require("./fs");
 const process_1 = require("./process");
 class Git {
+    options;
     constructor(options) {
         this.options = options;
     }

package/utils/ObjectVault.js

@@ -2,11 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObjectVault = void 0;
 class ObjectVault {
-    constructor() {
-        this.counter = 0;
-        this.ids = {};
-        this.objects = {};
-    }
+    counter = 0;
+    ids = {};
+    objects = {};
     static serializeKeys(keys) {
         return JSON.stringify(keys);
     }

package/utils/Restic.js

@@ -7,6 +7,7 @@ const string_1 = require("./string");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class Restic {
+    options;
     constructor(options) {
         this.options = options;
     }

package/utils/datatruck/config.d.ts

@@ -59,8 +59,8 @@ export declare const params: {
         action: string;
     };
     pkgRestorePath: {
-        packageName: string;
         path: string;
+        packageName: string;
         snapshotId: string;
         temp: string;
         snapshotDate: string;

package/utils/fs.d.ts

@@ -2,7 +2,9 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { Dirent, Stats } from "fs";
+import { Progress } from "./progress";
+import { Entry, Options } from "fast-glob";
+import { Dirent, ReadStream, Stats } from "fs";
 import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
@@ -51,7 +53,7 @@ export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
     outDir: string;
 }): Promise<string>;
-export declare function waitForClose(stream: WriteStream): Promise<WriteStream>;
+export declare function waitForClose(stream: WriteStream | ReadStream): Promise<void>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;
 export declare function updateFileStats(path: string, fileInfo: Stats): Promise<void>;
 export declare function isNotFoundError(error: unknown): boolean;
@@ -97,4 +99,17 @@ export declare function cpy(options: {
     files: number;
     dirs: number;
 }>;
+export declare function createFileScanner(options: {
+    glob: Options & {
+        include: string[];
+    };
+    onProgress: (data: Progress) => Promise<void>;
+}): Promise<{
+    disposed: boolean;
+    total: number;
+    current: number;
+    progress: (description: string, path?: string) => Promise<void>;
+    end: () => Promise<void>;
+    start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
+}>;
 export {};