@datatruck/cli 0.20.0 → 0.21.1

package/JsonSchema/DefinitionEnum.d.ts

@@ -19,7 +19,8 @@ export declare enum DefinitionEnum {
     sqlDumpTask = "sqldump-task",
     stringListUtil = "stringlist-util",
     prunePolicy = "prune-policy",
-    pathsObject = "paths-object"
+    pathsObject = "paths-object",
+    compressUtil = "compress-util"
 }
 export declare function makeRef(type: DefinitionEnum, subType?: string): {
     $ref: string;

package/JsonSchema/DefinitionEnum.js

@@ -24,6 +24,7 @@ var DefinitionEnum;
     DefinitionEnum["stringListUtil"] = "stringlist-util";
     DefinitionEnum["prunePolicy"] = "prune-policy";
     DefinitionEnum["pathsObject"] = "paths-object";
+    DefinitionEnum["compressUtil"] = "compress-util";
 })(DefinitionEnum || (exports.DefinitionEnum = DefinitionEnum = {}));
 function makeRef(type, subType) {
     return {

package/JsonSchema/JsonSchema.js

@@ -17,6 +17,7 @@ const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
 const ScriptTask_1 = require("../Task/ScriptTask");
 const SqlDumpTaskAbstract_1 = require("../Task/SqlDumpTaskAbstract");
+const tar_1 = require("../utils/tar");
 const DefinitionEnum_1 = require("./DefinitionEnum");
 exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.stringListUtil]: {
@@ -45,6 +46,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.config]: Config_1.configDefinition,
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
     [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
+    [DefinitionEnum_1.DefinitionEnum.compressUtil]: tar_1.compressDefinition,
 };
 for (const key in exports.definitions) {
     const schemaKey = key;

package/Repository/DatatruckRepository.d.ts

@@ -1,3 +1,4 @@
+import { CompressOptions } from "../utils/tar";
 import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
 import type { JSONSchema7 } from "json-schema";
 export type MetaDataType = {
@@ -8,20 +9,23 @@ export type MetaDataType = {
     tags: string[];
     version: string;
     size: number;
+    tarStats?: Record<string, {
+        files: number;
+    }>;
 };
 export type DatatruckRepositoryConfigType = {
     outPath: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
 };
 type PackObject = {
     name?: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     include: string[];
     exclude?: string[];
     onePackByResult?: boolean;
 };
 export type DatatruckPackageRepositoryConfigType = {
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     packs?: PackObject[];
 };
 export declare const datatruckRepositoryName = "datatruck";

package/Repository/DatatruckRepository.js

@@ -20,7 +20,9 @@ exports.datatruckRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         outPath: { type: "string" },
-        compress: { type: "boolean" },
+        compress: {
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+        },
     },
 };
 exports.datatruckPackageRepositoryDefinition = {
@@ -28,7 +30,7 @@ exports.datatruckPackageRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         compress: {
-            type: "boolean",
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
         },
         packs: {
             type: "array",
@@ -38,6 +40,9 @@ exports.datatruckPackageRepositoryDefinition = {
                 required: ["include"],
                 properties: {
                     name: { type: "string" },
+                    compress: {
+                        anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+                    },
                     include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     onePackByResult: { type: "boolean" },
@@ -208,22 +213,26 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         });
         await stream.end();
         let packIndex = 0;
+        const tarStats = {};
         for (const pack of packs) {
             const packBasename = [`pack`, packIndex.toString(), pack.name]
                 .filter((v) => typeof v === "string")
                 .map((v) => encodeURIComponent(v.toString().replace(/[\\/]/g, "-")))
-                .join("-");
-            const ext = pack.compress ? `.tar.gz` : `.tar`;
+                .join("-") + (pack.compress ? `.tar.gz` : `.tar`);
             const includeList = stream.path(packIndex);
-            if (includeList)
+            if (includeList) {
+                tarStats[packBasename] = {
+                    files: stream.lines(packIndex),
+                };
                 await (0, tar_1.createTar)({
                     compress: pack.compress,
                     verbose: data.options.verbose,
                     includeList,
                     path: sourcePath,
-                    output: (0, path_1.join)(outPath, packBasename) + ext,
+                    output: (0, path_1.join)(outPath, packBasename),
                     onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
                 });
+            }
             packIndex++;
         }
         await scanner.end();
@@ -238,6 +247,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             task: data.package.task?.name,
             version: nodePkg.version,
             size: await (0, fs_1.fastFolderSizeAsync)(outPath),
+            tarStats,
         };
         if (data.options.verbose)
             (0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
@@ -293,6 +303,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
+        const metaPath = (0, path_1.join)(sourcePath, "meta.json");
+        const meta = await DatatruckRepository.parseMetaData(metaPath);
         const scanner = await (0, fs_1.createFileScanner)({
             onProgress: data.onProgress,
             glob: {
@@ -301,29 +313,40 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             },
         });
         const tarFiles = [];
+        const tarStats = meta?.tarStats || {};
         await scanner.start(async (entry) => {
             const path = (0, path_1.join)(sourcePath, entry.name);
             const isTar = entry.name.endsWith(".tar");
             const isTarGz = entry.name.endsWith(".tar.gz");
             if (isTar || isTarGz) {
                 tarFiles.push(path);
-                await (0, tar_1.listTar)({
-                    input: path,
-                    verbose: data.options.verbose,
-                    onEntry: () => {
-                        scanner.total++;
-                    },
-                });
+                if (typeof tarStats[entry.name]?.files === "number") {
+                    scanner.total += tarStats[entry.name].files;
+                }
+                else {
+                    scanner.progress("Scanning", entry.name, false);
+                    const selfTarStats = (tarStats[entry.name] = { files: 0 });
+                    await (0, tar_1.listTar)({
+                        input: path,
+                        verbose: data.options.verbose,
+                        onEntry: () => {
+                            scanner.total++;
+                            selfTarStats.files++;
+                        },
+                    });
+                }
             }
             return false;
         });
         if (data.options.verbose)
             (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
         for (const tarFile of tarFiles) {
+            const entryName = (0, path_1.basename)(tarFile);
             await (0, tar_1.extractTar)({
+                total: tarStats[entryName].files,
                 input: tarFile,
                 output: restorePath,
-                uncompress: tarFile.endsWith(".tar.gz"),
+                decompress: tarFile.endsWith(".tar.gz"),
                 verbose: data.options.verbose,
                 onEntry: async (data) => await scanner.progress(tarFile.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
             });

package/Task/MariadbTask.js

@@ -258,7 +258,7 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
             });
             await (0, tar_1.extractTar)({
                 input: (0, path_1.join)(restorePath, zipFile),
-                uncompress: true,
+                decompress: true,
                 output: restorePath,
                 verbose: this.verbose,
                 async onEntry(item) {

package/Task/MysqlDumpTask.d.ts

@@ -4,7 +4,7 @@ export declare const mysqlDumpTaskName = "mysql-dump";
 export type MysqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
 export declare const mysqlDumpTaskDefinition: JSONSchema7;
 export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConfigType> {
-    buildConnectionArgs(database?: boolean): Promise<string[]>;
+    buildConnectionArgs(database?: string): Promise<string[]>;
     onDatabaseIsEmpty(name: string): Promise<boolean>;
     onCreateDatabase(database: TargetDatabaseType): Promise<void>;
     onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;

package/Task/MysqlDumpTask.js

@@ -13,14 +13,14 @@ exports.mysqlDumpTaskDefinition = {
     allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
 };
 class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
-    async buildConnectionArgs(database = true) {
+    async buildConnectionArgs(database) {
         const password = await this.fetchPassword();
         return [
             `--host=${this.config.hostname}`,
             ...(this.config.port ? [`--port=${this.config.port}`] : []),
             `--user=${this.config.username}`,
             `--password=${password ?? ""}`,
-            ...(database && this.config.database ? [this.config.database] : []),
+            ...(database ? [database] : []),
         ];
     }
     async onDatabaseIsEmpty(name) {
@@ -78,7 +78,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 "--lock-tables=false",
                 "--skip-add-drop-table=false",
                 ...tableNames,
@@ -120,7 +120,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 "--lock-tables=false",
                 "--routines",
                 "--events",
@@ -143,7 +143,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
         ]);
     }
     async onImport(path, database) {
-        await (0, process_1.exec)("mysql", [...(await this.buildConnectionArgs(false)), database], null, {
+        await (0, process_1.exec)("mysql", await this.buildConnectionArgs(database), null, {
             pipe: {
                 stream: (0, fs_3.createReadStream)(path),
                 onReadProgress: (data) => {

package/Task/PostgresqlDumpTask.js

@@ -81,7 +81,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             (0, process_1.exec)("pg_dump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 ...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
             ], null, {
                 pipe: { stream, onWriteProgress: onProgress },

package/config.schema.json

@@ -449,7 +449,14 @@
           "type": "string"
         },
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         }
       }
     },
@@ -458,7 +465,14 @@
       "additionalProperties": false,
       "properties": {
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         },
         "packs": {
           "type": "array",
@@ -472,6 +486,16 @@
               "name": {
                 "type": "string"
               },
+              "compress": {
+                "anyOf": [
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "$ref": "#/definitions/compress-util"
+                  }
+                ]
+              },
               "include": {
                 "$ref": "#/definitions/stringlist-util"
               },
@@ -975,6 +999,33 @@
         }
       ]
     },
+    "compress-util": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "level": {
+          "type": "integer"
+        },
+        "cores": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "object",
+              "required": [
+                "percent"
+              ],
+              "properties": {
+                "percent": {
+                  "type": "integer"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
     "script-task_step": {
       "type": "object",
       "required": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.20.0",
+  "version": "0.21.1",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",

package/utils/fs.d.ts

@@ -96,12 +96,13 @@ export declare function createFileScanner(options: {
     disposed: boolean;
     total: number;
     current: number;
-    progress: (description: string, path?: string) => Promise<void>;
+    progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
     end: () => Promise<void>;
     start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
 type StreamItem = {
     key: string;
+    lines: number;
     stream: WriteStream;
     finished: boolean;
     error?: Error;
@@ -113,6 +114,7 @@ export declare function createWriteStreamPool(options: {
 }): {
     pool: Record<string, StreamItem>;
     path(key: string | number): string | undefined;
+    lines(key: string | number): number;
     writeLine(key: string | number, v: string): boolean;
     end(): Promise<void>;
 };

package/utils/fs.js

@@ -410,10 +410,10 @@ async function createFileScanner(options) {
         disposed: false,
         total: 0,
         current: 0,
-        progress: async (description, path) => {
+        progress: async (description, path, increment = true) => {
             if (object.disposed)
                 return;
-            if (path)
+            if (path && increment)
                 object.current++;
             await options.onProgress({
                 relative: {
@@ -463,6 +463,7 @@ function createWriteStreamPool(options) {
     const create = (key) => {
         const item = {
             key,
+            lines: 0,
             stream: (0, fs_2.createWriteStream)((0, path_2.join)(options.path, options.onStreamPath ? options.onStreamPath(key) : key)),
             finished: false,
         };
@@ -484,16 +485,22 @@ function createWriteStreamPool(options) {
                 throw new Error(`Stream path is not defined: ${key}`);
             return item.stream.path;
         },
+        lines(key) {
+            const item = pool[key];
+            return item?.lines;
+        },
         writeLine(key, v) {
             const item = pool[key] || create(key.toString());
             if (item.finished) {
                 return false;
             }
             else if (item.written) {
+                item.lines++;
                 return item.stream.write(`\n${v}`);
             }
             else {
                 item.written = true;
+                item.lines++;
                 return item.stream.write(`${v}`);
             }
         },

package/utils/tar.d.ts

@@ -1,3 +1,4 @@
+import type { JSONSchema7 } from "json-schema";
 export type Progress = {
     percent: number;
     current: number;
@@ -7,22 +8,39 @@ export type TarEntry = {
     path: string;
     progress: Progress;
 };
+export type CoresOptions = number | {
+    percent: number;
+};
+export type CompressOptions = {
+    level?: number;
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
+export type DecompressOptions = {
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
 export interface CreateTarOptions {
     path: string;
     verbose?: boolean;
     output: string;
     includeList: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     onEntry?: (entry: TarEntry) => void;
 }
 export interface ExtractOptions {
     input: string;
     output: string;
     verbose?: boolean;
-    uncompress?: boolean;
+    decompress?: boolean | DecompressOptions;
     total?: number;
     onEntry?: (entry: TarEntry) => void;
 }
+export declare const compressDefinition: JSONSchema7;
 export type TarVendor = "busybox" | "bsdtar" | "gnu";
 export declare function getTarVendor(cache?: boolean, log?: boolean): Promise<TarVendor | null>;
 export type ListTarOptions = {
@@ -31,5 +49,6 @@ export type ListTarOptions = {
     verbose?: boolean;
 };
 export declare function listTar(options: ListTarOptions): Promise<number>;
+export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
 export declare function createTar(options: CreateTarOptions): Promise<void>;
 export declare function extractTar(options: ExtractOptions): Promise<void>;

package/utils/tar.js

@@ -1,12 +1,29 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTar = exports.createTar = exports.listTar = exports.getTarVendor = void 0;
+exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
 const process_1 = require("./process");
 const promises_1 = require("fs/promises");
 const os_1 = require("os");
+exports.compressDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        level: { type: "integer" },
+        cores: {
+            anyOf: [
+                { type: "integer" },
+                {
+                    type: "object",
+                    required: ["percent"],
+                    properties: { percent: { type: "integer" } },
+                },
+            ],
+        },
+    },
+};
 let tarVendor;
 async function getTarVendor(cache = true, log = false) {
     if (cache && typeof tarVendor !== "undefined")
@@ -49,37 +66,94 @@ async function listTar(options) {
     return total;
 }
 exports.listTar = listTar;
+let pigzLib;
+async function checkPigzLib(cache = true) {
+    if (cache && pigzLib !== undefined)
+        return pigzLib;
+    try {
+        return !(await (0, process_1.exec)("pigz", ["-V"])).exitCode;
+    }
+    catch {
+        return false;
+    }
+}
+exports.checkPigzLib = checkPigzLib;
+async function resolveCores(input) {
+    if (!(await checkPigzLib()))
+        return 1;
+    const total = (0, os_1.cpus)().length;
+    return Math.min(total, typeof input === "number"
+        ? input
+        : Math.max(0, Math.round(((input?.percent || 50) * total) / 100)));
+}
+async function ifX(input, cb) {
+    return input ? await cb((input === true ? {} : input)) : undefined;
+}
 async function createTar(options) {
     const vendor = await getTarVendor(true, options.verbose);
     const total = await (0, fs_1.countFileLines)(options.includeList);
+    const compress = await ifX(options.compress, async (compress) => ({
+        ...compress,
+        cores: await resolveCores(compress.cores),
+    }));
     let current = 0;
+    const env = {
+        ...(compress?.cores === 1 &&
+            compress.level && {
+            GZIP_OPT: compress.level.toString(),
+        }),
+    };
     await (0, process_1.exec)("tar", [
         "-C",
         toLocalPath(options.path),
-        options.compress ? "-czvf" : "-cvf",
+        compress?.cores === 1 ? "-czvf" : "-cvf",
         toLocalPath(options.output),
         "-T",
         toLocalPath(options.includeList),
         "--ignore-failed-read",
         "--force-local",
-    ], {}, {
-        log: options.verbose,
-        stderr: { toExitCode: true },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (line) => {
-                current++;
-                const path = vendor === "bsdtar" ? line.slice(2) : line;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
+        ...(compress && compress.cores > 1
+            ? [
+                "-I",
+                `"${[
+                    "pigz",
+                    "-r",
+                    !!compress.level && `-${compress.level}`,
+                    `-p ${compress.cores}`,
+                ]
+                    .filter(Boolean)
+                    .join(" ")}"`,
+            ]
+            : []),
+    ], {
+        ...(compress &&
+            compress.cores > 1 && {
+            shell: true,
+        }),
+        env: {
+            ...process.env,
+            ...env,
         },
+    }, {
+        log: options.verbose ? { envNames: Object.keys(env) } : false,
+        stderr: { toExitCode: true },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (line) => {
+                    current++;
+                    const path = vendor === "bsdtar" ? line.slice(2) : line;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.createTar = createTar;
@@ -92,36 +166,51 @@ function toLocalPath(path) {
 async function extractTar(options) {
     let total = options.total ??
         (await listTar({ input: options.input, verbose: options.verbose }));
-    if (options.verbose)
-        (0, cli_1.logExec)("mkdir", ["-p", options.output]);
-    await (0, promises_1.mkdir)(options.output, { recursive: true });
-    await (0, fs_1.ensureEmptyDir)(options.output);
+    if (!(await (0, fs_1.existsDir)(options.output))) {
+        if (options.verbose)
+            (0, cli_1.logExec)("mkdir", ["-p", options.output]);
+        await (0, promises_1.mkdir)(options.output, { recursive: true });
+    }
+    const decompress = await ifX(options.decompress, async (decompress) => ({
+        ...decompress,
+        cores: await resolveCores(decompress.cores),
+    }));
     let current = 0;
     await (0, process_1.exec)("tar", [
-        options.uncompress ? "-xzvpf" : "-xvpf",
+        decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
         toLocalPath(options.input),
         "-C",
         toLocalPath(options.output),
         "--force-local",
-    ], {}, {
+        ...(decompress && decompress.cores > 1
+            ? ["-I", `"pigz -p ${decompress.cores}"`]
+            : []),
+    ], {
+        ...(decompress &&
+            decompress.cores > 1 && {
+            shell: true,
+        }),
+    }, {
         log: options.verbose,
         stderr: {
             toExitCode: true,
         },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (path) => {
-                current++;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
-        },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (path) => {
+                    current++;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.extractTar = extractTar;