@datatruck/cli 0.20.0 → 0.21.0

package/JsonSchema/DefinitionEnum.d.ts

@@ -19,7 +19,8 @@ export declare enum DefinitionEnum {
     sqlDumpTask = "sqldump-task",
     stringListUtil = "stringlist-util",
     prunePolicy = "prune-policy",
-    pathsObject = "paths-object"
+    pathsObject = "paths-object",
+    compressUtil = "compress-util"
 }
 export declare function makeRef(type: DefinitionEnum, subType?: string): {
     $ref: string;

package/JsonSchema/DefinitionEnum.js

@@ -24,6 +24,7 @@ var DefinitionEnum;
     DefinitionEnum["stringListUtil"] = "stringlist-util";
     DefinitionEnum["prunePolicy"] = "prune-policy";
     DefinitionEnum["pathsObject"] = "paths-object";
+    DefinitionEnum["compressUtil"] = "compress-util";
 })(DefinitionEnum || (exports.DefinitionEnum = DefinitionEnum = {}));
 function makeRef(type, subType) {
     return {

package/JsonSchema/JsonSchema.js

@@ -17,6 +17,7 @@ const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
 const ScriptTask_1 = require("../Task/ScriptTask");
 const SqlDumpTaskAbstract_1 = require("../Task/SqlDumpTaskAbstract");
+const tar_1 = require("../utils/tar");
 const DefinitionEnum_1 = require("./DefinitionEnum");
 exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.stringListUtil]: {
@@ -45,6 +46,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.config]: Config_1.configDefinition,
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
     [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
+    [DefinitionEnum_1.DefinitionEnum.compressUtil]: tar_1.compressDefinition,
 };
 for (const key in exports.definitions) {
     const schemaKey = key;

package/Repository/DatatruckRepository.d.ts

@@ -1,3 +1,4 @@
+import { CompressOptions } from "../utils/tar";
 import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
 import type { JSONSchema7 } from "json-schema";
 export type MetaDataType = {
@@ -11,17 +12,17 @@ export type MetaDataType = {
 };
 export type DatatruckRepositoryConfigType = {
     outPath: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
 };
 type PackObject = {
     name?: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     include: string[];
     exclude?: string[];
     onePackByResult?: boolean;
 };
 export type DatatruckPackageRepositoryConfigType = {
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     packs?: PackObject[];
 };
 export declare const datatruckRepositoryName = "datatruck";

package/Repository/DatatruckRepository.js

@@ -20,7 +20,9 @@ exports.datatruckRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         outPath: { type: "string" },
-        compress: { type: "boolean" },
+        compress: {
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+        },
     },
 };
 exports.datatruckPackageRepositoryDefinition = {
@@ -28,7 +30,7 @@ exports.datatruckPackageRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         compress: {
-            type: "boolean",
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
         },
         packs: {
             type: "array",
@@ -38,6 +40,9 @@ exports.datatruckPackageRepositoryDefinition = {
                 required: ["include"],
                 properties: {
                     name: { type: "string" },
+                    compress: {
+                        anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+                    },
                     include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     onePackByResult: { type: "boolean" },
@@ -323,7 +328,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             await (0, tar_1.extractTar)({
                 input: tarFile,
                 output: restorePath,
-                uncompress: tarFile.endsWith(".tar.gz"),
+                decompress: tarFile.endsWith(".tar.gz"),
                 verbose: data.options.verbose,
                 onEntry: async (data) => await scanner.progress(tarFile.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
             });

package/Task/MariadbTask.js

@@ -258,7 +258,7 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
             });
             await (0, tar_1.extractTar)({
                 input: (0, path_1.join)(restorePath, zipFile),
-                uncompress: true,
+                decompress: true,
                 output: restorePath,
                 verbose: this.verbose,
                 async onEntry(item) {

package/config.schema.json

@@ -449,7 +449,14 @@
           "type": "string"
         },
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         }
       }
     },
@@ -458,7 +465,14 @@
       "additionalProperties": false,
       "properties": {
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         },
         "packs": {
           "type": "array",
@@ -472,6 +486,16 @@
               "name": {
                 "type": "string"
               },
+              "compress": {
+                "anyOf": [
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "$ref": "#/definitions/compress-util"
+                  }
+                ]
+              },
               "include": {
                 "$ref": "#/definitions/stringlist-util"
               },
@@ -975,6 +999,33 @@
         }
       ]
     },
+    "compress-util": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "level": {
+          "type": "integer"
+        },
+        "cores": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "object",
+              "required": [
+                "percent"
+              ],
+              "properties": {
+                "percent": {
+                  "type": "integer"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
     "script-task_step": {
       "type": "object",
       "required": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",

package/utils/tar.d.ts

@@ -1,3 +1,4 @@
+import type { JSONSchema7 } from "json-schema";
 export type Progress = {
     percent: number;
     current: number;
@@ -7,22 +8,39 @@ export type TarEntry = {
     path: string;
     progress: Progress;
 };
+export type CoresOptions = number | {
+    percent: number;
+};
+export type CompressOptions = {
+    level?: number;
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
+export type DecompressOptions = {
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
 export interface CreateTarOptions {
     path: string;
     verbose?: boolean;
     output: string;
     includeList: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     onEntry?: (entry: TarEntry) => void;
 }
 export interface ExtractOptions {
     input: string;
     output: string;
     verbose?: boolean;
-    uncompress?: boolean;
+    decompress?: boolean | DecompressOptions;
     total?: number;
     onEntry?: (entry: TarEntry) => void;
 }
+export declare const compressDefinition: JSONSchema7;
 export type TarVendor = "busybox" | "bsdtar" | "gnu";
 export declare function getTarVendor(cache?: boolean, log?: boolean): Promise<TarVendor | null>;
 export type ListTarOptions = {
@@ -31,5 +49,6 @@ export type ListTarOptions = {
     verbose?: boolean;
 };
 export declare function listTar(options: ListTarOptions): Promise<number>;
+export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
 export declare function createTar(options: CreateTarOptions): Promise<void>;
 export declare function extractTar(options: ExtractOptions): Promise<void>;

package/utils/tar.js

@@ -1,12 +1,29 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTar = exports.createTar = exports.listTar = exports.getTarVendor = void 0;
+exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
 const process_1 = require("./process");
 const promises_1 = require("fs/promises");
 const os_1 = require("os");
+exports.compressDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        level: { type: "integer" },
+        cores: {
+            anyOf: [
+                { type: "integer" },
+                {
+                    type: "object",
+                    required: ["percent"],
+                    properties: { percent: { type: "integer" } },
+                },
+            ],
+        },
+    },
+};
 let tarVendor;
 async function getTarVendor(cache = true, log = false) {
     if (cache && typeof tarVendor !== "undefined")
@@ -49,21 +66,76 @@ async function listTar(options) {
     return total;
 }
 exports.listTar = listTar;
+let pigzLib;
+async function checkPigzLib(cache = true) {
+    if (cache && pigzLib !== undefined)
+        return pigzLib;
+    try {
+        return !(await (0, process_1.exec)("pigz", ["-V"])).exitCode;
+    }
+    catch {
+        return false;
+    }
+}
+exports.checkPigzLib = checkPigzLib;
+async function resolveCores(input) {
+    if (!(await checkPigzLib()))
+        return 1;
+    const total = (0, os_1.cpus)().length;
+    return Math.min(total, typeof input === "number"
+        ? input
+        : Math.max(0, Math.round(((input?.percent || 50) * total) / 100)));
+}
+async function ifX(input, cb) {
+    return input ? await cb((input === true ? {} : input)) : undefined;
+}
 async function createTar(options) {
     const vendor = await getTarVendor(true, options.verbose);
     const total = await (0, fs_1.countFileLines)(options.includeList);
+    const compress = await ifX(options.compress, async (compress) => ({
+        ...compress,
+        cores: await resolveCores(compress.cores),
+    }));
     let current = 0;
+    const env = {
+        ...(compress?.cores === 1 &&
+            compress.level && {
+            GZIP_OPT: compress.level.toString(),
+        }),
+    };
     await (0, process_1.exec)("tar", [
         "-C",
         toLocalPath(options.path),
-        options.compress ? "-czvf" : "-cvf",
+        compress?.cores === 1 ? "-czvf" : "-cvf",
         toLocalPath(options.output),
         "-T",
         toLocalPath(options.includeList),
         "--ignore-failed-read",
         "--force-local",
-    ], {}, {
-        log: options.verbose,
+        ...(compress && compress.cores > 1
+            ? [
+                "-I",
+                `"${[
+                    "pigz",
+                    "-r",
+                    !!compress.level && `-${compress.level}`,
+                    `-p ${compress.cores}`,
+                ]
+                    .filter(Boolean)
+                    .join(" ")}"`,
+            ]
+            : []),
+    ], {
+        ...(compress &&
+            compress.cores > 1 && {
+            shell: true,
+        }),
+        env: {
+            ...process.env,
+            ...env,
+        },
+    }, {
+        log: options.verbose ? { envNames: Object.keys(env) } : false,
         stderr: { toExitCode: true },
         stdout: {
             parseLines: "skip-empty",
@@ -96,14 +168,26 @@ async function extractTar(options) {
         (0, cli_1.logExec)("mkdir", ["-p", options.output]);
     await (0, promises_1.mkdir)(options.output, { recursive: true });
     await (0, fs_1.ensureEmptyDir)(options.output);
+    const decompress = await ifX(options.decompress, async (decompress) => ({
+        ...decompress,
+        cores: await resolveCores(decompress.cores),
+    }));
     let current = 0;
     await (0, process_1.exec)("tar", [
-        options.uncompress ? "-xzvpf" : "-xvpf",
+        decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
         toLocalPath(options.input),
         "-C",
         toLocalPath(options.output),
         "--force-local",
-    ], {}, {
+        ...(decompress && decompress.cores > 1
+            ? ["-I", `"pigz -p ${decompress.cores}"`]
+            : []),
+    ], {
+        ...(decompress &&
+            decompress.cores > 1 && {
+            shell: true,
+        }),
+    }, {
         log: options.verbose,
         stderr: {
             toExitCode: true,