@datatruck/cli 0.19.0 → 0.21.0

package/Action/RestoreAction.js

@@ -149,7 +149,7 @@ class RestoreAction {
             if (typeof pkg.restorePath !== "string")
                 throw new AppError_1.AppError("Restore path is not defined");
             await (0, fs_1.mkdirIfNotExists)(pkg.restorePath);
-            if (!(await (0, fs_1.isDirEmpty)(pkg.restorePath)))
+            if (!(await (0, fs_1.isEmptyDir)(pkg.restorePath)))
                 throw new AppError_1.AppError(`Restore path is not empty: ${pkg.restorePath}`);
             if (this.options.verbose)
                 (0, cli_1.logExec)(`restorePath=${pkg.restorePath}`);

package/JsonSchema/DefinitionEnum.d.ts

@@ -19,7 +19,8 @@ export declare enum DefinitionEnum {
     sqlDumpTask = "sqldump-task",
     stringListUtil = "stringlist-util",
     prunePolicy = "prune-policy",
-    pathsObject = "paths-object"
+    pathsObject = "paths-object",
+    compressUtil = "compress-util"
 }
 export declare function makeRef(type: DefinitionEnum, subType?: string): {
     $ref: string;

package/JsonSchema/DefinitionEnum.js

@@ -24,6 +24,7 @@ var DefinitionEnum;
     DefinitionEnum["stringListUtil"] = "stringlist-util";
     DefinitionEnum["prunePolicy"] = "prune-policy";
     DefinitionEnum["pathsObject"] = "paths-object";
+    DefinitionEnum["compressUtil"] = "compress-util";
 })(DefinitionEnum || (exports.DefinitionEnum = DefinitionEnum = {}));
 function makeRef(type, subType) {
     return {

package/JsonSchema/JsonSchema.js

@@ -17,6 +17,7 @@ const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
 const ScriptTask_1 = require("../Task/ScriptTask");
 const SqlDumpTaskAbstract_1 = require("../Task/SqlDumpTaskAbstract");
+const tar_1 = require("../utils/tar");
 const DefinitionEnum_1 = require("./DefinitionEnum");
 exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.stringListUtil]: {
@@ -45,6 +46,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.config]: Config_1.configDefinition,
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
     [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
+    [DefinitionEnum_1.DefinitionEnum.compressUtil]: tar_1.compressDefinition,
 };
 for (const key in exports.definitions) {
     const schemaKey = key;

package/Repository/DatatruckRepository.d.ts

@@ -1,3 +1,4 @@
+import { CompressOptions } from "../utils/tar";
 import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
 import type { JSONSchema7 } from "json-schema";
 export type MetaDataType = {
@@ -11,17 +12,17 @@ export type MetaDataType = {
 };
 export type DatatruckRepositoryConfigType = {
     outPath: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
 };
 type PackObject = {
     name?: string;
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     include: string[];
     exclude?: string[];
     onePackByResult?: boolean;
 };
 export type DatatruckPackageRepositoryConfigType = {
-    compress?: boolean;
+    compress?: boolean | CompressOptions;
     packs?: PackObject[];
 };
 export declare const datatruckRepositoryName = "datatruck";

package/Repository/DatatruckRepository.js

@@ -20,7 +20,9 @@ exports.datatruckRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         outPath: { type: "string" },
-        compress: { type: "boolean" },
+        compress: {
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+        },
     },
 };
 exports.datatruckPackageRepositoryDefinition = {
@@ -28,7 +30,7 @@ exports.datatruckPackageRepositoryDefinition = {
     additionalProperties: false,
     properties: {
         compress: {
-            type: "boolean",
+            anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
         },
         packs: {
             type: "array",
@@ -38,6 +40,9 @@ exports.datatruckPackageRepositoryDefinition = {
                 required: ["include"],
                 properties: {
                     name: { type: "string" },
+                    compress: {
+                        anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
+                    },
                     include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                     onePackByResult: { type: "boolean" },
@@ -98,13 +103,13 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
         if (data.options.verbose)
             (0, cli_1.logExec)(`Deleting ${snapshotPath}`);
-        if (await (0, fs_1.checkDir)(snapshotPath))
+        if (await (0, fs_1.existsDir)(snapshotPath))
             await (0, promises_1.rm)(snapshotPath, {
                 recursive: true,
             });
     }
     async onSnapshots(data) {
-        if (!(await (0, fs_1.checkDir)(this.config.outPath)))
+        if (!(await (0, fs_1.existsDir)(this.config.outPath)))
             throw new Error(`Repository (${this.repository.name}) out path does not exist: ${this.config.outPath}`);
         const snapshotNames = await (0, fs_1.readDir)(this.config.outPath);
         const snapshots = [];
@@ -175,49 +180,56 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const configPacks = (data.packageConfig?.packs ?? []).map((p) => ({
             ...p,
             compress: p.compress ?? data.packageConfig?.compress ?? this.config.compress,
-            includeResult: [],
         }));
         const defaultsPack = {
             name: "defaults",
             compress: data.packageConfig?.compress ?? this.config.compress,
             include: [],
-            includeResult: [],
         };
         const packs = [...configPacks, defaultsPack];
+        const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
+        const stream = (0, fs_1.createWriteStreamPool)({
+            path: await this.mkTmpDir("files"),
+            onStreamPath: (key) => `files-${key}.txt`,
+        });
         await scanner.start(async (entry) => {
             if (entry.dirent.isDirectory() &&
                 !(await (0, fs_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))))
                 return false;
-            const pack = configPacks.find((pack) => (0, string_1.checkPath)(entry.path, pack.include, pack.exclude)) || defaultsPack;
+            let packIndex = configPacks.findIndex((pack) => (0, string_1.checkPath)(entry.path, pack.include, pack.exclude));
+            if (packIndex === -1)
+                packIndex = defaultsPackIndex;
+            const pack = packs[packIndex];
             if (pack.onePackByResult) {
                 const subname = (0, path_1.basename)(entry.path);
                 packs.push({
                     ...pack,
                     name: pack.name ? `${pack.name}-${subname}` : subname,
-                    includeResult: [entry.path],
                 });
+                packIndex = packs.length - 1;
             }
-            else {
-                pack.includeResult.push(entry.path);
-            }
+            stream.writeLine(packIndex, entry.path);
             return true;
         });
+        await stream.end();
         let packIndex = 0;
         for (const pack of packs) {
-            const packBasename = [`pack`, (packIndex++).toString(), pack.name]
+            const packBasename = [`pack`, packIndex.toString(), pack.name]
                 .filter((v) => typeof v === "string")
                 .map((v) => encodeURIComponent(v.toString().replace(/[\\/]/g, "-")))
                 .join("-");
             const ext = pack.compress ? `.tar.gz` : `.tar`;
-            if (pack.includeResult.length)
+            const includeList = stream.path(packIndex);
+            if (includeList)
                 await (0, tar_1.createTar)({
                     compress: pack.compress,
                     verbose: data.options.verbose,
-                    include: pack.includeResult,
+                    includeList,
                     path: sourcePath,
                     output: (0, path_1.join)(outPath, packBasename) + ext,
                     onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
                 });
+            packIndex++;
         }
         await scanner.end();
         // Meta
@@ -316,6 +328,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             await (0, tar_1.extractTar)({
                 input: tarFile,
                 output: restorePath,
+                decompress: tarFile.endsWith(".tar.gz"),
                 verbose: data.options.verbose,
                 onEntry: async (data) => await scanner.progress(tarFile.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
             });

package/Task/GitTask.js

@@ -231,14 +231,14 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
         await incrementProgress();
         // Config
         const configPath = (0, path_1.join)(targetPath, "repo.config");
-        if (await (0, fs_1.checkFile)(configPath)) {
+        if (await (0, fs_1.existsFile)(configPath)) {
             await (0, promises_1.copyFile)(configPath, (0, path_1.join)(restorePath, ".git", "config"));
             await incrementProgress();
         }
         // ls-files
         for (const name of ["untracked", "modified", "ignored"]) {
             const sourcePath = (0, path_1.join)(targetPath, `repo.${name}`);
-            if (await (0, fs_1.checkDir)(sourcePath)) {
+            if (await (0, fs_1.existsDir)(sourcePath)) {
                 if (data.options.verbose)
                     (0, cli_1.logExec)(`Copying ${name} files to ${restorePath}`);
                 await (0, fs_1.cpy)({

package/Task/MariadbTask.js

@@ -258,6 +258,7 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
             });
             await (0, tar_1.extractTar)({
                 input: (0, path_1.join)(restorePath, zipFile),
+                decompress: true,
                 output: restorePath,
                 verbose: this.verbose,
                 async onEntry(item) {

package/Task/SqlDumpTaskAbstract.js

@@ -115,7 +115,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             return true;
         });
         (0, assert_1.ok)(typeof outputPath === "string");
-        if (!(await (0, fs_1.checkDir)(outputPath)))
+        if (!(await (0, fs_1.existsDir)(outputPath)))
             await (0, promises_1.mkdir)(outputPath, { recursive: true });
         if (!this.config.oneFileByTable) {
             const outPath = (0, path_1.join)(outputPath, serializeSqlFile({ database: this.config.database }));

package/config.schema.json

@@ -449,7 +449,14 @@
           "type": "string"
         },
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         }
       }
     },
@@ -458,7 +465,14 @@
       "additionalProperties": false,
       "properties": {
         "compress": {
-          "type": "boolean"
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/compress-util"
+            }
+          ]
         },
         "packs": {
           "type": "array",
@@ -472,6 +486,16 @@
               "name": {
                 "type": "string"
               },
+              "compress": {
+                "anyOf": [
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "$ref": "#/definitions/compress-util"
+                  }
+                ]
+              },
               "include": {
                 "$ref": "#/definitions/stringlist-util"
               },
@@ -975,6 +999,33 @@
         }
       ]
     },
+    "compress-util": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "level": {
+          "type": "integer"
+        },
+        "cores": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "object",
+              "required": [
+                "percent"
+              ],
+              "properties": {
+                "percent": {
+                  "type": "integer"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
     "script-task_step": {
       "type": "object",
       "required": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",
@@ -13,8 +13,7 @@
     "micromatch": "^4.0.5",
     "pretty-bytes": "^5.6.0",
     "sqlite": "^5.0.1",
-    "sqlite3": "^5.1.6",
-    "tar": "^6.2.0"
+    "sqlite3": "^5.1.6"
   },
   "optionalDependencies": {
     "ts-node": "^10.9.1",

package/utils/Git.js

@@ -16,7 +16,7 @@ class Git {
     }
     async canBeInit(repo) {
         return ((0, fs_1.isLocalDir)(repo) &&
-            (!(await (0, fs_1.checkDir)(repo)) || !(await (0, fs_1.readDir)(repo)).length));
+            (!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.readDir)(repo)).length));
     }
     async clone(options) {
         return await this.exec([

package/utils/datatruck/paths.js

@@ -9,7 +9,11 @@ async function parsePaths(values, options) {
             paths.push(value);
         }
         else if (value.type === "spawn") {
-            const spawnResult = await (0, process_1.exec)(value.command, value.args, { cwd: options.cwd }, { log: options.verbose, stderr: { save: true }, stdout: { save: true } });
+            const spawnResult = await (0, process_1.exec)(value.command, value.args, { cwd: options.cwd }, {
+                log: options.verbose,
+                stderr: { save: true },
+                stdout: { save: true },
+            });
             const spawnFiles = [spawnResult.stderr, spawnResult.stdout].flatMap((text) => text
                 .split(/\r?\n/)
                 .map((v) => v.trim())

package/utils/fs.d.ts

@@ -1,29 +1,20 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-/// <reference types="node" />
 import { Progress } from "./progress";
 import { Entry, Options } from "fast-glob";
-import { Dirent, ReadStream, Stats } from "fs";
+import { ReadStream, Stats } from "fs";
 import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
 export declare function isEmptyDir(path: string): Promise<boolean>;
-type EntryObject = {
-    name: string;
-    path: string;
-    dirent: Dirent;
-    stats: Stats;
-};
-export declare function applyPermissions(baseDir: string, permissionsPath: string): Promise<void>;
-export declare function pathIterator(stream: AsyncIterable<string | Buffer>): AsyncIterable<EntryObject>;
 export declare function isLocalDir(path: string): boolean;
-export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
 export declare function ensureEmptyDir(path: string): Promise<void>;
+export declare function safeStat(path: string): Promise<Stats | undefined>;
 export declare function existsDir(path: string): Promise<boolean>;
+export declare function existsFile(path: string): Promise<boolean>;
 export declare function writeJSONFile<T = any>(path: string, json: T): Promise<void>;
-export declare function readdirIfExists(path: string): Promise<string[]>;
 export declare const parseFileExtensions: string[];
 export declare function parseFile(path: string, jsKey?: string): Promise<any>;
 export declare function parsePackageFile(): {
@@ -32,7 +23,6 @@ export declare function parsePackageFile(): {
     description: string;
 };
 export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
-export declare function existsFile(path: string): Promise<boolean>;
 export declare function parentTmpDir(): string;
 export declare function sessionTmpDir(): string;
 export declare function isTmpDir(path: string): boolean;
@@ -41,8 +31,6 @@ export declare function tmpDir(prefix: string, id?: string): string;
 export declare function fastFolderSizeAsync(path: string): Promise<number>;
 export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
 export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
-export declare function checkFile(path: string): Promise<boolean>;
-export declare function checkDir(path: string): Promise<boolean>;
 export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
 /**
@@ -112,4 +100,21 @@ export declare function createFileScanner(options: {
     end: () => Promise<void>;
     start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
+type StreamItem = {
+    key: string;
+    stream: WriteStream;
+    finished: boolean;
+    error?: Error;
+    written?: boolean;
+};
+export declare function createWriteStreamPool(options: {
+    path: string;
+    onStreamPath?: (key: string) => string;
+}): {
+    pool: Record<string, StreamItem>;
+    path(key: string | number): string | undefined;
+    writeLine(key: string | number, v: string): boolean;
+    end(): Promise<void>;
+};
+export declare function countFileLines(path: string): Promise<number>;
 export {};

package/utils/fs.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const math_1 = require("./math");
 const path_1 = require("./path");
@@ -35,37 +35,13 @@ async function isEmptyDir(path) {
     }
 }
 exports.isEmptyDir = isEmptyDir;
-async function applyPermissions(baseDir, permissionsPath) {
-    const singleReader = (0, readline_1.createInterface)({
-        input: (0, fs_1.createReadStream)(permissionsPath),
-    });
-    for await (const line of singleReader) {
-        const [rpath, rawUid, rawGui, rawMode] = line.split(":");
-        const path = (0, path_2.join)(baseDir, rpath);
-        if (!path.startsWith(baseDir)) {
-            throw new Error(`Entry path is out of the base dir: (${path}, ${baseDir})`);
-        }
-        const uid = Number(rawUid);
-        const guid = Number(rawGui);
-        await (0, promises_1.chown)(path, uid, guid);
-        const mode = Number(rawMode);
-        await (0, promises_1.chmod)(path, mode);
-    }
-}
-exports.applyPermissions = applyPermissions;
 function pathIterator(stream) {
     return stream;
 }
-exports.pathIterator = pathIterator;
 function isLocalDir(path) {
     return /^[\/\.]|([A-Z]:)/i.test(path);
 }
 exports.isLocalDir = isLocalDir;
-async function isDirEmpty(path) {
-    const files = await readDir(path);
-    return !files.length;
-}
-exports.isDirEmpty = isDirEmpty;
 async function mkdirIfNotExists(path) {
     await (0, promises_1.mkdir)(path, {
         recursive: true,
@@ -74,28 +50,29 @@ async function mkdirIfNotExists(path) {
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
 async function ensureEmptyDir(path) {
-    if (!(await isDirEmpty(path)))
+    if (!(await isEmptyDir(path)))
         throw new Error(`Dir is not empty: ${path}`);
 }
 exports.ensureEmptyDir = ensureEmptyDir;
-async function existsDir(path) {
+async function safeStat(path) {
     try {
-        const info = await (0, promises_1.stat)(path);
-        return info.isDirectory();
-    }
-    catch (e) {
-        return false;
+        return await (0, promises_1.stat)(path);
     }
+    catch (e) { }
+}
+exports.safeStat = safeStat;
+async function existsDir(path) {
+    return (await safeStat(path))?.isDirectory() ?? false;
 }
 exports.existsDir = existsDir;
+async function existsFile(path) {
+    return (await safeStat(path))?.isFile() ?? false;
+}
+exports.existsFile = existsFile;
 async function writeJSONFile(path, json) {
     await (0, promises_1.writeFile)(path, JSON.stringify(json));
 }
 exports.writeJSONFile = writeJSONFile;
-async function readdirIfExists(path) {
-    return await readDir(path, true);
-}
-exports.readdirIfExists = readdirIfExists;
 exports.parseFileExtensions = ["json", "js", "ts", "yaml", "yml"];
 async function parseFile(path, jsKey) {
     if (!(0, path_3.isAbsolute)(path))
@@ -140,16 +117,6 @@ async function findFile(sourcePath, baseName, extensions, errorMessage = "Path n
     return path;
 }
 exports.findFile = findFile;
-async function existsFile(path) {
-    try {
-        const info = await (0, promises_1.stat)(path);
-        return info.isFile();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.existsFile = existsFile;
 function parentTmpDir() {
     return (0, path_2.join)(globalData_1.default.tempDir, "datatruck-temp");
 }
@@ -219,24 +186,6 @@ async function readPartialFile(path, positions) {
     });
 }
 exports.readPartialFile = readPartialFile;
-async function checkFile(path) {
-    try {
-        return (await (0, promises_1.stat)(path)).isFile();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.checkFile = checkFile;
-async function checkDir(path) {
-    try {
-        return (await (0, promises_1.stat)(path)).isDirectory();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.checkDir = checkDir;
 async function readDir(path, optional) {
     try {
         return await (0, promises_1.readdir)(path);
@@ -509,3 +458,74 @@ async function createFileScanner(options) {
     return object;
 }
 exports.createFileScanner = createFileScanner;
+function createWriteStreamPool(options) {
+    const pool = {};
+    const create = (key) => {
+        const item = {
+            key,
+            stream: (0, fs_2.createWriteStream)((0, path_2.join)(options.path, options.onStreamPath ? options.onStreamPath(key) : key)),
+            finished: false,
+        };
+        item.stream
+            .once("error", (error) => {
+            item.finished = true;
+            item.error = error;
+        })
+            .once("close", () => (item.finished = true));
+        return (pool[item.key] = item);
+    };
+    return {
+        pool,
+        path(key) {
+            const item = pool[key];
+            if (!item)
+                return;
+            if (typeof item.stream.path !== "string")
+                throw new Error(`Stream path is not defined: ${key}`);
+            return item.stream.path;
+        },
+        writeLine(key, v) {
+            const item = pool[key] || create(key.toString());
+            if (item.finished) {
+                return false;
+            }
+            else if (item.written) {
+                return item.stream.write(`\n${v}`);
+            }
+            else {
+                item.written = true;
+                return item.stream.write(`${v}`);
+            }
+        },
+        async end() {
+            const items = Object.values(pool);
+            for (const item of items)
+                if (!item.finished)
+                    item.stream.end();
+            const itemWithErrors = items.filter((v) => v.error);
+            if (itemWithErrors.length) {
+                const keys = itemWithErrors.map((item) => item.key);
+                throw new AggregateError(itemWithErrors.map((item) => item.error), `Streams faileds: ${keys.join(", ")}`);
+            }
+            await Promise.all(items
+                .filter((item) => !item.finished)
+                .map((item) => waitForClose(item.stream)));
+        },
+    };
+}
+exports.createWriteStreamPool = createWriteStreamPool;
+function countFileLines(path) {
+    let lines = 0;
+    const rl = (0, readline_1.createInterface)({
+        input: (0, fs_1.createReadStream)(path),
+    });
+    return new Promise((resolve, reject) => {
+        rl.on("line", (line) => {
+            if (!line.length)
+                lines++;
+        });
+        rl.on("close", () => resolve(lines));
+        rl.on("error", reject);
+    });
+}
+exports.countFileLines = countFileLines;

package/utils/process.d.ts

@@ -33,7 +33,7 @@ export interface ExecSettingsInterface {
     onSpawn?: (p: ChildProcess) => any;
     stdout?: {
         save?: boolean;
-        parseLines?: boolean;
+        parseLines?: boolean | "skip-empty";
         onData?: (data: string) => void;
     };
     stderr?: {

package/utils/process.js

@@ -188,7 +188,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 toStderr: log.allToStderr,
             });
         }
-        if (typeof options?.cwd === "string" && !(await (0, fs_1.checkDir)(options.cwd)))
+        if (typeof options?.cwd === "string" && !(await (0, fs_1.existsDir)(options.cwd)))
             throw new Error(`Current working directory does not exist: ${options.cwd}`);
         if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
             const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
@@ -225,13 +225,13 @@ async function exec(command, argv = [], options = null, settings = {}) {
             if (spawnData.exitCode) {
                 let exitCodeError;
                 if (settings.stderr?.toExitCode) {
-                    exitCodeError = new Error(`Exit code ${spawnData.exitCode}: ${spawnData.stderr
+                    exitCodeError = new Error(`Exit code ${spawnData.exitCode}: ${command} ${argv.join(" ")} | ${spawnData.stderr
                         .split(/\r?\n/g)
                         .filter((v) => !!v.length)
                         .join(" | ")}`);
                 }
                 else {
-                    exitCodeError = new Error(`Exit code: ${spawnData.exitCode} (${command} ${argv.join(" ")})`);
+                    exitCodeError = new Error(`Exit code ${spawnData.exitCode}: ${command} ${argv.join(" ")}`);
                 }
                 const exitCodeErrorResult = settings.onExitCodeError?.(spawnData, exitCodeError);
                 if (exitCodeErrorResult instanceof Error) {
@@ -286,17 +286,24 @@ async function exec(command, argv = [], options = null, settings = {}) {
             if (!p.stdout)
                 throw new Error(`stdout is not defined`);
             const parseLines = settings.stdout?.parseLines;
-            const onData = (data) => {
+            const skipEmptyLines = parseLines === "skip-empty";
+            const onData = (inData) => {
+                let data = inData.toString();
+                if (parseLines) {
+                    if (skipEmptyLines && !data.trim().length)
+                        return;
+                    data = `${inData}\n`;
+                }
                 if (log.stdout)
                     logExecStdout({
-                        data: parseLines ? `${data}\n` : data.toString(),
+                        data,
                         stderr: log.allToStderr,
                         colorize: log.colorize,
                     });
                 if (settings.stdout?.save)
-                    spawnData.stdout += data.toString();
+                    spawnData.stdout += data;
                 if (settings.stdout?.onData)
-                    settings.stdout.onData(data.toString());
+                    settings.stdout.onData(inData.toString());
             };
             if (parseLines) {
                 const rl = (0, readline_1.createInterface)({
@@ -305,7 +312,9 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 rl.on("line", onData);
                 rl.on("close", tryFinish);
             }
-            else {
+            else if (log.stdout ||
+                settings.stdout?.save ||
+                settings.stdout?.onData) {
                 p.stdout.on("data", onData);
             }
         }

package/utils/string.d.ts

@@ -1,6 +1,4 @@
 export declare function serialize(message: string, data?: Object): string;
-export declare function ucfirst(value: string): string;
-export declare function lcfirst(value: string): string;
 export declare function snakeCase(value: string, char?: string): string;
 export declare function render(subject: string, vars: Record<string, string | undefined>): string;
 export declare function parseStringList(value: string, validValues?: string[]): string[];

package/utils/string.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
+exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -9,14 +9,6 @@ function serialize(message, data) {
     return message;
 }
 exports.serialize = serialize;
-function ucfirst(value) {
-    return value.charAt(0).toUpperCase() + value.slice(1);
-}
-exports.ucfirst = ucfirst;
-function lcfirst(value) {
-    return value.charAt(0).toLowerCase() + value.slice(1);
-}
-exports.lcfirst = lcfirst;
 function snakeCase(value, char = "_") {
     return value.replace(/[A-Z]/g, (letter) => `${char}${letter.toLowerCase()}`);
 }

package/utils/tar.d.ts

@@ -1,3 +1,4 @@
+import type { JSONSchema7 } from "json-schema";
 export type Progress = {
     percent: number;
     current: number;
@@ -7,26 +8,47 @@ export type TarEntry = {
     path: string;
     progress: Progress;
 };
+export type CoresOptions = number | {
+    percent: number;
+};
+export type CompressOptions = {
+    level?: number;
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
+export type DecompressOptions = {
+    /**
+     * @default {percent:50}
+     */
+    cores?: CoresOptions;
+};
 export interface CreateTarOptions {
     path: string;
     verbose?: boolean;
     output: string;
-    include: string[];
-    compress?: boolean;
+    includeList: string;
+    compress?: boolean | CompressOptions;
     onEntry?: (entry: TarEntry) => void;
 }
 export interface ExtractOptions {
     input: string;
     output: string;
     verbose?: boolean;
+    decompress?: boolean | DecompressOptions;
     total?: number;
     onEntry?: (entry: TarEntry) => void;
 }
+export declare const compressDefinition: JSONSchema7;
+export type TarVendor = "busybox" | "bsdtar" | "gnu";
+export declare function getTarVendor(cache?: boolean, log?: boolean): Promise<TarVendor | null>;
 export type ListTarOptions = {
     input: string;
     onEntry?: (entry: Pick<TarEntry, "path">) => void;
     verbose?: boolean;
 };
 export declare function listTar(options: ListTarOptions): Promise<number>;
+export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
 export declare function createTar(options: CreateTarOptions): Promise<void>;
 export declare function extractTar(options: ExtractOptions): Promise<void>;

package/utils/tar.js

@@ -1,91 +1,210 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTar = exports.createTar = exports.listTar = void 0;
+exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
-const fs_2 = require("fs");
+const process_1 = require("./process");
 const promises_1 = require("fs/promises");
-const tar_1 = __importDefault(require("tar"));
+const os_1 = require("os");
+exports.compressDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        level: { type: "integer" },
+        cores: {
+            anyOf: [
+                { type: "integer" },
+                {
+                    type: "object",
+                    required: ["percent"],
+                    properties: { percent: { type: "integer" } },
+                },
+            ],
+        },
+    },
+};
+let tarVendor;
+async function getTarVendor(cache = true, log = false) {
+    if (cache && typeof tarVendor !== "undefined")
+        return tarVendor;
+    const p = await (0, process_1.exec)("tar", ["--help"], {}, {
+        log,
+        stdout: {
+            save: true,
+        },
+    });
+    const find = () => {
+        if (p.stdout.includes("BusyBox")) {
+            return "busybox";
+        }
+        else if (p.stdout.includes("bsdtar")) {
+            return "bsdtar";
+        }
+        else if (p.stdout.includes("GNU")) {
+            return "gnu";
+        }
+        else {
+            return null;
+        }
+    };
+    return (tarVendor = find());
+}
+exports.getTarVendor = getTarVendor;
 async function listTar(options) {
-    if (options.verbose)
-        (0, cli_1.logExec)("tar", ["-ztvf", options.input]);
     let total = 0;
-    await tar_1.default.list({
-        file: options.input,
-        onentry(entry) {
-            options.onEntry?.({ path: entry.path });
-            total++;
+    await (0, process_1.exec)("tar", ["-tf", toLocalPath(options.input), "--force-local"], {}, {
+        log: options.verbose,
+        stdout: {
+            parseLines: "skip-empty",
+            onData: (path) => {
+                options.onEntry?.({ path });
+                total++;
+            },
         },
     });
     return total;
 }
 exports.listTar = listTar;
+let pigzLib;
+async function checkPigzLib(cache = true) {
+    if (cache && pigzLib !== undefined)
+        return pigzLib;
+    try {
+        return !(await (0, process_1.exec)("pigz", ["-V"])).exitCode;
+    }
+    catch {
+        return false;
+    }
+}
+exports.checkPigzLib = checkPigzLib;
+async function resolveCores(input) {
+    if (!(await checkPigzLib()))
+        return 1;
+    const total = (0, os_1.cpus)().length;
+    return Math.min(total, typeof input === "number"
+        ? input
+        : Math.max(0, Math.round(((input?.percent || 50) * total) / 100)));
+}
+async function ifX(input, cb) {
+    return input ? await cb((input === true ? {} : input)) : undefined;
+}
 async function createTar(options) {
-    if (options.verbose)
-        (0, cli_1.logExec)("tar", [
-            options.compress ? "-czvf" : "-cvf",
-            options.output,
-            options.path,
-        ]);
-    let total = options.include.length;
-    if (!total)
-        throw new Error("'include' option is empty");
+    const vendor = await getTarVendor(true, options.verbose);
+    const total = await (0, fs_1.countFileLines)(options.includeList);
+    const compress = await ifX(options.compress, async (compress) => ({
+        ...compress,
+        cores: await resolveCores(compress.cores),
+    }));
     let current = 0;
-    let progressPromise;
-    const inStream = tar_1.default.create({
-        gzip: options.compress,
-        cwd: options.path,
-        filter(path) {
-            current++;
-            options.onEntry?.({
-                path: path,
-                progress: {
-                    total,
-                    current,
-                    percent: (0, math_1.progressPercent)(total, current),
-                },
-            });
-            return true;
+    const env = {
+        ...(compress?.cores === 1 &&
+            compress.level && {
+            GZIP_OPT: compress.level.toString(),
+        }),
+    };
+    await (0, process_1.exec)("tar", [
+        "-C",
+        toLocalPath(options.path),
+        compress?.cores === 1 ? "-czvf" : "-cvf",
+        toLocalPath(options.output),
+        "-T",
+        toLocalPath(options.includeList),
+        "--ignore-failed-read",
+        "--force-local",
+        ...(compress && compress.cores > 1
+            ? [
+                "-I",
+                `"${[
+                    "pigz",
+                    "-r",
+                    !!compress.level && `-${compress.level}`,
+                    `-p ${compress.cores}`,
+                ]
+                    .filter(Boolean)
+                    .join(" ")}"`,
+            ]
+            : []),
+    ], {
+        ...(compress &&
+            compress.cores > 1 && {
+            shell: true,
+        }),
+        env: {
+            ...process.env,
+            ...env,
+        },
+    }, {
+        log: options.verbose ? { envNames: Object.keys(env) } : false,
+        stderr: { toExitCode: true },
+        stdout: {
+            parseLines: "skip-empty",
+            onData: (line) => {
+                current++;
+                const path = vendor === "bsdtar" ? line.slice(2) : line;
+                options.onEntry?.({
+                    path,
+                    progress: {
+                        total,
+                        current,
+                        percent: (0, math_1.progressPercent)(total, current),
+                    },
+                });
+            },
         },
-    }, options.include);
-    const outStream = (0, fs_2.createWriteStream)(options.output);
-    await new Promise((resolve, reject) => {
-        inStream.on("error", reject);
-        outStream.on("error", reject);
-        inStream.pipe(outStream);
-        outStream.on("close", resolve);
     });
-    await progressPromise;
 }
 exports.createTar = createTar;
+/**
+ * Fix `tar.exe: Option --force-local is not supported`
+ */
+function toLocalPath(path) {
+    return (0, os_1.platform)() === "win32" ? path.replace(/\\/g, "/") : path;
+}
 async function extractTar(options) {
     let total = options.total ??
         (await listTar({ input: options.input, verbose: options.verbose }));
-    if (options.verbose) {
-        (0, cli_1.logExec)("tar", ["-xzvfp", options.input, "-C", options.output]);
+    if (options.verbose)
         (0, cli_1.logExec)("mkdir", ["-p", options.output]);
-    }
-    let current = 0;
     await (0, promises_1.mkdir)(options.output, { recursive: true });
     await (0, fs_1.ensureEmptyDir)(options.output);
-    await tar_1.default.extract({
-        file: options.input,
-        cwd: options.output,
-        preserveOwner: true,
-        onentry(entry) {
-            current++;
-            options.onEntry?.({
-                path: entry.path,
-                progress: {
-                    total,
-                    current,
-                    percent: (0, math_1.progressPercent)(total, current),
-                },
-            });
+    const decompress = await ifX(options.decompress, async (decompress) => ({
+        ...decompress,
+        cores: await resolveCores(decompress.cores),
+    }));
+    let current = 0;
+    await (0, process_1.exec)("tar", [
+        decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
+        toLocalPath(options.input),
+        "-C",
+        toLocalPath(options.output),
+        "--force-local",
+        ...(decompress && decompress.cores > 1
+            ? ["-I", `"pigz -p ${decompress.cores}"`]
+            : []),
+    ], {
+        ...(decompress &&
+            decompress.cores > 1 && {
+            shell: true,
+        }),
+    }, {
+        log: options.verbose,
+        stderr: {
+            toExitCode: true,
+        },
+        stdout: {
+            parseLines: "skip-empty",
+            onData: (path) => {
+                current++;
+                options.onEntry?.({
+                    path,
+                    progress: {
+                        total,
+                        current,
+                        percent: (0, math_1.progressPercent)(total, current),
+                    },
+                });
+            },
         },
     });
 }