@datatruck/cli 0.17.2 → 0.18.0

package/Repository/DatatruckRepository.d.ts

@@ -49,7 +49,6 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
     static stringifyMetaData(data: MetaDataType): string;
     onGetSource(): string;
     onInit(data: InitDataType): Promise<void>;
-    private createFileScanner;
     onPrune(data: PruneDataType): Promise<void>;
     onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
     private normalizeCompressConfig;

package/Repository/DatatruckRepository.js

@@ -1,7 +1,4 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exports.datatruckRepositoryDefinition = exports.datatruckRepositoryName = void 0;
 const AppError_1 = require("../Error/AppError");
@@ -9,17 +6,14 @@ const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
 const cli_1 = require("../utils/cli");
 const paths_1 = require("../utils/datatruck/paths");
 const fs_1 = require("../utils/fs");
-const math_1 = require("../utils/math");
 const string_1 = require("../utils/string");
 const zip_1 = require("../utils/zip");
 const RepositoryAbstract_1 = require("./RepositoryAbstract");
 const assert_1 = require("assert");
-const fast_glob_1 = __importDefault(require("fast-glob"));
 const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
-const perf_hooks_1 = require("perf_hooks");
 const readline_1 = require("readline");
 exports.datatruckRepositoryName = "datatruck";
 exports.datatruckRepositoryDefinition = {
@@ -110,66 +104,6 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     async onInit(data) {
         await (0, fs_1.mkdirIfNotExists)(this.config.outPath);
     }
-    async createFileScanner(options) {
-        const object = {
-            total: 0,
-            current: 0,
-            progress: async (description, data) => {
-                await options.onProgress({
-                    relative: {
-                        description,
-                        payload: data.path,
-                        percent: data.percent,
-                    },
-                    absolute: {
-                        total: object.total,
-                        current: object.current + data.current,
-                        percent: (0, math_1.progressPercent)(object.total, object.current + data.current),
-                    },
-                });
-                if (data.type === "end") {
-                    object.current += data.current;
-                }
-            },
-            updateProgress: async (end) => {
-                const currentTime = perf_hooks_1.performance.now();
-                const diff = currentTime - lastTime;
-                if (end || diff > 1000) {
-                    await options.onProgress({
-                        relative: {
-                            description: end ? "Scanned files" : "Scanning files",
-                            payload: object.total.toString(),
-                        },
-                    });
-                    lastTime = currentTime;
-                }
-            },
-            start: async (cb) => {
-                for await (const entry of (0, fs_1.pathIterator)(stream)) {
-                    if (!options.disableCounting)
-                        object.total++;
-                    await object.updateProgress();
-                    if (cb)
-                        await cb(entry);
-                }
-                if (!options.disableEndProgress)
-                    await object.updateProgress(true);
-            },
-        };
-        await options.onProgress({
-            relative: {
-                description: "Scanning files",
-            },
-        });
-        const stream = fast_glob_1.default.stream(options.glob.include, {
-            dot: true,
-            markDirectories: true,
-            stats: true,
-            ...options.glob,
-        });
-        let lastTime = perf_hooks_1.performance.now();
-        return object;
-    }
     async onPrune(data) {
         const snapshotName = DatatruckRepository.buildSnapshotName({
             snapshotId: data.snapshot.id,
@@ -273,7 +207,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const streams = [unpackedStream, singlePackStream, ...packStreams];
         if (data.options.verbose)
             (0, cli_1.logExec)(`Writing file lists in ${tmpDir}`);
-        const scanner = await this.createFileScanner({
+        const scanner = await (0, fs_1.createFileScanner)({
             glob: {
                 include,
                 cwd: sourcePath,
@@ -404,7 +338,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (data.options.verbose)
             (0, cli_1.logExec)(`Copying backup files to ${targetPath}`);
         await (0, promises_1.mkdir)(targetPath);
-        const scanner = await this.createFileScanner({
+        const scanner = await (0, fs_1.createFileScanner)({
             glob: {
                 include: ["**/*"],
                 cwd: sourcePath,
@@ -438,7 +372,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
-        const scanner = await this.createFileScanner({
+        const scanner = await (0, fs_1.createFileScanner)({
             glob: {
                 include: ["unpacked/**/*"],
                 cwd: sourcePath,

package/Task/MariadbTask.d.ts

@@ -11,6 +11,17 @@ export type MariadbTaskConfigType = {
     excludeTables?: string[];
     includeDatabases?: string[];
     excludeDatabases?: string[];
+    /**
+     * @default "auto"
+     */
+    parallel?: number | "auto";
+    compress?: {
+        command: string;
+        args?: string[];
+    } | {
+        type: "gzip" | "pigz";
+        args?: string[];
+    };
 };
 export declare const mariadbTaskName = "mariadb";
 export declare const mariadbTaskDefinition: JSONSchema7;

package/Task/MariadbTask.js

@@ -6,11 +6,13 @@ const cli_1 = require("../utils/cli");
 const fs_1 = require("../utils/fs");
 const math_1 = require("../utils/math");
 const process_1 = require("../utils/process");
+const zip_1 = require("../utils/zip");
 const TaskAbstract_1 = require("./TaskAbstract");
 const assert_1 = require("assert");
+const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
+const os_1 = require("os");
 const path_1 = require("path");
-const posix_1 = require("path/posix");
 exports.mariadbTaskName = "mariadb";
 exports.mariadbTaskDefinition = {
     type: "object",
@@ -39,8 +41,65 @@ exports.mariadbTaskDefinition = {
         excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         includeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         excludeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        parallel: {
+            anyOf: [{ type: "integer", minimum: 1 }, { enum: ["auto"] }],
+        },
+        compress: {
+            anyOf: [
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        command: { type: "string" },
+                        args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        type: { enum: ["gzip", "pigz"] },
+                        args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                    },
+                },
+            ],
+        },
     },
 };
+function normalizeConfig(input) {
+    let parallel = input.parallel ?? "auto";
+    let cores = (0, os_1.cpus)().length;
+    if (parallel === "auto") {
+        parallel = input.compress ? Math.round(cores / 2) : cores;
+        cores = Math.max(1, cores - parallel);
+    }
+    let compress;
+    if (input.compress && "type" in input.compress) {
+        if (input.compress.type === "pigz") {
+            compress = { command: "pigz", args: [] };
+            if (!input.compress.args?.includes("-p")) {
+                compress.args.push("-p", cores.toString());
+            }
+        }
+        else if (input.compress.type === "gzip") {
+            compress = { command: "gzip" };
+        }
+    }
+    else {
+        compress = input.compress;
+    }
+    return { parallel, compress };
+}
+const parseLineRegex = /^(?:\[(?<step>\d+)\] )?(?<dateTime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})?\s+(?<text>.+)\s*$/;
+function parseLine(line) {
+    const result = parseLineRegex.exec(line);
+    if (result) {
+        return result.groups;
+    }
+    else {
+        return { text: line };
+    }
+}
 class MariadbTask extends TaskAbstract_1.TaskAbstract {
     get command() {
         return this.config.command ?? "mariabackup";
@@ -58,18 +117,25 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         const targetPath = data.targetPath;
         (0, assert_1.ok)(typeof sourcePath === "string");
         (0, assert_1.ok)(typeof targetPath === "string");
+        const { parallel, compress } = normalizeConfig(config);
         const args = [
             `--backup`,
             `--datadir=${sourcePath}`,
-            `--target-dir=${targetPath}`,
             `--host=${config.hostname}`,
             `--user=${config.username}`,
+            `--parallel=${parallel}`,
             `--password=${typeof config.password === "string"
                 ? config.password
                 : config.password
                     ? (await (0, promises_1.readFile)(config.password.path)).toString()
                     : ""}`,
         ];
+        if (compress) {
+            args.push(`--stream=xbstream`);
+        }
+        else {
+            args.push(`--target-dir=${targetPath}`);
+        }
         if (config.includeDatabases)
             args.push(`--databases=${config.includeDatabases.join(" ")}`);
         if (config.excludeDatabases)
@@ -83,34 +149,23 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
         await (0, fs_1.forEachFile)(sourcePath, () => {
             total++;
         });
-        let childProcess;
-        const onData = async (strLines) => {
-            const paths = [];
-            const pathRegex = /\[\d{1,}\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} Copying (.+) to/;
-            const lines = strLines.split(/\r?\n/);
-            let fatalError = false;
-            for (const line of lines) {
-                if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
-                    line.includes("Error: cannot read redo log header")) {
-                    fatalError = true;
-                }
-                else {
-                    const matches = pathRegex.exec(line);
-                    if (matches) {
-                        current++;
-                        paths.push(matches[1]);
-                    }
-                }
-            }
-            if (fatalError) {
-                childProcess.kill();
+        let p1;
+        let lastLineText;
+        const pathRegex = /((Copying|Streaming) .+) to/;
+        const onData = async (line) => {
+            const { text } = parseLine(line);
+            lastLineText = text;
+            if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
+                line.includes("Error: cannot read redo log header")) {
+                p1.kill();
             }
-            else if (paths.length) {
-                const path = (0, posix_1.normalize)(paths[0]);
+            else {
+                const matches = pathRegex.exec(text);
+                if (matches)
+                    current++;
                 await data.onProgress({
                     relative: {
-                        description: "Copying file",
-                        payload: path,
+                        payload: matches ? matches[1] : text,
                     },
                     absolute: {
                         current,
@@ -120,36 +175,163 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
                 });
             }
         };
-        await (0, process_1.exec)(command, args, undefined, {
-            log: this.verbose,
-            onSpawn: (p) => {
-                childProcess = p;
-            },
-            stdout: {
-                onData,
-            },
-            stderr: {
-                onData,
-            },
-        });
-        await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
-            log: this.verbose,
-            stderr: { onData: () => { } },
-        });
+        const stats = { xbFiles: total };
+        await (0, promises_1.writeFile)((0, path_1.join)(targetPath, "stats.dtt.json"), JSON.stringify(stats));
+        if (compress) {
+            const p0 = (0, fs_2.createWriteStream)((0, path_1.join)(targetPath, "db.xb.gz"));
+            p1 = (0, process_1.createProcess)(command, args, {
+                $log: {
+                    exec: this.verbose,
+                    stderr: this.verbose,
+                },
+                $stderr: {
+                    parseLines: true,
+                    onData,
+                },
+                $onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
+            });
+            const p2 = (0, process_1.createProcess)(compress.command, compress.args);
+            p1.stdout.pipe(p2.stdin, { end: true });
+            p2.stdout.pipe(p0, { end: true });
+            await Promise.all([p1, p2, (0, fs_1.waitForClose)(p0)]);
+        }
+        else {
+            p1 = (0, process_1.createProcess)(command, args, {
+                $log: this.verbose,
+                $stdout: {
+                    parseLines: true,
+                    onData,
+                },
+                $stderr: {
+                    parseLines: true,
+                    onData,
+                },
+                $onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
+            });
+            await p1;
+            await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
+                log: this.verbose,
+                stderr: { onData: () => { } },
+            });
+        }
     }
     async onRestore(data) {
         this.verbose = data.options.verbose;
         const restorePath = data.package.restorePath;
         (0, assert_1.ok)(typeof restorePath === "string");
         await (0, fs_1.mkdirIfNotExists)(restorePath);
-        const files = await (0, fs_1.readDir)(restorePath);
-        for (const file of files) {
-            if (file.startsWith("ib_logfile")) {
-                const filePath = (0, path_1.join)(restorePath, file);
-                if (this.verbose)
-                    (0, cli_1.logExec)("rm", [filePath]);
-                await (0, promises_1.rm)(filePath);
-            }
+        const removeFiles = [];
+        let files = [];
+        const reloadFiles = async (data = {}) => {
+            if (data.removeFile)
+                removeFiles.push(data.removeFile);
+            return (files = (await (0, fs_1.readDir)(restorePath)).filter((v) => !removeFiles.includes(v)));
+        };
+        await reloadFiles();
+        const absolute = {
+            format: "amount",
+            total: 3,
+            current: 0,
+            description: undefined,
+            payload: undefined,
+            percent: undefined,
+        };
+        // Stats
+        const statsFile = files.find((file) => file === "stats.dtt.json");
+        let stats;
+        if (statsFile) {
+            const statsFilePath = (0, path_1.join)(restorePath, statsFile);
+            const statsBuffer = await (0, promises_1.readFile)(statsFilePath);
+            stats = JSON.parse(statsBuffer.toString());
+            await reloadFiles({ removeFile: statsFile });
+        }
+        const zipFile = files.find((file) => file.endsWith(".gz"));
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        // Extract
+        if (files.length === 1 && zipFile) {
+            absolute.description = "Extracting";
+            absolute.payload = zipFile;
+            await data.onProgress({
+                absolute,
+            });
+            await (0, zip_1.unzip)({
+                input: (0, path_1.join)(restorePath, zipFile),
+                output: restorePath,
+                verbose: this.verbose,
+                async onProgress(item) {
+                    await data.onProgress({
+                        absolute,
+                        relative: {
+                            payload: item.path,
+                            format: "amount",
+                            percent: item.percent,
+                        },
+                    });
+                },
+            });
+            await reloadFiles({ removeFile: zipFile });
+        }
+        // Extract stream
+        const xbFile = files.find((file) => file.endsWith(".xb"));
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        if (files.length === 1 && xbFile) {
+            const xbFilePath = (0, path_1.join)(restorePath, xbFile);
+            const xbStream = (0, fs_2.createReadStream)(xbFilePath);
+            removeFiles.push(xbFile);
+            absolute.description = "Extracting stream";
+            absolute.payload = xbFile;
+            await data.onProgress({
+                absolute,
+            });
+            let currentXbFiles = 0;
+            const { parallel } = normalizeConfig({ parallel: this.config.parallel });
+            const p1 = (0, process_1.createProcess)("mbstream", ["-x", "-C", restorePath, "-v", "-p", parallel], {
+                $log: this.verbose,
+                $stderr: {
+                    parseLines: true,
+                    async onData(line) {
+                        const { text: path } = parseLine(line);
+                        await data.onProgress({
+                            absolute,
+                            relative: {
+                                payload: path,
+                                format: "amount",
+                                current: ++currentXbFiles,
+                                total: stats?.xbFiles,
+                                percent: stats?.xbFiles
+                                    ? (0, math_1.progressPercent)(stats.xbFiles, currentXbFiles)
+                                    : undefined,
+                            },
+                        });
+                    },
+                },
+            });
+            xbStream.pipe(p1.stdin, { end: true });
+            await Promise.all([(0, fs_1.waitForClose)(xbStream), p1]);
+        }
+        // Prepare
+        absolute.current++;
+        absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
+        if (files.length === 1 && xbFile) {
+            absolute.description = "Preparing";
+            await data.onProgress({
+                absolute,
+            });
+            await (0, process_1.exec)(this.command, [`--prepare`, `--target-dir=${restorePath}`], undefined, {
+                log: this.verbose,
+                stderr: { onData: () => { } },
+            });
+        }
+        await reloadFiles();
+        removeFiles.push(...files.filter((file) => file.startsWith("ib_logfile")));
+        // Remove files
+        for (const file of removeFiles) {
+            const filePath = (0, path_1.join)(restorePath, file);
+            if (this.verbose)
+                (0, cli_1.logExec)("rm", [filePath]);
+            await (0, promises_1.rm)(filePath);
         }
     }
 }

package/config.schema.json

@@ -798,6 +798,50 @@
         },
         "excludeDatabases": {
           "$ref": "#/definitions/stringlist-util"
+        },
+        "parallel": {
+          "anyOf": [
+            {
+              "type": "integer",
+              "minimum": 1
+            },
+            {
+              "enum": [
+                "auto"
+              ]
+            }
+          ]
+        },
+        "compress": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "command": {
+                  "type": "string"
+                },
+                "args": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "type": {
+                  "enum": [
+                    "gzip",
+                    "pigz"
+                  ]
+                },
+                "args": {
+                  "$ref": "#/definitions/stringlist-util"
+                }
+              }
+            }
+          ]
         }
       }
     },

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.17.2",
+  "version": "0.18.0",
   "dependencies": {
-    "ajv": "^8.11.2",
+    "ajv": "^8.12.0",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
     "cli-table3": "^0.6.3",
@@ -17,7 +17,7 @@
   },
   "optionalDependencies": {
     "ts-node": "^10.9.1",
-    "yaml": "^2.2.0"
+    "yaml": "^2.2.1"
   },
   "engine": {
     "node": ">=16.0.0"

package/utils/datatruck/config.d.ts

@@ -59,8 +59,8 @@ export declare const params: {
         action: string;
     };
     pkgRestorePath: {
-        packageName: string;
         path: string;
+        packageName: string;
         snapshotId: string;
         temp: string;
         snapshotDate: string;

package/utils/fs.d.ts

@@ -2,7 +2,9 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { Dirent, Stats } from "fs";
+import { Progress } from "./progress";
+import { Entry, Options } from "fast-glob";
+import { Dirent, ReadStream, Stats } from "fs";
 import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
@@ -51,7 +53,7 @@ export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
     outDir: string;
 }): Promise<string>;
-export declare function waitForClose(stream: WriteStream): Promise<WriteStream>;
+export declare function waitForClose(stream: WriteStream | ReadStream): Promise<void>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;
 export declare function updateFileStats(path: string, fileInfo: Stats): Promise<void>;
 export declare function isNotFoundError(error: unknown): boolean;
@@ -97,4 +99,23 @@ export declare function cpy(options: {
     files: number;
     dirs: number;
 }>;
+export declare function createFileScanner(options: {
+    glob: Options & {
+        include: string[];
+    };
+    onProgress: (data: Progress) => Promise<void>;
+    disableCounting?: boolean;
+    disableEndProgress?: boolean;
+}): Promise<{
+    total: number;
+    current: number;
+    progress: (description: string, data: {
+        path?: string;
+        current: number;
+        type?: "start" | "end";
+        percent?: number;
+    }) => Promise<void>;
+    updateProgress: (end?: boolean) => Promise<void>;
+    start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
+}>;
 export {};

package/utils/fs.js

@@ -3,8 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
+const math_1 = require("./math");
 const path_1 = require("./path");
 const async_1 = require("async");
 const crypto_1 = require("crypto");
@@ -66,12 +67,9 @@ async function isDirEmpty(path) {
 }
 exports.isDirEmpty = isDirEmpty;
 async function mkdirIfNotExists(path) {
-    try {
-        await (0, promises_1.mkdir)(path, {
-            recursive: true,
-        });
-    }
-    catch (e) { }
+    await (0, promises_1.mkdir)(path, {
+        recursive: true,
+    });
     return path;
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
@@ -458,3 +456,64 @@ async function cpy(options) {
     return stats;
 }
 exports.cpy = cpy;
+async function createFileScanner(options) {
+    const object = {
+        total: 0,
+        current: 0,
+        progress: async (description, data) => {
+            await options.onProgress({
+                relative: {
+                    description,
+                    payload: data.path,
+                    percent: data.percent,
+                },
+                absolute: {
+                    total: object.total,
+                    current: object.current + data.current,
+                    percent: (0, math_1.progressPercent)(object.total, object.current + data.current),
+                },
+            });
+            if (data.type === "end") {
+                object.current += data.current;
+            }
+        },
+        updateProgress: async (end) => {
+            const currentTime = performance.now();
+            const diff = currentTime - lastTime;
+            if (end || diff > 1000) {
+                await options.onProgress({
+                    relative: {
+                        description: end ? "Scanned files" : "Scanning files",
+                        payload: object.total.toString(),
+                    },
+                });
+                lastTime = currentTime;
+            }
+        },
+        start: async (cb) => {
+            for await (const entry of pathIterator(stream)) {
+                if (!options.disableCounting)
+                    object.total++;
+                await object.updateProgress();
+                if (cb)
+                    await cb(entry);
+            }
+            if (!options.disableEndProgress)
+                await object.updateProgress(true);
+        },
+    };
+    await options.onProgress({
+        relative: {
+            description: "Scanning files",
+        },
+    });
+    const stream = fast_glob_1.default.stream(options.glob.include, {
+        dot: true,
+        markDirectories: true,
+        stats: true,
+        ...options.glob,
+    });
+    let lastTime = performance.now();
+    return object;
+}
+exports.createFileScanner = createFileScanner;

package/utils/process.d.ts

@@ -1,7 +1,9 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { SpawnOptions, ChildProcess } from "child_process";
+/// <reference types="node" />
+import { SpawnOptions, ChildProcess, ChildProcessByStdio } from "child_process";
 import { ReadStream, WriteStream } from "fs";
+import { Readable, Writable } from "stream";
 export type ExecLogSettingsType = {
     colorize?: boolean;
     exec?: boolean;
@@ -13,18 +15,22 @@ export type ExecLogSettingsType = {
 export interface ExecSettingsInterface {
     exec?: boolean;
     pipe?: {
-        stream: WriteStream | ReadStream;
+        stream: WriteStream;
         onWriteProgress?: (data: {
             totalBytes: number;
         }) => void;
+    } | {
+        stream: ReadStream;
         onReadProgress?: (data: {
             totalBytes: number;
             currentBytes: number;
             progress: number;
         }) => void;
+    } | {
+        stream: Readable;
     };
     log?: ExecLogSettingsType | boolean;
-    onSpawn?: (p: ChildProcess) => void;
+    onSpawn?: (p: ChildProcess) => any;
     stdout?: {
         save?: boolean;
         parseLines?: boolean;
@@ -49,6 +55,52 @@ export type ExecResultType = {
     stderr: string;
     exitCode: number;
 };
+export type ParseStreamDataOptions<S extends boolean = boolean> = {
+    save?: S;
+    parseLines?: boolean;
+    log?: LogProcessOptions | boolean;
+    onData?: (data: string) => void;
+};
+export declare function parseStreamData<S extends boolean>(stream: Readable, options?: ParseStreamDataOptions<S>): Promise<S extends true ? string : undefined>;
+type OnExitCode = OnExitCodeValue | ((code: number) => OnExitCodeValue | void | undefined);
+type OnExitCodeValue = Error | string | number | boolean;
+export declare function waitForClose<O extends boolean, E extends boolean>(p: ChildProcess, options?: {
+    strict?: boolean;
+    stdout?: O;
+    stderr?: E;
+    onExitCode?: OnExitCode;
+}): Promise<{
+    exitCode: number;
+} & (O extends true ? {
+    stdout: string;
+} : {}) & (E extends true ? {
+    stderr: string;
+} : {})>;
+export type LogProcessOptions = {
+    envNames?: string[];
+    env?: Record<string, any>;
+    pipe?: Readable | Writable;
+    toStderr?: boolean;
+    colorize?: boolean;
+};
+export declare function logProcessExec(command: string, argv: any[], options: LogProcessOptions): Promise<void>;
+export type ProcessOptions<O1 extends boolean, O2 extends boolean> = {
+    $stdout?: Omit<ParseStreamDataOptions<O1>, "log">;
+    $stderr?: Omit<ParseStreamDataOptions<O2>, "log">;
+    $onExitCode?: OnExitCode;
+    $log?: boolean | {
+        exec?: boolean | LogProcessOptions;
+        stdout?: boolean | LogProcessOptions;
+        stderr?: boolean | LogProcessOptions;
+    };
+};
+export declare function createProcess<O1 extends boolean, O2 extends boolean>(command: string, argv?: (string | number)[], options?: SpawnOptions & ProcessOptions<O1, O2>): ChildProcessByStdio<Writable, Readable, Readable> & PromiseLike<{
+    exitCode: number;
+} & (O1 extends true ? {
+    stdout: string;
+} : {}) & (O2 extends true ? {
+    stderr: string;
+} : {})>;
 export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
 type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
 export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void): void;

package/utils/process.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.onExit = exports.exec = exports.logExecStderr = exports.logExecStdout = void 0;
+exports.onExit = exports.exec = exports.createProcess = exports.logProcessExec = exports.waitForClose = exports.parseStreamData = exports.logExecStderr = exports.logExecStdout = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
@@ -12,6 +12,7 @@ const child_process_1 = require("child_process");
 const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
 const readline_1 = require("readline");
+const stream_1 = require("stream");
 function logExecStdout(input) {
     let text = input.colorize ? chalk_1.default.grey(input.data) : input.data;
     if (input.lineSalt)
@@ -23,6 +24,152 @@ function logExecStderr(data, colorize) {
     process.stdout.write(colorize ? chalk_1.default.red(data) : data);
 }
 exports.logExecStderr = logExecStderr;
+function parseStreamData(stream, options = {}) {
+    const log = options.log === true ? {} : options.log;
+    let result;
+    if (options.save)
+        result = "";
+    return new Promise((resolve, reject) => {
+        const lines = options.parseLines;
+        const onData = (data) => {
+            if (options.onData)
+                options.onData(data.toString());
+            if (log)
+                logExecStdout({
+                    data: lines ? `${data}\n` : data.toString(),
+                    stderr: log.toStderr,
+                    colorize: log.colorize,
+                });
+            if (options?.save)
+                result += data.toString();
+        };
+        if (lines) {
+            const rl = (0, readline_1.createInterface)({
+                input: stream,
+            });
+            rl.on("line", onData).on("close", () => resolve(result));
+        }
+        else {
+            stream
+                .on("data", onData)
+                .on("error", reject)
+                .once("close", () => resolve(result));
+        }
+    });
+}
+exports.parseStreamData = parseStreamData;
+function waitForClose(p, options = {}) {
+    return new Promise((resolve, reject) => {
+        let result = {
+            exitCode: 1,
+        };
+        if (options.stdout) {
+            result.stdout = "";
+            p.stdout.on("data", (data) => (result.stdout += data.toString()));
+        }
+        p.once("error", reject).once("close", (exitCode) => {
+            if (exitCode) {
+                let onExitCode = options.onExitCode ?? true;
+                if (typeof onExitCode === "function") {
+                    onExitCode = onExitCode(exitCode);
+                }
+                if (typeof onExitCode === "string") {
+                    reject(new Error(onExitCode));
+                }
+                else if (typeof onExitCode === "number") {
+                    reject(new Error(`Exit code: ${onExitCode}`));
+                }
+                else if (onExitCode instanceof Error) {
+                    reject(onExitCode);
+                }
+                else if (onExitCode === false) {
+                    resolve({ ...result, exitCode: exitCode });
+                }
+                else {
+                    reject(new Error(`Exit code: ${exitCode}`));
+                }
+            }
+            else {
+                resolve({ ...result, exitCode: exitCode });
+            }
+        });
+    });
+}
+exports.waitForClose = waitForClose;
+async function logProcessExec(command, argv, options) {
+    const logEnv = options.envNames?.reduce((env, key) => {
+        const value = options?.env?.[key];
+        if (typeof value !== "undefined")
+            env[key] = value;
+        return env;
+    }, {});
+    (0, cli_1.logExec)(command, options.pipe
+        ? [
+            ...argv,
+            options.pipe instanceof stream_1.Readable ? "<" : ">",
+            "path" in options.pipe ? String(options.pipe.path) : "[stream]",
+        ]
+        : argv, logEnv, options.toStderr);
+}
+exports.logProcessExec = logProcessExec;
+function createProcess(command, argv = [], options = {}) {
+    const $log = options.$log === true
+        ? { exec: {}, stdout: {}, stderr: {} }
+        : options.$log || {};
+    if ($log.exec)
+        logProcessExec(command, argv, $log.exec === true ? {} : $log.exec);
+    if (typeof options.cwd === "string") {
+        let isDir = false;
+        try {
+            isDir = (0, fs_2.statSync)(options.cwd).isDirectory();
+        }
+        catch (error) { }
+        if (!isDir)
+            throw new Error(`Current working directory does not exist: ${options.cwd}`);
+    }
+    const handler = (0, child_process_1.spawn)(command, argv.map((v) => (typeof v === "number" ? v.toString() : v)), options ?? {});
+    const { $stdout, $stderr, $onExitCode } = options;
+    async function exec() {
+        const [stdout, stderr, result] = await Promise.all([
+            (!!$log.stdout || !!$stdout) &&
+                parseStreamData(handler.stdout, {
+                    log: $log.stdout,
+                    ...$stdout,
+                }),
+            (!!$log.stderr || !!$stderr) &&
+                parseStreamData(handler.stderr, {
+                    log: $log.stderr,
+                    ...$stderr,
+                }),
+            waitForClose(handler, {
+                onExitCode: $onExitCode,
+            }),
+        ]);
+        const endResult = {
+            exitCode: result.exitCode,
+        };
+        if (typeof stdout === "string")
+            endResult.stdout = stdout;
+        if (typeof stderr === "string")
+            endResult.stderr = stderr;
+        return endResult;
+    }
+    const promise = {
+        [Symbol.toStringTag]: "process",
+        then: function (onfulfilled, onrejected) {
+            return exec().then(onfulfilled, onrejected);
+        },
+        catch: function (onrejected) {
+            return exec().catch(onrejected);
+        },
+        finally: function (onfinally) {
+            return exec().finally(onfinally);
+        },
+    };
+    Object.assign(handler, promise);
+    return handler;
+}
+exports.createProcess = createProcess;
 async function exec(command, argv = [], options = null, settings = {}) {
     const pipe = settings.pipe;
     let log = {};
@@ -34,23 +181,16 @@ async function exec(command, argv = [], options = null, settings = {}) {
     }
     return new Promise(async (resolve, reject) => {
         if (log.exec) {
-            const logEnv = log.envNames?.reduce((env, key) => {
-                const value = options?.env?.[key];
-                if (typeof value !== "undefined")
-                    env[key] = value;
-                return env;
-            }, {});
-            (0, cli_1.logExec)(command, pipe
-                ? [
-                    ...argv,
-                    pipe.stream instanceof fs_2.ReadStream ? "<" : ">",
-                    String(pipe.stream.path),
-                ]
-                : argv, logEnv, log.allToStderr);
+            logProcessExec(command, argv, {
+                env: options?.env,
+                envNames: log.envNames,
+                pipe: pipe?.stream,
+                toStderr: log.allToStderr,
+            });
         }
         if (typeof options?.cwd === "string" && !(await (0, fs_1.checkDir)(options.cwd)))
-            return reject(new Error(`Current working directory does not exist: ${options.cwd}`));
-        if (pipe?.onReadProgress && pipe.stream instanceof fs_2.ReadStream) {
+            throw new Error(`Current working directory does not exist: ${options.cwd}`);
+        if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
             const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
             const totalBytes = fileInfo.size;
             let currentBytes = 0;
@@ -64,7 +204,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
             });
         }
         const p = (0, child_process_1.spawn)(command, argv, options ?? {});
-        settings.onSpawn?.(p);
+        await settings.onSpawn?.(p);
         let spawnError;
         const spawnData = {
             stdout: "",
@@ -121,7 +261,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     throw new Error(`stdout is not defined`);
                 if (!p.stderr)
                     throw new Error(`stderr is not defined`);
-                if (pipe.onWriteProgress) {
+                if ("onWriteProgress" in pipe && pipe.onWriteProgress) {
                     let totalBytes = 0;
                     p.stdout.on("data", (chunk) => {
                         totalBytes += chunk.length;
@@ -136,7 +276,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 p.stderr.pipe(pipe.stream, { end: false });
                 p.on("close", tryFinish);
             }
-            else if (pipe.stream instanceof fs_2.ReadStream) {
+            else if (pipe.stream instanceof stream_1.Readable) {
                 if (!p.stdin)
                     throw new Error(`stdin is not defined`);
                 pipe.stream.pipe(p.stdin);