@datatruck/cli 0.17.2 → 0.19.0

package/utils/fs.js

@@ -3,8 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
+const math_1 = require("./math");
 const path_1 = require("./path");
 const async_1 = require("async");
 const crypto_1 = require("crypto");
@@ -66,12 +67,9 @@ async function isDirEmpty(path) {
 }
 exports.isDirEmpty = isDirEmpty;
 async function mkdirIfNotExists(path) {
-    try {
-        await (0, promises_1.mkdir)(path, {
-            recursive: true,
-        });
-    }
-    catch (e) { }
+    await (0, promises_1.mkdir)(path, {
+        recursive: true,
+    });
     return path;
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
@@ -458,3 +456,56 @@ async function cpy(options) {
     return stats;
 }
 exports.cpy = cpy;
+async function createFileScanner(options) {
+    const object = {
+        disposed: false,
+        total: 0,
+        current: 0,
+        progress: async (description, path) => {
+            if (object.disposed)
+                return;
+            if (path)
+                object.current++;
+            await options.onProgress({
+                relative: {
+                    description,
+                    payload: path,
+                },
+                absolute: {
+                    total: object.total,
+                    current: object.current,
+                    percent: (0, math_1.progressPercent)(object.total, object.current),
+                },
+            });
+        },
+        end: async () => {
+            if (!object.disposed)
+                await object.progress("Finished");
+            object.disposed = true;
+        },
+        start: async (cb) => {
+            let lastTime = performance.now();
+            await object.progress("Scanning files");
+            for await (const entry of pathIterator(stream)) {
+                if (cb) {
+                    if (await cb(entry))
+                        object.total++;
+                }
+                else {
+                    object.total++;
+                }
+                if (lastTime - performance.now() > 500)
+                    await object.progress("Scanning files");
+            }
+            await object.progress("Scanned files");
+        },
+    };
+    const stream = fast_glob_1.default.stream(options.glob.include, {
+        dot: true,
+        markDirectories: true,
+        stats: true,
+        ...options.glob,
+    });
+    return object;
+}
+exports.createFileScanner = createFileScanner;

package/utils/process.d.ts

@@ -1,7 +1,9 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { SpawnOptions, ChildProcess } from "child_process";
+/// <reference types="node" />
+import { SpawnOptions, ChildProcess, ChildProcessByStdio } from "child_process";
 import { ReadStream, WriteStream } from "fs";
+import { Readable, Writable } from "stream";
 export type ExecLogSettingsType = {
     colorize?: boolean;
     exec?: boolean;
@@ -13,18 +15,22 @@ export type ExecLogSettingsType = {
 export interface ExecSettingsInterface {
     exec?: boolean;
     pipe?: {
-        stream: WriteStream | ReadStream;
+        stream: WriteStream;
         onWriteProgress?: (data: {
             totalBytes: number;
         }) => void;
+    } | {
+        stream: ReadStream;
         onReadProgress?: (data: {
             totalBytes: number;
             currentBytes: number;
             progress: number;
         }) => void;
+    } | {
+        stream: Readable;
     };
     log?: ExecLogSettingsType | boolean;
-    onSpawn?: (p: ChildProcess) => void;
+    onSpawn?: (p: ChildProcess) => any;
     stdout?: {
         save?: boolean;
         parseLines?: boolean;
@@ -49,6 +55,52 @@ export type ExecResultType = {
     stderr: string;
     exitCode: number;
 };
+export type ParseStreamDataOptions<S extends boolean = boolean> = {
+    save?: S;
+    parseLines?: boolean;
+    log?: LogProcessOptions | boolean;
+    onData?: (data: string) => void;
+};
+export declare function parseStreamData<S extends boolean>(stream: Readable, options?: ParseStreamDataOptions<S>): Promise<S extends true ? string : undefined>;
+type OnExitCode = OnExitCodeValue | ((code: number) => OnExitCodeValue | void | undefined);
+type OnExitCodeValue = Error | string | number | boolean;
+export declare function waitForClose<O extends boolean, E extends boolean>(p: ChildProcess, options?: {
+    strict?: boolean;
+    stdout?: O;
+    stderr?: E;
+    onExitCode?: OnExitCode;
+}): Promise<{
+    exitCode: number;
+} & (O extends true ? {
+    stdout: string;
+} : {}) & (E extends true ? {
+    stderr: string;
+} : {})>;
+export type LogProcessOptions = {
+    envNames?: string[];
+    env?: Record<string, any>;
+    pipe?: Readable | Writable;
+    toStderr?: boolean;
+    colorize?: boolean;
+};
+export declare function logProcessExec(command: string, argv: any[], options: LogProcessOptions): Promise<void>;
+export type ProcessOptions<O1 extends boolean, O2 extends boolean> = {
+    $stdout?: Omit<ParseStreamDataOptions<O1>, "log">;
+    $stderr?: Omit<ParseStreamDataOptions<O2>, "log">;
+    $onExitCode?: OnExitCode;
+    $log?: boolean | {
+        exec?: boolean | LogProcessOptions;
+        stdout?: boolean | LogProcessOptions;
+        stderr?: boolean | LogProcessOptions;
+    };
+};
+export declare function createProcess<O1 extends boolean, O2 extends boolean>(command: string, argv?: (string | number)[], options?: SpawnOptions & ProcessOptions<O1, O2>): ChildProcessByStdio<Writable, Readable, Readable> & PromiseLike<{
+    exitCode: number;
+} & (O1 extends true ? {
+    stdout: string;
+} : {}) & (O2 extends true ? {
+    stderr: string;
+} : {})>;
 export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
 type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
 export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void): void;

package/utils/process.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.onExit = exports.exec = exports.logExecStderr = exports.logExecStdout = void 0;
+exports.onExit = exports.exec = exports.createProcess = exports.logProcessExec = exports.waitForClose = exports.parseStreamData = exports.logExecStderr = exports.logExecStdout = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
@@ -12,6 +12,7 @@ const child_process_1 = require("child_process");
 const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
 const readline_1 = require("readline");
+const stream_1 = require("stream");
 function logExecStdout(input) {
     let text = input.colorize ? chalk_1.default.grey(input.data) : input.data;
     if (input.lineSalt)
@@ -23,6 +24,152 @@ function logExecStderr(data, colorize) {
     process.stdout.write(colorize ? chalk_1.default.red(data) : data);
 }
 exports.logExecStderr = logExecStderr;
+function parseStreamData(stream, options = {}) {
+    const log = options.log === true ? {} : options.log;
+    let result;
+    if (options.save)
+        result = "";
+    return new Promise((resolve, reject) => {
+        const lines = options.parseLines;
+        const onData = (data) => {
+            if (options.onData)
+                options.onData(data.toString());
+            if (log)
+                logExecStdout({
+                    data: lines ? `${data}\n` : data.toString(),
+                    stderr: log.toStderr,
+                    colorize: log.colorize,
+                });
+            if (options?.save)
+                result += data.toString();
+        };
+        if (lines) {
+            const rl = (0, readline_1.createInterface)({
+                input: stream,
+            });
+            rl.on("line", onData).on("close", () => resolve(result));
+        }
+        else {
+            stream
+                .on("data", onData)
+                .on("error", reject)
+                .once("close", () => resolve(result));
+        }
+    });
+}
+exports.parseStreamData = parseStreamData;
+function waitForClose(p, options = {}) {
+    return new Promise((resolve, reject) => {
+        let result = {
+            exitCode: 1,
+        };
+        if (options.stdout) {
+            result.stdout = "";
+            p.stdout.on("data", (data) => (result.stdout += data.toString()));
+        }
+        p.once("error", reject).once("close", (exitCode) => {
+            if (exitCode) {
+                let onExitCode = options.onExitCode ?? true;
+                if (typeof onExitCode === "function") {
+                    onExitCode = onExitCode(exitCode);
+                }
+                if (typeof onExitCode === "string") {
+                    reject(new Error(onExitCode));
+                }
+                else if (typeof onExitCode === "number") {
+                    reject(new Error(`Exit code: ${onExitCode}`));
+                }
+                else if (onExitCode instanceof Error) {
+                    reject(onExitCode);
+                }
+                else if (onExitCode === false) {
+                    resolve({ ...result, exitCode: exitCode });
+                }
+                else {
+                    reject(new Error(`Exit code: ${exitCode}`));
+                }
+            }
+            else {
+                resolve({ ...result, exitCode: exitCode });
+            }
+        });
+    });
+}
+exports.waitForClose = waitForClose;
+async function logProcessExec(command, argv, options) {
+    const logEnv = options.envNames?.reduce((env, key) => {
+        const value = options?.env?.[key];
+        if (typeof value !== "undefined")
+            env[key] = value;
+        return env;
+    }, {});
+    (0, cli_1.logExec)(command, options.pipe
+        ? [
+            ...argv,
+            options.pipe instanceof stream_1.Readable ? "<" : ">",
+            "path" in options.pipe ? String(options.pipe.path) : "[stream]",
+        ]
+        : argv, logEnv, options.toStderr);
+}
+exports.logProcessExec = logProcessExec;
+function createProcess(command, argv = [], options = {}) {
+    const $log = options.$log === true
+        ? { exec: {}, stdout: {}, stderr: {} }
+        : options.$log || {};
+    if ($log.exec)
+        logProcessExec(command, argv, $log.exec === true ? {} : $log.exec);
+    if (typeof options.cwd === "string") {
+        let isDir = false;
+        try {
+            isDir = (0, fs_2.statSync)(options.cwd).isDirectory();
+        }
+        catch (error) { }
+        if (!isDir)
+            throw new Error(`Current working directory does not exist: ${options.cwd}`);
+    }
+    const handler = (0, child_process_1.spawn)(command, argv.map((v) => (typeof v === "number" ? v.toString() : v)), options ?? {});
+    const { $stdout, $stderr, $onExitCode } = options;
+    async function exec() {
+        const [stdout, stderr, result] = await Promise.all([
+            (!!$log.stdout || !!$stdout) &&
+                parseStreamData(handler.stdout, {
+                    log: $log.stdout,
+                    ...$stdout,
+                }),
+            (!!$log.stderr || !!$stderr) &&
+                parseStreamData(handler.stderr, {
+                    log: $log.stderr,
+                    ...$stderr,
+                }),
+            waitForClose(handler, {
+                onExitCode: $onExitCode,
+            }),
+        ]);
+        const endResult = {
+            exitCode: result.exitCode,
+        };
+        if (typeof stdout === "string")
+            endResult.stdout = stdout;
+        if (typeof stderr === "string")
+            endResult.stderr = stderr;
+        return endResult;
+    }
+    const promise = {
+        [Symbol.toStringTag]: "process",
+        then: function (onfulfilled, onrejected) {
+            return exec().then(onfulfilled, onrejected);
+        },
+        catch: function (onrejected) {
+            return exec().catch(onrejected);
+        },
+        finally: function (onfinally) {
+            return exec().finally(onfinally);
+        },
+    };
+    Object.assign(handler, promise);
+    return handler;
+}
+exports.createProcess = createProcess;
 async function exec(command, argv = [], options = null, settings = {}) {
     const pipe = settings.pipe;
     let log = {};
@@ -34,23 +181,16 @@ async function exec(command, argv = [], options = null, settings = {}) {
     }
     return new Promise(async (resolve, reject) => {
         if (log.exec) {
-            const logEnv = log.envNames?.reduce((env, key) => {
-                const value = options?.env?.[key];
-                if (typeof value !== "undefined")
-                    env[key] = value;
-                return env;
-            }, {});
-            (0, cli_1.logExec)(command, pipe
-                ? [
-                    ...argv,
-                    pipe.stream instanceof fs_2.ReadStream ? "<" : ">",
-                    String(pipe.stream.path),
-                ]
-                : argv, logEnv, log.allToStderr);
+            logProcessExec(command, argv, {
+                env: options?.env,
+                envNames: log.envNames,
+                pipe: pipe?.stream,
+                toStderr: log.allToStderr,
+            });
         }
         if (typeof options?.cwd === "string" && !(await (0, fs_1.checkDir)(options.cwd)))
-            return reject(new Error(`Current working directory does not exist: ${options.cwd}`));
-        if (pipe?.onReadProgress && pipe.stream instanceof fs_2.ReadStream) {
+            throw new Error(`Current working directory does not exist: ${options.cwd}`);
+        if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
             const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
             const totalBytes = fileInfo.size;
             let currentBytes = 0;
@@ -64,7 +204,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
             });
         }
         const p = (0, child_process_1.spawn)(command, argv, options ?? {});
-        settings.onSpawn?.(p);
+        await settings.onSpawn?.(p);
         let spawnError;
         const spawnData = {
             stdout: "",
@@ -121,7 +261,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     throw new Error(`stdout is not defined`);
                 if (!p.stderr)
                     throw new Error(`stderr is not defined`);
-                if (pipe.onWriteProgress) {
+                if ("onWriteProgress" in pipe && pipe.onWriteProgress) {
                     let totalBytes = 0;
                     p.stdout.on("data", (chunk) => {
                         totalBytes += chunk.length;
@@ -136,7 +276,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 p.stderr.pipe(pipe.stream, { end: false });
                 p.on("close", tryFinish);
             }
-            else if (pipe.stream instanceof fs_2.ReadStream) {
+            else if (pipe.stream instanceof stream_1.Readable) {
                 if (!p.stdin)
                     throw new Error(`stdin is not defined`);
                 pipe.stream.pipe(p.stdin);

package/utils/tar.d.ts

@@ -0,0 +1,32 @@
+export type Progress = {
+    percent: number;
+    current: number;
+    total: number;
+};
+export type TarEntry = {
+    path: string;
+    progress: Progress;
+};
+export interface CreateTarOptions {
+    path: string;
+    verbose?: boolean;
+    output: string;
+    include: string[];
+    compress?: boolean;
+    onEntry?: (entry: TarEntry) => void;
+}
+export interface ExtractOptions {
+    input: string;
+    output: string;
+    verbose?: boolean;
+    total?: number;
+    onEntry?: (entry: TarEntry) => void;
+}
+export type ListTarOptions = {
+    input: string;
+    onEntry?: (entry: Pick<TarEntry, "path">) => void;
+    verbose?: boolean;
+};
+export declare function listTar(options: ListTarOptions): Promise<number>;
+export declare function createTar(options: CreateTarOptions): Promise<void>;
+export declare function extractTar(options: ExtractOptions): Promise<void>;

package/utils/tar.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractTar = exports.createTar = exports.listTar = void 0;
+const cli_1 = require("./cli");
+const fs_1 = require("./fs");
+const math_1 = require("./math");
+const fs_2 = require("fs");
+const promises_1 = require("fs/promises");
+const tar_1 = __importDefault(require("tar"));
+async function listTar(options) {
+    if (options.verbose)
+        (0, cli_1.logExec)("tar", ["-ztvf", options.input]);
+    let total = 0;
+    await tar_1.default.list({
+        file: options.input,
+        onentry(entry) {
+            options.onEntry?.({ path: entry.path });
+            total++;
+        },
+    });
+    return total;
+}
+exports.listTar = listTar;
+async function createTar(options) {
+    if (options.verbose)
+        (0, cli_1.logExec)("tar", [
+            options.compress ? "-czvf" : "-cvf",
+            options.output,
+            options.path,
+        ]);
+    let total = options.include.length;
+    if (!total)
+        throw new Error("'include' option is empty");
+    let current = 0;
+    let progressPromise;
+    const inStream = tar_1.default.create({
+        gzip: options.compress,
+        cwd: options.path,
+        filter(path) {
+            current++;
+            options.onEntry?.({
+                path: path,
+                progress: {
+                    total,
+                    current,
+                    percent: (0, math_1.progressPercent)(total, current),
+                },
+            });
+            return true;
+        },
+    }, options.include);
+    const outStream = (0, fs_2.createWriteStream)(options.output);
+    await new Promise((resolve, reject) => {
+        inStream.on("error", reject);
+        outStream.on("error", reject);
+        inStream.pipe(outStream);
+        outStream.on("close", resolve);
+    });
+    await progressPromise;
+}
+exports.createTar = createTar;
+async function extractTar(options) {
+    let total = options.total ??
+        (await listTar({ input: options.input, verbose: options.verbose }));
+    if (options.verbose) {
+        (0, cli_1.logExec)("tar", ["-xzvfp", options.input, "-C", options.output]);
+        (0, cli_1.logExec)("mkdir", ["-p", options.output]);
+    }
+    let current = 0;
+    await (0, promises_1.mkdir)(options.output, { recursive: true });
+    await (0, fs_1.ensureEmptyDir)(options.output);
+    await tar_1.default.extract({
+        file: options.input,
+        cwd: options.output,
+        preserveOwner: true,
+        onentry(entry) {
+            current++;
+            options.onEntry?.({
+                path: entry.path,
+                progress: {
+                    total,
+                    current,
+                    percent: (0, math_1.progressPercent)(total, current),
+                },
+            });
+        },
+    });
+}
+exports.extractTar = extractTar;

package/utils/zip.d.ts

@@ -1,97 +0,0 @@
-export interface ZipDataFilterType {
-    recursive?: boolean;
-    exclude?: boolean;
-    patterns: string[];
-}
-export interface ZipDataType {
-    command?: string;
-    path: string;
-    filter?: (ZipDataFilterType | string)[];
-    output: string;
-    deleteOnZip?: boolean;
-    includeList?: string;
-    excludeList?: string;
-    verbose?: boolean;
-    onProgress?: (data: {
-        percent: number;
-        total: number;
-        current: number;
-        path?: string;
-        type?: "start" | "end";
-    }) => void | Promise<void>;
-    onStream?: (data: ZipStream) => void | Promise<void>;
-}
-export interface UnzipDataType {
-    command?: string;
-    input: string;
-    files?: (ZipDataFilterType | string)[];
-    output: string;
-    verbose?: boolean;
-    onProgress?: (data: {
-        percent: number;
-        current: number;
-        path?: string;
-        type?: "start" | "end";
-    }) => void | Promise<void>;
-    onStream?: (data: UnzipStream) => void | Promise<void>;
-}
-export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
-export declare function checkSSEOption(command?: string): Promise<boolean>;
-type ListZipStream = {
-    Path?: string;
-    Folder?: string;
-    Size?: string;
-    "Packed Size"?: string;
-    Modified?: string;
-    Created?: string;
-    Accessed?: string;
-    Attributes?: string;
-    Encrypted?: string;
-    Comment?: string;
-    CRC?: string;
-    Method?: string;
-    Characteristics?: string;
-    "Host OS"?: string;
-    Version?: string;
-    Volume?: string;
-    Offset?: string;
-};
-export declare function listZip(data: {
-    command?: string;
-    path: string;
-    onStream: (item: ListZipStream) => Promise<void>;
-    verbose?: boolean;
-}): Promise<void>;
-export type ZipStream = {
-    type: "progress";
-    data: {
-        progress: number;
-        files: number;
-        path: string;
-    };
-} | {
-    type: "summary";
-    data: {
-        folders: number;
-        files: number;
-    };
-};
-export declare function zip(data: ZipDataType): Promise<{
-    folders: number;
-    files: number;
-}>;
-export type UnzipStream = {
-    type: "progress";
-    data: {
-        percent: number;
-        files: number;
-        path: string;
-    };
-};
-export declare function unzip(data: UnzipDataType): Promise<{
-    files: number;
-    stdout: string;
-    stderr: string;
-    exitCode: number;
-}>;
-export {};