@datatruck/cli 0.13.1 → 0.16.0

package/util/process-util.js

@@ -11,6 +11,7 @@ const chalk_1 = __importDefault(require("chalk"));
 const child_process_1 = require("child_process");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
+const readline_1 = require("readline");
 function logExecStdout(input) {
     let text = input.colorize ? chalk_1.default.grey(input.data) : input.data;
     if (input.lineSalt)
@@ -70,7 +71,11 @@ async function exec(command, argv = [], options = null, settings = {}) {
             stderr: "",
             exitCode: 0,
         };
-        let finishListeners = pipe?.stream instanceof fs_1.WriteStream ? 2 : 1;
+        let finishListeners = 1;
+        if (pipe?.stream instanceof fs_1.WriteStream)
+            finishListeners++;
+        if (settings.stdout?.parseLines)
+            finishListeners++;
         let streamError;
         const tryFinish = () => {
             if (!--finishListeners)
@@ -116,6 +121,17 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     throw new Error(`stdout is not defined`);
                 if (!p.stderr)
                     throw new Error(`stderr is not defined`);
+                if (pipe.onWriteProgress) {
+                    let totalBytes = 0;
+                    p.stdout.on("data", (chunk) => {
+                        totalBytes += chunk.length;
+                        pipe.onWriteProgress({ totalBytes });
+                    });
+                    p.stderr.on("data", (chunk) => {
+                        totalBytes += chunk.length;
+                        pipe.onWriteProgress({ totalBytes });
+                    });
+                }
                 p.stdout.pipe(pipe.stream, { end: false });
                 p.stderr.pipe(pipe.stream, { end: false });
                 p.on("close", tryFinish);
@@ -129,10 +145,11 @@ async function exec(command, argv = [], options = null, settings = {}) {
         if (log.stdout || settings.stdout) {
             if (!p.stdout)
                 throw new Error(`stdout is not defined`);
-            p.stdout.on("data", (data) => {
+            const parseLines = settings.stdout?.parseLines;
+            const onData = (data) => {
                 if (log.stdout)
                     logExecStdout({
-                        data: data.toString(),
+                        data: parseLines ? `${data}\n` : data.toString(),
                         stderr: log.allToStderr,
                         colorize: log.colorize,
                     });
@@ -140,7 +157,17 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     spawnData.stdout += data.toString();
                 if (settings.stdout?.onData)
                     settings.stdout.onData(data.toString());
-            });
+            };
+            if (parseLines) {
+                const rl = (0, readline_1.createInterface)({
+                    input: p.stdout,
+                });
+                rl.on("line", onData);
+                rl.on("close", tryFinish);
+            }
+            else {
+                p.stdout.on("data", onData);
+            }
         }
         if (log.stderr || settings.stderr) {
             if (!p.stderr)

package/util/progress.d.ts

@@ -0,0 +1,12 @@
+export declare type ProgressStats = {
+    percent?: number;
+    total?: number;
+    current?: number;
+    description?: string;
+    payload?: string;
+    format?: "amount" | "size";
+};
+export declare type Progress = {
+    absolute?: ProgressStats;
+    relative?: ProgressStats;
+};

package/util/progress.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/util/string-util.d.ts

@@ -15,5 +15,6 @@ export declare type UriType = {
 export declare function formatUri(input: UriType, hidePassword?: boolean): string;
 export declare function formatSeconds(seconds: number): string;
 export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
+export declare function checkPath(path: string, include: string[], exclude?: string[]): boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;

package/util/string-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.checkMatch = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
+exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -99,6 +99,13 @@ function makePathPatterns(values) {
     });
 }
 exports.makePathPatterns = makePathPatterns;
+function checkPath(path, include, exclude) {
+    return ((0, micromatch_1.isMatch)(path, include, {
+        dot: true,
+    }) &&
+        (!exclude || !(0, micromatch_1.isMatch)(path, exclude, { dot: true })));
+}
+exports.checkPath = checkPath;
 function checkMatch(subject, patterns) {
     if (!subject?.length)
         subject = "<empty>";

package/util/zip-util.d.ts

@@ -3,20 +3,6 @@ export interface ZipDataFilterType {
     exclude?: boolean;
     patterns: string[];
 }
-export declare type ZipStreamDataType = {
-    type: "progress";
-    data: {
-        progress: number;
-        files: number;
-        path: string;
-    };
-} | {
-    type: "summary";
-    data: {
-        folders: number;
-        files: number;
-    };
-};
 export interface ZipDataType {
     command?: string;
     path: string;
@@ -26,7 +12,14 @@ export interface ZipDataType {
     includeList?: string;
     excludeList?: string;
     verbose?: boolean;
-    onStream?: (data: ZipStreamDataType) => void;
+    onProgress?: (data: {
+        percent: number;
+        total: number;
+        current: number;
+        path?: string;
+        type?: "start" | "end";
+    }) => void | Promise<void>;
+    onStream?: (data: ZipStream) => void | Promise<void>;
 }
 export interface UnzipDataType {
     command?: string;
@@ -34,20 +27,71 @@ export interface UnzipDataType {
     files?: (ZipDataFilterType | string)[];
     output: string;
     verbose?: boolean;
-    onStream?: (data: UnzipStreamDataType) => void;
+    onProgress?: (data: {
+        percent: number;
+        current: number;
+        path?: string;
+        type?: "start" | "end";
+    }) => void | Promise<void>;
+    onStream?: (data: UnzipStream) => void | Promise<void>;
 }
-export declare type UnzipStreamDataType = {
+export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
+export declare function checkSSEOption(command?: string): Promise<boolean>;
+declare type ListZipStream = {
+    Path?: string;
+    Folder?: string;
+    Size?: string;
+    "Packed Size"?: string;
+    Modified?: string;
+    Created?: string;
+    Accessed?: string;
+    Attributes?: string;
+    Encrypted?: string;
+    Comment?: string;
+    CRC?: string;
+    Method?: string;
+    Characteristics?: string;
+    "Host OS"?: string;
+    Version?: string;
+    Volume?: string;
+    Offset?: string;
+};
+export declare function listZip(data: {
+    command?: string;
+    path: string;
+    onStream: (item: ListZipStream) => Promise<void>;
+    verbose?: boolean;
+}): Promise<void>;
+export declare type ZipStream = {
     type: "progress";
     data: {
         progress: number;
         files: number;
         path: string;
     };
+} | {
+    type: "summary";
+    data: {
+        folders: number;
+        files: number;
+    };
 };
-export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
-export declare function checkSSEOption(command?: string): Promise<boolean>;
 export declare function zip(data: ZipDataType): Promise<{
     folders: number;
     files: number;
 }>;
-export declare function unzip(data: UnzipDataType): Promise<import("./process-util").ExecResultType>;
+export declare type UnzipStream = {
+    type: "progress";
+    data: {
+        percent: number;
+        files: number;
+        path: string;
+    };
+};
+export declare function unzip(data: UnzipDataType): Promise<{
+    files: number;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}>;
+export {};

package/util/zip-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unzip = exports.zip = exports.checkSSEOption = exports.buildArguments = void 0;
+exports.unzip = exports.zip = exports.listZip = exports.checkSSEOption = exports.buildArguments = void 0;
 const process_util_1 = require("./process-util");
 const path_1 = require("path");
 function buildArguments(filters) {
@@ -30,50 +30,100 @@ function buildArguments(filters) {
     return args;
 }
 exports.buildArguments = buildArguments;
-function parseZipStream(chunk, buffer, cb) {
-    const lines = chunk.replaceAll("\b", "").trim().split(/\r?\n/);
-    for (const line of lines) {
-        let matches = null;
-        if ((matches = /^(\d+)% (\d+ )?\+/.exec(line))) {
-            const path = line.slice(line.indexOf("+") + 1).trim();
-            const progress = Number(matches[1]);
-            if (!buffer.currentPaths)
-                buffer.currentPaths = 0;
-            if (path !== buffer.lastPath)
-                buffer.currentPaths++;
-            buffer.lastPath = path;
-            cb({
-                type: "progress",
-                data: { progress, path, files: buffer.currentPaths },
-            });
-        }
-        else if (line.startsWith("Add new data to archive:")) {
-            const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
-            const [, files] = /(\d+) files?/i.exec(line) || [, 0];
-            cb({
-                type: "summary",
-                data: {
-                    folders: Number(folders),
-                    files: Number(files),
-                },
-            });
-        }
-    }
-}
 let checkSSEOptionResult;
 async function checkSSEOption(command = "7z") {
-    const result = await (0, process_util_1.exec)(command);
+    const result = await (0, process_util_1.exec)(command, [], {}, {
+        stdout: {
+            save: true,
+        },
+    });
     if (typeof checkSSEOptionResult === "boolean")
         return checkSSEOptionResult;
     return (checkSSEOptionResult = result.stdout.includes(" -sse"));
 }
 exports.checkSSEOption = checkSSEOption;
+const listZipLineEqChar = " = ";
+function parseListZipLine(line, buffer) {
+    if (buffer.started) {
+        if (line === "") {
+            if (buffer.opened) {
+                const { stream } = buffer;
+                buffer.stream = {};
+                buffer.opened = false;
+                return stream;
+            }
+        }
+        else {
+            const separator = line.indexOf(listZipLineEqChar);
+            const key = line.slice(0, separator);
+            const value = line.slice(separator + listZipLineEqChar.length);
+            buffer.opened = true;
+            buffer.stream[key] = value;
+        }
+    }
+    else if (line.startsWith("----------")) {
+        buffer.started = true;
+        buffer.stream = {};
+    }
+}
+async function listZip(data) {
+    const buffer = {};
+    await (0, process_util_1.exec)(data.command ?? "7z", ["l", data.path, "-slt"], {}, {
+        log: {
+            exec: data.verbose ?? false,
+            stderr: data.verbose ?? false,
+            stdout: false,
+        },
+        onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
+        stdout: {
+            parseLines: true,
+            onData: async (line) => {
+                const stream = parseListZipLine(line, buffer);
+                if (stream) {
+                    await data.onStream?.(stream);
+                }
+            },
+        },
+    });
+}
+exports.listZip = listZip;
+function parseZipLine(line) {
+    let matches = null;
+    line = line.trim();
+    if (!line.length)
+        return;
+    if ((matches = /^(\d+)% (\d+ )?\+/.exec(line))) {
+        const path = line.slice(line.indexOf("+") + 1).trim();
+        const progress = Number(matches[1]);
+        const files = matches[2] ? Number(matches[2]) : 1;
+        return {
+            type: "progress",
+            data: { progress, path, files },
+        };
+    }
+    else if (line.startsWith("Add new data to archive:")) {
+        const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
+        const [, files] = /(\d+) files?/i.exec(line) || [, 0];
+        return {
+            type: "summary",
+            data: {
+                folders: Number(folders),
+                files: Number(files),
+            },
+        };
+    }
+}
 async function zip(data) {
-    let result = {
+    let summary = {
         folders: 0,
         files: 0,
     };
-    let buffer = {};
+    await data.onProgress?.({
+        current: 0,
+        percent: 0,
+        total: 0,
+        type: "start",
+    });
     await (0, process_util_1.exec)(data.command ?? "7z", [
         "a",
         // https://sourceforge.net/p/sevenzip/bugs/2099/,
@@ -91,35 +141,58 @@ async function zip(data) {
         log: data.verbose ?? false,
         onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
         stdout: {
-            onData: (chunk) => {
-                parseZipStream(chunk, buffer, (stream) => {
-                    data.onStream?.(stream);
-                    if (stream.type === "summary")
-                        result = stream.data;
-                });
+            onData: async (lines) => {
+                for (const line of lines.replaceAll("\b", "").split(/\r?\n/)) {
+                    const stream = parseZipLine(line);
+                    if (stream) {
+                        if (stream.type === "summary")
+                            summary = stream.data;
+                        if (stream.type === "progress") {
+                            const current = Math.max(0, stream.data.files - 1);
+                            await data.onProgress?.({
+                                total: summary.files,
+                                current,
+                                path: stream.data.path,
+                                percent: stream.data.progress,
+                            });
+                        }
+                        await data.onStream?.(stream);
+                    }
+                }
             },
         },
     });
-    return result;
+    await data.onProgress?.({
+        total: summary.files,
+        current: summary.files,
+        percent: 100,
+        type: "end",
+    });
+    return summary;
 }
 exports.zip = zip;
-function parseUnzipStream(chunk, cb) {
-    const lines = chunk.trim().split(/\r?\n/g);
-    for (const line of lines) {
-        let matches = null;
-        if ((matches = /^(\d+)% (\d+) \-/.exec(line))) {
-            const progress = Number(matches[1]);
-            const files = Number(matches[2]);
-            const path = line.slice(line.indexOf("-") + 1).trim();
-            cb({
-                type: "progress",
-                data: { progress, path, files },
-            });
-        }
+function parseUnzipLine(line) {
+    let matches = null;
+    if ((matches = /^\s*(\d+)% (\d+) \-/.exec(line))) {
+        const progress = Number(matches[1]);
+        const files = Number(matches[2]);
+        const path = line.slice(line.indexOf("-") + 1).trim();
+        return {
+            type: "progress",
+            data: { percent: progress, path, files },
+        };
     }
 }
 async function unzip(data) {
-    return await (0, process_util_1.exec)(data.command ?? "7z", [
+    let summary = {
+        files: 0,
+    };
+    await data.onProgress?.({
+        current: summary.files,
+        percent: 0,
+        type: "start",
+    });
+    const result = await (0, process_util_1.exec)(data.command ?? "7z", [
         "x",
         "-bsp1",
         (0, path_1.normalize)(data.input),
@@ -130,13 +203,34 @@ async function unzip(data) {
         log: data.verbose ?? false,
         stderr: { toExitCode: true },
         stdout: {
-            ...(data.onStream && {
-                onData: (chunk) => {
-                    if (data.onStream)
-                        parseUnzipStream(chunk, data.onStream);
+            ...((data.onStream || data.onProgress) && {
+                parseLines: true,
+                onData: async (line) => {
+                    const stream = parseUnzipLine(line);
+                    if (stream) {
+                        if (stream.type === "progress") {
+                            const current = Math.max(0, stream.data.files - 1);
+                            summary.files = stream.data.files;
+                            await data.onProgress?.({
+                                current,
+                                percent: stream.data.percent,
+                                path: stream.data.path,
+                            });
+                        }
+                        await data.onStream?.(stream);
+                    }
                 },
             }),
         },
     });
+    await data.onProgress?.({
+        current: summary.files,
+        percent: 100,
+        type: "end",
+    });
+    return {
+        ...result,
+        ...summary,
+    };
 }
 exports.unzip = unzip;