@datatruck/cli 0.14.0 → 0.16.1

package/cli.js

@@ -81,7 +81,7 @@ program.usage("dtt");
 program.option("-v,--verbose", "Verbose", (_, previous) => previous + 1, 0);
 program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFIG"] ?? (cwd.endsWith(path_1.sep) ? cwd : `${cwd}${path_1.sep}`));
 program.option("--progress <value>", "Progress type (auto, plain, tty)", "auto");
-program.option("--progress-interval <ms>", "Progress interval");
+program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
 program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
 makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
 makeCommand(CommandFactory_1.CommandEnum.init).alias("i");

package/migrations/001-initial.sql

@@ -11,11 +11,7 @@ CREATE TABLE "backup_session" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"snapshotId"	TEXT NOT NULL,
 	"packageName"	TEXT NOT NULL,
@@ -32,11 +28,7 @@ CREATE TABLE "backup_session_task" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"sessionId"	INTEGER NOT NULL,
 	"taskName"	TEXT NOT NULL
@@ -51,11 +43,7 @@ CREATE TABLE "backup_session_repository" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"sessionId"	INTEGER NOT NULL,
 	"repositoryName"	TEXT NOT NULL,
@@ -71,11 +59,7 @@ CREATE TABLE "restore_session" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"snapshotId"	TEXT NOT NULL,
 	"packageName"	TEXT NOT NULL -- ,
@@ -91,11 +75,7 @@ CREATE TABLE "restore_session_task" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"sessionId"	INTEGER NOT NULL,
 	"taskName"	TEXT NOT NULL
@@ -110,11 +90,7 @@ CREATE TABLE "restore_session_repository" (
 	"endDate"	TEXT,
 	"state"	TEXT,
 	"error"	TEXT,
-	"progressCurrent" INTEGER,
-	"progressTotal" INTEGER,
-	"progressPercent" INTEGER,
-	"progressStep" TEXT,
-	"progressStepPercent" INTEGER,
+	"progress" TEXT,
 
 	"sessionId"	INTEGER NOT NULL,
 	"repositoryName"	TEXT NOT NULL,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.14.0",
+  "version": "0.16.1",
   "dependencies": {
     "ajv": "^8.11.0",
     "async": "^3.2.4",

package/util/cli-util.d.ts

@@ -5,7 +5,7 @@ export declare const showCursorCommand = "\u001B[?25h";
 export declare const clearCommand = "\r\u001B[K";
 export declare const hideCursorCommand = "\u001B[?25l";
 export declare function renderSpinner(counter: number): string;
-export declare function renderProgressBar(progress: number, size?: number): string;
+export declare function renderProgressBar(progress: number, size?: number, subprogress?: number): string;
 export declare function logVars(data: Record<string, any>): void;
 export declare function logExec(command: string, argv?: string[], env?: NodeJS.ProcessEnv, logToStderr?: boolean): void;
 export declare function resultColumn(error: Error | null | string, state?: "started" | "ended"): "❌" | " ? " | "✅";

package/util/cli-util.js

@@ -20,12 +20,27 @@ function renderSpinner(counter) {
     return exports.spinnerChars[counter % (exports.spinnerChars.length - 1)];
 }
 exports.renderSpinner = renderSpinner;
-function renderProgressBar(progress, size = 10) {
+function renderProgressBar(progress, size = 10, subprogress) {
     const completeChar = "\u2588";
     const incompleteChar = "\u2591";
     const completedSize = Math.round((progress * size) / 100);
     const restSize = Math.max(size - completedSize, 0);
-    return completeChar.repeat(completedSize) + incompleteChar.repeat(restSize);
+    let result = completeChar.repeat(completedSize) + incompleteChar.repeat(restSize);
+    if (typeof subprogress === "number") {
+        const subprogressChar = Math.round((subprogress * size) / 100);
+        if (subprogressChar === size) {
+            result =
+                result.slice(0, subprogressChar - 1) +
+                    chalk_1.default.white(result[Math.max(0, subprogressChar - 1)]);
+        }
+        else {
+            result =
+                result.slice(0, subprogressChar) +
+                    chalk_1.default.white(result[Math.max(0, subprogressChar - 1)]) +
+                    result.slice(subprogressChar + 1);
+        }
+    }
+    return (0, chalk_2.cyan)(result);
 }
 exports.renderProgressBar = renderProgressBar;
 function logVars(data) {

package/util/fs-util.d.ts

@@ -73,6 +73,11 @@ export declare function cpy(options: {
      */
     concurrency?: number;
     skipNotFoundError?: boolean;
+    onProgress?: (data: {
+        current: number;
+        path?: string;
+        type?: "start" | "end";
+    }) => Promise<boolean | void>;
     onPath?: (data: {
         isDir: boolean;
         entryPath: string;
@@ -84,5 +89,9 @@ export declare function cpy(options: {
             dirs: number;
         };
     }) => Promise<boolean | void>;
-}): Promise<void>;
+}): Promise<{
+    paths: number;
+    files: number;
+    dirs: number;
+}>;
 export {};

package/util/fs-util.js

@@ -162,7 +162,7 @@ function sessionTmpDir() {
 exports.sessionTmpDir = sessionTmpDir;
 function tmpDir(prefix, id) {
     if (!id)
-        id = (0, crypto_1.randomBytes)(8).toString("hex");
+        id = (0, crypto_1.randomUUID)().slice(0, 8);
     return (0, path_1.join)(sessionTmpDir(), `${prefix}-${id}`);
 }
 exports.tmpDir = tmpDir;
@@ -369,6 +369,10 @@ async function cpy(options) {
         else {
             const dir = (0, path_1.dirname)(entryTargetPath);
             await makeRecursiveDir(dir);
+            await options.onProgress?.({
+                current: stats.files,
+                path: entryPath,
+            });
             stats.files++;
             // https://github.com/nodejs/node/issues/44261
             if (exports.isWSLSystem) {
@@ -429,5 +433,10 @@ async function cpy(options) {
             },
         });
     }
+    await options.onProgress?.({
+        current: stats.files,
+        type: "end",
+    });
+    return stats;
 }
 exports.cpy = cpy;

package/util/process-util.d.ts

@@ -14,6 +14,9 @@ export interface ExecSettingsInterface {
     exec?: boolean;
     pipe?: {
         stream: WriteStream | ReadStream;
+        onWriteProgress?: (data: {
+            totalBytes: number;
+        }) => void;
         onReadProgress?: (data: {
             totalBytes: number;
             currentBytes: number;

package/util/process-util.js

@@ -121,6 +121,17 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     throw new Error(`stdout is not defined`);
                 if (!p.stderr)
                     throw new Error(`stderr is not defined`);
+                if (pipe.onWriteProgress) {
+                    let totalBytes = 0;
+                    p.stdout.on("data", (chunk) => {
+                        totalBytes += chunk.length;
+                        pipe.onWriteProgress({ totalBytes });
+                    });
+                    p.stderr.on("data", (chunk) => {
+                        totalBytes += chunk.length;
+                        pipe.onWriteProgress({ totalBytes });
+                    });
+                }
                 p.stdout.pipe(pipe.stream, { end: false });
                 p.stderr.pipe(pipe.stream, { end: false });
                 p.on("close", tryFinish);

package/util/progress.d.ts

@@ -0,0 +1,12 @@
+export declare type ProgressStats = {
+    percent?: number;
+    total?: number;
+    current?: number;
+    description?: string;
+    payload?: string;
+    format?: "amount" | "size";
+};
+export declare type Progress = {
+    absolute?: ProgressStats;
+    relative?: ProgressStats;
+};

package/util/progress.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/util/zip-util.d.ts

@@ -12,7 +12,14 @@ export interface ZipDataType {
     includeList?: string;
     excludeList?: string;
     verbose?: boolean;
-    onStream?: (data: ZipStream) => void;
+    onProgress?: (data: {
+        percent: number;
+        total: number;
+        current: number;
+        path?: string;
+        type?: "start" | "end";
+    }) => void | Promise<void>;
+    onStream?: (data: ZipStream) => void | Promise<void>;
 }
 export interface UnzipDataType {
     command?: string;
@@ -20,7 +27,13 @@ export interface UnzipDataType {
     files?: (ZipDataFilterType | string)[];
     output: string;
     verbose?: boolean;
-    onStream?: (data: UnzipStream) => void;
+    onProgress?: (data: {
+        percent: number;
+        current: number;
+        path?: string;
+        type?: "start" | "end";
+    }) => void | Promise<void>;
+    onStream?: (data: UnzipStream) => void | Promise<void>;
 }
 export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
 export declare function checkSSEOption(command?: string): Promise<boolean>;
@@ -46,7 +59,7 @@ declare type ListZipStream = {
 export declare function listZip(data: {
     command?: string;
     path: string;
-    onStream: (item: ListZipStream) => void;
+    onStream: (item: ListZipStream) => Promise<void>;
     verbose?: boolean;
 }): Promise<void>;
 export declare type ZipStream = {
@@ -70,10 +83,15 @@ export declare function zip(data: ZipDataType): Promise<{
 export declare type UnzipStream = {
     type: "progress";
     data: {
-        progress: number;
+        percent: number;
         files: number;
         path: string;
     };
 };
-export declare function unzip(data: UnzipDataType): Promise<import("./process-util").ExecResultType>;
+export declare function unzip(data: UnzipDataType): Promise<{
+    files: number;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}>;
 export {};

package/util/zip-util.js

@@ -32,7 +32,11 @@ function buildArguments(filters) {
 exports.buildArguments = buildArguments;
 let checkSSEOptionResult;
 async function checkSSEOption(command = "7z") {
-    const result = await (0, process_util_1.exec)(command);
+    const result = await (0, process_util_1.exec)(command, [], {}, {
+        stdout: {
+            save: true,
+        },
+    });
     if (typeof checkSSEOptionResult === "boolean")
         return checkSSEOptionResult;
     return (checkSSEOptionResult = result.stdout.includes(" -sse"));
@@ -73,10 +77,10 @@ async function listZip(data) {
         onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
         stdout: {
             parseLines: true,
-            onData: (line) => {
+            onData: async (line) => {
                 const stream = parseListZipLine(line, buffer);
                 if (stream) {
-                    data.onStream?.(stream);
+                    await data.onStream?.(stream);
                 }
             },
         },
@@ -85,12 +89,13 @@ async function listZip(data) {
 exports.listZip = listZip;
 function parseZipLine(line) {
     let matches = null;
-    if (!line.trim().length)
+    line = line.trim();
+    if (!line.length)
         return;
-    if ((matches = /^\s*(\d+)% (\d+ )?\+/.exec(line))) {
+    if ((matches = /^(\d+)% (\d+ )?\+/.exec(line))) {
         const path = line.slice(line.indexOf("+") + 1).trim();
         const progress = Number(matches[1]);
-        const files = Number(matches[2]);
+        const files = matches[2] ? Number(matches[2]) : 1;
         return {
             type: "progress",
             data: { progress, path, files },
@@ -109,10 +114,16 @@ function parseZipLine(line) {
     }
 }
 async function zip(data) {
-    let result = {
+    let summary = {
         folders: 0,
         files: 0,
     };
+    await data.onProgress?.({
+        current: 0,
+        percent: 0,
+        total: 0,
+        type: "start",
+    });
     await (0, process_util_1.exec)(data.command ?? "7z", [
         "a",
         // https://sourceforge.net/p/sevenzip/bugs/2099/,
@@ -130,33 +141,58 @@ async function zip(data) {
         log: data.verbose ?? false,
         onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
         stdout: {
-            onData: (line) => {
-                const stream = parseZipLine(line);
-                if (stream) {
-                    data.onStream?.(stream);
-                    if (stream.type === "summary")
-                        result = stream.data;
+            onData: async (lines) => {
+                for (const line of lines.replaceAll("\b", "").split(/\r?\n/)) {
+                    const stream = parseZipLine(line);
+                    if (stream) {
+                        if (stream.type === "summary")
+                            summary = stream.data;
+                        if (stream.type === "progress") {
+                            const current = Math.max(0, stream.data.files - 1);
+                            await data.onProgress?.({
+                                total: summary.files,
+                                current,
+                                path: stream.data.path,
+                                percent: stream.data.progress,
+                            });
+                        }
+                        await data.onStream?.(stream);
+                    }
                 }
             },
         },
     });
-    return result;
+    await data.onProgress?.({
+        total: summary.files,
+        current: summary.files,
+        percent: 100,
+        type: "end",
+    });
+    return summary;
 }
 exports.zip = zip;
 function parseUnzipLine(line) {
     let matches = null;
-    if ((matches = /^\s*(\d+)% (\d+) \-/.exec(line))) {
-        const progress = Number(matches[1]);
-        const files = Number(matches[2]);
+    if ((matches = /^\s*(?<percent>\d+)%(?: (?<files>\d+))? \-/.exec(line))) {
+        const progress = Number(matches.groups["percent"]);
+        const files = matches.groups["files"] ? Number(matches[2]) : 1;
         const path = line.slice(line.indexOf("-") + 1).trim();
         return {
             type: "progress",
-            data: { progress, path, files },
+            data: { percent: progress, path, files },
         };
     }
 }
 async function unzip(data) {
-    return await (0, process_util_1.exec)(data.command ?? "7z", [
+    let summary = {
+        files: 0,
+    };
+    await data.onProgress?.({
+        current: summary.files,
+        percent: 0,
+        type: "start",
+    });
+    const result = await (0, process_util_1.exec)(data.command ?? "7z", [
         "x",
         "-bsp1",
         (0, path_1.normalize)(data.input),
@@ -167,15 +203,36 @@ async function unzip(data) {
         log: data.verbose ?? false,
         stderr: { toExitCode: true },
         stdout: {
-            ...(data.onStream && {
-                parseLines: true,
-                onData: (line) => {
-                    const stream = parseUnzipLine(line);
-                    if (stream)
-                        data.onStream(stream);
+            ...((data.onStream || data.onProgress) && {
+                onData: async (chunk) => {
+                    const lines = chunk.replaceAll("\b", "").split(/\r?\n/);
+                    for (const line of lines) {
+                        const stream = parseUnzipLine(line);
+                        if (stream) {
+                            if (stream.type === "progress") {
+                                const current = Math.max(0, stream.data.files - 1);
+                                summary.files = stream.data.files;
+                                await data.onProgress?.({
+                                    current,
+                                    percent: stream.data.percent,
+                                    path: stream.data.path,
+                                });
+                            }
+                            await data.onStream?.(stream);
+                        }
+                    }
                 },
             }),
         },
     });
+    await data.onProgress?.({
+        current: summary.files,
+        percent: 100,
+        type: "end",
+    });
+    return {
+        ...result,
+        ...summary,
+    };
 }
 exports.unzip = unzip;