@datatruck/cli 0.14.0 → 0.15.0

package/Action/BackupAction.js

@@ -77,11 +77,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressTask({
                             id: taskId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -123,11 +119,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressRepository({
                             id: repositoryId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -167,11 +159,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressRepository({
                             id: repositoryId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });

package/Action/RestoreAction.js

@@ -114,11 +114,7 @@ class RestoreAction {
                     onProgress: async (data) => {
                         await session.progressTask({
                             id: taskId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -163,11 +159,7 @@ class RestoreAction {
                 onProgress: async (data) => {
                     await session.progressRepository({
                         id: repositoryId,
-                        progressCurrent: data.current,
-                        progressPercent: data.percent,
-                        progressStep: data.step,
-                        progressStepPercent: data.stepPercent,
-                        progressTotal: data.total,
+                        ...data,
                     });
                 },
             });

package/Command/BackupCommand.js

@@ -68,9 +68,10 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
                 new ConsoleSessionDriver_1.ConsoleSessionDriver({
                     verbose: verbose > 0,
                     progress: this.globalOptions.progress,
-                    progressInterval: this.globalOptions.progressInterval,
                 }),
             ],
+            verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const result = await backup.exec(sessionManager);
         if (result.errors) {

package/Command/BackupSessionsCommand.js

@@ -50,6 +50,7 @@ class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
                 verbose: verbose > 1,
             }),
             verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const items = await action.exec(manager);
         const dataFormat = new DataFormat_1.DataFormat({

package/Command/RestoreCommand.js

@@ -68,10 +68,10 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
                 new ConsoleSessionDriver_1.ConsoleSessionDriver({
                     verbose: verbose > 0,
                     progress: this.globalOptions.progress,
-                    progressInterval: this.globalOptions.progressInterval,
                 }),
             ],
             verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const result = await restore.exec(sessionManager);
         return result ? 0 : 1;

package/Command/RestoreSessionsCommand.js

@@ -49,6 +49,7 @@ class RestoreSessionsCommand extends CommandAbstract_1.CommandAbstract {
             driver: new SqliteSessionDriver_1.SqliteSessionDriver({
                 verbose: verbose > 1,
             }),
+            progressInterval: this.globalOptions.progressInterval,
         });
         const items = await action.exec(manager);
         const dataFormat = new DataFormat_1.DataFormat({

package/Entity/StateEntityAbstract.d.ts

@@ -7,6 +7,7 @@ export declare abstract class StateEntityAbstract extends CrudEntityAbstract {
     progressTotal?: number | null;
     progressCurrent?: number | null;
     progressPercent?: number | null;
-    progressStep?: string | null;
+    progressStepDescription?: string | null;
+    progressStepItem?: string | null;
     progressStepPercent?: number | null;
 }

package/Repository/DatatruckRepository.d.ts

@@ -49,6 +49,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
     static stringifyMetaData(data: MetaDataType): string;
     onGetSource(): string;
     onInit(data: InitDataType): Promise<void>;
+    private createFileScanner;
     onPrune(data: PruneDataType): Promise<void>;
     onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
     private normalizeCompressConfig;

package/Repository/DatatruckRepository.js

@@ -19,6 +19,7 @@ const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
+const perf_hooks_1 = require("perf_hooks");
 const readline_1 = require("readline");
 exports.datatruckRepositoryName = "datatruck";
 exports.datatruckRepositoryDefinition = {
@@ -103,6 +104,67 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     async onInit(data) {
         await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
     }
+    async createFileScanner(options) {
+        const object = {
+            total: 0,
+            current: 0,
+            progress: async (description, data) => {
+                await options.onProgress({
+                    step: {
+                        description,
+                        item: data.path,
+                        percent: data.percent,
+                    },
+                    stats: {
+                        total: object.total,
+                        current: object.current + data.current,
+                        percent: (0, math_util_1.progressPercent)(object.total, object.current + data.current),
+                    },
+                });
+                if (data.type === "end") {
+                    object.current += data.current;
+                }
+            },
+            start: async (cb) => {
+                for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
+                    if (!options.disableCounting)
+                        object.total++;
+                    const currentTime = perf_hooks_1.performance.now();
+                    const diff = currentTime - lastTime;
+                    if (diff > 1000) {
+                        await options.onProgress({
+                            step: {
+                                description: "Scanning files",
+                                item: object.total.toString(),
+                            },
+                        });
+                        lastTime = currentTime;
+                    }
+                    if (cb)
+                        await cb(entry);
+                }
+                await options.onProgress({
+                    step: {
+                        description: "Scanned files",
+                        item: object.total.toString(),
+                    },
+                });
+            },
+        };
+        await options.onProgress({
+            step: {
+                description: "Scanning files",
+            },
+        });
+        const stream = fast_glob_1.default.stream(options.glob.include, {
+            dot: true,
+            markDirectories: true,
+            stats: true,
+            ...options.glob,
+        });
+        let lastTime = perf_hooks_1.performance.now();
+        return object;
+    }
     async onPrune(data) {
         const snapshotName = DatatruckRepository.buildSnapshotName({
             snapshotId: data.snapshot.id,
@@ -199,30 +261,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 verbose: data.options.verbose,
             })
             : undefined;
-        const stream = fast_glob_1.default.stream(include, {
-            cwd: sourcePath,
-            ignore: exclude,
-            dot: true,
-            onlyFiles: false,
-            markDirectories: true,
-            stats: true,
-        });
         const packs = compress?.packs || [];
         const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
         const nonPackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "nonpack.txt"));
         const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
         const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
-        let totalFiles = 0;
         const streams = [nonPackStream, singlePackStream, ...packStreams];
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
-        await data.onProgress({
-            step: "Writing the file lists...",
+        const scanner = await this.createFileScanner({
+            glob: {
+                include,
+                cwd: sourcePath,
+                ignore: exclude,
+                onlyFiles: false,
+            },
+            onProgress: data.onProgress,
+            disableCounting: true,
         });
         await Promise.all([
             ...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
             (async () => {
-                for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
+                await scanner.start(async (entry) => {
                     const pathSubject = entry.stats.isDirectory()
                         ? entry.path.slice(0, -1)
                         : entry.path;
@@ -254,23 +314,22 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                             value += `:${successPackIndex}`;
                         }
                         if (!entry.stats.isDirectory())
-                            totalFiles++;
+                            scanner.total++;
                         stream.write(`${value}\n`);
                     }
-                }
+                });
                 for (const stream of streams) {
                     stream.end();
                 }
             })(),
         ]);
-        let currentFiles = 0;
         const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
         const dttPath = (0, path_1.join)(outPath, dttFolder);
         await (0, promises_1.mkdir)(dttPath);
+        await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
         // Non pack
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
-        await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
+            (0, cli_util_1.logExec)(`Copying files from ${nonPackStream.path.toString()} to ${outPath}`);
         await (0, fs_util_1.cpy)({
             input: {
                 type: "pathList",
@@ -280,17 +339,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             targetPath: outPath,
             skipNotFoundError: true,
             concurrency: this.config.fileCopyConcurrency,
-            async onPath({ isDir, entryPath }) {
-                if (isDir)
-                    return;
-                currentFiles++;
-                await data.onProgress({
-                    total: totalFiles,
-                    current: currentFiles,
-                    percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
-                    step: entryPath,
-                });
-            },
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
         });
         // Single pack
         const singleReader = (0, readline_1.createInterface)({
@@ -305,44 +354,29 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 `-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
                 `.zip`).slice(0, 255);
             const target = (0, path_1.join)(dttPath, outBasename);
-            const stats = await (0, zip_util_1.zip)({
+            await (0, zip_util_1.zip)({
                 path: pkg.path,
                 output: target,
                 filter: [{ patterns: [packPath] }],
                 verbose: data.options.verbose,
-                onStream: async (stream) => {
-                    if (stream.type === "progress")
-                        await data.onProgress({
-                            total: totalFiles,
-                            current: currentFiles + stream.data.files,
-                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
-                            step: stream.data.path,
-                            stepPercent: stream.data.progress,
-                        });
-                },
+                onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                    ? "Starting compressing"
+                    : "Compressing file", progress),
             });
-            currentFiles += stats.files;
         }
         // Packs
         for (const [packIndex, packStream] of packStreams.entries()) {
             const pack = packs[packIndex];
             const target = (0, path_1.join)(dttPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
-            const stats = await (0, zip_util_1.zip)({
+            await (0, zip_util_1.zip)({
                 path: sourcePath,
                 output: target,
                 includeList: packStream.path.toString(),
                 verbose: data.options.verbose,
-                onStream: async (stream) => {
-                    if (stream.type === "progress")
-                        await data.onProgress({
-                            total: totalFiles,
-                            current: currentFiles + stream.data.files,
-                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
-                            step: stream.data.path,
-                        });
-                },
+                onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                    ? "Starting compressing"
+                    : "Compressing file", progress),
             });
-            currentFiles += stats.files;
         }
         // Meta
         const metaPath = `${outPath}.json`;
@@ -374,12 +408,21 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
         await (0, promises_1.mkdir)(targetPath);
+        const scanner = await this.createFileScanner({
+            glob: {
+                include: ["**/*"],
+                cwd: sourcePath,
+            },
+            onProgress: data.onProgress,
+        });
+        await scanner.start();
         await (0, fs_util_1.cpy)({
             input: {
                 type: "glob",
                 sourcePath,
             },
             targetPath,
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
         });
         await (0, promises_1.copyFile)(sourceMetaPath, targetMetaPath);
     }
@@ -400,43 +443,38 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
-        let totalFiles = 0;
-        let currentFiles = 0;
-        await data.onProgress({
-            step: "Counting files...",
-        });
         const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
         const dttPath = (0, path_1.join)(sourcePath, dttFolder);
+        const scanner = await this.createFileScanner({
+            glob: {
+                include: ["**/*"],
+                cwd: sourcePath,
+                ignore: [dttFolder],
+            },
+            onProgress: data.onProgress,
+        });
+        await scanner.start();
         const dttPathExists = await (0, fs_util_1.checkDir)(dttPath);
         if (dttPathExists) {
             const it = await (0, promises_1.opendir)(dttPath);
             for await (const dirent of it) {
                 const path = (0, path_1.join)(dttPath, dirent.name);
                 if (dirent.name === "permissions.txt") {
-                    totalFiles++;
+                    scanner.total++;
                 }
                 else if (dirent.name.endsWith(".zip")) {
                     await (0, zip_util_1.listZip)({
                         path,
                         verbose: data.options.verbose,
-                        onStream: (item) => {
+                        onStream: async (item) => {
                             const isDir = item.Folder === "+";
                             if (!isDir)
-                                totalFiles++;
+                                scanner.total++;
                         },
                     });
                 }
             }
         }
-        const allFiles = fast_glob_1.default.stream(["**"], {
-            cwd: sourcePath,
-            ignore: [dttFolder],
-            dot: true,
-            onlyFiles: true,
-        });
-        for await (const _file of allFiles) {
-            totalFiles++;
-        }
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
         await (0, fs_util_1.cpy)({
@@ -447,17 +485,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             },
             targetPath: restorePath,
             concurrency: this.config.fileCopyConcurrency,
-            onPath: async ({ entryPath, isDir }) => {
-                if (!isDir) {
-                    currentFiles++;
-                    await data.onProgress({
-                        total: totalFiles,
-                        current: Math.max(currentFiles, 0),
-                        percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
-                        step: entryPath,
-                    });
-                }
-            },
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
         });
         if (dttPathExists) {
             const it = await (0, promises_1.opendir)(dttPath);
@@ -466,31 +494,23 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 if (dirent.name === "permissions.txt") {
                     if (data.options.verbose)
                         (0, cli_util_1.logExec)(`Applying permissions (${path})`);
-                    currentFiles++;
-                    await data.onProgress({
-                        total: totalFiles,
-                        current: currentFiles,
-                        percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
-                        step: "Applying permissions",
+                    await scanner.progress("Applying permissions", {
+                        current: 0,
                     });
                     await (0, fs_util_1.applyPermissions)(restorePath, path);
+                    await scanner.progress("Permissions applied", {
+                        current: 1,
+                        type: "end",
+                    });
                 }
                 else if (dirent.name.endsWith(".zip")) {
                     await (0, zip_util_1.unzip)({
                         input: path,
                         output: restorePath,
                         verbose: data.options.verbose,
-                        onStream: async (stream) => {
-                            await data.onProgress({
-                                total: totalFiles,
-                                current: currentFiles + stream.data.files,
-                                percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
-                                step: stream.type === "progress"
-                                    ? `Extracting ${stream.data.path}`
-                                    : "",
-                                stepPercent: stream.data.progress,
-                            });
-                        },
+                        onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                            ? "Starting extracting"
+                            : "Extracting file", progress),
                     });
                 }
             }

package/Repository/RepositoryAbstract.d.ts

@@ -16,11 +16,16 @@ export declare type SnapshotResultType = SnapshotType & {
     size: number;
 };
 export declare type ProgressDataType = {
-    total?: number;
-    current?: number;
-    percent?: number;
-    step?: string;
-    stepPercent?: number | null;
+    stats?: {
+        total?: number;
+        current?: number;
+        percent?: number;
+    };
+    step?: {
+        description?: string;
+        item?: string;
+        percent?: number | null;
+    };
 };
 export declare type InitDataType = {
     options: InitActionOptionsType;

package/Repository/ResticRepository.js

@@ -195,7 +195,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 verbose: data.options.verbose,
             });
             await data.onProgress({
-                step: "Writing excluded paths list...",
+                step: {
+                    description: "Writing excluded paths list",
+                },
             });
             const tmpDir = await (0, fs_util_1.mkTmpDir)("restic-exclude");
             const ignoredContents = (0, fs_util_1.fastglobToGitIgnore)(exclude, sourcePath).join("\n");
@@ -223,7 +225,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             if (data.options.verbose)
                 (0, cli_util_1.logExec)(`Writing paths lists`);
             await data.onProgress({
-                step: "Writing excluded paths list...",
+                step: {
+                    description: "Writing excluded paths list",
+                },
             });
             gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
                 paths: stream,
@@ -233,7 +237,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
         await data.onProgress({
-            step: "Fetching last snapshot...",
+            step: {
+                description: "Fetching last snapshot",
+            },
         });
         const [lastSnapshot] = await restic.snapshots({
             json: true,
@@ -245,7 +251,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         let totalFilesChanges = 0;
         const totalFilesChangesLimit = 10;
         await data.onProgress({
-            step: "Executing backup action...",
+            step: {
+                description: "Executing backup action",
+            },
         });
         let resticSnapshotId;
         let resticTotalBytes;
@@ -278,19 +286,24 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
                     if (totalFilesChanges > totalFilesChangesLimit) {
                         showProgressBar = true;
                     }
-                    else if (lastProgress?.total !== streamData.total_files) {
+                    else if (lastProgress?.stats?.total !== streamData.total_files) {
                         totalFilesChanges = 0;
                     }
                     else {
                         totalFilesChanges++;
                     }
                     await data.onProgress((lastProgress = {
-                        total: Math.max(lastProgress?.total || 0, streamData.total_files || 0),
-                        current: Math.max(lastProgress?.current || 0, streamData.files_done ?? 0),
-                        percent: showProgressBar
-                            ? Number((streamData.percent_done * 100).toFixed(2))
-                            : 0,
-                        step: streamData.current_files?.join(", ") ?? "-",
+                        step: {
+                            description: "Copying file",
+                            item: streamData.current_files?.join(", ") ?? "-",
+                        },
+                        stats: {
+                            total: Math.max(lastProgress?.stats?.total || 0, streamData.total_files || 0),
+                            current: Math.max(lastProgress?.stats?.current || 0, streamData.files_done ?? 0),
+                            percent: showProgressBar
+                                ? Number((streamData.percent_done * 100).toFixed(2))
+                                : 0,
+                        },
                     }));
                 }
                 else if (streamData.message_type === "summary") {
@@ -306,9 +319,11 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const sizeTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SIZE, resticTotalBytes.toString());
         await restic.exec(["tag", "--add", sizeTag, resticSnapshotId]);
         await data.onProgress({
-            total: lastProgress?.total || 0,
-            current: lastProgress?.total || 0,
-            percent: 100,
+            stats: {
+                total: lastProgress?.stats?.total || 0,
+                current: lastProgress?.stats?.total || 0,
+                percent: 100,
+            },
         });
     }
     async onCopyBackup(data) {
@@ -357,9 +372,11 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 if (streamData.message_type === "restore-status") {
                     const current = Math.min(streamData.total_bytes, snapshot.size);
                     await data.onProgress({
-                        total: snapshot.size,
-                        current,
-                        percent: (0, math_util_1.progressPercent)(snapshot.size, current),
+                        stats: {
+                            total: snapshot.size,
+                            current,
+                            percent: (0, math_util_1.progressPercent)(snapshot.size, current),
+                        },
                     });
                 }
             },

package/SessionDriver/ConsoleSessionDriver.d.ts

@@ -15,17 +15,16 @@ declare type MessageType = {
     progressCurrent?: number | null;
     progressTotal?: number | null;
     progressPercent?: number | null;
-    progressStep?: string | null;
+    progressStepDescription?: string | null;
+    progressStepItem?: string | null;
     progressStepPercent?: number | null;
 };
 declare type ConsoleSessionDriverOptions = SessionDriverOptions & {
     progress?: "auto" | "tty" | "plain";
-    progressInterval?: number;
 };
 export declare class ConsoleSessionDriver extends SessionDriverAbstract<ConsoleSessionDriverOptions> {
     protected lastMessage: MessageType | undefined;
     protected lastMessageText: string | undefined;
-    protected lastProgressDate: number | undefined;
     protected prints: number;
     protected renderInterval: NodeJS.Timeout;
     protected rendering?: boolean;