@datatruck/cli 0.13.0 → 0.15.0

package/Action/BackupAction.js

@@ -77,11 +77,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressTask({
                             id: taskId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -123,11 +119,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressRepository({
                             id: repositoryId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -167,11 +159,7 @@ class BackupAction {
                     onProgress: async (data) => {
                         await session.progressRepository({
                             id: repositoryId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });

package/Action/RestoreAction.js

@@ -114,11 +114,7 @@ class RestoreAction {
                     onProgress: async (data) => {
                         await session.progressTask({
                             id: taskId,
-                            progressCurrent: data.current,
-                            progressPercent: data.percent,
-                            progressStep: data.step,
-                            progressStepPercent: data.stepPercent,
-                            progressTotal: data.total,
+                            ...data,
                         });
                     },
                 });
@@ -163,11 +159,7 @@ class RestoreAction {
                 onProgress: async (data) => {
                     await session.progressRepository({
                         id: repositoryId,
-                        progressCurrent: data.current,
-                        progressPercent: data.percent,
-                        progressStep: data.step,
-                        progressStepPercent: data.stepPercent,
-                        progressTotal: data.total,
+                        ...data,
                     });
                 },
             });

package/Command/BackupCommand.js

@@ -68,9 +68,10 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
                 new ConsoleSessionDriver_1.ConsoleSessionDriver({
                     verbose: verbose > 0,
                     progress: this.globalOptions.progress,
-                    progressInterval: this.globalOptions.progressInterval,
                 }),
             ],
+            verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const result = await backup.exec(sessionManager);
         if (result.errors) {

package/Command/BackupSessionsCommand.js

@@ -50,6 +50,7 @@ class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
                 verbose: verbose > 1,
             }),
             verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const items = await action.exec(manager);
         const dataFormat = new DataFormat_1.DataFormat({

package/Command/RestoreCommand.js

@@ -68,10 +68,10 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
                 new ConsoleSessionDriver_1.ConsoleSessionDriver({
                     verbose: verbose > 0,
                     progress: this.globalOptions.progress,
-                    progressInterval: this.globalOptions.progressInterval,
                 }),
             ],
             verbose: verbose > 1,
+            progressInterval: this.globalOptions.progressInterval,
         });
         const result = await restore.exec(sessionManager);
         return result ? 0 : 1;

package/Command/RestoreSessionsCommand.js

@@ -49,6 +49,7 @@ class RestoreSessionsCommand extends CommandAbstract_1.CommandAbstract {
             driver: new SqliteSessionDriver_1.SqliteSessionDriver({
                 verbose: verbose > 1,
             }),
+            progressInterval: this.globalOptions.progressInterval,
         });
         const items = await action.exec(manager);
         const dataFormat = new DataFormat_1.DataFormat({

package/Entity/StateEntityAbstract.d.ts

@@ -7,6 +7,7 @@ export declare abstract class StateEntityAbstract extends CrudEntityAbstract {
     progressTotal?: number | null;
     progressCurrent?: number | null;
     progressPercent?: number | null;
-    progressStep?: string | null;
+    progressStepDescription?: string | null;
+    progressStepItem?: string | null;
     progressStepPercent?: number | null;
 }

package/Repository/DatatruckRepository.d.ts

@@ -19,6 +19,7 @@ export declare type DatatruckRepositoryConfigType = {
 };
 declare type CompressObjectType = {
     packs?: {
+        name?: string;
         include: string[];
         exclude?: string[];
         onePackByResult?: boolean;
@@ -48,6 +49,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
     static stringifyMetaData(data: MetaDataType): string;
     onGetSource(): string;
     onInit(data: InitDataType): Promise<void>;
+    private createFileScanner;
     onPrune(data: PruneDataType): Promise<void>;
     onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
     private normalizeCompressConfig;

package/Repository/DatatruckRepository.js

@@ -19,6 +19,7 @@ const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
+const perf_hooks_1 = require("perf_hooks");
 const readline_1 = require("readline");
 exports.datatruckRepositoryName = "datatruck";
 exports.datatruckRepositoryDefinition = {
@@ -50,6 +51,7 @@ exports.datatruckPackageRepositoryDefinition = {
                                 additionalProperties: false,
                                 required: ["include"],
                                 properties: {
+                                    name: { type: "string" },
                                     include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                                     exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                                     onePackByResult: { type: "boolean" },
@@ -102,6 +104,67 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     async onInit(data) {
         await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
     }
+    async createFileScanner(options) {
+        const object = {
+            total: 0,
+            current: 0,
+            progress: async (description, data) => {
+                await options.onProgress({
+                    step: {
+                        description,
+                        item: data.path,
+                        percent: data.percent,
+                    },
+                    stats: {
+                        total: object.total,
+                        current: object.current + data.current,
+                        percent: (0, math_util_1.progressPercent)(object.total, object.current + data.current),
+                    },
+                });
+                if (data.type === "end") {
+                    object.current += data.current;
+                }
+            },
+            start: async (cb) => {
+                for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
+                    if (!options.disableCounting)
+                        object.total++;
+                    const currentTime = perf_hooks_1.performance.now();
+                    const diff = currentTime - lastTime;
+                    if (diff > 1000) {
+                        await options.onProgress({
+                            step: {
+                                description: "Scanning files",
+                                item: object.total.toString(),
+                            },
+                        });
+                        lastTime = currentTime;
+                    }
+                    if (cb)
+                        await cb(entry);
+                }
+                await options.onProgress({
+                    step: {
+                        description: "Scanned files",
+                        item: object.total.toString(),
+                    },
+                });
+            },
+        };
+        await options.onProgress({
+            step: {
+                description: "Scanning files",
+            },
+        });
+        const stream = fast_glob_1.default.stream(options.glob.include, {
+            dot: true,
+            markDirectories: true,
+            stats: true,
+            ...options.glob,
+        });
+        let lastTime = perf_hooks_1.performance.now();
+        return object;
+    }
     async onPrune(data) {
         const snapshotName = DatatruckRepository.buildSnapshotName({
             snapshotId: data.snapshot.id,
@@ -160,7 +223,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     }
     normalizeCompressConfig(packageConfig) {
         let compress = packageConfig?.compress ?? this.config.compress;
-        if (compress === true) {
+        if (compress === true || (compress && !Array.isArray(compress.packs))) {
             return {
                 packs: [
                     {
@@ -198,94 +261,124 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 verbose: data.options.verbose,
             })
             : undefined;
-        const stream = fast_glob_1.default.stream(include, {
-            cwd: sourcePath,
-            ignore: exclude,
-            dot: true,
-            onlyFiles: compress ? false : true,
-            markDirectories: true,
-        });
+        const packs = compress?.packs || [];
+        const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
+        const nonPackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "nonpack.txt"));
+        const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
+        const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
+        const streams = [nonPackStream, singlePackStream, ...packStreams];
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Writing paths lists`);
-        const pathLists = await (0, fs_util_1.writePathLists)({
-            paths: stream,
-            packs: compress?.packs,
+            (0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
+        const scanner = await this.createFileScanner({
+            glob: {
+                include,
+                cwd: sourcePath,
+                ignore: exclude,
+                onlyFiles: false,
+            },
+            onProgress: data.onProgress,
+            disableCounting: true,
         });
-        if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
-        let currentFiles = 0;
-        if (compress?.packs) {
-            let packIndex = 0;
-            for (const packsPath of pathLists.includedPackPaths) {
-                const pack = compress.packs[packIndex];
-                if (pack.onePackByResult) {
-                    const reader = (0, readline_1.createInterface)({
-                        input: (0, fs_1.createReadStream)(packsPath),
-                    });
-                    let multipleIndex = 0;
-                    for await (let packPath of reader) {
-                        if (packPath.endsWith("/"))
-                            packPath = packPath.slice(0, -1);
-                        const target = (0, path_1.join)(outPath, `.${packIndex}-${multipleIndex++}-${encodeURIComponent(packPath.replace(/[\\/]/g, "-")).slice(0, 255)}.dd.zip`);
-                        const stats = await (0, zip_util_1.zip)({
-                            path: pkg.path,
-                            output: target,
-                            filter: [{ patterns: [packPath] }],
-                            excludeList: pathLists.excludedPackPaths[packIndex],
-                            verbose: data.options.verbose,
-                            onStream: async (stream) => await data.onProgress({
-                                total: pathLists.total.all,
-                                current: currentFiles + stream.data.files,
-                                percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
-                                step: stream.type === "progress" ? stream.data.path : "",
-                                stepPercent: stream.type === "progress" ? stream.data.progress : null,
-                            }),
-                        });
-                        currentFiles += stats.files;
+        await Promise.all([
+            ...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
+            (async () => {
+                await scanner.start(async (entry) => {
+                    const pathSubject = entry.stats.isDirectory()
+                        ? entry.path.slice(0, -1)
+                        : entry.path;
+                    let stream = nonPackStream;
+                    let successPackIndex;
+                    for (const [packIndex, pack] of packs.entries()) {
+                        if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
+                            stream = pack.onePackByResult
+                                ? singlePackStream
+                                : packStreams[packIndex];
+                            successPackIndex = packIndex;
+                            break;
+                        }
                     }
+                    const isNonPackStream = stream === nonPackStream;
+                    const isPackStream = stream !== nonPackStream;
+                    const isSinglePackStream = stream === singlePackStream;
+                    const include = isPackStream
+                        ? entry.stats.isDirectory()
+                            ? await (0, fs_util_1.isEmptyDir)(entry.path)
+                            : true
+                        : true;
+                    if (include) {
+                        let value = entry.path;
+                        if (isNonPackStream) {
+                            value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
+                        }
+                        else if (isSinglePackStream) {
+                            value += `:${successPackIndex}`;
+                        }
+                        if (!entry.stats.isDirectory())
+                            scanner.total++;
+                        stream.write(`${value}\n`);
+                    }
+                });
+                for (const stream of streams) {
+                    stream.end();
                 }
-                else {
-                    const target = (0, path_1.join)(outPath, `.${packIndex}.dd.zip`);
-                    const stats = await (0, zip_util_1.zip)({
-                        path: sourcePath,
-                        output: target,
-                        includeList: packsPath,
-                        excludeList: pathLists.excludedPackPaths[packIndex],
-                        verbose: data.options.verbose,
-                        onStream: async (stream) => await data.onProgress({
-                            total: pathLists.total.all,
-                            current: currentFiles + stream.data.files,
-                            percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
-                            step: stream.type === "progress" ? stream.data.path : "",
-                        }),
-                    });
-                    currentFiles += stats.files;
-                }
-                packIndex++;
-            }
-        }
+            })(),
+        ]);
+        const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
+        const dttPath = (0, path_1.join)(outPath, dttFolder);
+        await (0, promises_1.mkdir)(dttPath);
+        await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
+        // Non pack
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
+            (0, cli_util_1.logExec)(`Copying files from ${nonPackStream.path.toString()} to ${outPath}`);
         await (0, fs_util_1.cpy)({
             input: {
                 type: "pathList",
-                path: pathLists.path,
+                path: nonPackStream.path.toString(),
                 basePath: sourcePath,
             },
             targetPath: outPath,
+            skipNotFoundError: true,
             concurrency: this.config.fileCopyConcurrency,
-            async onPath({ isDir, entryPath }) {
-                if (isDir)
-                    return;
-                currentFiles++;
-                await data.onProgress({
-                    total: pathLists.total.all,
-                    current: currentFiles,
-                    percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
-                    step: entryPath,
-                });
-            },
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
+        });
+        // Single pack
+        const singleReader = (0, readline_1.createInterface)({
+            input: (0, fs_1.createReadStream)(singlePackStream.path),
         });
+        for await (const line of singleReader) {
+            let [packPath, packIndex] = line.split(":");
+            const pack = packs[packIndex];
+            if (packPath.endsWith("/"))
+                packPath = packPath.slice(0, -1);
+            const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
+                `-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
+                `.zip`).slice(0, 255);
+            const target = (0, path_1.join)(dttPath, outBasename);
+            await (0, zip_util_1.zip)({
+                path: pkg.path,
+                output: target,
+                filter: [{ patterns: [packPath] }],
+                verbose: data.options.verbose,
+                onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                    ? "Starting compressing"
+                    : "Compressing file", progress),
+            });
+        }
+        // Packs
+        for (const [packIndex, packStream] of packStreams.entries()) {
+            const pack = packs[packIndex];
+            const target = (0, path_1.join)(dttPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
+            await (0, zip_util_1.zip)({
+                path: sourcePath,
+                output: target,
+                includeList: packStream.path.toString(),
+                verbose: data.options.verbose,
+                onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                    ? "Starting compressing"
+                    : "Compressing file", progress),
+            });
+        }
+        // Meta
         const metaPath = `${outPath}.json`;
         const nodePkg = (0, fs_util_1.parsePackageFile)();
         const meta = {
@@ -295,7 +388,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             package: data.package.name,
             task: data.package.task?.name,
             version: nodePkg.version,
-            size: await (0, fs_util_1.fastFolderSizeAsync)(outPath),
+            size: (await (0, fs_util_1.fastFolderSizeAsync)(outPath)) -
+                (await (0, fs_util_1.fastFolderSizeAsync)(dttPath)),
         };
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
@@ -312,14 +406,23 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const sourceMetaPath = `${sourcePath}.json`;
         const targetMetaPath = `${targetPath}.json`;
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
+            (0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
         await (0, promises_1.mkdir)(targetPath);
+        const scanner = await this.createFileScanner({
+            glob: {
+                include: ["**/*"],
+                cwd: sourcePath,
+            },
+            onProgress: data.onProgress,
+        });
+        await scanner.start();
         await (0, fs_util_1.cpy)({
             input: {
                 type: "glob",
                 sourcePath,
             },
             targetPath,
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
         });
         await (0, promises_1.copyFile)(sourceMetaPath, targetMetaPath);
     }
@@ -340,50 +443,78 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
-        let totalFiles = 0;
-        let currentFiles = -1;
-        await (0, fs_util_1.forEachFile)(sourcePath, () => {
-            totalFiles++;
-        }, true);
+        const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
+        const dttPath = (0, path_1.join)(sourcePath, dttFolder);
+        const scanner = await this.createFileScanner({
+            glob: {
+                include: ["**/*"],
+                cwd: sourcePath,
+                ignore: [dttFolder],
+            },
+            onProgress: data.onProgress,
+        });
+        await scanner.start();
+        const dttPathExists = await (0, fs_util_1.checkDir)(dttPath);
+        if (dttPathExists) {
+            const it = await (0, promises_1.opendir)(dttPath);
+            for await (const dirent of it) {
+                const path = (0, path_1.join)(dttPath, dirent.name);
+                if (dirent.name === "permissions.txt") {
+                    scanner.total++;
+                }
+                else if (dirent.name.endsWith(".zip")) {
+                    await (0, zip_util_1.listZip)({
+                        path,
+                        verbose: data.options.verbose,
+                        onStream: async (item) => {
+                            const isDir = item.Folder === "+";
+                            if (!isDir)
+                                scanner.total++;
+                        },
+                    });
+                }
+            }
+        }
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
         await (0, fs_util_1.cpy)({
             input: {
                 type: "glob",
                 sourcePath,
+                exclude: [dttFolder],
             },
             targetPath: restorePath,
             concurrency: this.config.fileCopyConcurrency,
-            onPath: async ({ entryPath, entrySourcePath }) => {
-                const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
-                const isZipFile = isRootFile &&
-                    entryPath.startsWith(".") &&
-                    entryPath.endsWith(".dd.zip");
-                await data.onProgress({
-                    total: totalFiles,
-                    current: Math.max(currentFiles, 0),
-                    percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
-                    step: entryPath,
-                });
-                if (isZipFile) {
+            onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
+        });
+        if (dttPathExists) {
+            const it = await (0, promises_1.opendir)(dttPath);
+            for await (const dirent of it) {
+                const path = (0, path_1.join)(dttPath, dirent.name);
+                if (dirent.name === "permissions.txt") {
+                    if (data.options.verbose)
+                        (0, cli_util_1.logExec)(`Applying permissions (${path})`);
+                    await scanner.progress("Applying permissions", {
+                        current: 0,
+                    });
+                    await (0, fs_util_1.applyPermissions)(restorePath, path);
+                    await scanner.progress("Permissions applied", {
+                        current: 1,
+                        type: "end",
+                    });
+                }
+                else if (dirent.name.endsWith(".zip")) {
                     await (0, zip_util_1.unzip)({
-                        input: entrySourcePath,
+                        input: path,
                         output: restorePath,
                         verbose: data.options.verbose,
-                        onStream: async (stream) => await data.onProgress({
-                            total: totalFiles,
-                            current: currentFiles + 1,
-                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
-                            step: stream.type === "progress"
-                                ? `Extracting ${stream.data.path}`
-                                : "",
-                        }),
+                        onProgress: async (progress) => await scanner.progress(progress.type === "start"
+                            ? "Starting extracting"
+                            : "Extracting file", progress),
                     });
                 }
-                currentFiles++;
-                return isZipFile ? false : true;
-            },
-        });
+            }
+        }
     }
 }
 exports.DatatruckRepository = DatatruckRepository;

package/Repository/RepositoryAbstract.d.ts

@@ -16,11 +16,16 @@ export declare type SnapshotResultType = SnapshotType & {
     size: number;
 };
 export declare type ProgressDataType = {
-    total?: number;
-    current?: number;
-    percent?: number;
-    step?: string;
-    stepPercent?: number | null;
+    stats?: {
+        total?: number;
+        current?: number;
+        percent?: number;
+    };
+    step?: {
+        description?: string;
+        item?: string;
+        percent?: number | null;
+    };
 };
 export declare type InitDataType = {
     options: InitActionOptionsType;