@datatruck/cli 0.13.1 → 0.14.0

package/Repository/DatatruckRepository.d.ts

@@ -19,6 +19,7 @@ export declare type DatatruckRepositoryConfigType = {
 };
 declare type CompressObjectType = {
     packs?: {
+        name?: string;
         include: string[];
         exclude?: string[];
         onePackByResult?: boolean;

package/Repository/DatatruckRepository.js

@@ -50,6 +50,7 @@ exports.datatruckPackageRepositoryDefinition = {
                                 additionalProperties: false,
                                 required: ["include"],
                                 properties: {
+                                    name: { type: "string" },
                                     include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                                     exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
                                     onePackByResult: { type: "boolean" },
@@ -160,7 +161,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
     }
     normalizeCompressConfig(packageConfig) {
         let compress = packageConfig?.compress ?? this.config.compress;
-        if (compress === true) {
+        if (compress === true || (compress && !Array.isArray(compress.packs))) {
             return {
                 packs: [
                     {
@@ -202,74 +203,78 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             cwd: sourcePath,
             ignore: exclude,
             dot: true,
-            onlyFiles: compress ? false : true,
+            onlyFiles: false,
             markDirectories: true,
+            stats: true,
         });
+        const packs = compress?.packs || [];
+        const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
+        const nonPackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "nonpack.txt"));
+        const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
+        const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
+        let totalFiles = 0;
+        const streams = [nonPackStream, singlePackStream, ...packStreams];
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Writing paths lists`);
-        const pathLists = await (0, fs_util_1.writePathLists)({
-            paths: stream,
-            packs: compress?.packs,
+            (0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
+        await data.onProgress({
+            step: "Writing the file lists...",
         });
-        if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
-        let currentFiles = 0;
-        if (compress?.packs) {
-            let packIndex = 0;
-            for (const packsPath of pathLists.includedPackPaths) {
-                const pack = compress.packs[packIndex];
-                if (pack.onePackByResult) {
-                    const reader = (0, readline_1.createInterface)({
-                        input: (0, fs_1.createReadStream)(packsPath),
-                    });
-                    let multipleIndex = 0;
-                    for await (let packPath of reader) {
-                        if (packPath.endsWith("/"))
-                            packPath = packPath.slice(0, -1);
-                        const target = (0, path_1.join)(outPath, `.${packIndex}-${multipleIndex++}-${encodeURIComponent(packPath.replace(/[\\/]/g, "-")).slice(0, 255)}.dd.zip`);
-                        const stats = await (0, zip_util_1.zip)({
-                            path: pkg.path,
-                            output: target,
-                            filter: [{ patterns: [packPath] }],
-                            excludeList: pathLists.excludedPackPaths[packIndex],
-                            verbose: data.options.verbose,
-                            onStream: async (stream) => await data.onProgress({
-                                total: pathLists.total.all,
-                                current: currentFiles + stream.data.files,
-                                percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
-                                step: stream.type === "progress" ? stream.data.path : "",
-                                stepPercent: stream.type === "progress" ? stream.data.progress : null,
-                            }),
-                        });
-                        currentFiles += stats.files;
+        await Promise.all([
+            ...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
+            (async () => {
+                for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
+                    const pathSubject = entry.stats.isDirectory()
+                        ? entry.path.slice(0, -1)
+                        : entry.path;
+                    let stream = nonPackStream;
+                    let successPackIndex;
+                    for (const [packIndex, pack] of packs.entries()) {
+                        if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
+                            stream = pack.onePackByResult
+                                ? singlePackStream
+                                : packStreams[packIndex];
+                            successPackIndex = packIndex;
+                            break;
+                        }
+                    }
+                    const isNonPackStream = stream === nonPackStream;
+                    const isPackStream = stream !== nonPackStream;
+                    const isSinglePackStream = stream === singlePackStream;
+                    const include = isPackStream
+                        ? entry.stats.isDirectory()
+                            ? await (0, fs_util_1.isEmptyDir)(entry.path)
+                            : true
+                        : true;
+                    if (include) {
+                        let value = entry.path;
+                        if (isNonPackStream) {
+                            value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
+                        }
+                        else if (isSinglePackStream) {
+                            value += `:${successPackIndex}`;
+                        }
+                        if (!entry.stats.isDirectory())
+                            totalFiles++;
+                        stream.write(`${value}\n`);
                     }
                 }
-                else {
-                    const target = (0, path_1.join)(outPath, `.${packIndex}.dd.zip`);
-                    const stats = await (0, zip_util_1.zip)({
-                        path: sourcePath,
-                        output: target,
-                        includeList: packsPath,
-                        excludeList: pathLists.excludedPackPaths[packIndex],
-                        verbose: data.options.verbose,
-                        onStream: async (stream) => await data.onProgress({
-                            total: pathLists.total.all,
-                            current: currentFiles + stream.data.files,
-                            percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
-                            step: stream.type === "progress" ? stream.data.path : "",
-                        }),
-                    });
-                    currentFiles += stats.files;
+                for (const stream of streams) {
+                    stream.end();
                 }
-                packIndex++;
-            }
-        }
+            })(),
+        ]);
+        let currentFiles = 0;
+        const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
+        const dttPath = (0, path_1.join)(outPath, dttFolder);
+        await (0, promises_1.mkdir)(dttPath);
+        // Non pack
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
+        await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
         await (0, fs_util_1.cpy)({
             input: {
                 type: "pathList",
-                path: pathLists.path,
+                path: nonPackStream.path.toString(),
                 basePath: sourcePath,
             },
             targetPath: outPath,
@@ -280,13 +285,66 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                     return;
                 currentFiles++;
                 await data.onProgress({
-                    total: pathLists.total.all,
+                    total: totalFiles,
                     current: currentFiles,
-                    percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
+                    percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
                     step: entryPath,
                 });
             },
         });
+        // Single pack
+        const singleReader = (0, readline_1.createInterface)({
+            input: (0, fs_1.createReadStream)(singlePackStream.path),
+        });
+        for await (const line of singleReader) {
+            let [packPath, packIndex] = line.split(":");
+            const pack = packs[packIndex];
+            if (packPath.endsWith("/"))
+                packPath = packPath.slice(0, -1);
+            const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
+                `-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
+                `.zip`).slice(0, 255);
+            const target = (0, path_1.join)(dttPath, outBasename);
+            const stats = await (0, zip_util_1.zip)({
+                path: pkg.path,
+                output: target,
+                filter: [{ patterns: [packPath] }],
+                verbose: data.options.verbose,
+                onStream: async (stream) => {
+                    if (stream.type === "progress")
+                        await data.onProgress({
+                            total: totalFiles,
+                            current: currentFiles + stream.data.files,
+                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
+                            step: stream.data.path,
+                            stepPercent: stream.data.progress,
+                        });
+                },
+            });
+            currentFiles += stats.files;
+        }
+        // Packs
+        for (const [packIndex, packStream] of packStreams.entries()) {
+            const pack = packs[packIndex];
+            const target = (0, path_1.join)(dttPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
+            const stats = await (0, zip_util_1.zip)({
+                path: sourcePath,
+                output: target,
+                includeList: packStream.path.toString(),
+                verbose: data.options.verbose,
+                onStream: async (stream) => {
+                    if (stream.type === "progress")
+                        await data.onProgress({
+                            total: totalFiles,
+                            current: currentFiles + stream.data.files,
+                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
+                            step: stream.data.path,
+                        });
+                },
+            });
+            currentFiles += stats.files;
+        }
+        // Meta
         const metaPath = `${outPath}.json`;
         const nodePkg = (0, fs_util_1.parsePackageFile)();
         const meta = {
@@ -296,7 +354,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             package: data.package.name,
             task: data.package.task?.name,
             version: nodePkg.version,
-            size: await (0, fs_util_1.fastFolderSizeAsync)(outPath),
+            size: (await (0, fs_util_1.fastFolderSizeAsync)(outPath)) -
+                (await (0, fs_util_1.fastFolderSizeAsync)(dttPath)),
         };
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
@@ -313,7 +372,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const sourceMetaPath = `${sourcePath}.json`;
         const targetMetaPath = `${targetPath}.json`;
         if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
+            (0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
         await (0, promises_1.mkdir)(targetPath);
         await (0, fs_util_1.cpy)({
             input: {
@@ -342,49 +401,100 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
         let totalFiles = 0;
-        let currentFiles = -1;
-        await (0, fs_util_1.forEachFile)(sourcePath, () => {
+        let currentFiles = 0;
+        await data.onProgress({
+            step: "Counting files...",
+        });
+        const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
+        const dttPath = (0, path_1.join)(sourcePath, dttFolder);
+        const dttPathExists = await (0, fs_util_1.checkDir)(dttPath);
+        if (dttPathExists) {
+            const it = await (0, promises_1.opendir)(dttPath);
+            for await (const dirent of it) {
+                const path = (0, path_1.join)(dttPath, dirent.name);
+                if (dirent.name === "permissions.txt") {
+                    totalFiles++;
+                }
+                else if (dirent.name.endsWith(".zip")) {
+                    await (0, zip_util_1.listZip)({
+                        path,
+                        verbose: data.options.verbose,
+                        onStream: (item) => {
+                            const isDir = item.Folder === "+";
+                            if (!isDir)
+                                totalFiles++;
+                        },
+                    });
+                }
+            }
+        }
+        const allFiles = fast_glob_1.default.stream(["**"], {
+            cwd: sourcePath,
+            ignore: [dttFolder],
+            dot: true,
+            onlyFiles: true,
+        });
+        for await (const _file of allFiles) {
             totalFiles++;
-        }, true);
+        }
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
         await (0, fs_util_1.cpy)({
             input: {
                 type: "glob",
                 sourcePath,
+                exclude: [dttFolder],
             },
             targetPath: restorePath,
             concurrency: this.config.fileCopyConcurrency,
-            onPath: async ({ entryPath, entrySourcePath }) => {
-                const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
-                const isZipFile = isRootFile &&
-                    entryPath.startsWith(".") &&
-                    entryPath.endsWith(".dd.zip");
-                await data.onProgress({
-                    total: totalFiles,
-                    current: Math.max(currentFiles, 0),
-                    percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
-                    step: entryPath,
-                });
-                if (isZipFile) {
+            onPath: async ({ entryPath, isDir }) => {
+                if (!isDir) {
+                    currentFiles++;
+                    await data.onProgress({
+                        total: totalFiles,
+                        current: Math.max(currentFiles, 0),
+                        percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
+                        step: entryPath,
+                    });
+                }
+            },
+        });
+        if (dttPathExists) {
+            const it = await (0, promises_1.opendir)(dttPath);
+            for await (const dirent of it) {
+                const path = (0, path_1.join)(dttPath, dirent.name);
+                if (dirent.name === "permissions.txt") {
+                    if (data.options.verbose)
+                        (0, cli_util_1.logExec)(`Applying permissions (${path})`);
+                    currentFiles++;
+                    await data.onProgress({
+                        total: totalFiles,
+                        current: currentFiles,
+                        percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
+                        step: "Applying permissions",
+                    });
+                    await (0, fs_util_1.applyPermissions)(restorePath, path);
+                }
+                else if (dirent.name.endsWith(".zip")) {
                     await (0, zip_util_1.unzip)({
-                        input: entrySourcePath,
+                        input: path,
                         output: restorePath,
                         verbose: data.options.verbose,
-                        onStream: async (stream) => await data.onProgress({
-                            total: totalFiles,
-                            current: currentFiles + 1,
-                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
-                            step: stream.type === "progress"
-                                ? `Extracting ${stream.data.path}`
-                                : "",
-                        }),
+                        onStream: async (stream) => {
+                            await data.onProgress({
+                                total: totalFiles,
+                                current: currentFiles + stream.data.files,
+                                percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
+                                step: stream.type === "progress"
+                                    ? `Extracting ${stream.data.path}`
+                                    : "",
+                                stepPercent: stream.data.progress,
+                            });
+                        },
                     });
                 }
-                currentFiles++;
-                return isZipFile ? false : true;
-            },
-        });
+            }
+        }
     }
 }
 exports.DatatruckRepository = DatatruckRepository;

package/config.schema.json

@@ -475,6 +475,9 @@
                       "include"
                     ],
                     "properties": {
+                      "name": {
+                        "type": "string"
+                      },
                       "include": {
                         "$ref": "#/definitions/stringlist-util"
                       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.13.1",
+  "version": "0.14.0",
   "dependencies": {
     "ajv": "^8.11.0",
     "async": "^3.2.4",

package/util/fs-util.d.ts

@@ -1,9 +1,20 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { Stats } from "fs";
+/// <reference types="node" />
+import { Dirent, Stats } from "fs";
+import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
+export declare function isEmptyDir(path: string): Promise<boolean>;
+declare type EntryObject = {
+    name: string;
+    path: string;
+    dirent: Dirent;
+    stats: Stats;
+};
+export declare function applyPermissions(baseDir: string, permissionsPath: string): Promise<void>;
+export declare function pathIterator(stream: AsyncIterable<string | Buffer>): AsyncIterable<EntryObject>;
 export declare function isLocalDir(path: string): boolean;
 export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
@@ -28,7 +39,7 @@ export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
 export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
 export declare function checkFile(path: string): Promise<boolean>;
 export declare function checkDir(path: string): Promise<boolean>;
-export declare function readDir(path: string): Promise<string[]>;
+export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
 /**
  * @experimental
@@ -37,24 +48,7 @@ export declare function fastglobToGitIgnore(patterns: string[], baseDir: string)
 export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
 }): Promise<string>;
-export declare function writePathLists(options: {
-    paths: NodeJS.ReadableStream | string[];
-    packs?: {
-        include: string[];
-        exclude?: string[];
-        multiple?: boolean;
-    }[];
-}): Promise<{
-    path: string;
-    includedPackPaths: string[];
-    excludedPackPaths: string[];
-    total: {
-        all: number;
-        path: number;
-        packsPaths: number[];
-        multipleStats: Record<string, number>;
-    };
-}>;
+export declare function waitForClose(stream: WriteStream): Promise<WriteStream>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;
 export declare function updateFileStats(path: string, fileInfo: Stats): Promise<void>;
 export declare function isNotFoundError(error: unknown): boolean;
@@ -91,3 +85,4 @@ export declare function cpy(options: {
         };
     }) => Promise<boolean | void>;
 }): Promise<void>;
+export {};

package/util/fs-util.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.writePathLists = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.isWSLSystem = void 0;
+exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const path_util_1 = require("./path-util");
 const async_1 = require("async");
@@ -13,13 +13,49 @@ const fast_glob_1 = __importDefault(require("fast-glob"));
 const fs_1 = require("fs");
 const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
-const micromatch_1 = require("micromatch");
 const os_1 = require("os");
 const path_1 = require("path");
 const path_2 = require("path");
 const readline_1 = require("readline");
 const util_1 = require("util");
 exports.isWSLSystem = (0, os_1.release)().includes("microsoft-standard-WSL");
+async function isEmptyDir(path) {
+    const iterator = await (0, promises_1.opendir)(path);
+    let done = false;
+    try {
+        const next = await iterator[Symbol.asyncIterator]().next();
+        done = !!next.done;
+        return done;
+    }
+    finally {
+        if (!done) {
+            await iterator.close();
+        }
+    }
+}
+exports.isEmptyDir = isEmptyDir;
+async function applyPermissions(baseDir, permissionsPath) {
+    const singleReader = (0, readline_1.createInterface)({
+        input: (0, fs_1.createReadStream)(permissionsPath),
+    });
+    for await (const line of singleReader) {
+        const [rpath, rawUid, rawGui, rawMode] = line.split(":");
+        const path = (0, path_1.join)(baseDir, rpath);
+        if (!path.startsWith(baseDir)) {
+            throw new Error(`Entry path is out of the base dir: (${path}, ${baseDir})`);
+        }
+        const uid = Number(rawUid);
+        const guid = Number(rawGui);
+        await (0, promises_1.chown)(path, uid, guid);
+        const mode = Number(rawMode);
+        await (0, promises_1.chmod)(path, mode);
+    }
+}
+exports.applyPermissions = applyPermissions;
+function pathIterator(stream) {
+    return stream;
+}
+exports.pathIterator = pathIterator;
 function isLocalDir(path) {
     return /^[\/\.]|([A-Z]:)/i.test(path);
 }
@@ -59,9 +95,7 @@ async function writeJSONFile(path, json) {
 }
 exports.writeJSONFile = writeJSONFile;
 async function readdirIfExists(path) {
-    if (!(await existsDir(path)))
-        return [];
-    return await readDir(path);
+    return await readDir(path, true);
 }
 exports.readdirIfExists = readdirIfExists;
 exports.parseFileExtensions = ["json", "js", "ts", "yaml", "yml"];
@@ -187,13 +221,15 @@ async function checkDir(path) {
     }
 }
 exports.checkDir = checkDir;
-async function readDir(path) {
+async function readDir(path, optional) {
     try {
         return await (0, promises_1.readdir)(path);
     }
     catch (anyError) {
         const nodeError = anyError;
         if (nodeError.code === "ENOENT") {
+            if (optional)
+                return [];
             const error = new Error(nodeError.message);
             error.code = nodeError.code;
             error.errno = nodeError.errno;
@@ -262,93 +298,14 @@ async function writeGitIgnoreList(options) {
     return path;
 }
 exports.writeGitIgnoreList = writeGitIgnoreList;
-async function writePathLists(options) {
-    const tempDir = await mkTmpDir("path-lists");
-    const includedPaths = [];
-    const excludedPaths = [];
-    const included = [];
-    const excluded = [];
-    const multipleStats = {};
-    const total = new Array((options.packs?.length || 0) + 1).fill(0);
-    await Promise.all([
-        ...new Array((options.packs?.length || 0) + 1).fill(null).map((_, index) => new Promise((resolve, reject) => {
-            const path = (0, path_1.join)(tempDir, `${index}-included.txt`);
-            const stream = (0, fs_2.createWriteStream)(path);
-            includedPaths.push(path);
-            included.push(stream);
-            stream.on("close", resolve);
-            stream.on("error", reject);
-        })),
-        ...new Array(options.packs?.length || 0).fill(null).map((_, index) => new Promise((resolve, reject) => {
-            const path = (0, path_1.join)(tempDir, `${index}-excluded.txt`);
-            const stream = (0, fs_2.createWriteStream)(path);
-            excludedPaths.push(path);
-            excluded.push(stream);
-            stream.on("close", resolve);
-            stream.on("error", reject);
-        })),
-        new Promise(async (resolve) => {
-            const packDirectories = [];
-            for await (const value of options.paths) {
-                const entry = value.toString();
-                const isDir = entry.endsWith("/");
-                const matchEntry = isDir ? entry.slice(0, -1) : entry;
-                let packIndex = 1;
-                let matches = false;
-                for (const pack of options.packs || []) {
-                    if ((0, micromatch_1.isMatch)(matchEntry, pack.include) &&
-                        (!pack.exclude || !(0, micromatch_1.isMatch)(matchEntry, pack.exclude))) {
-                        if (isDir)
-                            packDirectories.push([packIndex - 1, entry]);
-                        included[packIndex].write(`${entry}\n`);
-                        if (!isDir)
-                            total[packIndex]++;
-                        matches = true;
-                        break;
-                    }
-                    packIndex++;
-                }
-                if (!matches) {
-                    const packDir = packDirectories.find(([, p]) => entry.startsWith(p));
-                    if (packDir) {
-                        const [i, v] = packDir;
-                        const multipleExclude = options.packs?.[i].exclude;
-                        if (multipleExclude && (0, micromatch_1.isMatch)(matchEntry, multipleExclude)) {
-                            included[0].write(`${entry}\n`);
-                            excluded[i].write(`${entry}\n`);
-                        }
-                        else {
-                            if (!multipleStats[v])
-                                multipleStats[v] = 0;
-                            multipleStats[v]++;
-                        }
-                    }
-                    else {
-                        included[0].write(`${entry}\n`);
-                    }
-                    if (!isDir)
-                        total[0]++;
-                }
-            }
-            for (const stream of [...included, ...excluded]) {
-                stream.end();
-            }
-            resolve();
-        }),
-    ]);
-    return {
-        path: includedPaths[0],
-        includedPackPaths: includedPaths.slice(1),
-        excludedPackPaths: excludedPaths,
-        total: {
-            all: total.reduce((p, v) => p + v, 0),
-            path: total[0],
-            packsPaths: total.slice(1),
-            multipleStats,
-        },
-    };
+async function waitForClose(stream) {
+    return new Promise(async (resolve, reject) => {
+        stream.on("close", resolve);
+        stream.on("error", reject);
+        return stream;
+    });
 }
-exports.writePathLists = writePathLists;
+exports.waitForClose = waitForClose;
 async function copyFileWithStreams(source, target) {
     const r = (0, fs_1.createReadStream)(source);
     const w = (0, fs_2.createWriteStream)(target);
@@ -391,6 +348,7 @@ async function cpy(options) {
         }
     };
     const task = async (rawEntryPath, basePath) => {
+        [rawEntryPath] = rawEntryPath.split(":");
         const isDir = rawEntryPath.endsWith("/");
         const entryPath = (0, path_1.normalize)(rawEntryPath);
         const entrySourcePath = (0, path_1.resolve)((0, path_1.join)(basePath, rawEntryPath));

package/util/math-util.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.progressPercent = void 0;
 function progressPercent(total, current) {
+    if (total === 0 && current === 0)
+        return 0;
     return Number(((current / total) * 100).toFixed(2));
 }
 exports.progressPercent = progressPercent;

package/util/process-util.d.ts

@@ -24,6 +24,7 @@ export interface ExecSettingsInterface {
     onSpawn?: (p: ChildProcess) => void;
     stdout?: {
         save?: boolean;
+        parseLines?: boolean;
         onData?: (data: string) => void;
     };
     stderr?: {

package/util/process-util.js

@@ -11,6 +11,7 @@ const chalk_1 = __importDefault(require("chalk"));
 const child_process_1 = require("child_process");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
+const readline_1 = require("readline");
 function logExecStdout(input) {
     let text = input.colorize ? chalk_1.default.grey(input.data) : input.data;
     if (input.lineSalt)
@@ -70,7 +71,11 @@ async function exec(command, argv = [], options = null, settings = {}) {
             stderr: "",
             exitCode: 0,
         };
-        let finishListeners = pipe?.stream instanceof fs_1.WriteStream ? 2 : 1;
+        let finishListeners = 1;
+        if (pipe?.stream instanceof fs_1.WriteStream)
+            finishListeners++;
+        if (settings.stdout?.parseLines)
+            finishListeners++;
         let streamError;
         const tryFinish = () => {
             if (!--finishListeners)
@@ -129,10 +134,11 @@ async function exec(command, argv = [], options = null, settings = {}) {
         if (log.stdout || settings.stdout) {
             if (!p.stdout)
                 throw new Error(`stdout is not defined`);
-            p.stdout.on("data", (data) => {
+            const parseLines = settings.stdout?.parseLines;
+            const onData = (data) => {
                 if (log.stdout)
                     logExecStdout({
-                        data: data.toString(),
+                        data: parseLines ? `${data}\n` : data.toString(),
                         stderr: log.allToStderr,
                         colorize: log.colorize,
                     });
@@ -140,7 +146,17 @@ async function exec(command, argv = [], options = null, settings = {}) {
                     spawnData.stdout += data.toString();
                 if (settings.stdout?.onData)
                     settings.stdout.onData(data.toString());
-            });
+            };
+            if (parseLines) {
+                const rl = (0, readline_1.createInterface)({
+                    input: p.stdout,
+                });
+                rl.on("line", onData);
+                rl.on("close", tryFinish);
+            }
+            else {
+                p.stdout.on("data", onData);
+            }
         }
         if (log.stderr || settings.stderr) {
             if (!p.stderr)

package/util/string-util.d.ts

@@ -15,5 +15,6 @@ export declare type UriType = {
 export declare function formatUri(input: UriType, hidePassword?: boolean): string;
 export declare function formatSeconds(seconds: number): string;
 export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
+export declare function checkPath(path: string, include: string[], exclude?: string[]): boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;

package/util/string-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.checkMatch = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
+exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -99,6 +99,13 @@ function makePathPatterns(values) {
     });
 }
 exports.makePathPatterns = makePathPatterns;
+function checkPath(path, include, exclude) {
+    return ((0, micromatch_1.isMatch)(path, include, {
+        dot: true,
+    }) &&
+        (!exclude || !(0, micromatch_1.isMatch)(path, exclude, { dot: true })));
+}
+exports.checkPath = checkPath;
 function checkMatch(subject, patterns) {
     if (!subject?.length)
         subject = "<empty>";

package/util/zip-util.d.ts

@@ -3,20 +3,6 @@ export interface ZipDataFilterType {
     exclude?: boolean;
     patterns: string[];
 }
-export declare type ZipStreamDataType = {
-    type: "progress";
-    data: {
-        progress: number;
-        files: number;
-        path: string;
-    };
-} | {
-    type: "summary";
-    data: {
-        folders: number;
-        files: number;
-    };
-};
 export interface ZipDataType {
     command?: string;
     path: string;
@@ -26,7 +12,7 @@ export interface ZipDataType {
     includeList?: string;
     excludeList?: string;
     verbose?: boolean;
-    onStream?: (data: ZipStreamDataType) => void;
+    onStream?: (data: ZipStream) => void;
 }
 export interface UnzipDataType {
     command?: string;
@@ -34,20 +20,60 @@ export interface UnzipDataType {
     files?: (ZipDataFilterType | string)[];
     output: string;
     verbose?: boolean;
-    onStream?: (data: UnzipStreamDataType) => void;
+    onStream?: (data: UnzipStream) => void;
 }
-export declare type UnzipStreamDataType = {
+export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
+export declare function checkSSEOption(command?: string): Promise<boolean>;
+declare type ListZipStream = {
+    Path?: string;
+    Folder?: string;
+    Size?: string;
+    "Packed Size"?: string;
+    Modified?: string;
+    Created?: string;
+    Accessed?: string;
+    Attributes?: string;
+    Encrypted?: string;
+    Comment?: string;
+    CRC?: string;
+    Method?: string;
+    Characteristics?: string;
+    "Host OS"?: string;
+    Version?: string;
+    Volume?: string;
+    Offset?: string;
+};
+export declare function listZip(data: {
+    command?: string;
+    path: string;
+    onStream: (item: ListZipStream) => void;
+    verbose?: boolean;
+}): Promise<void>;
+export declare type ZipStream = {
     type: "progress";
     data: {
         progress: number;
         files: number;
         path: string;
     };
+} | {
+    type: "summary";
+    data: {
+        folders: number;
+        files: number;
+    };
 };
-export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
-export declare function checkSSEOption(command?: string): Promise<boolean>;
 export declare function zip(data: ZipDataType): Promise<{
     folders: number;
     files: number;
 }>;
+export declare type UnzipStream = {
+    type: "progress";
+    data: {
+        progress: number;
+        files: number;
+        path: string;
+    };
+};
 export declare function unzip(data: UnzipDataType): Promise<import("./process-util").ExecResultType>;
+export {};

package/util/zip-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unzip = exports.zip = exports.checkSSEOption = exports.buildArguments = void 0;
+exports.unzip = exports.zip = exports.listZip = exports.checkSSEOption = exports.buildArguments = void 0;
 const process_util_1 = require("./process-util");
 const path_1 = require("path");
 function buildArguments(filters) {
@@ -30,36 +30,6 @@ function buildArguments(filters) {
     return args;
 }
 exports.buildArguments = buildArguments;
-function parseZipStream(chunk, buffer, cb) {
-    const lines = chunk.replaceAll("\b", "").trim().split(/\r?\n/);
-    for (const line of lines) {
-        let matches = null;
-        if ((matches = /^(\d+)% (\d+ )?\+/.exec(line))) {
-            const path = line.slice(line.indexOf("+") + 1).trim();
-            const progress = Number(matches[1]);
-            if (!buffer.currentPaths)
-                buffer.currentPaths = 0;
-            if (path !== buffer.lastPath)
-                buffer.currentPaths++;
-            buffer.lastPath = path;
-            cb({
-                type: "progress",
-                data: { progress, path, files: buffer.currentPaths },
-            });
-        }
-        else if (line.startsWith("Add new data to archive:")) {
-            const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
-            const [, files] = /(\d+) files?/i.exec(line) || [, 0];
-            cb({
-                type: "summary",
-                data: {
-                    folders: Number(folders),
-                    files: Number(files),
-                },
-            });
-        }
-    }
-}
 let checkSSEOptionResult;
 async function checkSSEOption(command = "7z") {
     const result = await (0, process_util_1.exec)(command);
@@ -68,12 +38,81 @@ async function checkSSEOption(command = "7z") {
     return (checkSSEOptionResult = result.stdout.includes(" -sse"));
 }
 exports.checkSSEOption = checkSSEOption;
+const listZipLineEqChar = " = ";
+function parseListZipLine(line, buffer) {
+    if (buffer.started) {
+        if (line === "") {
+            if (buffer.opened) {
+                const { stream } = buffer;
+                buffer.stream = {};
+                buffer.opened = false;
+                return stream;
+            }
+        }
+        else {
+            const separator = line.indexOf(listZipLineEqChar);
+            const key = line.slice(0, separator);
+            const value = line.slice(separator + listZipLineEqChar.length);
+            buffer.opened = true;
+            buffer.stream[key] = value;
+        }
+    }
+    else if (line.startsWith("----------")) {
+        buffer.started = true;
+        buffer.stream = {};
+    }
+}
+async function listZip(data) {
+    const buffer = {};
+    await (0, process_util_1.exec)(data.command ?? "7z", ["l", data.path, "-slt"], {}, {
+        log: {
+            exec: data.verbose ?? false,
+            stderr: data.verbose ?? false,
+            stdout: false,
+        },
+        onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
+        stdout: {
+            parseLines: true,
+            onData: (line) => {
+                const stream = parseListZipLine(line, buffer);
+                if (stream) {
+                    data.onStream?.(stream);
+                }
+            },
+        },
+    });
+}
+exports.listZip = listZip;
+function parseZipLine(line) {
+    let matches = null;
+    if (!line.trim().length)
+        return;
+    if ((matches = /^\s*(\d+)% (\d+ )?\+/.exec(line))) {
+        const path = line.slice(line.indexOf("+") + 1).trim();
+        const progress = Number(matches[1]);
+        const files = Number(matches[2]);
+        return {
+            type: "progress",
+            data: { progress, path, files },
+        };
+    }
+    else if (line.startsWith("Add new data to archive:")) {
+        const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
+        const [, files] = /(\d+) files?/i.exec(line) || [, 0];
+        return {
+            type: "summary",
+            data: {
+                folders: Number(folders),
+                files: Number(files),
+            },
+        };
+    }
+}
 async function zip(data) {
     let result = {
         folders: 0,
         files: 0,
     };
-    let buffer = {};
     await (0, process_util_1.exec)(data.command ?? "7z", [
         "a",
         // https://sourceforge.net/p/sevenzip/bugs/2099/,
@@ -91,31 +130,29 @@ async function zip(data) {
         log: data.verbose ?? false,
         onExitCodeError: (data, error) => (data.exitCode > 2 ? error : false),
         stdout: {
-            onData: (chunk) => {
-                parseZipStream(chunk, buffer, (stream) => {
+            onData: (line) => {
+                const stream = parseZipLine(line);
+                if (stream) {
                     data.onStream?.(stream);
                     if (stream.type === "summary")
                         result = stream.data;
-                });
+                }
             },
         },
     });
     return result;
 }
 exports.zip = zip;
-function parseUnzipStream(chunk, cb) {
-    const lines = chunk.trim().split(/\r?\n/g);
-    for (const line of lines) {
-        let matches = null;
-        if ((matches = /^(\d+)% (\d+) \-/.exec(line))) {
-            const progress = Number(matches[1]);
-            const files = Number(matches[2]);
-            const path = line.slice(line.indexOf("-") + 1).trim();
-            cb({
-                type: "progress",
-                data: { progress, path, files },
-            });
-        }
+function parseUnzipLine(line) {
+    let matches = null;
+    if ((matches = /^\s*(\d+)% (\d+) \-/.exec(line))) {
+        const progress = Number(matches[1]);
+        const files = Number(matches[2]);
+        const path = line.slice(line.indexOf("-") + 1).trim();
+        return {
+            type: "progress",
+            data: { progress, path, files },
+        };
     }
 }
 async function unzip(data) {
@@ -131,9 +168,11 @@ async function unzip(data) {
         stderr: { toExitCode: true },
         stdout: {
             ...(data.onStream && {
-                onData: (chunk) => {
-                    if (data.onStream)
-                        parseUnzipStream(chunk, data.onStream);
+                parseLines: true,
+                onData: (line) => {
+                    const stream = parseUnzipLine(line);
+                    if (stream)
+                        data.onStream(stream);
                 },
             }),
         },