@datatruck/cli 0.16.0 → 0.16.1

package/Action/BackupAction.d.ts

@@ -23,10 +23,17 @@ export declare class BackupAction<TRequired extends boolean = true> {
     protected repoErrors: Record<string, Error[]>;
     constructor(config: ConfigType, options?: IfRequireKeys<TRequired, BackupActionOptionsType>);
     protected init(session: BackupSessionManager): Promise<[SnapshotType, PackageConfigType[]]>;
-    protected execTask(session: BackupSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
-    protected execRepository(session: BackupSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
-    protected execCopyRepository(session: BackupSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, mirrorRepo: RepositoryConfigType, snapshot: SnapshotType): Promise<boolean>;
+    protected task(session: BackupSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected backup(session: BackupSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected copyBackup(session: BackupSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, mirrorRepo: RepositoryConfigType, snapshot: SnapshotType): Promise<boolean>;
     protected getError(pkg: PackageConfigType): AppError | null;
+    protected splitRepositories(repositoryNames: string[]): {
+        repoNames: string[];
+        mirrors: {
+            sourceName: string;
+            name: string;
+        }[];
+    };
     exec(session: BackupSessionManager): Promise<{
         total: number;
         errors: number;

package/Action/BackupAction.js

@@ -53,7 +53,7 @@ class BackupAction {
         }
         return [snapshot, packages];
     }
-    async execTask(session, pkg, task, snapshot, targetPath) {
+    async task(session, pkg, task, snapshot, targetPath) {
         const taskId = session.findTaskId({
             packageName: pkg.name,
             taskName: task.name,
@@ -94,7 +94,7 @@ class BackupAction {
         });
         return error ? false : true;
     }
-    async execRepository(session, pkg, repo, snapshot, targetPath) {
+    async backup(session, pkg, repo, snapshot, targetPath) {
         const repositoryId = session.findRepositoryId({
             packageName: pkg.name,
             repositoryName: repo.name,
@@ -136,7 +136,7 @@ class BackupAction {
         });
         return error ? false : true;
     }
-    async execCopyRepository(session, pkg, repo, mirrorRepo, snapshot) {
+    async copyBackup(session, pkg, repo, mirrorRepo, snapshot) {
         const repositoryId = session.findRepositoryId({
             packageName: pkg.name,
             repositoryName: mirrorRepo.name,
@@ -192,12 +192,33 @@ class BackupAction {
             return null;
         }
     }
+    splitRepositories(repositoryNames) {
+        const mirrorRepoMap = {};
+        const allMirrorRepoNames = [];
+        const repoNames = repositoryNames ?? [];
+        for (const repoName of repoNames) {
+            const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repoName);
+            if (repo.mirrorRepoNames)
+                mirrorRepoMap[repoName] = repo.mirrorRepoNames.filter((mirrorRepoName) => {
+                    allMirrorRepoNames.push(mirrorRepoName);
+                    return repoNames.includes(mirrorRepoName);
+                });
+        }
+        return {
+            repoNames: repoNames.filter((v) => !allMirrorRepoNames.includes(v)),
+            mirrors: repoNames.flatMap((sourceName) => {
+                const mirrorNames = mirrorRepoMap[sourceName] || [];
+                return mirrorNames.map((name) => ({
+                    sourceName,
+                    name,
+                }));
+            }),
+        };
+    }
     async exec(session) {
         const [snapshot, packages] = await this.init(session);
-        let total = 0;
         let errors = 0;
         for (const pkg of packages) {
-            total++;
             const id = session.findId({
                 packageName: pkg.name,
             });
@@ -212,34 +233,17 @@ class BackupAction {
                     package: pkg,
                     snapshot,
                 });
-                await this.execTask(session, pkg, pkg.task, snapshot, (targetPath = result?.targetPath));
+                await this.task(session, pkg, pkg.task, snapshot, (targetPath = result?.targetPath));
             }
-            const mirrorRepoMap = {};
-            const allMirrorRepoNames = [];
-            const repoNames = pkg.repositoryNames ?? [];
+            const { repoNames, mirrors } = this.splitRepositories(pkg.repositoryNames ?? []);
             for (const repoName of repoNames) {
                 const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repoName);
-                if (repo.mirrorRepoNames)
-                    mirrorRepoMap[repoName] = repo.mirrorRepoNames.filter((mirrorRepoName) => {
-                        allMirrorRepoNames.push(mirrorRepoName);
-                        return repoNames.includes(mirrorRepoName);
-                    });
+                await this.backup(session, pkg, repo, snapshot, targetPath);
             }
-            for (const repoName of repoNames) {
-                if (allMirrorRepoNames.includes(repoName))
-                    continue;
-                const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repoName);
-                await this.execRepository(session, pkg, repo, snapshot, targetPath);
-            }
-            for (const repoName of repoNames) {
-                const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repoName);
-                const mirrorRepoNames = mirrorRepoMap[repoName];
-                if (mirrorRepoNames) {
-                    for (const mirrorRepoName of mirrorRepoNames) {
-                        const mirrorRepo = (0, config_util_1.findRepositoryOrFail)(this.config, mirrorRepoName);
-                        await this.execCopyRepository(session, pkg, repo, mirrorRepo, snapshot);
-                    }
-                }
+            for (const mirror of mirrors) {
+                const repo = (0, config_util_1.findRepositoryOrFail)(this.config, mirror.sourceName);
+                const mirrorRepo = (0, config_util_1.findRepositoryOrFail)(this.config, mirror.name);
+                await this.copyBackup(session, pkg, repo, mirrorRepo, snapshot);
             }
             const error = this.getError(pkg);
             if (error)
@@ -253,7 +257,7 @@ class BackupAction {
             snapshotId: snapshot.id.slice(0, 8),
         });
         return {
-            total: total,
+            total: packages.length,
             errors: errors,
         };
     }

package/Action/RestoreAction.d.ts

@@ -30,8 +30,8 @@ export declare class RestoreAction<TRequired extends boolean = true> {
     protected init(session: RestoreSessionManager, snapshotId: string, snapshots: SnapshotAndConfigType[]): Promise<void>;
     protected findSnapshots(): Promise<SnapshotType[]>;
     protected groupSnapshots(snapshots: SnapshotType[]): SnapshotType[];
-    protected execTask(session: RestoreSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
-    protected execRepository(session: RestoreSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected task(session: RestoreSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected restore(session: RestoreSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
     protected getError(pkg: PackageConfigType): AppError | null;
     exec(session: RestoreSessionManager): Promise<boolean>;
 }

package/Action/RestoreAction.js

@@ -88,7 +88,7 @@ class RestoreAction {
             return true;
         });
     }
-    async execTask(session, pkg, task, snapshot, targetPath) {
+    async task(session, pkg, task, snapshot, targetPath) {
         const taskId = session.findTaskId({
             packageName: pkg.name,
             taskName: task.name,
@@ -131,7 +131,7 @@ class RestoreAction {
         });
         return error ? false : true;
     }
-    async execRepository(session, pkg, repo, snapshot, targetPath) {
+    async restore(session, pkg, repo, snapshot, targetPath) {
         const repositoryId = session.findRepositoryId({
             packageName: pkg.name,
             repositoryName: repo.name,
@@ -232,9 +232,9 @@ class RestoreAction {
                 });
                 targetPath = result?.targetPath;
             }
-            await this.execRepository(session, pkg, repo, snapshot, targetPath);
+            await this.restore(session, pkg, repo, snapshot, targetPath);
             if (pkg.task)
-                await this.execTask(session, pkg, pkg.task, snapshot, targetPath);
+                await this.task(session, pkg, pkg.task, snapshot, targetPath);
             const error = this.getError(pkg);
             await session.end({
                 id,

package/Repository/DatatruckRepository.js

@@ -123,30 +123,29 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                     object.current += data.current;
                 }
             },
+            updateProgress: async (end) => {
+                const currentTime = perf_hooks_1.performance.now();
+                const diff = currentTime - lastTime;
+                if (end || diff > 1000) {
+                    await options.onProgress({
+                        relative: {
+                            description: end ? "Scanned files" : "Scanning files",
+                            payload: object.total.toString(),
+                        },
+                    });
+                    lastTime = currentTime;
+                }
+            },
             start: async (cb) => {
                 for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
                     if (!options.disableCounting)
                         object.total++;
-                    const currentTime = perf_hooks_1.performance.now();
-                    const diff = currentTime - lastTime;
-                    if (diff > 1000) {
-                        await options.onProgress({
-                            relative: {
-                                description: "Scanning files",
-                                payload: object.total.toString(),
-                            },
-                        });
-                        lastTime = currentTime;
-                    }
+                    await object.updateProgress();
                     if (cb)
                         await cb(entry);
                 }
-                await options.onProgress({
-                    relative: {
-                        description: "Scanned files",
-                        payload: object.total.toString(),
-                    },
-                });
+                if (!options.disableEndProgress)
+                    await object.updateProgress(true);
             },
         };
         await options.onProgress({
@@ -439,6 +438,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                 cwd: sourcePath,
             },
             onProgress: data.onProgress,
+            disableEndProgress: true,
         });
         await scanner.start();
         const it = await (0, promises_1.opendir)(sourcePath);
@@ -446,6 +446,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             const path = (0, path_1.join)(sourcePath, dirent.name);
             if (dirent.name === "permissions.txt") {
                 scanner.total++;
+                await scanner.updateProgress();
             }
             else if (dirent.name.endsWith(".zip")) {
                 await (0, zip_util_1.listZip)({
@@ -455,10 +456,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
                         const isDir = item.Folder === "+";
                         if (!isDir)
                             scanner.total++;
+                        await scanner.updateProgress();
                     },
                 });
             }
         }
+        await scanner.updateProgress(true);
         if (data.options.verbose)
             (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
         await (0, fs_util_1.cpy)({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.16.0",
+  "version": "0.16.1",
   "dependencies": {
     "ajv": "^8.11.0",
     "async": "^3.2.4",

package/util/zip-util.js

@@ -173,9 +173,9 @@ async function zip(data) {
 exports.zip = zip;
 function parseUnzipLine(line) {
     let matches = null;
-    if ((matches = /^\s*(\d+)% (\d+) \-/.exec(line))) {
-        const progress = Number(matches[1]);
-        const files = Number(matches[2]);
+    if ((matches = /^\s*(?<percent>\d+)%(?: (?<files>\d+))? \-/.exec(line))) {
+        const progress = Number(matches.groups["percent"]);
+        const files = matches.groups["files"] ? Number(matches[2]) : 1;
         const path = line.slice(line.indexOf("-") + 1).trim();
         return {
             type: "progress",
@@ -204,20 +204,22 @@ async function unzip(data) {
         stderr: { toExitCode: true },
         stdout: {
             ...((data.onStream || data.onProgress) && {
-                parseLines: true,
-                onData: async (line) => {
-                    const stream = parseUnzipLine(line);
-                    if (stream) {
-                        if (stream.type === "progress") {
-                            const current = Math.max(0, stream.data.files - 1);
-                            summary.files = stream.data.files;
-                            await data.onProgress?.({
-                                current,
-                                percent: stream.data.percent,
-                                path: stream.data.path,
-                            });
+                onData: async (chunk) => {
+                    const lines = chunk.replaceAll("\b", "").split(/\r?\n/);
+                    for (const line of lines) {
+                        const stream = parseUnzipLine(line);
+                        if (stream) {
+                            if (stream.type === "progress") {
+                                const current = Math.max(0, stream.data.files - 1);
+                                summary.files = stream.data.files;
+                                await data.onProgress?.({
+                                    current,
+                                    percent: stream.data.percent,
+                                    path: stream.data.path,
+                                });
+                            }
+                            await data.onStream?.(stream);
                         }
-                        await data.onStream?.(stream);
                     }
                 },
             }),