@datatruck/cli 0.21.0 → 0.22.0

package/Action/BackupAction.d.ts

@@ -1,7 +1,6 @@
 import type { ConfigType } from "../Config/Config";
 import { PackageConfigType } from "../Config/PackageConfig";
 import { RepositoryConfigType } from "../Config/RepositoryConfig";
-import { AppError } from "../Error/AppError";
 import { SnapshotType } from "../Repository/RepositoryAbstract";
 import { BackupSessionManager } from "../SessionManager/BackupSessionManager";
 import { TaskAbstract } from "../Task/TaskAbstract";
@@ -35,7 +34,7 @@ export declare class BackupAction<TRequired extends boolean = true> {
         error: boolean;
         tmpDirs: string[];
     }>;
-    protected getError(pkg: PackageConfigType): AppError | null;
+    protected getError(pkg: PackageConfigType): Error | undefined;
     protected splitRepositories(repositoryNames: string[]): {
         repoNames: string[];
         mirrors: {
@@ -45,6 +44,6 @@ export declare class BackupAction<TRequired extends boolean = true> {
     };
     exec(session: BackupSessionManager): Promise<{
         total: number;
-        errors: number;
+        errors: Error[];
     }>;
 }

package/Action/BackupAction.js

@@ -67,7 +67,7 @@ class BackupAction {
         const key = `${pkg.name}`;
         let error;
         if (this.taskErrors[key]?.length) {
-            error = new AppError_1.AppError("Previous task failed");
+            error = AppError_1.AppError.create("Previous task failed", this.taskErrors[key]);
         }
         else {
             try {
@@ -110,7 +110,7 @@ class BackupAction {
         let error;
         let repoInstance;
         if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Task failed");
+            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -156,7 +156,7 @@ class BackupAction {
         let error;
         let repoInstance;
         if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Task failed");
+            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -190,20 +190,16 @@ class BackupAction {
         };
     }
     getError(pkg) {
-        const taskErrors = this.taskErrors[pkg.name]?.length;
-        const repoErrors = this.repoErrors[pkg.name]?.length;
-        if (taskErrors && repoErrors) {
-            return new AppError_1.AppError("Task and repository failed");
-        }
-        else if (taskErrors && !repoErrors) {
-            return new AppError_1.AppError("Task failed");
-        }
-        else if (!taskErrors && repoErrors) {
-            return new AppError_1.AppError("Repository failed");
-        }
-        else {
-            return null;
-        }
+        const taskErrors = this.taskErrors[pkg.name] || [];
+        const repoErrors = this.repoErrors[pkg.name] || [];
+        const errors = [...taskErrors, ...repoErrors];
+        if (!errors.length)
+            return;
+        return AppError_1.AppError.create(taskErrors.length && repoErrors.length
+            ? "Task and repository failed"
+            : taskErrors.length && !repoErrors.length
+                ? "Task failed"
+                : "Repository failed", errors);
     }
     splitRepositories(repositoryNames) {
         const mirrorRepoMap = {};
@@ -230,7 +226,7 @@ class BackupAction {
     }
     async exec(session) {
         const [snapshot, packages] = await this.init(session);
-        let errors = 0;
+        const errors = [];
         for (const pkg of packages) {
             const id = session.findId({
                 packageName: pkg.name,
@@ -272,7 +268,7 @@ class BackupAction {
             }
             const error = this.getError(pkg);
             if (error)
-                errors++;
+                errors.push(error);
             await session.end({
                 id: id,
                 error: error?.message,
@@ -283,7 +279,7 @@ class BackupAction {
         });
         return {
             total: packages.length,
-            errors: errors,
+            errors,
         };
     }
 }

package/Action/RestoreAction.d.ts

@@ -39,6 +39,8 @@ export declare class RestoreAction<TRequired extends boolean = true> {
         tmpDirs: string[];
     }>;
     protected getError(pkg: PackageConfigType): AppError | null;
-    exec(session: RestoreSessionManager): Promise<boolean>;
+    exec(session: RestoreSessionManager): Promise<{
+        errors: Error[];
+    }>;
 }
 export {};

package/Action/RestoreAction.js

@@ -100,10 +100,10 @@ class RestoreAction {
         });
         let error;
         if (this.repoErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Repository failed");
+            error = AppError_1.AppError.create("Repository failed", this.repoErrors[pkg.name]);
         }
         else if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Previous task failed");
+            error = AppError_1.AppError.create("Previous task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -222,7 +222,7 @@ class RestoreAction {
         });
         const snapshotAndConfigs = this.assocConfigs(packages, snapshots);
         await this.init(session, this.options.snapshotId, snapshotAndConfigs);
-        let sessionErrors = 0;
+        const errors = [];
         for (const [snapshot, pkg] of snapshotAndConfigs) {
             (0, assert_1.ok)(pkg);
             const repo = (0, config_1.findRepositoryOrFail)(this.config, snapshot.repositoryName);
@@ -257,10 +257,10 @@ class RestoreAction {
                 error: error?.message,
             });
             if (error)
-                sessionErrors++;
+                errors.push(error);
         }
         await session.endDrivers();
-        return !sessionErrors;
+        return { errors };
     }
 }
 exports.RestoreAction = RestoreAction;

package/Command/BackupCommand.d.ts

@@ -12,5 +12,5 @@ export type BackupCommandOptionsType<TResolved = false> = {
 };
 export declare class BackupCommand extends CommandAbstract<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>>;
-    onExec(): Promise<0 | 1>;
+    onExec(): Promise<1 | 0>;
 }

package/Command/BackupCommand.js

@@ -74,7 +74,7 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
             progressInterval: this.globalOptions.progressInterval,
         });
         const result = await backup.exec(sessionManager);
-        if (result.errors) {
+        if (result.errors.length) {
             return 1;
         }
         else if (!result.total) {

package/Command/RestoreCommand.d.ts

@@ -12,5 +12,5 @@ export type RestoreCommandOptionsType<TResolved = false> = {
 };
 export declare class RestoreCommand extends CommandAbstract<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>>;
-    onExec(): Promise<0 | 1>;
+    onExec(): Promise<1 | 0>;
 }

package/Command/RestoreCommand.js

@@ -74,7 +74,7 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
             progressInterval: this.globalOptions.progressInterval,
         });
         const result = await restore.exec(sessionManager);
-        return result ? 0 : 1;
+        return result.errors.length ? 1 : 0;
     }
 }
 exports.RestoreCommand = RestoreCommand;

package/Error/AppError.d.ts

@@ -1,3 +1,4 @@
 export declare class AppError extends Error {
     constructor(message: string);
+    static create(message: string, errors: Error[]): Error;
 }

package/Error/AppError.js

@@ -6,5 +6,13 @@ class AppError extends Error {
         super(message);
         this.name = AppError.name;
     }
+    static create(message, errors) {
+        if (errors.length === 1) {
+            return errors[0];
+        }
+        else {
+            return new AggregateError(errors, message);
+        }
+    }
 }
 exports.AppError = AppError;

package/JsonSchema/JsonSchema.js

@@ -38,7 +38,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.resticPackageRepository]: ResticRepository_1.resticPackageRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.gitTask]: GitTask_1.gitTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.scriptTask]: ScriptTask_1.scriptTaskDefinition,
-    [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: SqlDumpTaskAbstract_1.sqlDumpTaskDefinition,
+    [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: (0, SqlDumpTaskAbstract_1.sqlDumpTaskDefinition)(),
     [DefinitionEnum_1.DefinitionEnum.mariadbTask]: MariadbTask_1.mariadbTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mssqlTask]: MssqlTask_1.mssqlTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mysqlDumpTask]: MysqlDumpTask_1.mysqlDumpTaskDefinition,

package/Repository/DatatruckRepository.d.ts

@@ -9,6 +9,9 @@ export type MetaDataType = {
     tags: string[];
     version: string;
     size: number;
+    tarStats?: Record<string, {
+        files: number;
+    }>;
 };
 export type DatatruckRepositoryConfigType = {
     outPath: string;

package/Repository/DatatruckRepository.js

@@ -196,7 +196,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             if (entry.dirent.isDirectory() &&
                 !(await (0, fs_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))))
                 return false;
-            let packIndex = configPacks.findIndex((pack) => (0, string_1.checkPath)(entry.path, pack.include, pack.exclude));
+            let packIndex = configPacks.findIndex((pack) => (0, string_1.match)(entry.path, pack.include, pack.exclude));
             if (packIndex === -1)
                 packIndex = defaultsPackIndex;
             const pack = packs[packIndex];
@@ -213,22 +213,26 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         });
         await stream.end();
         let packIndex = 0;
+        const tarStats = {};
         for (const pack of packs) {
             const packBasename = [`pack`, packIndex.toString(), pack.name]
                 .filter((v) => typeof v === "string")
                 .map((v) => encodeURIComponent(v.toString().replace(/[\\/]/g, "-")))
-                .join("-");
-            const ext = pack.compress ? `.tar.gz` : `.tar`;
+                .join("-") + (pack.compress ? `.tar.gz` : `.tar`);
             const includeList = stream.path(packIndex);
-            if (includeList)
+            if (includeList) {
+                tarStats[packBasename] = {
+                    files: stream.lines(packIndex),
+                };
                 await (0, tar_1.createTar)({
                     compress: pack.compress,
                     verbose: data.options.verbose,
                     includeList,
                     path: sourcePath,
-                    output: (0, path_1.join)(outPath, packBasename) + ext,
+                    output: (0, path_1.join)(outPath, packBasename),
                     onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
                 });
+            }
             packIndex++;
         }
         await scanner.end();
@@ -243,6 +247,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             task: data.package.task?.name,
             version: nodePkg.version,
             size: await (0, fs_1.fastFolderSizeAsync)(outPath),
+            tarStats,
         };
         if (data.options.verbose)
             (0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
@@ -298,6 +303,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
+        const metaPath = (0, path_1.join)(sourcePath, "meta.json");
+        const meta = await DatatruckRepository.parseMetaData(metaPath);
         const scanner = await (0, fs_1.createFileScanner)({
             onProgress: data.onProgress,
             glob: {
@@ -306,26 +313,37 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             },
         });
         const tarFiles = [];
+        const tarStats = meta?.tarStats || {};
         await scanner.start(async (entry) => {
             const path = (0, path_1.join)(sourcePath, entry.name);
             const isTar = entry.name.endsWith(".tar");
             const isTarGz = entry.name.endsWith(".tar.gz");
             if (isTar || isTarGz) {
                 tarFiles.push(path);
-                await (0, tar_1.listTar)({
-                    input: path,
-                    verbose: data.options.verbose,
-                    onEntry: () => {
-                        scanner.total++;
-                    },
-                });
+                if (typeof tarStats[entry.name]?.files === "number") {
+                    scanner.total += tarStats[entry.name].files;
+                }
+                else {
+                    scanner.progress("Scanning", entry.name, false);
+                    const selfTarStats = (tarStats[entry.name] = { files: 0 });
+                    await (0, tar_1.listTar)({
+                        input: path,
+                        verbose: data.options.verbose,
+                        onEntry: () => {
+                            scanner.total++;
+                            selfTarStats.files++;
+                        },
+                    });
+                }
             }
             return false;
         });
         if (data.options.verbose)
             (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
         for (const tarFile of tarFiles) {
+            const entryName = (0, path_1.basename)(tarFile);
             await (0, tar_1.extractTar)({
+                total: tarStats[entryName].files,
                 input: tarFile,
                 output: restorePath,
                 decompress: tarFile.endsWith(".tar.gz"),

package/Task/MysqlDumpTask.d.ts

@@ -1,17 +1,15 @@
-import { SqlDumpTaskAbstract, SqlDumpTaskConfigType, TargetDatabaseType } from "./SqlDumpTaskAbstract";
-import { JSONSchema7 } from "json-schema";
+import { SqlDumpTaskConfigType } from "./SqlDumpTaskAbstract";
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
 export declare const mysqlDumpTaskName = "mysql-dump";
-export type MysqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
-export declare const mysqlDumpTaskDefinition: JSONSchema7;
-export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConfigType> {
-    buildConnectionArgs(database?: boolean): Promise<string[]>;
-    onDatabaseIsEmpty(name: string): Promise<boolean>;
-    onCreateDatabase(database: TargetDatabaseType): Promise<void>;
-    onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;
-    onFetchTableNames(database: string): Promise<string[]>;
-    onExportTables(tableNames: string[], output: string, onProgress: (progress: {
-        totalBytes: number;
-    }) => void): Promise<void>;
-    onExportStoredPrograms(output: string): Promise<void>;
-    onImport(path: string, database: string): Promise<void>;
+export type MysqlDumpTaskConfigType = {
+    /**
+     * @default "sql"
+     */
+    dataFormat?: "csv" | "sql";
+    csvSharedPath?: string;
+} & SqlDumpTaskConfigType;
+export declare const mysqlDumpTaskDefinition: import("json-schema").JSONSchema7;
+export declare class MysqlDumpTask extends TaskAbstract<MysqlDumpTaskConfigType> {
+    onBackup(data: BackupDataType): Promise<void>;
+    onRestore(data: RestoreDataType): Promise<void>;
 }