@datatruck/cli 0.21.1 → 0.22.1

package/Action/BackupAction.d.ts

@@ -1,7 +1,6 @@
 import type { ConfigType } from "../Config/Config";
 import { PackageConfigType } from "../Config/PackageConfig";
 import { RepositoryConfigType } from "../Config/RepositoryConfig";
-import { AppError } from "../Error/AppError";
 import { SnapshotType } from "../Repository/RepositoryAbstract";
 import { BackupSessionManager } from "../SessionManager/BackupSessionManager";
 import { TaskAbstract } from "../Task/TaskAbstract";
@@ -35,7 +34,7 @@ export declare class BackupAction<TRequired extends boolean = true> {
         error: boolean;
         tmpDirs: string[];
     }>;
-    protected getError(pkg: PackageConfigType): AppError | null;
+    protected getError(pkg: PackageConfigType): Error | undefined;
     protected splitRepositories(repositoryNames: string[]): {
         repoNames: string[];
         mirrors: {
@@ -45,6 +44,6 @@ export declare class BackupAction<TRequired extends boolean = true> {
     };
     exec(session: BackupSessionManager): Promise<{
         total: number;
-        errors: number;
+        errors: Error[];
     }>;
 }

package/Action/BackupAction.js

@@ -67,7 +67,7 @@ class BackupAction {
         const key = `${pkg.name}`;
         let error;
         if (this.taskErrors[key]?.length) {
-            error = new AppError_1.AppError("Previous task failed");
+            error = AppError_1.AppError.create("Previous task failed", this.taskErrors[key]);
         }
         else {
             try {
@@ -110,7 +110,7 @@ class BackupAction {
         let error;
         let repoInstance;
         if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Task failed");
+            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -156,7 +156,7 @@ class BackupAction {
         let error;
         let repoInstance;
         if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Task failed");
+            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -190,20 +190,16 @@ class BackupAction {
         };
     }
     getError(pkg) {
-        const taskErrors = this.taskErrors[pkg.name]?.length;
-        const repoErrors = this.repoErrors[pkg.name]?.length;
-        if (taskErrors && repoErrors) {
-            return new AppError_1.AppError("Task and repository failed");
-        }
-        else if (taskErrors && !repoErrors) {
-            return new AppError_1.AppError("Task failed");
-        }
-        else if (!taskErrors && repoErrors) {
-            return new AppError_1.AppError("Repository failed");
-        }
-        else {
-            return null;
-        }
+        const taskErrors = this.taskErrors[pkg.name] || [];
+        const repoErrors = this.repoErrors[pkg.name] || [];
+        const errors = [...taskErrors, ...repoErrors];
+        if (!errors.length)
+            return;
+        return AppError_1.AppError.create(taskErrors.length && repoErrors.length
+            ? "Task and repository failed"
+            : taskErrors.length && !repoErrors.length
+                ? "Task failed"
+                : "Repository failed", errors);
     }
     splitRepositories(repositoryNames) {
         const mirrorRepoMap = {};
@@ -230,7 +226,7 @@ class BackupAction {
     }
     async exec(session) {
         const [snapshot, packages] = await this.init(session);
-        let errors = 0;
+        const errors = [];
         for (const pkg of packages) {
             const id = session.findId({
                 packageName: pkg.name,
@@ -272,7 +268,7 @@ class BackupAction {
             }
             const error = this.getError(pkg);
             if (error)
-                errors++;
+                errors.push(error);
             await session.end({
                 id: id,
                 error: error?.message,
@@ -283,7 +279,7 @@ class BackupAction {
         });
         return {
             total: packages.length,
-            errors: errors,
+            errors,
         };
     }
 }

package/Action/RestoreAction.d.ts

@@ -39,6 +39,8 @@ export declare class RestoreAction<TRequired extends boolean = true> {
         tmpDirs: string[];
     }>;
     protected getError(pkg: PackageConfigType): AppError | null;
-    exec(session: RestoreSessionManager): Promise<boolean>;
+    exec(session: RestoreSessionManager): Promise<{
+        errors: Error[];
+    }>;
 }
 export {};

package/Action/RestoreAction.js

@@ -100,10 +100,10 @@ class RestoreAction {
         });
         let error;
         if (this.repoErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Repository failed");
+            error = AppError_1.AppError.create("Repository failed", this.repoErrors[pkg.name]);
         }
         else if (this.taskErrors[pkg.name]?.length) {
-            error = new AppError_1.AppError("Previous task failed");
+            error = AppError_1.AppError.create("Previous task failed", this.taskErrors[pkg.name]);
         }
         else {
             try {
@@ -222,7 +222,7 @@ class RestoreAction {
         });
         const snapshotAndConfigs = this.assocConfigs(packages, snapshots);
         await this.init(session, this.options.snapshotId, snapshotAndConfigs);
-        let sessionErrors = 0;
+        const errors = [];
         for (const [snapshot, pkg] of snapshotAndConfigs) {
             (0, assert_1.ok)(pkg);
             const repo = (0, config_1.findRepositoryOrFail)(this.config, snapshot.repositoryName);
@@ -257,10 +257,10 @@ class RestoreAction {
                 error: error?.message,
             });
             if (error)
-                sessionErrors++;
+                errors.push(error);
         }
         await session.endDrivers();
-        return !sessionErrors;
+        return { errors };
     }
 }
 exports.RestoreAction = RestoreAction;

package/Command/BackupCommand.d.ts

@@ -12,5 +12,5 @@ export type BackupCommandOptionsType<TResolved = false> = {
 };
 export declare class BackupCommand extends CommandAbstract<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>>;
-    onExec(): Promise<0 | 1>;
+    onExec(): Promise<1 | 0>;
 }

package/Command/BackupCommand.js

@@ -74,7 +74,7 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
             progressInterval: this.globalOptions.progressInterval,
         });
         const result = await backup.exec(sessionManager);
-        if (result.errors) {
+        if (result.errors.length) {
             return 1;
         }
         else if (!result.total) {

package/Command/RestoreCommand.d.ts

@@ -12,5 +12,5 @@ export type RestoreCommandOptionsType<TResolved = false> = {
 };
 export declare class RestoreCommand extends CommandAbstract<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>>;
-    onExec(): Promise<0 | 1>;
+    onExec(): Promise<1 | 0>;
 }

package/Command/RestoreCommand.js

@@ -74,7 +74,7 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
             progressInterval: this.globalOptions.progressInterval,
         });
         const result = await restore.exec(sessionManager);
-        return result ? 0 : 1;
+        return result.errors.length ? 1 : 0;
     }
 }
 exports.RestoreCommand = RestoreCommand;

package/Error/AppError.d.ts

@@ -1,3 +1,4 @@
 export declare class AppError extends Error {
     constructor(message: string);
+    static create(message: string, errors: Error[]): Error;
 }

package/Error/AppError.js

@@ -6,5 +6,13 @@ class AppError extends Error {
         super(message);
         this.name = AppError.name;
     }
+    static create(message, errors) {
+        if (errors.length === 1) {
+            return errors[0];
+        }
+        else {
+            return new AggregateError(errors, message);
+        }
+    }
 }
 exports.AppError = AppError;

package/JsonSchema/JsonSchema.js

@@ -38,7 +38,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.resticPackageRepository]: ResticRepository_1.resticPackageRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.gitTask]: GitTask_1.gitTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.scriptTask]: ScriptTask_1.scriptTaskDefinition,
-    [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: SqlDumpTaskAbstract_1.sqlDumpTaskDefinition,
+    [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: (0, SqlDumpTaskAbstract_1.sqlDumpTaskDefinition)(),
     [DefinitionEnum_1.DefinitionEnum.mariadbTask]: MariadbTask_1.mariadbTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mssqlTask]: MssqlTask_1.mssqlTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mysqlDumpTask]: MysqlDumpTask_1.mysqlDumpTaskDefinition,

package/Repository/DatatruckRepository.js

@@ -196,7 +196,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             if (entry.dirent.isDirectory() &&
                 !(await (0, fs_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))))
                 return false;
-            let packIndex = configPacks.findIndex((pack) => (0, string_1.checkPath)(entry.path, pack.include, pack.exclude));
+            let packIndex = configPacks.findIndex((pack) => (0, string_1.match)(entry.path, pack.include, pack.exclude));
             if (packIndex === -1)
                 packIndex = defaultsPackIndex;
             const pack = packs[packIndex];

package/Task/MysqlDumpTask.d.ts

@@ -1,17 +1,15 @@
-import { SqlDumpTaskAbstract, SqlDumpTaskConfigType, TargetDatabaseType } from "./SqlDumpTaskAbstract";
-import { JSONSchema7 } from "json-schema";
+import { SqlDumpTaskConfigType } from "./SqlDumpTaskAbstract";
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
 export declare const mysqlDumpTaskName = "mysql-dump";
-export type MysqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
-export declare const mysqlDumpTaskDefinition: JSONSchema7;
-export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConfigType> {
-    buildConnectionArgs(database?: string): Promise<string[]>;
-    onDatabaseIsEmpty(name: string): Promise<boolean>;
-    onCreateDatabase(database: TargetDatabaseType): Promise<void>;
-    onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;
-    onFetchTableNames(database: string): Promise<string[]>;
-    onExportTables(tableNames: string[], output: string, onProgress: (progress: {
-        totalBytes: number;
-    }) => void): Promise<void>;
-    onExportStoredPrograms(output: string): Promise<void>;
-    onImport(path: string, database: string): Promise<void>;
+export type MysqlDumpTaskConfigType = {
+    /**
+     * @default "sql"
+     */
+    dataFormat?: "csv" | "sql";
+    csvSharedPath?: string;
+} & SqlDumpTaskConfigType;
+export declare const mysqlDumpTaskDefinition: import("json-schema").JSONSchema7;
+export declare class MysqlDumpTask extends TaskAbstract<MysqlDumpTaskConfigType> {
+    onBackup(data: BackupDataType): Promise<void>;
+    onRestore(data: RestoreDataType): Promise<void>;
 }

package/Task/MysqlDumpTask.js

@@ -2,165 +2,226 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MysqlDumpTask = exports.mysqlDumpTaskDefinition = exports.mysqlDumpTaskName = void 0;
 const AppError_1 = require("../Error/AppError");
-const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_1 = require("../utils/cli");
+const config_1 = require("../utils/datatruck/config");
 const fs_1 = require("../utils/fs");
-const process_1 = require("../utils/process");
+const math_1 = require("../utils/math");
+const mysql_1 = require("../utils/mysql");
+const string_1 = require("../utils/string");
 const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
-const fs_2 = require("fs");
-const fs_3 = require("fs");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
 exports.mysqlDumpTaskName = "mysql-dump";
-exports.mysqlDumpTaskDefinition = {
-    allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
+exports.mysqlDumpTaskDefinition = (0, SqlDumpTaskAbstract_1.sqlDumpTaskDefinition)({
+    dataFormat: { enum: ["csv", "sql"] },
+    csvSharedPath: { type: "string" },
+});
+const suffix = {
+    database: ".database.sql",
+    stored: ".stored-programs.sql",
+    table: ".table.sql",
+    tableData: ".table-data.csv",
+    tableSchema: ".table-schema.sql",
 };
-class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
-    async buildConnectionArgs(database) {
-        const password = await this.fetchPassword();
-        return [
-            `--host=${this.config.hostname}`,
-            ...(this.config.port ? [`--port=${this.config.port}`] : []),
-            `--user=${this.config.username}`,
-            `--password=${password ?? ""}`,
-            ...(database ? [database] : []),
-        ];
-    }
-    async onDatabaseIsEmpty(name) {
-        const [total] = await this.fetchValues(`
-      SELECT
-        COUNT(*) AS total 
-      FROM
-        information_schema.tables
-      WHERE
-        table_schema = '${name}'
-    `);
-        return Number(total) ? false : true;
-    }
-    async onCreateDatabase(database) {
-        const query = `
-      CREATE DATABASE IF NOT EXISTS \`${database.name}\`
-      CHARACTER SET ${database.charset ?? "utf8"}
-      COLLATE ${database.charset ?? "utf8_general_ci"}
-    `;
-        await this.onExecQuery(query);
-    }
-    async onExecQuery(query) {
-        return await (0, process_1.exec)("mysql", [
-            ...(await this.buildConnectionArgs()),
-            "-e",
-            query.replace(/\s{1,}/g, " "),
-            "-N",
-        ], undefined, {
-            log: this.verbose,
-            stderr: {
-                toExitCode: true,
-            },
-            stdout: {
-                save: true,
-            },
+class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
+    async onBackup(data) {
+        const sql = (0, mysql_1.createMysqlCli)({
+            ...this.config,
+            verbose: data.options.verbose,
         });
+        const tableNames = await sql.fetchTableNames(this.config.database, this.config.includeTables, this.config.excludeTables);
+        const outputPath = data.package.path;
+        (0, assert_1.ok)(typeof outputPath === "string");
+        const dataFormat = this.config.dataFormat ?? "sql";
+        await (0, promises_1.mkdir)(outputPath, { recursive: true });
+        const sharedDir = dataFormat === "csv"
+            ? await sql.initSharedDir(this.config.csvSharedPath)
+            : undefined;
+        if (this.config.oneFileByTable || sharedDir) {
+            let current = 0;
+            for (const tableName of tableNames) {
+                await data.onProgress({
+                    relative: {
+                        description: "Exporting",
+                        payload: tableName,
+                    },
+                    absolute: {
+                        total: tableNames.length,
+                        current,
+                        percent: (0, math_1.progressPercent)(tableNames.length, current),
+                    },
+                });
+                if (sharedDir) {
+                    const tableSharedPath = (0, path_1.join)(sharedDir, `tmp-dtt-backup-${data.snapshot.id.slice(0, 8)}-${tableName}`);
+                    if (data.options.verbose) {
+                        (0, cli_1.logExec)("mkdir", ["-p", tableSharedPath]);
+                        (0, cli_1.logExec)("chmod", ["777", tableSharedPath]);
+                    }
+                    await (0, promises_1.mkdir)(tableSharedPath, { recursive: true });
+                    await (0, promises_1.chmod)(tableSharedPath, 0o777);
+                    try {
+                        await sql.csvDump({
+                            sharedPath: tableSharedPath,
+                            items: [tableName],
+                            database: this.config.database,
+                        });
+                        const files = await (0, promises_1.readdir)(tableSharedPath);
+                        const schemaFile = `${tableName}.sql`;
+                        const dataFile = `${tableName}.txt`;
+                        const successCsvDump = files.length === 2 &&
+                            files.every((file) => file === schemaFile || file === dataFile);
+                        if (!successCsvDump)
+                            throw new AppError_1.AppError(`Invalid csv dump files: ${files.join(", ")}`);
+                        await (0, promises_1.rename)((0, path_1.join)(tableSharedPath, schemaFile), (0, path_1.join)(outputPath, `${tableName}${suffix.tableSchema}`));
+                        await (0, promises_1.rename)((0, path_1.join)(tableSharedPath, dataFile), (0, path_1.join)(outputPath, `${tableName}${suffix.tableData}`));
+                    }
+                    finally {
+                        await (0, promises_1.rm)(tableSharedPath, { recursive: true });
+                    }
+                }
+                else {
+                    const outPath = (0, path_1.join)(outputPath, `${tableName}${suffix.table}`);
+                    await sql.dump({
+                        output: outPath,
+                        items: [tableName],
+                        database: this.config.database,
+                        onProgress(progress) {
+                            data.onProgress({
+                                relative: {
+                                    description: "Exporting",
+                                    payload: tableName,
+                                    current: progress.totalBytes,
+                                    format: "size",
+                                },
+                                absolute: {
+                                    total: tableNames.length,
+                                    current,
+                                    percent: (0, math_1.progressPercent)(tableNames.length, current),
+                                },
+                            });
+                        },
+                    });
+                }
+                current++;
+            }
+        }
+        else {
+            await data.onProgress({
+                relative: { description: "Exporting" },
+            });
+            await sql.dump({
+                output: (0, path_1.join)(outputPath, `${this.config.database}${suffix.database}`),
+                items: tableNames,
+                database: this.config.database,
+                onProgress: (progress) => data.onProgress({
+                    absolute: {
+                        description: "Exporting in single file",
+                        current: progress.totalBytes,
+                        format: "size",
+                    },
+                }),
+            });
+        }
+        if (this.config.storedPrograms ?? true) {
+            await data.onProgress({
+                relative: { description: "Exporting stored programs" },
+            });
+            await sql.dump({
+                database: this.config.database,
+                output: (0, path_1.join)(outputPath, `${this.config.database}${suffix.stored}`),
+                onlyStoredPrograms: true,
+            });
+        }
     }
-    async onFetchTableNames(database) {
-        return await this.fetchValues(`
-      SELECT
-        table_name 
-      FROM
-        information_schema.tables
-      WHERE
-        table_schema = '${database}'
-      ORDER BY
-        table_name
-	`);
-    }
-    async onExportTables(tableNames, output, onProgress) {
-        const stream = (0, fs_2.createWriteStream)(output);
-        await Promise.all([
-            new Promise((resolve, reject) => {
-                stream.on("close", resolve);
-                stream.on("error", reject);
-            }),
-            await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs(this.config.database)),
-                "--lock-tables=false",
-                "--skip-add-drop-table=false",
-                ...tableNames,
-            ], null, {
-                pipe: {
-                    stream,
-                    onWriteProgress: onProgress,
-                },
-                log: {
-                    exec: this.verbose,
-                    stderr: this.verbose,
-                    allToStderr: true,
-                },
-                stderr: {
-                    toExitCode: true,
-                },
-            }),
-        ]);
-        const headerContents = await (0, fs_1.readPartialFile)(output, [0, 100]);
-        const footerContents = await (0, fs_1.readPartialFile)(output, [-100]);
-        const successHeader = headerContents.split(/\r?\n/).some((line) => {
-            const firstLine = line.trim().toLowerCase();
-            return (firstLine.startsWith("-- mysql dump") ||
-                firstLine.startsWith("-- mariadb dump"));
+    async onRestore(data) {
+        const sql = (0, mysql_1.createMysqlCli)({
+            ...this.config,
+            verbose: data.options.verbose,
         });
-        if (!successHeader)
-            throw new AppError_1.AppError("No start line found");
-        const successFooter = footerContents
-            .split(/\r?\n/)
-            .some((line) => line.trim().toLowerCase().startsWith("-- dump completed"));
-        if (!successFooter)
-            throw new AppError_1.AppError("No end line found (incomplete backup)");
-    }
-    async onExportStoredPrograms(output) {
-        const stream = (0, fs_2.createWriteStream)(output);
-        await Promise.all([
-            new Promise((resolve, reject) => {
-                stream.on("close", resolve);
-                stream.on("error", reject);
-            }),
-            await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs(this.config.database)),
-                "--lock-tables=false",
-                "--routines",
-                "--events",
-                "--skip-triggers",
-                "--no-create-info",
-                "--no-data",
-                "--no-create-db",
-                "--skip-opt",
-            ], null, {
-                pipe: { stream: stream },
-                log: {
-                    exec: this.verbose,
-                    stderr: this.verbose,
-                    allToStderr: true,
+        const restorePath = data.package.restorePath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        const params = {
+            packageName: data.package.name,
+            snapshotId: data.options.snapshotId,
+            snapshotDate: data.snapshot.date,
+            action: "restore",
+            database: undefined,
+        };
+        const database = {
+            name: (0, config_1.resolveDatabaseName)(this.config.database, params),
+        };
+        if (this.config.targetDatabase)
+            database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
+                ...params,
+                database: database.name,
+            });
+        const suffixes = Object.values(suffix);
+        const files = (await (0, fs_1.readDir)(restorePath)).filter((f) => (0, string_1.endsWith)(f, suffixes));
+        // Database check
+        if (files.some((f) => f.endsWith(suffix.database)) &&
+            !(await sql.isDatabaseEmpty(database.name)))
+            throw new AppError_1.AppError(`Target database is not empty: ${database.name}`);
+        // Table check
+        const restoreTables = [
+            ...new Set(...files
+                .filter((f) => (0, string_1.endsWith)(f, [suffix.table, suffix.tableSchema, suffix.tableData]))
+                .map((f) => f.split(".")[0])),
+        ];
+        const serverTables = await sql.fetchTableNames(database.name);
+        const errorTables = restoreTables.filter((v) => serverTables.includes(v));
+        if (errorTables.length)
+            throw new AppError_1.AppError(`Target table already exists: ${errorTables.join(", ")}`);
+        // Data check
+        const dataFiles = files.filter((f) => f.endsWith(suffix.tableData));
+        const sharedDir = dataFiles.length
+            ? await sql.initSharedDir(this.config.csvSharedPath)
+            : undefined;
+        await sql.createDatabase(database);
+        if (data.options.verbose)
+            (0, cli_1.logExec)("readdir", [restorePath]);
+        let current = 0;
+        for (const file of files.filter((f) => !f.endsWith(suffix.tableData))) {
+            const path = (0, path_1.join)(restorePath, file);
+            data.onProgress({
+                relative: {
+                    description: "Importing",
+                    payload: file,
                 },
-                stderr: {
-                    toExitCode: true,
+                absolute: {
+                    total: files.length,
+                    current: current,
+                    percent: (0, math_1.progressPercent)(files.length, current),
                 },
-            }),
-        ]);
-    }
-    async onImport(path, database) {
-        await (0, process_1.exec)("mysql", await this.buildConnectionArgs(database), null, {
-            pipe: {
-                stream: (0, fs_3.createReadStream)(path),
-                onReadProgress: (data) => {
-                    if (this.verbose)
-                        (0, process_1.logExecStdout)({
-                            data: JSON.stringify(data),
-                            colorize: true,
-                            stderr: true,
-                            lineSalt: true,
-                        });
+            });
+            await sql.importFile(path, database.name);
+            current++;
+        }
+        for (const file of dataFiles) {
+            const filePath = (0, path_1.join)(restorePath, file);
+            const tableName = file.slice(0, suffix.tableData.length * -1);
+            data.onProgress({
+                relative: {
+                    description: "Importing",
+                    payload: file,
                 },
-            },
-            log: this.verbose,
-            stderr: {
-                toExitCode: true,
-            },
-        });
+                absolute: {
+                    total: files.length,
+                    current: current,
+                    percent: (0, math_1.progressPercent)(files.length, current),
+                },
+            });
+            const sharedFilePath = (0, path_1.join)(sharedDir, `tmp-dtt-restore-${data.snapshot.id.slice(0, 8)}-${tableName}.data.csv`);
+            try {
+                await (0, promises_1.rename)(filePath, sharedFilePath);
+                await sql.importCsvFile(sharedFilePath, database.name, tableName);
+            }
+            finally {
+                await (0, promises_1.rm)(sharedFilePath);
+            }
+            current++;
+        }
     }
 }
 exports.MysqlDumpTask = MysqlDumpTask;

package/Task/SqlDumpTaskAbstract.d.ts

@@ -1,6 +1,6 @@
 import { exec } from "../utils/process";
 import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
-import { JSONSchema7 } from "json-schema";
+import { JSONSchema7, JSONSchema7Definition } from "json-schema";
 export type TargetDatabaseType = {
     name: string;
     charset?: string;
@@ -14,13 +14,16 @@ export type SqlDumpTaskConfigType = {
     port?: number;
     database: string;
     username: string;
+    /**
+     * @default true
+     */
     storedPrograms?: boolean;
     targetDatabase?: TargetDatabaseType;
     includeTables?: string[];
     excludeTables?: string[];
     oneFileByTable?: boolean;
 };
-export declare const sqlDumpTaskDefinition: JSONSchema7;
+export declare const sqlDumpTaskDefinition: (props?: Record<string, JSONSchema7Definition>) => JSONSchema7;
 export declare abstract class SqlDumpTaskAbstract<TConfig extends SqlDumpTaskConfigType> extends TaskAbstract<TConfig> {
     protected verbose?: boolean;
     fetchPassword(): Promise<string | null>;

package/Task/SqlDumpTaskAbstract.js

@@ -12,11 +12,12 @@ const assert_1 = require("assert");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
-exports.sqlDumpTaskDefinition = {
+const sqlDumpTaskDefinition = (props = {}) => ({
     type: "object",
     required: ["password", "hostname", "username", "database"],
     additionalProperties: false,
     properties: {
+        ...props,
         password: {
             anyOf: [
                 {
@@ -50,7 +51,8 @@ exports.sqlDumpTaskDefinition = {
         excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         oneFileByTable: { type: "boolean" },
     },
-};
+});
+exports.sqlDumpTaskDefinition = sqlDumpTaskDefinition;
 function serializeSqlFile(input) {
     if (input.database && input.table) {
         return `${input.database}.${input.table}.table.sql`;

package/config.schema.json

@@ -880,11 +880,85 @@
       }
     },
     "mysql-dump-task": {
-      "allOf": [
-        {
-          "$ref": "#/definitions/sqldump-task"
+      "type": "object",
+      "required": [
+        "password",
+        "hostname",
+        "username",
+        "database"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "dataFormat": {
+          "enum": [
+            "csv",
+            "sql"
+          ]
+        },
+        "csvSharedPath": {
+          "type": "string"
+        },
+        "password": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "path"
+              ],
+              "properties": {
+                "path": {
+                  "type": "string"
+                }
+              }
+            }
+          ]
+        },
+        "hostname": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "username": {
+          "type": "string"
+        },
+        "database": {
+          "type": "string"
+        },
+        "targetDatabase": {
+          "type": "object",
+          "required": [
+            "name"
+          ],
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "charset": {
+              "type": "string"
+            },
+            "collate": {
+              "type": "string"
+            }
+          }
+        },
+        "storedPrograms": {
+          "type": "boolean"
+        },
+        "includeTables": {
+          "$ref": "#/definitions/stringlist-util"
+        },
+        "excludeTables": {
+          "$ref": "#/definitions/stringlist-util"
+        },
+        "oneFileByTable": {
+          "type": "boolean"
         }
-      ]
+      }
     },
     "postgresql-dump-task": {
       "allOf": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.21.1",
+  "version": "0.22.1",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",

package/utils/cli.js

@@ -65,7 +65,9 @@ function logExec(command, argv = [], env, logToStderr) {
             .join(" ")
         : "";
     const text = `+ ${envText ? envText + " " : ""}${chalk_1.default.yellow(`${command} ${argv.join(" ")}`)}`;
-    logToStderr ? process.stderr.write(`${text}\n`) : console.info(text);
+    logToStderr /* && process.env.VITEST !== "true"*/
+        ? process.stderr.write(`${text}\n`)
+        : console.info(text);
 }
 exports.logExec = logExec;
 function resultColumn(error, state) {

package/utils/datatruck/config.d.ts

@@ -15,7 +15,7 @@ type ResolvePackagePathParamsType = ResolvePackageParamsType & {
     path: string | undefined;
 };
 export declare function resolvePackagePath(value: string, params: ResolvePackagePathParamsType): string;
-type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
+export type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
     packageName: string;
     database: string | undefined;
 };

package/utils/fs.d.ts

@@ -119,4 +119,5 @@ export declare function createWriteStreamPool(options: {
     end(): Promise<void>;
 };
 export declare function countFileLines(path: string): Promise<number>;
+export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
 export {};

package/utils/fs.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const math_1 = require("./math");
 const path_1 = require("./path");
@@ -43,9 +43,8 @@ function isLocalDir(path) {
 }
 exports.isLocalDir = isLocalDir;
 async function mkdirIfNotExists(path) {
-    await (0, promises_1.mkdir)(path, {
-        recursive: true,
-    });
+    if (!(await existsDir(path)))
+        await (0, promises_1.mkdir)(path, { recursive: true });
     return path;
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
@@ -536,3 +535,12 @@ function countFileLines(path) {
     });
 }
 exports.countFileLines = countFileLines;
+async function fetchData(input, onPath) {
+    if (typeof input === "string")
+        return input;
+    const path = onPath?.(input);
+    if (typeof path === "string")
+        return (await (0, promises_1.readFile)(path)).toString();
+    return null;
+}
+exports.fetchData = fetchData;

package/utils/mysql.d.ts

@@ -0,0 +1,39 @@
+export type MysqlCliOptions = {
+    password: string | {
+        path: string;
+    };
+    hostname: string;
+    port?: number;
+    username: string;
+    verbose?: boolean;
+};
+export declare function createMysqlCli(options: MysqlCliOptions): {
+    options: MysqlCliOptions;
+    initSharedDir: (sharedDir?: string) => Promise<string>;
+    args: () => Promise<string[]>;
+    run: (query: string, database?: string, extra?: string[]) => Promise<import("./process").ExecResultType>;
+    fetchAll: (query: string, database?: string) => Promise<string[][]>;
+    dump: (input: {
+        output: string;
+        database: string;
+        items?: string[] | undefined;
+        onlyStoredPrograms?: boolean | undefined;
+        onProgress?: ((data: {
+            totalBytes: number;
+        }) => void) | undefined;
+    }) => Promise<[void, import("./process").ExecResultType]>;
+    fetchTableNames: (database: string, include?: string[], exclude?: string[]) => Promise<string[]>;
+    importFile: (path: string, database: string) => Promise<import("./process").ExecResultType>;
+    isDatabaseEmpty: (database: string) => Promise<boolean>;
+    createDatabase: (database: {
+        name: string;
+        charset?: string;
+    }) => Promise<void>;
+    csvDump: (input: {
+        database: string;
+        sharedPath: string;
+        items?: string[];
+    }) => Promise<void>;
+    importCsvFile: (path: string, database: string, table: string) => Promise<import("./process").ExecResultType>;
+    fetchVariable: (name: string) => Promise<string | undefined>;
+};

package/utils/mysql.js

@@ -0,0 +1,224 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMysqlCli = void 0;
+const AppError_1 = require("../Error/AppError");
+const fs_1 = require("./fs");
+const process_1 = require("./process");
+const string_1 = require("./string");
+const crypto_1 = require("crypto");
+const fs_2 = require("fs");
+const promises_1 = require("fs/promises");
+const os_1 = require("os");
+const path_1 = require("path");
+function createMysqlCli(options) {
+    let defaultsFilePath;
+    async function getDefaultsFilePath() {
+        if (defaultsFilePath)
+            return defaultsFilePath;
+        const dir = await (0, fs_1.mkTmpDir)("mysql-cli");
+        const password = await (0, fs_1.fetchData)(options.password, (p) => p.path);
+        const data = [
+            `[client]`,
+            `host = "${options.hostname}"`,
+            ...(options.port ? [`port = "${options.port}"`] : []),
+            `user = "${options.username}"`,
+            `password = "${password}"`,
+        ];
+        console.log(data.join("\n"));
+        await (0, promises_1.writeFile)((defaultsFilePath = (0, path_1.join)(dir, "mysql.conf")), data.join("\n"));
+        return defaultsFilePath;
+    }
+    async function args() {
+        return [`--defaults-file=${await getDefaultsFilePath()}`];
+    }
+    async function run(query, database, extra = []) {
+        return await (0, process_1.exec)("mysql", [
+            ...(await args()),
+            ...(database ? [database] : []),
+            ...(extra || []),
+            "-e",
+            query.replace(/\s{1,}/g, " "),
+            "-N",
+            "--silent",
+        ], undefined, {
+            log: options.verbose,
+            stderr: { toExitCode: true },
+            stdout: { save: true },
+        });
+    }
+    async function fetchAll(query, database) {
+        return (0, string_1.splitLines)((await run(query, database)).stdout).map((line) => line.split("\t"));
+    }
+    async function fetchTableNames(database, include, exclude) {
+        return (await fetchAll(`
+      SELECT
+        table_name 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${database}'
+      ORDER BY
+        table_name
+  `))
+            .map((r) => r[0])
+            .filter((0, string_1.createMatchFilter)(include, exclude));
+    }
+    async function dump(input) {
+        const stream = (0, fs_2.createWriteStream)(input.output);
+        return await Promise.all([
+            new Promise((resolve, reject) => {
+                stream.on("close", resolve);
+                stream.on("error", reject);
+            }),
+            await (0, process_1.exec)("mysqldump", [
+                ...(await args()),
+                input.database,
+                "--lock-tables=false",
+                "--skip-add-drop-table=false",
+                ...(input.onlyStoredPrograms
+                    ? [
+                        "--routines",
+                        "--events",
+                        "--skip-triggers",
+                        "--no-create-info",
+                        "--no-data",
+                        "--no-create-db",
+                        "--skip-opt",
+                    ]
+                    : []),
+                ...(input.items || []),
+            ], null, {
+                stderr: { toExitCode: true },
+                pipe: {
+                    stream,
+                    onWriteProgress: input.onProgress,
+                },
+                log: {
+                    exec: options.verbose,
+                    stderr: options.verbose,
+                    allToStderr: true,
+                },
+            }),
+        ]);
+    }
+    async function csvDump(input) {
+        await (0, process_1.exec)("mysqldump", [
+            ...(await args()),
+            input.database,
+            "--lock-tables=false",
+            "--skip-add-drop-table=false",
+            "-T",
+            input.sharedPath,
+            ...(input.items || []),
+        ], null, {
+            stderr: { toExitCode: true },
+            log: {
+                exec: options.verbose,
+                stderr: options.verbose,
+                allToStderr: true,
+            },
+        });
+    }
+    async function importFile(path, database) {
+        return await (0, process_1.exec)("mysql", [
+            ...(await args()),
+            `--init-command=SET ${[
+                "autocommit=0",
+                "unique_checks=0",
+                "foreign_key_checks=0",
+            ].join(",")};`,
+            database,
+        ], null, {
+            pipe: {
+                stream: (0, fs_2.createReadStream)(path),
+                onReadProgress: (data) => {
+                    if (options.verbose)
+                        (0, process_1.logExecStdout)({
+                            data: JSON.stringify(data),
+                            colorize: true,
+                            stderr: true,
+                            lineSalt: true,
+                        });
+                },
+            },
+            stderr: { toExitCode: true },
+            log: options.verbose,
+        });
+    }
+    async function importCsvFile(path, database, table) {
+        return run(`
+      LOAD DATA LOCAL INFILE '${path.replaceAll("\\", "/")}'
+      INTO TABLE ${table}
+      FIELDS TERMINATED BY ','
+      ENCLOSED BY '"'
+      LINES TERMINATED BY '\\n'`, database, ["--local-infile"]);
+    }
+    async function isDatabaseEmpty(database) {
+        const [total] = await fetchAll(`
+      SELECT
+        COUNT(*) AS total 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${database}'
+    `);
+        return Number(total) ? false : true;
+    }
+    async function createDatabase(database) {
+        await run(`
+      CREATE DATABASE IF NOT EXISTS \`${database.name}\`
+      CHARACTER SET ${database.charset ?? "utf8"}
+      COLLATE ${database.charset ?? "utf8_general_ci"}
+    `);
+    }
+    async function fetchVariable(name) {
+        const stdout = (0, string_1.undefIfEmpty)((await run(`SHOW VARIABLES LIKE "${name}"`)).stdout.trim());
+        return stdout ? (0, string_1.undefIfEmpty)(stdout.slice(name.length).trim()) : undefined;
+    }
+    async function initSharedDir(sharedDir) {
+        const secure_file_priv = await fetchVariable("secure_file_priv");
+        if (secure_file_priv?.toUpperCase() === "NULL")
+            throw new AppError_1.AppError("'secure_file_priv' is null in MySQL Server");
+        const dir = sharedDir ??
+            secure_file_priv ??
+            (await fetchVariable("tmpdir")) ??
+            (0, os_1.tmpdir)();
+        await checkSharedDir(dir);
+        return dir;
+    }
+    async function checkSharedDir(dir) {
+        const id = (0, crypto_1.randomBytes)(8).toString("hex");
+        const outFile = (0, path_1.join)(dir, `dtt_test_${id}`);
+        const outFileVar = JSON.stringify(outFile.replaceAll("\\", "/"));
+        try {
+            await (0, fs_1.mkdirIfNotExists)(dir);
+            await (0, promises_1.chmod)(dir, 0o777);
+            await run(`SELECT 1 INTO OUTFILE ${outFileVar}`);
+            const exists = await (0, fs_1.existsFile)(outFile);
+            if (!exists)
+                throw new AppError_1.AppError(`MySQL shared dir is not reached: ${dir}`);
+        }
+        finally {
+            try {
+                await (0, promises_1.rm)(outFile);
+            }
+            catch (e) { }
+        }
+    }
+    return {
+        options,
+        initSharedDir,
+        args,
+        run,
+        fetchAll,
+        dump,
+        fetchTableNames,
+        importFile,
+        isDatabaseEmpty,
+        createDatabase,
+        csvDump,
+        importCsvFile,
+        fetchVariable,
+    };
+}
+exports.createMysqlCli = createMysqlCli;

package/utils/string.d.ts

@@ -13,6 +13,10 @@ export type UriType = {
 export declare function formatUri(input: UriType, hidePassword?: boolean): string;
 export declare function formatSeconds(seconds: number): string;
 export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
-export declare function checkPath(path: string, include: string[], exclude?: string[]): boolean;
+export declare function match(path: string, include?: string[], exclude?: string[]): boolean;
+export declare function endsWith(input: string, patterns: string[]): boolean;
+export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;
+export declare function splitLines(input: string, satinize?: boolean): string[];
+export declare function undefIfEmpty(input: string): string | undefined;

package/utils/string.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
+exports.undefIfEmpty = exports.splitLines = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -91,13 +91,19 @@ function makePathPatterns(values) {
     });
 }
 exports.makePathPatterns = makePathPatterns;
-function checkPath(path, include, exclude) {
-    return ((0, micromatch_1.isMatch)(path, include, {
-        dot: true,
-    }) &&
+function match(path, include, exclude) {
+    return ((!include || (0, micromatch_1.isMatch)(path, include, { dot: true })) &&
         (!exclude || !(0, micromatch_1.isMatch)(path, exclude, { dot: true })));
 }
-exports.checkPath = checkPath;
+exports.match = match;
+function endsWith(input, patterns) {
+    return patterns.some((pattern) => input.endsWith(pattern));
+}
+exports.endsWith = endsWith;
+function createMatchFilter(include, exclude) {
+    return (input) => match(input, include, exclude);
+}
+exports.createMatchFilter = createMatchFilter;
 function checkMatch(subject, patterns) {
     if (!subject?.length)
         subject = "<empty>";
@@ -114,3 +120,19 @@ function formatDateTime(datetime) {
     return result;
 }
 exports.formatDateTime = formatDateTime;
+function splitLines(input, satinize = true) {
+    const lines = input.split(/\r?\n/);
+    return satinize
+        ? input.split(/\r?\n/).reduce((result, value) => {
+            value = value.trim();
+            if (value.length)
+                result.push(value);
+            return result;
+        }, [])
+        : lines;
+}
+exports.splitLines = splitLines;
+function undefIfEmpty(input) {
+    return input.length ? input : undefined;
+}
+exports.undefIfEmpty = undefIfEmpty;