@datatruck/cli 0.21.0 → 0.22.0

package/Task/MysqlDumpTask.js

@@ -2,165 +2,223 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MysqlDumpTask = exports.mysqlDumpTaskDefinition = exports.mysqlDumpTaskName = void 0;
 const AppError_1 = require("../Error/AppError");
-const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_1 = require("../utils/cli");
+const config_1 = require("../utils/datatruck/config");
 const fs_1 = require("../utils/fs");
-const process_1 = require("../utils/process");
+const math_1 = require("../utils/math");
+const mysql_1 = require("../utils/mysql");
+const string_1 = require("../utils/string");
 const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
-const fs_2 = require("fs");
-const fs_3 = require("fs");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
 exports.mysqlDumpTaskName = "mysql-dump";
-exports.mysqlDumpTaskDefinition = {
-    allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
+exports.mysqlDumpTaskDefinition = (0, SqlDumpTaskAbstract_1.sqlDumpTaskDefinition)({
+    dataFormat: { enum: ["csv", "sql"] },
+    csvSharedPath: { type: "string" },
+});
+const suffix = {
+    database: ".database.sql",
+    stored: ".stored-programs.sql",
+    table: ".table.sql",
+    tableData: ".table-data.csv",
+    tableSchema: ".table-schema.sql",
 };
-class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
-    async buildConnectionArgs(database = true) {
-        const password = await this.fetchPassword();
-        return [
-            `--host=${this.config.hostname}`,
-            ...(this.config.port ? [`--port=${this.config.port}`] : []),
-            `--user=${this.config.username}`,
-            `--password=${password ?? ""}`,
-            ...(database && this.config.database ? [this.config.database] : []),
-        ];
-    }
-    async onDatabaseIsEmpty(name) {
-        const [total] = await this.fetchValues(`
-      SELECT
-        COUNT(*) AS total 
-      FROM
-        information_schema.tables
-      WHERE
-        table_schema = '${name}'
-    `);
-        return Number(total) ? false : true;
-    }
-    async onCreateDatabase(database) {
-        const query = `
-      CREATE DATABASE IF NOT EXISTS \`${database.name}\`
-      CHARACTER SET ${database.charset ?? "utf8"}
-      COLLATE ${database.charset ?? "utf8_general_ci"}
-    `;
-        await this.onExecQuery(query);
-    }
-    async onExecQuery(query) {
-        return await (0, process_1.exec)("mysql", [
-            ...(await this.buildConnectionArgs()),
-            "-e",
-            query.replace(/\s{1,}/g, " "),
-            "-N",
-        ], undefined, {
-            log: this.verbose,
-            stderr: {
-                toExitCode: true,
-            },
-            stdout: {
-                save: true,
-            },
+class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
+    async onBackup(data) {
+        const sql = (0, mysql_1.createMysqlCli)({
+            ...this.config,
+            verbose: data.options.verbose,
         });
+        const tableNames = await sql.fetchTableNames(this.config.database, this.config.includeTables, this.config.excludeTables);
+        const outputPath = data.package.path;
+        (0, assert_1.ok)(typeof outputPath === "string");
+        const dataFormat = this.config.dataFormat ?? "sql";
+        await (0, promises_1.mkdir)(outputPath, { recursive: true });
+        const sharedDir = dataFormat === "csv"
+            ? await sql.initSharedDir(this.config.csvSharedPath)
+            : undefined;
+        if (this.config.oneFileByTable || sharedDir) {
+            let current = 0;
+            for (const tableName of tableNames) {
+                await data.onProgress({
+                    relative: {
+                        description: "Exporting",
+                        payload: tableName,
+                    },
+                    absolute: {
+                        total: tableNames.length,
+                        current,
+                        percent: (0, math_1.progressPercent)(tableNames.length, current),
+                    },
+                });
+                if (sharedDir) {
+                    const tableSharedPath = (0, path_1.join)(sharedDir, `tmp-dtt-backup-${data.snapshot.id.slice(0, 8)}-${tableName}`);
+                    if (data.options.verbose)
+                        (0, cli_1.logExec)("mkdir", [tableSharedPath]);
+                    await (0, promises_1.mkdir)(tableSharedPath, { recursive: true });
+                    try {
+                        await sql.csvDump({
+                            sharedPath: tableSharedPath,
+                            items: [tableName],
+                            database: this.config.database,
+                        });
+                        const files = await (0, promises_1.readdir)(tableSharedPath);
+                        const schemaFile = `${tableName}.sql`;
+                        const dataFile = `${tableName}.txt`;
+                        const successCsvDump = files.length === 2 &&
+                            files.every((file) => file === schemaFile || file === dataFile);
+                        if (!successCsvDump)
+                            throw new AppError_1.AppError(`Invalid csv dump files: ${files.join(", ")}`);
+                        await (0, promises_1.rename)((0, path_1.join)(tableSharedPath, schemaFile), (0, path_1.join)(outputPath, `${tableName}${suffix.tableSchema}`));
+                        await (0, promises_1.rename)((0, path_1.join)(tableSharedPath, dataFile), (0, path_1.join)(outputPath, `${tableName}${suffix.tableData}`));
+                    }
+                    finally {
+                        await (0, promises_1.rm)(tableSharedPath, { recursive: true });
+                    }
+                }
+                else {
+                    const outPath = (0, path_1.join)(outputPath, `${tableName}${suffix.table}`);
+                    await sql.dump({
+                        output: outPath,
+                        items: [tableName],
+                        database: this.config.database,
+                        onProgress(progress) {
+                            data.onProgress({
+                                relative: {
+                                    description: "Exporting",
+                                    payload: tableName,
+                                    current: progress.totalBytes,
+                                    format: "size",
+                                },
+                                absolute: {
+                                    total: tableNames.length,
+                                    current,
+                                    percent: (0, math_1.progressPercent)(tableNames.length, current),
+                                },
+                            });
+                        },
+                    });
+                }
+                current++;
+            }
+        }
+        else {
+            await data.onProgress({
+                relative: { description: "Exporting" },
+            });
+            await sql.dump({
+                output: (0, path_1.join)(outputPath, `${this.config.database}${suffix.database}`),
+                items: tableNames,
+                database: this.config.database,
+                onProgress: (progress) => data.onProgress({
+                    absolute: {
+                        description: "Exporting in single file",
+                        current: progress.totalBytes,
+                        format: "size",
+                    },
+                }),
+            });
+        }
+        if (this.config.storedPrograms ?? true) {
+            await data.onProgress({
+                relative: { description: "Exporting stored programs" },
+            });
+            await sql.dump({
+                database: this.config.database,
+                output: (0, path_1.join)(outputPath, `${this.config.database}${suffix.stored}`),
+                onlyStoredPrograms: true,
+            });
+        }
     }
-    async onFetchTableNames(database) {
-        return await this.fetchValues(`
-      SELECT
-        table_name 
-      FROM
-        information_schema.tables
-      WHERE
-        table_schema = '${database}'
-      ORDER BY
-        table_name
-	`);
-    }
-    async onExportTables(tableNames, output, onProgress) {
-        const stream = (0, fs_2.createWriteStream)(output);
-        await Promise.all([
-            new Promise((resolve, reject) => {
-                stream.on("close", resolve);
-                stream.on("error", reject);
-            }),
-            await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
-                "--lock-tables=false",
-                "--skip-add-drop-table=false",
-                ...tableNames,
-            ], null, {
-                pipe: {
-                    stream,
-                    onWriteProgress: onProgress,
-                },
-                log: {
-                    exec: this.verbose,
-                    stderr: this.verbose,
-                    allToStderr: true,
-                },
-                stderr: {
-                    toExitCode: true,
-                },
-            }),
-        ]);
-        const headerContents = await (0, fs_1.readPartialFile)(output, [0, 100]);
-        const footerContents = await (0, fs_1.readPartialFile)(output, [-100]);
-        const successHeader = headerContents.split(/\r?\n/).some((line) => {
-            const firstLine = line.trim().toLowerCase();
-            return (firstLine.startsWith("-- mysql dump") ||
-                firstLine.startsWith("-- mariadb dump"));
+    async onRestore(data) {
+        const sql = (0, mysql_1.createMysqlCli)({
+            ...this.config,
+            verbose: data.options.verbose,
         });
-        if (!successHeader)
-            throw new AppError_1.AppError("No start line found");
-        const successFooter = footerContents
-            .split(/\r?\n/)
-            .some((line) => line.trim().toLowerCase().startsWith("-- dump completed"));
-        if (!successFooter)
-            throw new AppError_1.AppError("No end line found (incomplete backup)");
-    }
-    async onExportStoredPrograms(output) {
-        const stream = (0, fs_2.createWriteStream)(output);
-        await Promise.all([
-            new Promise((resolve, reject) => {
-                stream.on("close", resolve);
-                stream.on("error", reject);
-            }),
-            await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
-                "--lock-tables=false",
-                "--routines",
-                "--events",
-                "--skip-triggers",
-                "--no-create-info",
-                "--no-data",
-                "--no-create-db",
-                "--skip-opt",
-            ], null, {
-                pipe: { stream: stream },
-                log: {
-                    exec: this.verbose,
-                    stderr: this.verbose,
-                    allToStderr: true,
+        const restorePath = data.package.restorePath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        const params = {
+            packageName: data.package.name,
+            snapshotId: data.options.snapshotId,
+            snapshotDate: data.snapshot.date,
+            action: "restore",
+            database: undefined,
+        };
+        const database = {
+            name: (0, config_1.resolveDatabaseName)(this.config.database, params),
+        };
+        if (this.config.targetDatabase)
+            database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
+                ...params,
+                database: database.name,
+            });
+        const suffixes = Object.values(suffix);
+        const files = (await (0, fs_1.readDir)(restorePath)).filter((f) => (0, string_1.endsWith)(f, suffixes));
+        // Database check
+        if (files.some((f) => f.endsWith(suffix.database)) &&
+            !(await sql.isDatabaseEmpty(database.name)))
+            throw new AppError_1.AppError(`Target database is not empty: ${database.name}`);
+        // Table check
+        const restoreTables = [
+            ...new Set(...files
+                .filter((f) => (0, string_1.endsWith)(f, [suffix.table, suffix.tableSchema, suffix.tableData]))
+                .map((f) => f.split(".")[0])),
+        ];
+        const serverTables = await sql.fetchTableNames(database.name);
+        const errorTables = restoreTables.filter((v) => serverTables.includes(v));
+        if (errorTables.length)
+            throw new AppError_1.AppError(`Target table already exists: ${errorTables.join(", ")}`);
+        // Data check
+        const dataFiles = files.filter((f) => f.endsWith(suffix.tableData));
+        const sharedDir = dataFiles.length
+            ? await sql.initSharedDir(this.config.csvSharedPath)
+            : undefined;
+        await sql.createDatabase(database);
+        if (data.options.verbose)
+            (0, cli_1.logExec)("readdir", [restorePath]);
+        let current = 0;
+        for (const file of files.filter((f) => !f.endsWith(suffix.tableData))) {
+            const path = (0, path_1.join)(restorePath, file);
+            data.onProgress({
+                relative: {
+                    description: "Importing",
+                    payload: file,
                 },
-                stderr: {
-                    toExitCode: true,
+                absolute: {
+                    total: files.length,
+                    current: current,
+                    percent: (0, math_1.progressPercent)(files.length, current),
                 },
-            }),
-        ]);
-    }
-    async onImport(path, database) {
-        await (0, process_1.exec)("mysql", [...(await this.buildConnectionArgs(false)), database], null, {
-            pipe: {
-                stream: (0, fs_3.createReadStream)(path),
-                onReadProgress: (data) => {
-                    if (this.verbose)
-                        (0, process_1.logExecStdout)({
-                            data: JSON.stringify(data),
-                            colorize: true,
-                            stderr: true,
-                            lineSalt: true,
-                        });
+            });
+            await sql.importFile(path, database.name);
+            current++;
+        }
+        for (const file of dataFiles) {
+            const filePath = (0, path_1.join)(restorePath, file);
+            const tableName = file.slice(0, suffix.tableData.length * -1);
+            data.onProgress({
+                relative: {
+                    description: "Importing",
+                    payload: file,
                 },
-            },
-            log: this.verbose,
-            stderr: {
-                toExitCode: true,
-            },
-        });
+                absolute: {
+                    total: files.length,
+                    current: current,
+                    percent: (0, math_1.progressPercent)(files.length, current),
+                },
+            });
+            const sharedFilePath = (0, path_1.join)(sharedDir, `tmp-dtt-restore-${data.snapshot.id.slice(0, 8)}-${tableName}.data.csv`);
+            try {
+                await (0, promises_1.rename)(filePath, sharedFilePath);
+                await sql.importCsvFile(sharedFilePath, database.name, tableName);
+            }
+            finally {
+                await (0, promises_1.rm)(sharedFilePath);
+            }
+            current++;
+        }
     }
 }
 exports.MysqlDumpTask = MysqlDumpTask;

package/Task/PostgresqlDumpTask.js

@@ -81,7 +81,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             (0, process_1.exec)("pg_dump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 ...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
             ], null, {
                 pipe: { stream, onWriteProgress: onProgress },

package/Task/SqlDumpTaskAbstract.d.ts

@@ -1,6 +1,6 @@
 import { exec } from "../utils/process";
 import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
-import { JSONSchema7 } from "json-schema";
+import { JSONSchema7, JSONSchema7Definition } from "json-schema";
 export type TargetDatabaseType = {
     name: string;
     charset?: string;
@@ -14,13 +14,16 @@ export type SqlDumpTaskConfigType = {
     port?: number;
     database: string;
     username: string;
+    /**
+     * @default true
+     */
     storedPrograms?: boolean;
     targetDatabase?: TargetDatabaseType;
     includeTables?: string[];
     excludeTables?: string[];
     oneFileByTable?: boolean;
 };
-export declare const sqlDumpTaskDefinition: JSONSchema7;
+export declare const sqlDumpTaskDefinition: (props?: Record<string, JSONSchema7Definition>) => JSONSchema7;
 export declare abstract class SqlDumpTaskAbstract<TConfig extends SqlDumpTaskConfigType> extends TaskAbstract<TConfig> {
     protected verbose?: boolean;
     fetchPassword(): Promise<string | null>;

package/Task/SqlDumpTaskAbstract.js

@@ -12,11 +12,12 @@ const assert_1 = require("assert");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
-exports.sqlDumpTaskDefinition = {
+const sqlDumpTaskDefinition = (props = {}) => ({
     type: "object",
     required: ["password", "hostname", "username", "database"],
     additionalProperties: false,
     properties: {
+        ...props,
         password: {
             anyOf: [
                 {
@@ -50,7 +51,8 @@ exports.sqlDumpTaskDefinition = {
         excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
         oneFileByTable: { type: "boolean" },
     },
-};
+});
+exports.sqlDumpTaskDefinition = sqlDumpTaskDefinition;
 function serializeSqlFile(input) {
     if (input.database && input.table) {
         return `${input.database}.${input.table}.table.sql`;

package/config.schema.json

@@ -880,11 +880,85 @@
       }
     },
     "mysql-dump-task": {
-      "allOf": [
-        {
-          "$ref": "#/definitions/sqldump-task"
+      "type": "object",
+      "required": [
+        "password",
+        "hostname",
+        "username",
+        "database"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "dataFormat": {
+          "enum": [
+            "csv",
+            "sql"
+          ]
+        },
+        "csvSharedPath": {
+          "type": "string"
+        },
+        "password": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "path"
+              ],
+              "properties": {
+                "path": {
+                  "type": "string"
+                }
+              }
+            }
+          ]
+        },
+        "hostname": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "username": {
+          "type": "string"
+        },
+        "database": {
+          "type": "string"
+        },
+        "targetDatabase": {
+          "type": "object",
+          "required": [
+            "name"
+          ],
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "charset": {
+              "type": "string"
+            },
+            "collate": {
+              "type": "string"
+            }
+          }
+        },
+        "storedPrograms": {
+          "type": "boolean"
+        },
+        "includeTables": {
+          "$ref": "#/definitions/stringlist-util"
+        },
+        "excludeTables": {
+          "$ref": "#/definitions/stringlist-util"
+        },
+        "oneFileByTable": {
+          "type": "boolean"
         }
-      ]
+      }
     },
     "postgresql-dump-task": {
       "allOf": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",

package/utils/cli.js

@@ -65,7 +65,9 @@ function logExec(command, argv = [], env, logToStderr) {
             .join(" ")
         : "";
     const text = `+ ${envText ? envText + " " : ""}${chalk_1.default.yellow(`${command} ${argv.join(" ")}`)}`;
-    logToStderr ? process.stderr.write(`${text}\n`) : console.info(text);
+    logToStderr /* && process.env.VITEST !== "true"*/
+        ? process.stderr.write(`${text}\n`)
+        : console.info(text);
 }
 exports.logExec = logExec;
 function resultColumn(error, state) {

package/utils/datatruck/config.d.ts

@@ -15,7 +15,7 @@ type ResolvePackagePathParamsType = ResolvePackageParamsType & {
     path: string | undefined;
 };
 export declare function resolvePackagePath(value: string, params: ResolvePackagePathParamsType): string;
-type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
+export type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
     packageName: string;
     database: string | undefined;
 };

package/utils/fs.d.ts

@@ -96,12 +96,13 @@ export declare function createFileScanner(options: {
     disposed: boolean;
     total: number;
     current: number;
-    progress: (description: string, path?: string) => Promise<void>;
+    progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
     end: () => Promise<void>;
     start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
 type StreamItem = {
     key: string;
+    lines: number;
     stream: WriteStream;
     finished: boolean;
     error?: Error;
@@ -113,8 +114,10 @@ export declare function createWriteStreamPool(options: {
 }): {
     pool: Record<string, StreamItem>;
     path(key: string | number): string | undefined;
+    lines(key: string | number): number;
     writeLine(key: string | number, v: string): boolean;
     end(): Promise<void>;
 };
 export declare function countFileLines(path: string): Promise<number>;
+export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
 export {};

package/utils/fs.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const math_1 = require("./math");
 const path_1 = require("./path");
@@ -43,9 +43,8 @@ function isLocalDir(path) {
 }
 exports.isLocalDir = isLocalDir;
 async function mkdirIfNotExists(path) {
-    await (0, promises_1.mkdir)(path, {
-        recursive: true,
-    });
+    if (!(await existsDir(path)))
+        await (0, promises_1.mkdir)(path, { recursive: true });
     return path;
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
@@ -410,10 +409,10 @@ async function createFileScanner(options) {
         disposed: false,
         total: 0,
         current: 0,
-        progress: async (description, path) => {
+        progress: async (description, path, increment = true) => {
             if (object.disposed)
                 return;
-            if (path)
+            if (path && increment)
                 object.current++;
             await options.onProgress({
                 relative: {
@@ -463,6 +462,7 @@ function createWriteStreamPool(options) {
     const create = (key) => {
         const item = {
             key,
+            lines: 0,
             stream: (0, fs_2.createWriteStream)((0, path_2.join)(options.path, options.onStreamPath ? options.onStreamPath(key) : key)),
             finished: false,
         };
@@ -484,16 +484,22 @@ function createWriteStreamPool(options) {
                 throw new Error(`Stream path is not defined: ${key}`);
             return item.stream.path;
         },
+        lines(key) {
+            const item = pool[key];
+            return item?.lines;
+        },
         writeLine(key, v) {
             const item = pool[key] || create(key.toString());
             if (item.finished) {
                 return false;
             }
             else if (item.written) {
+                item.lines++;
                 return item.stream.write(`\n${v}`);
             }
             else {
                 item.written = true;
+                item.lines++;
                 return item.stream.write(`${v}`);
             }
         },
@@ -529,3 +535,12 @@ function countFileLines(path) {
     });
 }
 exports.countFileLines = countFileLines;
+async function fetchData(input, onPath) {
+    if (typeof input === "string")
+        return input;
+    const path = onPath?.(input);
+    if (typeof path === "string")
+        return (await (0, promises_1.readFile)(path)).toString();
+    return null;
+}
+exports.fetchData = fetchData;