@datatruck/cli 0.0.1

package/lib/Task/MysqlDumpTask.js

@@ -0,0 +1,129 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MysqlDumpTask = exports.mysqlDumpTaskDefinition = exports.mysqlDumpTaskName = void 0;
+const AppError_1 = require("../Error/AppError");
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const fs_util_1 = require("../util/fs-util");
+const process_util_1 = require("../util/process-util");
+const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
+const fs_1 = require("fs");
+const fs_2 = require("fs");
+exports.mysqlDumpTaskName = "mysql-dump";
+exports.mysqlDumpTaskDefinition = {
+    allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
+};
+class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
+    async buildConnectionArgs(database = true) {
+        const password = await this.fetchPassword();
+        return [
+            `--host=${this.config.hostname}`,
+            `--user=${this.config.username}`,
+            `--password=${password ?? ""}`,
+            ...(database && this.config.database ? [this.config.database] : []),
+        ];
+    }
+    async onDatabaseIsEmpty(name) {
+        const [total] = await this.fetchValues(`
+      SELECT
+        COUNT(*) AS total 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${name}'
+    `);
+        return Number(total) ? false : true;
+    }
+    async onCreateDatabase(database) {
+        const query = `
+      CREATE DATABASE IF NOT EXISTS \`${database.name}\`
+      CHARACTER SET ${database.charset ?? "utf8"}
+      COLLATE ${database.charset ?? "utf8_general_ci"}
+    `;
+        await this.onExecQuery(query);
+    }
+    async onExecQuery(query) {
+        return await (0, process_util_1.exec)("mysql", [
+            ...(await this.buildConnectionArgs()),
+            "-e",
+            query.replace(/\s{1,}/g, " "),
+            "-N",
+        ], undefined, {
+            log: this.verbose,
+            stderr: {
+                toExitCode: true,
+            },
+            stdout: {
+                save: true,
+            },
+        });
+    }
+    async onFetchTableNames() {
+        return await this.fetchValues(`
+      SELECT
+        table_name 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${this.config.database}'
+	`);
+    }
+    async onExport(tableNames, output) {
+        const stream = (0, fs_1.createWriteStream)(output);
+        await Promise.all([
+            new Promise((resolve, reject) => {
+                stream.on("close", resolve);
+                stream.on("error", reject);
+            }),
+            await (0, process_util_1.exec)("mysqldump", [
+                ...(await this.buildConnectionArgs()),
+                "--lock-tables=false",
+                ...tableNames,
+            ], null, {
+                pipe: { stream: stream },
+                log: {
+                    exec: this.verbose,
+                    stderr: this.verbose,
+                    allToStderr: true,
+                },
+                stderr: {
+                    toExitCode: true,
+                },
+            }),
+        ]);
+        const headerContents = await (0, fs_util_1.readPartialFile)(output, [0, 100]);
+        const footerContents = await (0, fs_util_1.readPartialFile)(output, [-100]);
+        const successHeader = headerContents.split(/\r?\n/).some((line) => {
+            const firstLine = line.trim().toLowerCase();
+            return (firstLine.startsWith("-- mysql dump") ||
+                firstLine.startsWith("-- mariadb dump"));
+        });
+        if (!successHeader)
+            throw new AppError_1.AppError("No start line found");
+        const successFooter = footerContents
+            .split(/\r?\n/)
+            .some((line) => line.trim().toLowerCase().startsWith("-- dump completed"));
+        if (!successFooter)
+            throw new AppError_1.AppError("No end line found (incomplete backup)");
+    }
+    async onImport(path, database) {
+        await (0, process_util_1.exec)("mysql", [...(await this.buildConnectionArgs(false)), database], null, {
+            pipe: {
+                stream: (0, fs_2.createReadStream)(path),
+                onReadProgress: (data) => {
+                    if (this.verbose)
+                        (0, process_util_1.logExecStdout)({
+                            data: JSON.stringify(data),
+                            colorize: true,
+                            stderr: true,
+                            lineSalt: true,
+                        });
+                },
+            },
+            log: this.verbose,
+            stderr: {
+                toExitCode: true,
+            },
+        });
+    }
+}
+exports.MysqlDumpTask = MysqlDumpTask;

package/lib/Task/PostgresqlDumpTask.d.ts

@@ -0,0 +1,14 @@
+import { SqlDumpTaskAbstract, SqlDumpTaskConfigType, TargetDatabaseType } from "./SqlDumpTaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare const postgresqlDumpTaskName = "postgresql-dump";
+export declare type PostgresqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
+export declare const postgresqlDumpTaskDefinition: JSONSchema7;
+export declare class PostgresqlDumpTask extends SqlDumpTaskAbstract<PostgresqlDumpTaskConfigType> {
+    buildConnectionArgs(database?: string): Promise<string[]>;
+    onDatabaseIsEmpty(name: string): Promise<boolean>;
+    onCreateDatabase(database: TargetDatabaseType): Promise<void>;
+    onExecQuery(query: string): Promise<import("../util/process-util").ExecResultType>;
+    onFetchTableNames(): Promise<string[]>;
+    onExport(tableNames: string[], output: string): Promise<void>;
+    onImport(path: string, database: string): Promise<void>;
+}

package/lib/Task/PostgresqlDumpTask.js

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresqlDumpTask = exports.postgresqlDumpTaskDefinition = exports.postgresqlDumpTaskName = void 0;
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const process_util_1 = require("../util/process-util");
+const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
+const fs_1 = require("fs");
+const path_1 = require("path");
+exports.postgresqlDumpTaskName = "postgresql-dump";
+exports.postgresqlDumpTaskDefinition = {
+    allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
+};
+class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
+    async buildConnectionArgs(database) {
+        const password = await this.fetchPassword();
+        const config = this.config;
+        return [
+            "--no-password",
+            `--dbname=postgresql://${config.username}:${password ?? ""}@${config.hostname}:${config.port ?? 5432}/${database ?? config.database ?? ""}`,
+        ];
+    }
+    async onDatabaseIsEmpty(name) {
+        const [total] = await this.fetchValues(`
+      SELECT
+        COUNT(*) AS total
+      FROM
+        information_schema.tables
+      WHERE
+        table_catalog = '${name}' AND
+        table_schema NOT IN ('pg_catalog', 'information_schema')
+    `);
+        return Number(total) ? false : true;
+    }
+    async onCreateDatabase(database) {
+        let query = `CREATE DATABASE ${database.name}`;
+        if (database.charset || database.collate) {
+            query += ` WITH`;
+            if (database.charset) {
+                query += ` ENCONDING '${database.charset}'`;
+            }
+            if (database.collate) {
+                query += ` LC_COLLATE '${database.collate}'`;
+            }
+        }
+        await this.onExecQuery(query);
+    }
+    async onExecQuery(query) {
+        return await (0, process_util_1.exec)("psql", [
+            ...(await this.buildConnectionArgs()),
+            "-t",
+            "-c",
+            query.replace(/\s{1,}/g, " "),
+        ], undefined, {
+            log: this.verbose,
+            stderr: {
+                toExitCode: true,
+            },
+            stdout: {
+                save: true,
+            },
+        });
+    }
+    async onFetchTableNames() {
+        return await this.fetchValues(`
+      SELECT
+        CONCAT(table_schema, '.', table_name)
+      FROM
+        information_schema.tables
+      WHERE
+        table_catalog = '${this.config.database}' AND
+        table_schema NOT IN ('pg_catalog', 'information_schema')
+	  `);
+    }
+    async onExport(tableNames, output) {
+        const stream = (0, fs_1.createWriteStream)(output);
+        await Promise.all([
+            new Promise((resolve, reject) => {
+                stream.on("close", resolve);
+                stream.on("error", reject);
+            }),
+            (0, process_util_1.exec)("pg_dump", [
+                ...(await this.buildConnectionArgs()),
+                "--clean",
+                "--if-exists",
+                ...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
+            ], null, {
+                pipe: { stream: stream },
+                stderr: {
+                    toExitCode: true,
+                },
+                log: this.verbose,
+            }),
+        ]);
+    }
+    async onImport(path, database) {
+        await (0, process_util_1.exec)("psql", [...(await this.buildConnectionArgs(database)), "-f", (0, path_1.normalize)(path)], undefined, {
+            log: this.verbose,
+        });
+    }
+}
+exports.PostgresqlDumpTask = PostgresqlDumpTask;

package/lib/Task/SqlDumpTaskAbstract.d.ts

@@ -0,0 +1,36 @@
+import { exec } from "../util/process-util";
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type TargetDatabaseType = {
+    name: string;
+    charset?: string;
+    collate?: string;
+};
+export declare type SqlDumpTaskConfigType = {
+    password: string | {
+        path: string;
+    };
+    hostname: string;
+    port?: number;
+    database: string;
+    username: string;
+    targetDatabase?: TargetDatabaseType;
+    includeTables?: string[];
+    excludeTables?: string[];
+    oneFileByTable?: boolean;
+};
+export declare const sqlDumpTaskDefinition: JSONSchema7;
+export declare abstract class SqlDumpTaskAbstract<TConfig extends SqlDumpTaskConfigType> extends TaskAbstract<TConfig> {
+    protected verbose?: boolean;
+    fetchPassword(): Promise<string | null>;
+    fetchValues(query: string): Promise<string[]>;
+    abstract onCreateDatabase(database: TargetDatabaseType): Promise<void>;
+    abstract onDatabaseIsEmpty(databaseName: string): Promise<boolean>;
+    abstract onFetchTableNames(): Promise<string[]>;
+    abstract onExecQuery(query: string): ReturnType<typeof exec>;
+    abstract onExport(tableNames: string[], output: string): Promise<void>;
+    abstract onImport(path: string, database: string): Promise<void>;
+    fetchTableNames(): Promise<string[]>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/lib/Task/SqlDumpTaskAbstract.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqlDumpTaskAbstract = exports.sqlDumpTaskDefinition = void 0;
+const AppError_1 = require("../Error/AppError");
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_util_1 = require("../util/cli-util");
+const config_util_1 = require("../util/datatruck/config-util");
+const fs_util_1 = require("../util/fs-util");
+const math_util_1 = require("../util/math-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const micromatch_1 = require("micromatch");
+const path_1 = require("path");
+exports.sqlDumpTaskDefinition = {
+    type: "object",
+    required: ["password", "hostname", "username", "database"],
+    properties: {
+        password: {
+            anyOf: [
+                {
+                    type: "string",
+                },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    required: ["path"],
+                    properties: {
+                        path: { type: "string" },
+                    },
+                },
+            ],
+        },
+        hostname: { type: "string" },
+        port: { type: "integer" },
+        username: { type: "string" },
+        database: { type: "string" },
+        targetDatabase: {
+            type: "object",
+            required: ["name"],
+            properties: {
+                name: { type: "string" },
+                charset: { type: "string" },
+                collate: { type: "string" },
+            },
+        },
+        includeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        oneFileByTable: { type: "boolean" },
+    },
+};
+class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
+    async fetchPassword() {
+        if (typeof this.config.password === "string")
+            return this.config.password;
+        if (this.config.password)
+            return (await (0, promises_1.readFile)(this.config.password.path)).toString();
+        return null;
+    }
+    async fetchValues(query) {
+        const result = await this.onExecQuery(query);
+        return result.stdout.split(/\r?\n/).reduce((result, value) => {
+            value = value.trim();
+            if (value.length)
+                result.push(value);
+            return result;
+        }, []);
+    }
+    async fetchTableNames() {
+        const tableNames = await this.onFetchTableNames();
+        const config = this.config;
+        return tableNames.filter((tableName) => {
+            if (config.includeTables && !(0, micromatch_1.isMatch)(tableName, config.includeTables))
+                return false;
+            if (config.excludeTables && (0, micromatch_1.isMatch)(tableName, config.excludeTables))
+                return false;
+            return true;
+        });
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const outputPath = data.package.path;
+        const tableNames = await this.fetchTableNames();
+        (0, assert_1.ok)(typeof outputPath === "string");
+        if (!(await (0, fs_util_1.checkDir)(outputPath)))
+            await (0, promises_1.mkdir)(outputPath, { recursive: true });
+        if (!this.config.oneFileByTable) {
+            const outPath = (0, path_1.join)(outputPath, this.config.database) + ".database.sql";
+            await this.onExport(tableNames, outPath);
+        }
+        else {
+            let current = 0;
+            for (const tableName of tableNames) {
+                data.onProgress({
+                    total: tableNames.length,
+                    current: current,
+                    percent: (0, math_util_1.progressPercent)(tableNames.length, current),
+                    step: tableName,
+                });
+                current++;
+                const outPath = (0, path_1.join)(outputPath, tableName) + ".table.sql";
+                await this.onExport([tableName], outPath);
+            }
+        }
+    }
+    async onRestore(data) {
+        const restorePath = data.package.restorePath;
+        this.verbose = data.options.verbose;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        const database = {
+            name: (0, config_util_1.resolveDatabaseName)(this.config.database, {
+                packageName: data.package.name,
+                snapshotId: data.options.snapshotId,
+                action: "restore",
+                database: undefined,
+            }),
+        };
+        if (this.config.targetDatabase) {
+            database.name = (0, config_util_1.resolveDatabaseName)(this.config.targetDatabase.name, {
+                packageName: data.package.name,
+                snapshotId: data.options.snapshotId,
+                action: "restore",
+                database: database.name,
+            });
+        }
+        if (!(await this.onDatabaseIsEmpty(database.name)))
+            throw new AppError_1.AppError(`Target database is not empty: ${database.name}`);
+        await this.onCreateDatabase(database);
+        if (this.verbose)
+            (0, cli_util_1.logExec)("readdir", [restorePath]);
+        const files = (await (0, promises_1.readdir)(restorePath)).filter((name) => /\.sql$/i.test(name));
+        let current = 0;
+        for (const file of files) {
+            const path = (0, path_1.join)(restorePath, file);
+            data.onProgress({
+                total: files.length,
+                current: current,
+                percent: (0, math_util_1.progressPercent)(files.length, current),
+                step: file,
+            });
+            current++;
+            await this.onImport(path, database.name);
+        }
+    }
+}
+exports.SqlDumpTaskAbstract = SqlDumpTaskAbstract;

package/lib/Task/TaskAbstract.d.ts

@@ -0,0 +1,37 @@
+import { BackupActionOptionsType } from "../Action/BackupAction";
+import { RestoreActionOptionsType } from "../Action/RestoreAction";
+import { PackageConfigType } from "../Config/PackageConfig";
+import { SnapshotType } from "../Repository/RepositoryAbstract";
+export declare type ProgressDataType = {
+    total: number;
+    current: number;
+    percent: number;
+    step: string;
+    stepPercent?: number;
+};
+export declare type BackupDataType = {
+    onProgress: (data: ProgressDataType) => Promise<void>;
+    options: BackupActionOptionsType;
+    package: PackageConfigType;
+    targetPath: string | undefined;
+    snapshot: SnapshotType;
+};
+export declare type RestoreDataType = {
+    onProgress: (data: ProgressDataType) => Promise<void>;
+    options: RestoreActionOptionsType;
+    package: PackageConfigType;
+    targetPath: string | undefined;
+    snapshot: SnapshotType;
+};
+export declare abstract class TaskAbstract<TConfig = any> {
+    readonly config: TConfig;
+    constructor(config: TConfig);
+    onBeforeBackup(data: Omit<BackupDataType, "onProgress" | "targetPath">): Promise<{
+        targetPath?: string;
+    } | undefined>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onBeforeRestore(data: Omit<RestoreDataType, "onProgress" | "targetPath">): Promise<{
+        targetPath?: string;
+    } | undefined>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/lib/Task/TaskAbstract.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TaskAbstract = void 0;
+class TaskAbstract {
+    constructor(config) {
+        this.config = config;
+    }
+    async onBeforeBackup(data) {
+        return undefined;
+    }
+    async onBackup(data) { }
+    async onBeforeRestore(data) {
+        return undefined;
+    }
+    async onRestore(data) { }
+}
+exports.TaskAbstract = TaskAbstract;

package/lib/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/bin.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const cli_1 = require("./cli");
+(0, cli_1.parseArgs)(process.argv);

package/lib/cli.d.ts

@@ -0,0 +1,4 @@
+import { OptionsMapType } from "./Factory/CommandFactory";
+export declare function buildArgs<TCommand extends keyof OptionsMapType>(input: TCommand, options: OptionsMapType[TCommand]): any[];
+export declare function parseArgs(args: string[]): void;
+export declare function exec<TCommand extends keyof OptionsMapType>(input: TCommand, options: OptionsMapType[TCommand]): Promise<void>;

package/lib/cli.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exec = exports.parseArgs = exports.buildArgs = void 0;
+const AppError_1 = require("./Error/AppError");
+const CommandFactory_1 = require("./Factory/CommandFactory");
+const cli_util_1 = require("./util/cli-util");
+const fs_util_1 = require("./util/fs-util");
+const process_util_1 = require("./util/process-util");
+const string_util_1 = require("./util/string-util");
+const chalk_1 = require("chalk");
+const commander_1 = require("commander");
+const fs_1 = require("fs");
+const path_1 = require("path");
+function getGlobalOptions() {
+    return program.opts();
+}
+function makeCommand(command) {
+    const programCommand = program.command(command);
+    const instance = (0, CommandFactory_1.CommandFactory)(command, getGlobalOptions(), null);
+    const options = instance.onOptions();
+    for (const key in options) {
+        const option = options[key];
+        const description = `${option.description}${option.defaults ? ` (defaults: ${option.defaults})` : ""}`;
+        if (option.required) {
+            programCommand.requiredOption(option.option, description);
+        }
+        else {
+            programCommand.option(option.option, description);
+        }
+    }
+    return programCommand.action(makeCommandAction(command));
+}
+function makeCommandAction(command) {
+    return async function (options) {
+        let exitCode = 1;
+        const globalOptions = getGlobalOptions();
+        try {
+            exitCode = await (0, CommandFactory_1.CommandFactory)(command, getGlobalOptions(), options).onExec();
+        }
+        catch (e) {
+            const error = e;
+            if (globalOptions.verbose) {
+                console.error((0, chalk_1.red)(error.stack));
+            }
+            else {
+                if (error instanceof AppError_1.AppError) {
+                    console.error((0, chalk_1.red)(error.message));
+                }
+                else {
+                    console.error((0, chalk_1.red)(error.stack));
+                }
+            }
+        }
+        process.exit(exitCode);
+    };
+}
+const program = new commander_1.Command();
+const { name, version, description } = (0, fs_util_1.parsePackageFile)();
+const [, subname] = name.split("/");
+const cwd = process.cwd();
+program.name(subname);
+program.version(version);
+program.description(description);
+program.option("-v,--verbose", "Verbose", (_, previous) => previous + 1, 0);
+program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFIG"] ?? (cwd.endsWith(path_1.sep) ? cwd : `${cwd}${path_1.sep}`));
+program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table)", "table");
+makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
+makeCommand(CommandFactory_1.CommandEnum.init).alias("i");
+makeCommand(CommandFactory_1.CommandEnum.snapshots).alias("s");
+makeCommand(CommandFactory_1.CommandEnum.prune).alias("p");
+makeCommand(CommandFactory_1.CommandEnum.backup).alias("b");
+makeCommand(CommandFactory_1.CommandEnum.backupSessions).alias("bs");
+makeCommand(CommandFactory_1.CommandEnum.restore).alias("r");
+makeCommand(CommandFactory_1.CommandEnum.restoreSessions).alias("rs");
+makeCommand(CommandFactory_1.CommandEnum.cleanCache).alias("cc");
+function buildArgs(input, options) {
+    const optionsArray = Object.keys(options).flatMap((name) => [
+        `--${(0, string_util_1.snakeCase)(name, "-")}`,
+        options[name],
+    ]);
+    return [input, ...optionsArray];
+}
+exports.buildArgs = buildArgs;
+function parseArgs(args) {
+    program.parse(args);
+    const verbose = getGlobalOptions().verbose;
+    (0, process_util_1.onExit)((eventName) => {
+        if (eventName !== "exit") {
+            process.stdout.write(cli_util_1.showCursorCommand);
+            console.log("\nClosing...");
+        }
+        if (!verbose)
+            try {
+                (0, fs_1.rmSync)((0, fs_util_1.sessionTmpDir)(), {
+                    force: true,
+                    recursive: true,
+                    maxRetries: 5,
+                });
+            }
+            catch (error) { }
+        if (eventName !== "exit")
+            process.exit(1);
+    });
+}
+exports.parseArgs = parseArgs;
+async function exec(input, options) {
+    const argv = buildArgs(input, options);
+    return parseArgs(argv);
+}
+exports.exec = exec;

package/lib/index.js

@@ -0,0 +1 @@
+"use strict";

package/lib/util/DataFormat.d.ts

@@ -0,0 +1,24 @@
+export declare type FormatType = "json" | "pjson" | "table" | "yaml";
+export declare class DataFormat<TItem extends Record<string, unknown>> {
+    readonly options: {
+        items: TItem[];
+        json?: (item: TItem) => any;
+        table: {
+            labels: string[];
+            handler: (item: TItem) => (string | number | null | undefined)[];
+        };
+    };
+    constructor(options: {
+        items: TItem[];
+        json?: (item: TItem) => any;
+        table: {
+            labels: string[];
+            handler: (item: TItem) => (string | number | null | undefined)[];
+        };
+    });
+    protected formatToJson(): string;
+    protected formatToPrettyJson(): string;
+    protected formatToYaml(): any;
+    protected formatToTable(): string;
+    format(format: FormatType): any;
+}