@datatruck/cli 0.0.1

package/lib/Task/GitTask.d.ts

@@ -0,0 +1,35 @@
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type GitTaskConfigType = {
+    command?: string;
+    /**
+     * @default true
+     */
+    includeModified?: boolean | string[];
+    /**
+     * @default true
+     */
+    includeUntracked?: boolean | string[];
+    /**
+     * @default false
+     */
+    includeIgnored?: boolean | string[];
+    /**
+     * @default true
+     */
+    includeConfig?: boolean;
+};
+export declare const gitTaskName = "git";
+export declare const gitTaskDefinition: JSONSchema7;
+export declare class GitTask extends TaskAbstract<GitTaskConfigType> {
+    protected verbose?: boolean;
+    private get command();
+    onBeforeBackup(): Promise<{
+        targetPath: string;
+    }>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onBeforeRestore(): Promise<{
+        targetPath: string;
+    }>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/lib/Task/GitTask.js

@@ -0,0 +1,248 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GitTask = exports.gitTaskDefinition = exports.gitTaskName = void 0;
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_util_1 = require("../util/cli-util");
+const fs_util_1 = require("../util/fs-util");
+const math_util_1 = require("../util/math-util");
+const process_util_1 = require("../util/process-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const fs_1 = require("fs");
+const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
+const micromatch_1 = require("micromatch");
+const path_1 = require("path");
+const readline_1 = require("readline");
+exports.gitTaskName = "git";
+exports.gitTaskDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        command: {
+            type: "string",
+        },
+        includeModified: {
+            anyOf: [
+                {
+                    type: "boolean",
+                },
+                (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+            ],
+        },
+        includeUntracked: {
+            anyOf: [
+                {
+                    type: "boolean",
+                },
+                (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+            ],
+        },
+        includeIgnored: {
+            anyOf: [
+                {
+                    type: "boolean",
+                },
+                (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+            ],
+        },
+        includeConfig: {
+            type: "boolean",
+        },
+    },
+};
+class GitTask extends TaskAbstract_1.TaskAbstract {
+    get command() {
+        return this.config.command ?? "git";
+    }
+    async onBeforeBackup() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(GitTask.name),
+        };
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const path = data.package.path;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof path === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        // Bundle
+        const bundlePath = (0, path_1.join)(targetPath, "repo.bundle");
+        await (0, process_util_1.exec)(this.command, ["bundle", "create", bundlePath, "--all"], {
+            cwd: path,
+        }, {
+            log: this.verbose,
+        });
+        // Config
+        if (this.config.includeConfig ?? true) {
+            const configPath = (0, path_1.join)(targetPath, "repo.config");
+            await (0, promises_1.copyFile)((0, path_1.join)(path, ".git", "config"), configPath);
+        }
+        // git ls-files
+        const lsFilesConfig = [
+            {
+                name: "untracked",
+                argv: ["--others"],
+                include: config.includeUntracked ?? true,
+            },
+            {
+                name: "modified",
+                argv: ["--modified"],
+                include: config.includeModified ?? true,
+            },
+            {
+                name: "ignored",
+                argv: ["--others", "--ignored"],
+                include: config.includeIgnored,
+            },
+        ];
+        // Paths list
+        let total = 0;
+        let currentFiles = 0;
+        for (const option of lsFilesConfig) {
+            if (!option.include)
+                continue;
+            option.pathsPath = (0, path_1.join)(targetPath, `repo.${option.name}-paths.txt`);
+            const stream = (0, fs_1.createWriteStream)(option.pathsPath);
+            let streamError;
+            stream.on("error", (e) => (streamError = e));
+            try {
+                await (0, process_util_1.exec)(this.command, [
+                    "-c",
+                    "core.quotepath=off",
+                    "ls-files",
+                    ...option.argv,
+                    "--exclude-standard",
+                ], {
+                    cwd: data.package.path,
+                }, {
+                    log: {
+                        exec: this.verbose,
+                    },
+                    onSpawn: (p) => {
+                        const iface = (0, readline_1.createInterface)(p.stdout, stream);
+                        iface.on("close", () => stream.end());
+                        iface.on("line", (path) => {
+                            path = path.trim();
+                            if (!path.length)
+                                return;
+                            let found = false;
+                            if (option.include === true) {
+                                found = true;
+                            }
+                            else if (option.include) {
+                                found = (0, micromatch_1.isMatch)(path, option.include);
+                            }
+                            if (found) {
+                                total++;
+                                stream.write(`${path}\n`);
+                            }
+                        });
+                    },
+                });
+            }
+            finally {
+                await new Promise((resolve) => stream.end(resolve));
+                if (streamError)
+                    throw streamError;
+            }
+        }
+        // Copy
+        for (const option of lsFilesConfig) {
+            if (!option.include)
+                continue;
+            const createdPaths = [];
+            const outPath = (0, path_1.join)(targetPath, `repo.${option.name}`);
+            await (0, fs_util_1.mkdirIfNotExists)(outPath);
+            if (data.options.verbose)
+                (0, cli_util_1.logExec)(`Copying ${option.name} files to ${outPath}`);
+            const reader = (0, readline_1.createInterface)({
+                input: (0, fs_1.createReadStream)(option.pathsPath),
+            });
+            for await (const entry of reader) {
+                const source = (0, path_1.join)(path, entry);
+                const target = (0, path_1.join)(outPath, entry);
+                if (entry.endsWith("/")) {
+                    await (0, promises_1.mkdir)(target, {
+                        recursive: true,
+                    });
+                }
+                else {
+                    currentFiles++;
+                    await data.onProgress({
+                        total,
+                        current: currentFiles,
+                        percent: (0, math_util_1.progressPercent)(total, currentFiles),
+                        step: entry,
+                    });
+                    const dir = (0, path_1.dirname)(target);
+                    if (!createdPaths.includes(dir)) {
+                        await (0, promises_1.mkdir)(dir, {
+                            recursive: true,
+                        });
+                        createdPaths.push(dir);
+                    }
+                    await (0, promises_1.copyFile)(source, target);
+                }
+            }
+            await (0, promises_1.rm)(option.pathsPath);
+        }
+    }
+    async onBeforeRestore() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(GitTask.name),
+        };
+    }
+    async onRestore(data) {
+        this.verbose = data.options.verbose;
+        const restorePath = data.package.restorePath;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        await (0, fs_util_1.mkdirIfNotExists)(restorePath);
+        await (0, fs_util_1.ensureEmptyDir)(restorePath);
+        // Stats
+        let totalFiles = 0;
+        let currentFiles = 0;
+        await (0, fs_util_1.forEachFile)(targetPath, () => totalFiles++, true);
+        const incrementProgress = async (step = "") => {
+            await data.onProgress({
+                total: totalFiles,
+                current: Math.max(currentFiles, 0),
+                percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
+                step,
+            });
+            currentFiles++;
+        };
+        // Bundle
+        const bundlePath = (0, path_1.join)(targetPath, "repo.bundle");
+        await (0, process_util_1.exec)(this.command, ["clone", bundlePath, "."], {
+            cwd: restorePath,
+        }, {
+            log: this.verbose,
+        });
+        await incrementProgress();
+        // Config
+        const configPath = (0, path_1.join)(targetPath, "repo.config");
+        if (await (0, fs_util_1.checkFile)(configPath)) {
+            await (0, promises_1.copyFile)(configPath, (0, path_1.join)(restorePath, ".git", "config"));
+            await incrementProgress();
+        }
+        // ls-files
+        for (const name of ["untracked", "modified", "ignored"]) {
+            const sourcePath = (0, path_1.join)(targetPath, `repo.${name}`);
+            if (await (0, fs_util_1.checkDir)(sourcePath)) {
+                if (data.options.verbose)
+                    (0, cli_util_1.logExec)(`Copying ${name} files to ${restorePath}`);
+                await (0, fs_extra_1.copy)(sourcePath, restorePath, {
+                    filter: async (path) => {
+                        await incrementProgress((0, path_1.relative)(sourcePath, path));
+                        return true;
+                    },
+                });
+            }
+        }
+    }
+}
+exports.GitTask = GitTask;

package/lib/Task/MariadbTask.d.ts

@@ -0,0 +1,25 @@
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type MariadbTaskConfigType = {
+    command?: string;
+    hostname: string;
+    username: string;
+    password: string | {
+        path: string;
+    };
+    includeTables?: string[];
+    excludeTables?: string[];
+    includeDatabases?: string[];
+    excludeDatabases?: string[];
+};
+export declare const mariadbTaskName = "mariadb";
+export declare const mariadbTaskDefinition: JSONSchema7;
+export declare class MariadbTask extends TaskAbstract<MariadbTaskConfigType> {
+    protected verbose?: boolean;
+    private get command();
+    onBeforeBackup(): Promise<{
+        targetPath: string;
+    }>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/lib/Task/MariadbTask.js

@@ -0,0 +1,139 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MariadbTask = exports.mariadbTaskDefinition = exports.mariadbTaskName = void 0;
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_util_1 = require("../util/cli-util");
+const fs_util_1 = require("../util/fs-util");
+const math_util_1 = require("../util/math-util");
+const process_util_1 = require("../util/process-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const posix_1 = require("path/posix");
+exports.mariadbTaskName = "mariadb";
+exports.mariadbTaskDefinition = {
+    type: "object",
+    required: ["hostname", "username", "password"],
+    additionalProperties: false,
+    properties: {
+        command: { type: "string" },
+        hostname: { type: "string" },
+        username: { type: "string" },
+        password: {
+            anyOf: [
+                {
+                    type: "string",
+                },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    required: ["path"],
+                    properties: {
+                        path: { type: "string" },
+                    },
+                },
+            ],
+        },
+        includeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        includeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        excludeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+    },
+};
+class MariadbTask extends TaskAbstract_1.TaskAbstract {
+    get command() {
+        return this.config.command ?? "mariabackup";
+    }
+    async onBeforeBackup() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(MariadbTask.name),
+        };
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const command = this.command;
+        const sourcePath = data.package.path;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof sourcePath === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        const args = [
+            `--backup`,
+            `--datadir=${sourcePath}`,
+            `--target-dir=${targetPath}`,
+            `--host=${config.hostname}`,
+            `--user=${config.username}`,
+            `--password=${typeof config.password === "string"
+                ? config.password
+                : config.password
+                    ? (await (0, promises_1.readFile)(config.password.path)).toString()
+                    : ""}`,
+        ];
+        if (config.includeDatabases)
+            args.push(`--databases=${config.includeDatabases.join(" ")}`);
+        if (config.excludeDatabases)
+            args.push(`--databases-exclude=${config.excludeDatabases.join(" ")}`);
+        if (config.includeTables)
+            args.push(`--tables=^(${config.includeTables.join("|")})$`);
+        if (config.excludeTables)
+            args.push(`--tables-exclude=^(${config.excludeTables.join("|")})$`);
+        let total = 0;
+        let current = 0;
+        await (0, fs_util_1.forEachFile)(sourcePath, () => {
+            total++;
+        });
+        const onData = async (lines) => {
+            const regex = /\[\d{1,}\] \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} Copying (.+) to/;
+            let path = lines
+                .split(/\r?\n/)
+                .reduce((result, line) => {
+                const matches = regex.exec(line);
+                if (matches) {
+                    current++;
+                    result.push(matches[1]);
+                }
+                return result;
+            }, [])
+                .pop();
+            if (path) {
+                path = (0, posix_1.normalize)(path);
+                await data.onProgress({
+                    current,
+                    percent: (0, math_util_1.progressPercent)(total, current),
+                    total,
+                    step: `Copying ${path}`,
+                });
+            }
+        };
+        await (0, process_util_1.exec)(command, args, undefined, {
+            log: this.verbose,
+            stdout: {
+                onData,
+            },
+            stderr: {
+                onData,
+            },
+        });
+        await (0, process_util_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
+            log: this.verbose,
+            stderr: { onData: async () => { } },
+        });
+    }
+    async onRestore(data) {
+        this.verbose = data.options.verbose;
+        const restorePath = data.package.restorePath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        await (0, fs_util_1.mkdirIfNotExists)(restorePath);
+        const files = await (0, promises_1.readdir)(restorePath);
+        for (const file of files) {
+            if (file.startsWith("ib_logfile")) {
+                const filePath = (0, path_1.join)(restorePath, file);
+                if (this.verbose)
+                    (0, cli_util_1.logExec)("rm", [filePath]);
+                await (0, promises_1.rm)(filePath);
+            }
+        }
+    }
+}
+exports.MariadbTask = MariadbTask;

package/lib/Task/MssqlTask.d.ts

@@ -0,0 +1,22 @@
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type MssqlTaskConfigType = {
+    command?: string;
+    hostname?: string;
+    username?: string;
+    passwordFile?: string;
+    targetDatabase?: string;
+    includeDatabases?: string[];
+    excludeDatabases?: string[];
+};
+export declare const mssqlTaskName = "mssql";
+export declare const mssqlTaskDefinition: JSONSchema7;
+export declare class MssqlTask extends TaskAbstract<MssqlTaskConfigType> {
+    static SUFFIX: string;
+    protected verbose?: boolean;
+    private get command();
+    exec(query: string): Promise<string[][]>;
+    fetchDatabaseNames(name?: string): Promise<string[]>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/lib/Task/MssqlTask.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MssqlTask = exports.mssqlTaskDefinition = exports.mssqlTaskName = void 0;
+const AppError_1 = require("../Error/AppError");
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const config_util_1 = require("../util/datatruck/config-util");
+const fs_util_1 = require("../util/fs-util");
+const process_util_1 = require("../util/process-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
+const micromatch_1 = require("micromatch");
+const path_1 = require("path");
+exports.mssqlTaskName = "mssql";
+exports.mssqlTaskDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        command: { type: "string" },
+        hostname: { type: "string" },
+        username: { type: "string" },
+        passwordFile: { type: "string" },
+        includeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+        excludeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+    },
+};
+class MssqlTask extends TaskAbstract_1.TaskAbstract {
+    get command() {
+        return this.config.command ?? "sqlcmd";
+    }
+    async exec(query) {
+        const result = await (0, process_util_1.exec)(this.command, [
+            ...(this.config.hostname ? ["-S", this.config.hostname] : []),
+            ...(this.config.username ? ["-U", this.config.username] : []),
+            ...(this.config.passwordFile
+                ? ["-P", (await (0, promises_1.readFile)(this.config.passwordFile)).toString()]
+                : []),
+            "-E",
+            "-W",
+            "-s",
+            ",",
+            "-w",
+            "999",
+            "-Q",
+            `SET nocount ON; ${query.replace(/[\n\t]/g, " ")}`,
+        ], undefined, {
+            log: this.verbose,
+            stderr: {
+                toExitCode: true,
+            },
+            stdout: {
+                save: true,
+            },
+        });
+        return result.stdout
+            .split(/\n/g)
+            .map((row) => row.split(","))
+            .filter((row) => row.length);
+    }
+    async fetchDatabaseNames(name) {
+        const query = `SELECT name FROM master.dbo.sysdatabases ${name ? `WHERE name = '${name}'` : ""}`;
+        const rows = await this.exec(query);
+        const privateDatabases = ["master", "tempdb", "model", "msdb"];
+        return rows
+            .map(([database]) => database)
+            .filter((database) => !privateDatabases.includes(database));
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const targetPath = data.package.path;
+        (0, assert_1.ok)(typeof targetPath === "string");
+        const databaseNames = (await this.fetchDatabaseNames()).filter((databaseName) => (!this.config.includeDatabases ||
+            (0, micromatch_1.isMatch)(databaseName, this.config.includeDatabases)) &&
+            (!this.config.excludeDatabases ||
+                !(0, micromatch_1.isMatch)(databaseName, this.config.excludeDatabases)));
+        await (0, fs_util_1.mkdirIfNotExists)(targetPath);
+        for (const databaseName of databaseNames) {
+            const databasePath = (0, path_1.join)(targetPath, `${databaseName}${MssqlTask.SUFFIX}`);
+            await this.exec(`BACKUP DATABASE [${databaseName}] TO DISK='${databasePath}' WITH FORMAT`);
+        }
+    }
+    async onRestore(data) {
+        this.verbose = data.options.verbose;
+        const restorePath = data.package.restorePath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        await (0, fs_util_1.mkdirIfNotExists)(restorePath);
+        const files = await (0, fs_extra_1.readdir)(restorePath);
+        for (const file of files) {
+            if (!file.endsWith(MssqlTask.SUFFIX))
+                continue;
+            let databaseName = file.slice(0, MssqlTask.SUFFIX.length * -1);
+            if (this.config.targetDatabase)
+                databaseName = (0, config_util_1.resolveDatabaseName)(this.config.targetDatabase, {
+                    action: "restore",
+                    database: databaseName,
+                    packageName: data.package.name,
+                    snapshotId: data.options.snapshotId,
+                });
+            const databasePath = (0, path_1.join)(restorePath, file);
+            const exists = await this.fetchDatabaseNames(databaseName);
+            if (exists.length)
+                throw new AppError_1.AppError(`Target database already exists: ${databaseName}`);
+            await this.exec(`RESTORE DATABASE [${databaseName}] FROM disk='${databasePath}'`);
+        }
+    }
+}
+exports.MssqlTask = MssqlTask;
+MssqlTask.SUFFIX = ".BAK";

package/lib/Task/MysqlDumpTask.d.ts

@@ -0,0 +1,14 @@
+import { SqlDumpTaskAbstract, SqlDumpTaskConfigType, TargetDatabaseType } from "./SqlDumpTaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare const mysqlDumpTaskName = "mysql-dump";
+export declare type MysqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
+export declare const mysqlDumpTaskDefinition: JSONSchema7;
+export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConfigType> {
+    buildConnectionArgs(database?: boolean): Promise<string[]>;
+    onDatabaseIsEmpty(name: string): Promise<boolean>;
+    onCreateDatabase(database: TargetDatabaseType): Promise<void>;
+    onExecQuery(query: string): Promise<import("../util/process-util").ExecResultType>;
+    onFetchTableNames(): Promise<string[]>;
+    onExport(tableNames: string[], output: string): Promise<void>;
+    onImport(path: string, database: string): Promise<void>;
+}