@datatruck/cli 0.21.0 → 0.22.0

package/utils/mysql.d.ts

@@ -0,0 +1,39 @@
+export type MysqlCliOptions = {
+    password: string | {
+        path: string;
+    };
+    hostname: string;
+    port?: number;
+    username: string;
+    verbose?: boolean;
+};
+export declare function createMysqlCli(options: MysqlCliOptions): {
+    options: MysqlCliOptions;
+    initSharedDir: (sharedDir?: string) => Promise<string>;
+    args: () => Promise<string[]>;
+    run: (query: string, database?: string) => Promise<import("./process").ExecResultType>;
+    fetchAll: (query: string, database?: string) => Promise<string[][]>;
+    dump: (input: {
+        output: string;
+        database: string;
+        items?: string[] | undefined;
+        onlyStoredPrograms?: boolean | undefined;
+        onProgress?: ((data: {
+            totalBytes: number;
+        }) => void) | undefined;
+    }) => Promise<[void, import("./process").ExecResultType]>;
+    fetchTableNames: (database: string, include?: string[], exclude?: string[]) => Promise<string[]>;
+    importFile: (path: string, database: string) => Promise<import("./process").ExecResultType>;
+    isDatabaseEmpty: (database: string) => Promise<boolean>;
+    createDatabase: (database: {
+        name: string;
+        charset?: string;
+    }) => Promise<void>;
+    csvDump: (input: {
+        database: string;
+        sharedPath: string;
+        items?: string[];
+    }) => Promise<void>;
+    importCsvFile: (path: string, database: string, table: string) => Promise<import("./process").ExecResultType>;
+    fetchVariable: (name: string) => Promise<string | undefined>;
+};

package/utils/mysql.js

@@ -0,0 +1,211 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMysqlCli = void 0;
+const AppError_1 = require("../Error/AppError");
+const fs_1 = require("./fs");
+const process_1 = require("./process");
+const string_1 = require("./string");
+const crypto_1 = require("crypto");
+const fs_2 = require("fs");
+const promises_1 = require("fs/promises");
+const os_1 = require("os");
+const path_1 = require("path");
+function createMysqlCli(options) {
+    async function args() {
+        const password = await (0, fs_1.fetchData)(options.password, (p) => p.path);
+        return [
+            `--host=${options.hostname}`,
+            ...(options.port ? [`--port=${options.port}`] : []),
+            `--user=${options.username}`,
+            `--password=${password ?? ""}`,
+        ];
+    }
+    async function run(query, database) {
+        return await (0, process_1.exec)("mysql", [
+            ...(await args()),
+            ...(database ? [database] : []),
+            "-e",
+            query.replace(/\s{1,}/g, " "),
+            "-N",
+            "--silent",
+        ], undefined, {
+            log: options.verbose,
+            stderr: { toExitCode: true },
+            stdout: { save: true },
+        });
+    }
+    async function fetchAll(query, database) {
+        return (0, string_1.splitLines)((await run(query, database)).stdout).map((line) => line.split("\t"));
+    }
+    async function fetchTableNames(database, include, exclude) {
+        return (await fetchAll(`
+      SELECT
+        table_name 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${database}'
+      ORDER BY
+        table_name
+  `))
+            .map((r) => r[0])
+            .filter((0, string_1.createMatchFilter)(include, exclude));
+    }
+    async function dump(input) {
+        const stream = (0, fs_2.createWriteStream)(input.output);
+        return await Promise.all([
+            new Promise((resolve, reject) => {
+                stream.on("close", resolve);
+                stream.on("error", reject);
+            }),
+            await (0, process_1.exec)("mysqldump", [
+                ...(await args()),
+                input.database,
+                "--lock-tables=false",
+                "--skip-add-drop-table=false",
+                ...(input.onlyStoredPrograms
+                    ? [
+                        "--routines",
+                        "--events",
+                        "--skip-triggers",
+                        "--no-create-info",
+                        "--no-data",
+                        "--no-create-db",
+                        "--skip-opt",
+                    ]
+                    : []),
+                ...(input.items || []),
+            ], null, {
+                stderr: { toExitCode: true },
+                pipe: {
+                    stream,
+                    onWriteProgress: input.onProgress,
+                },
+                log: {
+                    exec: options.verbose,
+                    stderr: options.verbose,
+                    allToStderr: true,
+                },
+            }),
+        ]);
+    }
+    async function csvDump(input) {
+        await (0, process_1.exec)("mysqldump", [
+            ...(await args()),
+            input.database,
+            "--lock-tables=false",
+            "--skip-add-drop-table=false",
+            "-T",
+            input.sharedPath,
+            ...(input.items || []),
+        ], null, {
+            stderr: { toExitCode: true },
+            log: {
+                exec: options.verbose,
+                stderr: options.verbose,
+                allToStderr: true,
+            },
+        });
+    }
+    async function importFile(path, database) {
+        return await (0, process_1.exec)("mysql", [
+            `--init-command=SET ${[
+                "autocommit=0",
+                "unique_checks=0",
+                "foreign_key_checks=0",
+            ].join(",")};`,
+            ...(await args()),
+            database,
+        ], null, {
+            pipe: {
+                stream: (0, fs_2.createReadStream)(path),
+                onReadProgress: (data) => {
+                    if (options.verbose)
+                        (0, process_1.logExecStdout)({
+                            data: JSON.stringify(data),
+                            colorize: true,
+                            stderr: true,
+                            lineSalt: true,
+                        });
+                },
+            },
+            stderr: { toExitCode: true },
+            log: options.verbose,
+        });
+    }
+    async function importCsvFile(path, database, table) {
+        return run(`
+      LOAD DATA LOCAL INFILE '${path.replaceAll("\\", "/")}'
+      INTO TABLE ${table}
+      FIELDS TERMINATED BY ','
+      ENCLOSED BY '"'
+      LINES TERMINATED BY '\\n'`, database);
+    }
+    async function isDatabaseEmpty(database) {
+        const [total] = await fetchAll(`
+      SELECT
+        COUNT(*) AS total 
+      FROM
+        information_schema.tables
+      WHERE
+        table_schema = '${database}'
+    `);
+        return Number(total) ? false : true;
+    }
+    async function createDatabase(database) {
+        await run(`
+      CREATE DATABASE IF NOT EXISTS \`${database.name}\`
+      CHARACTER SET ${database.charset ?? "utf8"}
+      COLLATE ${database.charset ?? "utf8_general_ci"}
+    `);
+    }
+    async function fetchVariable(name) {
+        const stdout = (0, string_1.undefIfEmpty)((await run(`SHOW VARIABLES LIKE "${name}"`)).stdout.trim());
+        return stdout ? (0, string_1.undefIfEmpty)(stdout.slice(name.length).trim()) : undefined;
+    }
+    async function initSharedDir(sharedDir) {
+        const secure_file_priv = await fetchVariable("secure_file_priv");
+        if (secure_file_priv?.toUpperCase() === "NULL")
+            throw new AppError_1.AppError("'secure_file_priv' is null in MySQL Server");
+        const dir = sharedDir ??
+            secure_file_priv ??
+            (await fetchVariable("tmpdir")) ??
+            (0, os_1.tmpdir)();
+        await checkSharedDir(dir);
+        return dir;
+    }
+    async function checkSharedDir(dir) {
+        const id = (0, crypto_1.randomBytes)(8).toString("hex");
+        const outFile = (0, path_1.join)(dir, `dtt_test_${id}`);
+        const outFileVar = JSON.stringify(outFile.replaceAll("\\", "/"));
+        try {
+            await (0, fs_1.mkdirIfNotExists)(dir);
+            await run(`SELECT 1 INTO OUTFILE ${outFileVar}`);
+            const exists = await (0, fs_1.existsFile)(outFile);
+            if (!exists)
+                throw new AppError_1.AppError(`MySQL shared dir is not reached: ${dir}`);
+        }
+        finally {
+            try {
+                await (0, promises_1.rm)(outFile);
+            }
+            catch (e) { }
+        }
+    }
+    return {
+        options,
+        initSharedDir,
+        args,
+        run,
+        fetchAll,
+        dump,
+        fetchTableNames,
+        importFile,
+        isDatabaseEmpty,
+        createDatabase,
+        csvDump,
+        importCsvFile,
+        fetchVariable,
+    };
+}
+exports.createMysqlCli = createMysqlCli;

package/utils/string.d.ts

@@ -13,6 +13,10 @@ export type UriType = {
 export declare function formatUri(input: UriType, hidePassword?: boolean): string;
 export declare function formatSeconds(seconds: number): string;
 export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
-export declare function checkPath(path: string, include: string[], exclude?: string[]): boolean;
+export declare function match(path: string, include?: string[], exclude?: string[]): boolean;
+export declare function endsWith(input: string, patterns: string[]): boolean;
+export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;
+export declare function splitLines(input: string, satinize?: boolean): string[];
+export declare function undefIfEmpty(input: string): string | undefined;

package/utils/string.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.checkMatch = exports.checkPath = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
+exports.undefIfEmpty = exports.splitLines = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -91,13 +91,19 @@ function makePathPatterns(values) {
     });
 }
 exports.makePathPatterns = makePathPatterns;
-function checkPath(path, include, exclude) {
-    return ((0, micromatch_1.isMatch)(path, include, {
-        dot: true,
-    }) &&
+function match(path, include, exclude) {
+    return ((!include || (0, micromatch_1.isMatch)(path, include, { dot: true })) &&
         (!exclude || !(0, micromatch_1.isMatch)(path, exclude, { dot: true })));
 }
-exports.checkPath = checkPath;
+exports.match = match;
+function endsWith(input, patterns) {
+    return patterns.some((pattern) => input.endsWith(pattern));
+}
+exports.endsWith = endsWith;
+function createMatchFilter(include, exclude) {
+    return (input) => match(input, include, exclude);
+}
+exports.createMatchFilter = createMatchFilter;
 function checkMatch(subject, patterns) {
     if (!subject?.length)
         subject = "<empty>";
@@ -114,3 +120,19 @@ function formatDateTime(datetime) {
     return result;
 }
 exports.formatDateTime = formatDateTime;
+function splitLines(input, satinize = true) {
+    const lines = input.split(/\r?\n/);
+    return satinize
+        ? input.split(/\r?\n/).reduce((result, value) => {
+            value = value.trim();
+            if (value.length)
+                result.push(value);
+            return result;
+        }, [])
+        : lines;
+}
+exports.splitLines = splitLines;
+function undefIfEmpty(input) {
+    return input.length ? input : undefined;
+}
+exports.undefIfEmpty = undefIfEmpty;

package/utils/tar.js

@@ -137,21 +137,23 @@ async function createTar(options) {
     }, {
         log: options.verbose ? { envNames: Object.keys(env) } : false,
         stderr: { toExitCode: true },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (line) => {
-                current++;
-                const path = vendor === "bsdtar" ? line.slice(2) : line;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
-        },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (line) => {
+                    current++;
+                    const path = vendor === "bsdtar" ? line.slice(2) : line;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.createTar = createTar;
@@ -164,10 +166,11 @@ function toLocalPath(path) {
 async function extractTar(options) {
     let total = options.total ??
         (await listTar({ input: options.input, verbose: options.verbose }));
-    if (options.verbose)
-        (0, cli_1.logExec)("mkdir", ["-p", options.output]);
-    await (0, promises_1.mkdir)(options.output, { recursive: true });
-    await (0, fs_1.ensureEmptyDir)(options.output);
+    if (!(await (0, fs_1.existsDir)(options.output))) {
+        if (options.verbose)
+            (0, cli_1.logExec)("mkdir", ["-p", options.output]);
+        await (0, promises_1.mkdir)(options.output, { recursive: true });
+    }
     const decompress = await ifX(options.decompress, async (decompress) => ({
         ...decompress,
         cores: await resolveCores(decompress.cores),
@@ -192,20 +195,22 @@ async function extractTar(options) {
         stderr: {
             toExitCode: true,
         },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (path) => {
-                current++;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
-        },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (path) => {
+                    current++;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.extractTar = extractTar;