@verdnatura/myt 1.5.20 → 1.5.21

package/README.md

@@ -13,6 +13,11 @@ development, so any help is welcomed! Feel free to contribute.
 
 ## Installation
 
+Required libraries to build with *node-gyp*.
+```text
+# apt install libkrb5-dev libssl-dev
+```
+
 It's recommended to install the package globally.
 
 ```text
@@ -148,9 +153,10 @@ routines.
 
 You can create your local fixture and structure files.
 
-* *dump/beforeDump.sql*
-* *dump/afterDump.sql*
-* *dump/fixtures.sql*
+* *dump/dump.before.sql*
+* *dump/dump.after.sql*
+* *dump/fixtures.before.sql*
+* *dump/fixtures.after.sql*
 
 ## Versioning commands
 
@@ -180,7 +186,7 @@ When *--checkout* option is provided, it does the following before export:
 Applies versions and routine changes into database.
 
 ```text
-$ myt push [<remote>] [-f|--force] [-c|--commit] [-s|--sums]
+$ myt push [<remote>] [-f|--force] [-c|--commit] [-s|--sums] [-t|--triggers]
 ```
 
 Commit is saved into database only if *--commit* option is provided, it
@@ -221,7 +227,7 @@ Exports database structure and fixtures from remote into hidden files located
 in *dump* folder. If no remote is specified *production* is used.
 
 ```text
-$ myt dump [<remote>] [-l|--lock]
+$ myt dump [<remote>] [-l|--lock] [-t|--triggers]
 ```
 
 ### fixtures

package/assets/myt.default.yml

@@ -2,7 +2,24 @@ versionSchema: myt
 versionDigits: 5
 maxOldVersions: 20
 mockDate: false
+mockFunctions:
+  - mockTime
+  - mockUtcTime
 sumViews: true
+privileges:
+  userTable: global_priv
+  userWhere: >-
+    JSON_VALUE(`Priv`, '$.is_role')
+    AND JSON_VALUE(`Priv`, '$.authentication_string') IS NULL
+    AND JSON_VALUE(`Priv`, '$.mysql_old_password') IS NULL
+    AND JSON_VALUE(`Priv`, '$.mysql_native_password') IS NULL
+  tables:
+    - db
+    - tables_priv
+    - columns_priv
+    - procs_priv
+  where: >-
+    `Host` = ''
 schemas:
   - myt
 fixtures:

package/helpers/local-fixtures.js

@@ -0,0 +1,79 @@
+#!/bin/node
+
+require('require-yaml');
+const mysql = require('mysql2/promise');
+const {fixtures} = require(`${process.cwd()}/myt.config.yml`);
+
+const config = {
+  host: 'localhost',
+  user: 'root',
+  password: 'root'
+};
+
+async function connectAndQuery() {
+	try {
+		const connection = await mysql.createConnection(config);
+
+		// Get schemas
+
+		const [schemaResults] = await connection.query('SHOW DATABASES');
+		const schemaNames = schemaResults.map(r => r.Database);
+
+		// Filter system schemas
+
+		const systemSchemas = ['mysql', 'sys', 'information_schema', 'performance_schema'];
+		const userSchemas = schemaNames.filter(s => !systemSchemas.includes(s));
+
+		if (userSchemas.length === 0) {
+			console.log('There are no user schemas with tables that have records.');
+			connection.end();
+			return;
+		}
+
+		// Get tables with records
+
+		userSchemas.sort((a, b) => a.localeCompare(b));
+		for (const schemaName of userSchemas) {
+			const [tableResults] = await connection.query(
+				`SELECT TABLE_NAME tableName
+					FROM information_schema.TABLES
+					WHERE TABLE_SCHEMA = ?
+						AND TABLE_TYPE <> 'VIEW'`,
+				[schemaName]
+			);
+			const tableNames = tableResults.map(r => r.tableName);
+			if (tableNames.length === 0) continue;
+
+			const schemaFixtures = new Set(fixtures[schemaName]);
+			const nonEmptyTables = [];
+
+			for (const tableName of tableNames) {
+				if (schemaFixtures.has(tableName)) continue;
+				try {
+					const [[row]] = await connection.query(
+						`SELECT COUNT(*) \`count\` FROM ??.??`,
+						[schemaName, tableName]
+					);
+					if (row.count == 0) continue;
+					if (!fixtures[schemaName]?.[tableName])
+						nonEmptyTables.push(tableName);
+				} catch (err) {
+					console.error('Error:', err.message);
+				}
+			}
+
+			nonEmptyTables.sort((a, b) => a.localeCompare(b));
+			if (nonEmptyTables.length) {
+				console.log(`${schemaName}:`);
+				for (const tableName of nonEmptyTables)
+					console.log(`  - ${tableName}`);
+			}
+		}
+
+		connection.end();
+	} catch (err) {
+		console.error('Error:', err.message);
+	}
+}
+
+connectAndQuery();

package/lib/command.js

@@ -10,4 +10,10 @@ module.exports = class MytCommand {
     async run(myt, opts) {
         throw new Error('run command not defined');
     }
+
+    emit(event) {
+        const messages = this.constructor.messages;
+        if (messages && messages[event])
+            console.log(messages[event]);
+    }
 }

package/lib/dumper.js

@@ -8,12 +8,11 @@ module.exports = class Dumper {
         this.opts = opts;
     }
 
-    async init(dumpFile) {
-        const dumpDir = this.opts.dumpDir;
+    async init(dumpDir, dumpFile) {
         if (!await fs.pathExists(dumpDir))
-            await fs.mkdir(dumpDir);
+            await fs.mkdir(dumpDir, {recursive: true});
 
-        const dumpPath = path.join(dumpDir, dumpFile);
+        const dumpPath = path.join(dumpDir, `${dumpFile}.sql`);
 
         // FIXME: If it's called after docker.build() statement it creates an 
         // "invalid" WriteStream
@@ -28,6 +27,10 @@ module.exports = class Dumper {
         this.dumpStream = dumpStream;
     }
 
+    async end() {
+        await this.dumpStream.end();
+    }
+
     async use(schema) {
         await this.dumpStream.write(
             `USE ${SqlString.escapeId(schema, true)};\n`,
@@ -36,7 +39,7 @@ module.exports = class Dumper {
     }
 
     async dumpFixtures(tables, replace, args) {
-        const fixturesArgs = [
+        let fixturesArgs = [
             '--no-create-info',
             '--skip-triggers',
             '--skip-extended-insert',
@@ -45,18 +48,40 @@ module.exports = class Dumper {
             '--skip-set-charset',
             '--skip-comments',
             '--skip-tz-utc'
-        ].concat(args);
-
+        ]
+        if (args)
+            fixturesArgs = fixturesArgs.concat(args);
         if (replace)
             fixturesArgs.push('--replace');
 
         for (const schema in tables) {
-            await this.use(schema);
             const args = fixturesArgs.concat([schema], tables[schema]);
-            await this.runDump('mysqldump', args, this.dumpStream);
+            await this.use(schema);
+            await this.runDump('mysqldump', args);
         }
     }
 
+    async dumpPrivileges(tables, args, where) {
+        let privArgs = [
+            '--no-create-info',
+            '--skip-triggers',
+            '--insert-ignore',
+            '--skip-extended-insert',
+            '--skip-add-locks',
+            '--skip-set-charset',
+            '--skip-comments',
+            '--skip-tz-utc'
+        ];
+        if (args)
+            privArgs = privArgs.concat(args);
+        if (where)
+            privArgs.push('--where', where);
+        args = privArgs.concat(['mysql'], tables);
+
+        await this.use('mysql');
+        await this.runDump('mysqldump', args);
+    }
+
     async runDump(command, args) {
         const iniPath = path.join(this.opts.subdir || '', 'remotes', this.opts.iniFile);
         const myArgs = [
@@ -76,8 +101,4 @@ module.exports = class Dumper {
             rm: true
         }, execOptions);
     }
-
-    async end() {
-        await this.dumpStream.end();
-    }
 }

package/myt-dump.js

@@ -1,13 +1,15 @@
 const Myt = require('./myt');
 const Command = require('./lib/command');
-const fs = require('fs-extra');
 const Dumper = require('./lib/dumper');
+const fs = require('fs-extra');
+const path = require('path');
 
 class Dump extends Command {
     static usage = {
         description: 'Dumps structure and fixtures from remote',
         params: {
-            lock: 'Whether to lock tables on dump'
+            lock: 'Whether to lock tables on dump',
+            triggers: 'Wether to include triggers into dump'
         },
         operand: 'remote'
     };
@@ -17,54 +19,111 @@ class Dump extends Command {
             remote: 'production'
         },
         alias: {
-            lock: 'l'
+            lock: 'l',
+            triggers: 't'
         },
         boolean: [
-            'lock'
+            'lock',
+            'triggers'
         ]
     };
 
+    static messages = {
+        dumpStructure: 'Dumping structure.',
+        dumpData: 'Dumping data.',
+        dumpPrivileges: 'Dumping privileges.',
+        dumpTriggers: 'Dumping triggers.'
+    };
+
     async run(myt, opts) {
-        const dumper = new Dumper(opts);
-        await dumper.init('.dump.sql');
+        let dumper;
+        const dumpDataDir = path.join(opts.dumpDir, '.dump');
         const baseArgs = [
             `--lock-tables=${opts.lock ? 'true' : 'false'}`
         ];
 
-        console.log('Dumping structure.');
+        await fs.remove(dumpDataDir);
+
+        // Structure
+
+        this.emit('dumpStructure');
+
+        dumper = new Dumper(opts);
+        await dumper.init(dumpDataDir, 'structure');
         let dumpArgs = [
             '--default-character-set=utf8',
             '--no-data',
             '--comments',
             '--routines',
             '--events',
-            '--skip-triggers',
-            '--databases'
+            '--skip-triggers'
         ].concat(baseArgs);
 
+        dumpArgs.push('--databases');
         dumpArgs = dumpArgs.concat(opts.schemas);
         await dumper.runDump('docker-dump.sh', dumpArgs);
+        await dumper.end();
+
+        // Data
+
+        this.emit('dumpData');
 
-        console.log('Dumping fixtures.');
+        dumper = new Dumper(opts);
+        await dumper.init(dumpDataDir, 'data');
         await dumper.dumpFixtures(opts.fixtures, false, baseArgs);
+        await dumper.end();
+
+        // Privileges
 
-        console.log('Dumping privileges.');
         const privs = opts.privileges;
-        if (privs && Array.isArray(privs.tables)) {
-            let args = [
-                '--no-create-info',
-                '--skip-triggers',
-                '--insert-ignore',
-                '--complete-insert'
-            ].concat(baseArgs);
-            if (privs.where) args.push('--where', privs.where);
-            args = args.concat(['mysql'], privs.tables);
+        if (privs) {
+            this.emit('dumpPrivileges');
+
+            dumper = new Dumper(opts);
+            await dumper.init(dumpDataDir, 'privileges');
+
+            const {tables, userTable, where} = privs;
+
+            if (tables)
+                await dumper.dumpPrivileges(tables, baseArgs, where);
 
-            await dumper.use('mysql');
-            await dumper.runDump('mysqldump', args);
+            if (userTable) {
+                let userWhere = '';
+                for (const cond of [where, privs.userWhere]) {
+                    if (!cond) continue;
+                    if (userWhere) userWhere += ' AND ';
+                    userWhere += cond;
+                }
+                await dumper.dumpPrivileges([userTable], baseArgs, userWhere);
+            }
+
+            await dumper.end();
         }
 
         await dumper.end();
+
+        // Triggers
+
+        if (opts.triggers) {
+            this.emit('dumpTriggers');
+
+            const dumper = new Dumper(opts);
+            await dumper.init(dumpDataDir, 'triggers');
+
+            let dumpArgs = [
+                '--default-character-set=utf8',
+                '--no-create-info',
+                '--no-data',
+                '--no-create-db',
+                '--skip-opt',
+                '--comments'
+            ].concat(baseArgs);
+
+            dumpArgs.push('--databases');
+            dumpArgs = dumpArgs.concat(opts.schemas);
+            await dumper.runDump('mysqldump', dumpArgs);
+            await dumper.end();
+        }
     }
 }
 

package/myt-fixtures.js

@@ -16,8 +16,8 @@ class Fixtures extends Command {
 
     async run(myt, opts) {
         const dumper = new Dumper(opts);
-        await dumper.init('fixtures.sql');
-        await dumper.dumpFixtures(opts.localFixtures, true);
+        await dumper.init(opts.dumpDir, '.fixtures');
+        await dumper.dumpFixtures(opts.localFixtures, false);
         await dumper.end();
     }
 }
@@ -26,4 +26,3 @@ module.exports = Fixtures;
 
 if (require.main === module)
     new Myt().run(Fixtures);
-

package/myt-pull.js

@@ -98,7 +98,7 @@ class Pull extends Command {
         const schemas = await fs.readdir(routinesDir);
         for (const schema of schemas) {
             if (opts.schemas.indexOf(schema) == -1)
-                await fs.remove(`${routinesDir}/${schema}`, {recursive: true});
+                await fs.remove(`${routinesDir}/${schema}`);
         }
 
         for (const schema in shaSums) {

package/myt-push.js

@@ -311,7 +311,7 @@ class Push extends Command {
 
         for (const change of changes)
         try {
-            if (opts.trigger && change.type.name === 'TRIGGER')
+            if (opts.triggers && change.type.name === 'TRIGGER')
                 continue;
 
             const schema = change.schema;
@@ -325,7 +325,14 @@ class Push extends Command {
                 newSql = await fs.readFile(fullPath, 'utf8');
             const oldSql = await engine.fetchRoutine(type, schema, name);
             const oldSum = engine.getShaSum(type, schema, name);
-            const isEqual = newSql == oldSql;
+
+            const isMockFn = type == 'function'
+                && schema == opts.versionSchema
+                && opts.remote == 'local'
+                && opts.mockDate
+                && opts.mockFunctions
+                && opts.mockFunctions.indexOf(name) !== -1;
+            const ignore = newSql == oldSql || isMockFn;
 
             let statusMsg;
             if (exists && !oldSql)
@@ -336,7 +343,7 @@ class Push extends Command {
                 statusMsg = '[·]'.yellow;
 
             let actionMsg;
-            if (isEqual)
+            if (ignore)
                 actionMsg = '[I]'.blue;
             else
                 actionMsg = '[A]'.green;
@@ -348,7 +355,7 @@ class Push extends Command {
                 change.fullName
             );
 
-            if (!isEqual) {
+            if (!ignore) {
                 const scapedSchema = SqlString.escapeId(schema, true);
 
                 if (exists) {

package/myt-run.js

@@ -36,9 +36,10 @@ class Run extends Command {
 
     async run(myt, opts) {
         const dumpDir = opts.dumpDir;
+        const dumpDataDir = path.join(dumpDir, '.dump');
         const serverDir = path.join(__dirname, 'server');
 
-        if (!await fs.pathExists(`${dumpDir}/.dump.sql`))
+        if (!await fs.pathExists(`${dumpDataDir}/structure.sql`))
             throw new Error('To run local database you have to create a dump first');
 
         // Build base image
@@ -118,12 +119,29 @@ class Run extends Command {
         }
 
         await server.wait();
+        const conn = await myt.createConnection();
+
+        // Mock date functions
+
+        console.log('Mocking date functions.');
+        const mockDateScript = path.join(dumpDir, 'mockDate.sql');
+
+        if (opts.mockDate) {
+            if (!await fs.pathExists(mockDateScript))
+                throw new Error(`Date mock enabled but mock script does not exist: ${mockDateScript}`);
+
+            let sql = await fs.readFile(mockDateScript, 'utf8');
+            sql = sql.replace(/@mockDate/g, SqlString.escape(opts.mockDate));
+            await connExt.multiQuery(conn, sql);
+        }
 
         // Apply changes
 
+        const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
+
         Object.assign(opts, {
+            triggers: hasTriggers,
             commit: true,
-            trigger: true,
             dbConfig
         });
         await myt.runCommand(Push, opts);
@@ -131,40 +149,35 @@ class Run extends Command {
         // Apply fixtures
 
         console.log('Applying fixtures.');
-        await ct.exec(null,
-            'docker-import.sh',
-            ['/workspace/dump/fixtures'],
-            'spawn',
-            true
-        );
-
-        // Create triggers
-
-        console.log('Creating triggers.');
-        const conn = await myt.createConnection();
-
-        for (const schema of opts.schemas) {
-            const triggersPath = `${opts.routinesDir}/${schema}/triggers`;
-            if (!await fs.pathExists(triggersPath))
-                continue;
-
-            const triggersDir = await fs.readdir(triggersPath);
-            for (const triggerFile of triggersDir)
-                await connExt.queryFromFile(conn, `${triggersPath}/${triggerFile}`);
+        const fixturesFiles = [
+            'fixtures.before',
+            '.fixtures',
+            'fixtures.after',
+            'fixtures.local'
+        ]
+        for (const file of fixturesFiles) {
+            if (!await fs.exists(`${dumpDir}/${file}.sql`)) continue;
+            await ct.exec(null, 'docker-import.sh',
+                [`/workspace/dump/${file}`],
+                'spawn',
+                true
+            );
         }
 
-        // Mock date functions
+        // Create triggers
 
-        console.log('Mocking date functions.');
-        const mockDateScript = path.join(dumpDir, 'mockDate.sql');
+        if (!hasTriggers) {
+            console.log('Creating triggers.');
 
-        if (opts.mockDate) {
-            if (!await fs.pathExists(mockDateScript))
-                throw new Error(`Date mock enabled but mock script does not exist: ${mockDateScript}`);
+            for (const schema of opts.schemas) {
+                const triggersPath = `${opts.routinesDir}/${schema}/triggers`;
+                if (!await fs.pathExists(triggersPath))
+                    continue;
 
-            let sql = await fs.readFile(mockDateScript, 'utf8');
-            sql = sql.replace(/@mockDate/g, SqlString.escape(opts.mockDate));
-            await connExt.multiQuery(conn, sql);
+                const triggersDir = await fs.readdir(triggersPath);
+                for (const triggerFile of triggersDir)
+                    await connExt.queryFromFile(conn, `${triggersPath}/${triggerFile}`);
+            }
         }
 
         return server;

package/myt-version.js

@@ -123,7 +123,7 @@ class Version extends Command {
         } catch (err) {
             await conn.query('ROLLBACK');
             if (newVersionDir && await fs.pathExists(newVersionDir))
-                await fs.remove(newVersionDir, {recursive: true});
+                await fs.remove(newVersionDir);
             throw err;
         }
     }

package/myt.js

@@ -170,12 +170,29 @@ class Myt {
     async load(opts) {
         // Configuration file
 
-        const config = require(`${__dirname}/assets/myt.default.yml`);
+        const defaultConfig = require(`${__dirname}/assets/myt.default.yml`);
+        const config = Object.assign({}, defaultConfig);
         
         const configFile = 'myt.config.yml';
         const configPath = path.join(opts.workspace, configFile);
-        if (await fs.pathExists(configPath))
-            Object.assign(config, require(configPath));
+
+        if (await fs.pathExists(configPath)) {
+            const mergeKeys = new Set([
+                'privileges'
+            ]);
+
+            const wsConfig = require(configPath);
+            for (const key in wsConfig) {
+                if (!mergeKeys.has(key)) {
+                    config[key] = wsConfig[key];
+                } else {
+                    config[key] = Object.assign({},
+                        config[key],
+                        wsConfig[key]
+                    );
+                }
+            }
+        }
 
         Object.assign(opts, config);
         opts.configFile = configFile;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@verdnatura/myt",
-  "version": "1.5.20",
+  "version": "1.5.21",
   "author": "Verdnatura Levante SL",
   "description": "MySQL version control",
   "license": "GPL-3.0",
@@ -15,10 +15,10 @@
     "@sqltools/formatter": "^1.2.5",
     "colors": "^1.4.0",
     "ejs": "^3.1.9",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^11.1.1",
     "getopts": "^2.3.0",
-    "ini": "^1.3.8",
-    "mysql2": "^2.3.3",
+    "ini": "^4.1.1",
+    "mysql2": "^3.3.4",
     "nodegit": "^0.27.0",
     "require-yaml": "^0.0.1",
     "sha.js": "^2.4.11"

package/server/Dockerfile.dump

@@ -2,11 +2,8 @@ FROM myt/server
 
 USER root
 
-COPY \
-    .dump.sql  \
-    beforeDump.sql \
-    afterDump.sql \
-    dump/
+COPY .dump dump/.dump
+COPY dump.*.sql dump/
 
 RUN gosu mysql docker-init.sh
 

package/server/docker-fixtures.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 
 # FIXME: It can corrupt data
+# Currently not used, instead mysqldump --skip-extended-insert option is used
 mysqldump $@ | sed -E 's/(VALUES |\),)\(/\1\n\t\(/g'

package/server/docker-import.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 
 FILE="$1.sql"
+if [[ ! -f "$FILE" ]] ; then
+    exit
+fi
+
 echo "[LOG] -> Importing $FILE"
 export MYSQL_PWD=root
 mysql -u root --default-character-set=utf8 --comments -f < "$FILE"

package/server/docker-init.sh

@@ -13,8 +13,11 @@ docker_temp_server_start "$CMD"
 docker_setup_db
 docker_process_init_files /docker-entrypoint-initdb.d/*
 
-docker-import.sh dump/beforeDump
-docker-import.sh dump/.dump
-docker-import.sh dump/afterDump
+docker-import.sh dump/dump.before
+docker-import.sh dump/.dump/structure
+docker-import.sh dump/.dump/data
+docker-import.sh dump/.dump/triggers
+docker-import.sh dump/.dump/privileges
+docker-import.sh dump/dump.after
 
 docker_temp_server_stop

package/template/dump/dump.after.sql

@@ -0,0 +1 @@
+-- Executed after dump

package/template/dump/dump.before.sql

@@ -0,0 +1 @@
+-- Executed before dump

package/template/dump/fixtures.after.sql

@@ -0,0 +1 @@
+-- Executed after fixtures

package/template/dump/fixtures.before.sql

@@ -0,0 +1 @@
+-- Executed before fixtures

package/template/dump/afterDump.sql

@@ -1 +0,0 @@
--- Executed after data dump

package/template/dump/beforeDump.sql

@@ -1 +0,0 @@
--- Executed before data dump

package/template/dump/fixtures.sql

@@ -1 +0,0 @@
--- Place your local fixtures here