@verdnatura/myt 1.5.1

package/myt-pull.js

@@ -0,0 +1,129 @@
+
+const Myt = require('./myt');
+const Command = require('./lib/command');
+const fs = require('fs-extra');
+const nodegit = require('nodegit');
+const ExporterEngine = require('./lib/exporter-engine');
+
+class Pull extends Command {
+    static usage = {
+        description: 'Incorporate database routine changes into workspace',
+        params: {
+            force: 'Do it even if there are local changes',
+            checkout: 'Move to same database commit before pull',
+            update: 'Update all routines',
+            sums: 'Save SHA sums of all objects'
+        },
+        operand: 'remote'
+    };
+
+    static localOpts = {
+        alias: {
+            force: 'f',
+            checkout: 'c',
+            update: 'u',
+            sums: 's'
+        },
+        boolean: [
+            'force',
+            'checkout',
+            'update',
+            'sums'
+        ]
+    };
+
+    async run(myt, opts) {
+        const conn = await myt.dbConnect();
+        const repo = await myt.openRepo();
+
+        if (!opts.force) {
+            async function hasChanges(diff) {
+                if (diff)
+                for (const patch of await diff.patches()) {
+                    const match = patch
+                        .newFile()
+                        .path()
+                        .match(/^routines\/(.+)\.sql$/);
+                    if (match) return true;
+                }
+
+                return false;
+            }
+
+            // Check for unstaged changes
+
+            const unstagedDiff = await myt.getUnstaged(repo);
+
+            if (await hasChanges(unstagedDiff))
+                throw new Error('You have unstaged changes, save them before pull');
+
+            // Check for staged changes
+
+            const stagedDiff = await myt.getStaged(repo);
+ 
+            if (await hasChanges(stagedDiff))
+                throw new Error('You have staged changes, save them before pull');
+        }
+
+        // Checkout to remote commit
+
+        if (opts.checkout) {
+            const version = await myt.fetchDbVersion();
+
+            if (version && version.gitCommit) {
+                const now = parseInt(new Date().toJSON());
+                const branchName = `myt-pull_${now}`;
+                console.log(`Creating branch '${branchName}' from database commit.`);
+                const commit = await repo.getCommit(version.gitCommit);
+                const branch = await nodegit.Branch.create(repo,
+                    `myt-pull_${now}`, commit, () => {});
+                await repo.checkoutBranch(branch);
+            }
+        }
+
+        // Export routines to SQL files
+
+        console.log(`Incorporating routine changes.`);
+
+        const engine = new ExporterEngine(conn, opts.mytDir);
+        await engine.init();
+        const shaSums = engine.shaSums;
+
+        const routinesDir = opts.routinesDir;
+        if (!await fs.pathExists(routinesDir))
+            await fs.mkdir(routinesDir);
+
+        // Delete old schemas
+
+        const schemas = await fs.readdir(routinesDir);
+        for (const schema of schemas) {
+            if (opts.schemas.indexOf(schema) == -1)
+                await fs.remove(`${routinesDir}/${schema}`, {recursive: true});
+        }
+
+        for (const schema in shaSums) {
+            if (!await fs.pathExists(`${routinesDir}/${schema}`))
+                engine.deleteSchemaSums(schema);
+        }
+
+        // Export objects to SQL files
+
+        for (const schema of opts.schemas) {
+            let schemaDir = `${routinesDir}/${schema}`;
+            if (!await fs.pathExists(schemaDir))
+                await fs.mkdir(schemaDir);
+
+            for (const exporter of engine.exporters)
+                await exporter.export(routinesDir,
+                    schema, opts.update, opts.sums);
+        }
+
+        await engine.refreshPullDate();
+        await engine.saveInfo();
+    }
+}
+
+module.exports = Pull;
+
+if (require.main === module)
+    new Myt().run(Pull);

package/myt-push.js

@@ -0,0 +1,475 @@
+
+const Myt = require('./myt');
+const Command = require('./lib/command');
+const fs = require('fs-extra');
+const nodegit = require('nodegit');
+const ExporterEngine = require('./lib/exporter-engine');
+
+/**
+ * Pushes changes to remote.
+ */
+class Push extends Command {
+    static usage = {
+        description: 'Apply changes into database',
+        params: {
+            force: 'Answer yes to all questions',
+            commit: 'Wether to save the commit SHA into database',
+            sums: 'Save SHA sums of pushed objects'
+        },
+        operand: 'remote'
+    };
+
+    static localOpts = {
+        alias: {
+            force: 'f',
+            commit: 'c',
+            sums: 's'
+        },
+        boolean: [
+            'force',
+            'commit',
+            'sums'
+        ]
+    };
+
+    async run(myt, opts) {
+        const conn = await myt.dbConnect();
+        this.conn = conn;
+
+        if (opts.remote == 'local')
+            opts.commit = true;
+
+        // Obtain exclusive lock
+
+        const [[row]] = await conn.query(
+            `SELECT GET_LOCK('myt_push', 30) getLock`);
+
+        if (!row.getLock) {
+            let isUsed = 0;
+
+            if (row.getLock == 0) {
+                const [[row]] = await conn.query(
+                    `SELECT IS_USED_LOCK('myt_push') isUsed`);
+                isUsed = row.isUsed;
+            }
+
+            throw new Error(`Cannot obtain exclusive lock, used by connection ${isUsed}`);
+        }
+
+        async function releaseLock() {
+            await conn.query(`DO RELEASE_LOCK('myt_push')`);
+        }
+
+        try {
+            await this.push(myt, opts, conn);
+        } catch(err) {
+            try {
+                await releaseLock();
+            } catch (e) {
+                console.error(e);
+            }
+            throw err;
+        }
+
+        await releaseLock();
+    }
+
+    async push(myt, opts, conn) {
+        const pushConn = await myt.createConnection();
+
+        // Get database version
+
+        const version = await myt.fetchDbVersion() || {};
+
+        console.log(
+            `Database information:`
+            + `\n -> Version: ${version.number}`
+            + `\n -> Commit: ${version.gitCommit}`
+        );
+
+        if (!version.number)
+            version.number = String('0').padStart(opts.versionDigits, '0');
+        if (!/^[0-9]*$/.test(version.number))
+            throw new Error('Wrong database version');
+
+        // Prevent push to production by mistake
+
+        if (opts.remote == 'production') {
+            console.log(
+                '\n (   (       ) (                       (       )     ) '
+                + '\n )\\ ))\\ ) ( /( )\\ )          (        ))\\ ) ( /(  ( /( '
+                + '\n(()/(()/( )\\()|()/(     (    )\\   )  /(()/( )\\()) )\\())'
+                + '\n /(_))(_)|(_)\\ /(_))    )\\ (((_) ( )(_))(_)|(_)\\ ((_)\\ '
+                + '\n(_))(_))   ((_|_))_  _ ((_))\\___(_(_()|__)) ((_)  _((_)'
+                + '\n| _ \\ _ \\ / _ \\|   \\| | | ((/ __|_   _|_ _| / _ \\| \\| |'
+                + '\n|  _/   /| (_) | |) | |_| || (__  | |  | | | (_) | .  |'
+                + '\n|_| |_|_\\ \\___/|___/ \\___/  \\___| |_| |___| \\___/|_|\\_|'
+                + '\n'
+            );
+
+            if (!opts.force) {
+                const readline = require('readline');
+                const rl = readline.createInterface({
+                    input: process.stdin,
+                    output: process.stdout
+                });
+                const answer = await new Promise(resolve => {
+                    rl.question('Are you sure? (Default: no) [yes|no] ', resolve);
+                });
+                rl.close();
+
+                if (answer !== 'yes')
+                    throw new Error('Changes aborted');
+            }
+        }
+
+        // Apply versions
+
+        console.log('Applying versions.');
+
+        let nChanges = 0;
+        let silent = true;
+        const versionsDir = opts.versionsDir;
+
+        function logVersion(version, name, error) {
+            console.log('', version.bold, name);
+        }
+        function logScript(type, message, error) {
+            console.log(' ', type.bold, message);
+        }
+        function isUndoScript(script) {
+            return /\.undo\.sql$/.test(script);
+        }
+
+        if (await fs.pathExists(versionsDir)) {
+            const versionDirs = await fs.readdir(versionsDir);
+
+            for (const versionDir of versionDirs) {
+                if (versionDir == 'README.md')
+                    continue;
+
+                const dirVersion = myt.parseVersionDir(versionDir);
+                if (!dirVersion) {
+                    logVersion('[?????]'.yellow, versionDir,
+                        `Wrong directory name.`
+                    );
+                    continue;
+                }
+
+                const versionNumber = dirVersion.number;
+                const versionName = dirVersion.name;
+
+                if (versionNumber.length != version.number.length) {
+                    logVersion('[*****]'.gray, versionDir,
+                        `Bad version length, should have ${version.number.length} characters.`
+                    );
+                    continue;
+                }
+
+                const scriptsDir = `${versionsDir}/${versionDir}`;
+                const scripts = await fs.readdir(scriptsDir);
+
+                const [versionLog] = await conn.query(
+                    `SELECT file FROM versionLog
+                        WHERE code = ?
+                            AND number = ?
+                            AND errorNumber IS NULL`,
+                    [opts.code, versionNumber]
+                );
+
+                for (const script of scripts)
+                if (!isUndoScript(script)
+                && versionLog.findIndex(x => x.file == script) === -1) {
+                    silent = false;
+                    break;
+                }
+
+                if (silent) continue;
+                logVersion(`[${versionNumber}]`.cyan, versionName);
+
+                for (const script of scripts) {
+                    if (!/^[0-9]{2}-[a-zA-Z0-9_]+(.undo)?\.sql$/.test(script)) {
+                        logScript('[W]'.yellow, script, `Wrong file name.`);
+                        continue;
+                    }
+                    if (isUndoScript(script))
+                        continue;
+
+                    const [[row]] = await conn.query(
+                        `SELECT errorNumber FROM versionLog
+                            WHERE code = ?
+                                AND number = ?
+                                AND file = ?`,
+                        [
+                            opts.code,
+                            versionNumber,
+                            script
+                        ]
+                    );
+                    const apply = !row || row.errorNumber;
+                    const actionMsg = apply ? '[+]'.green : '[I]'.blue;
+                    
+                    logScript(actionMsg, script);
+                    if (!apply) continue;
+
+                    let err;
+                    try {
+                        await myt.queryFromFile(pushConn,
+                            `${scriptsDir}/${script}`);
+                    } catch (e) {
+                        err = e;
+                    }
+
+                    await conn.query(
+                        `INSERT INTO versionLog
+                            SET code = ?,
+                                number = ?,
+                                file = ?,
+                                user = USER(),
+                                updated = NOW(),
+                                errorNumber = ?,
+                                errorMessage = ?
+                            ON DUPLICATE KEY UPDATE
+                                updated = VALUES(updated),
+                                user = VALUES(user),
+                                errorNumber = VALUES(errorNumber),
+                                errorMessage = VALUES(errorMessage)`,
+                        [
+                            opts.code,
+                            versionNumber,
+                            script,
+                            err && err.errno,
+                            err && err.message
+                        ]
+                    );
+
+                    if (err) throw err;
+                    nChanges++;
+                }
+
+                await this.updateVersion('number', versionNumber);
+            }
+        }
+
+        // Apply routines
+
+        console.log('Applying changed routines.');
+    
+        const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
+
+        let nRoutines = 0;
+        let changes = await myt.changedRoutines(version.gitCommit);
+        changes = this.parseChanges(changes);
+
+        const routines = [];
+        for (const change of changes)
+            if (change.isRoutine)
+                routines.push([
+                    change.schema,
+                    change.name,
+                    change.type.name
+                ]);
+
+        if (routines.length) {
+            await conn.query(
+                `DROP TEMPORARY TABLE IF EXISTS tProcsPriv`
+            );
+            await conn.query(
+                `CREATE TEMPORARY TABLE tProcsPriv
+                    ENGINE = MEMORY
+                    SELECT * FROM mysql.procs_priv
+                        WHERE (Db, Routine_name, Routine_type) IN (?)`,
+                [routines]
+            );
+        }
+
+        const engine = new ExporterEngine(conn, opts.mytDir);
+        await engine.init();
+
+        async function finalize() {
+            await engine.saveInfo();
+
+            if (routines.length) {
+                await conn.query('FLUSH PRIVILEGES');
+                await conn.query(`DROP TEMPORARY TABLE tProcsPriv`);
+            }
+        }
+
+        for (const change of changes)
+        try {
+            const schema = change.schema;
+            const name = change.name;
+            const type = change.type.name.toLowerCase();
+            const fullPath = `${opts.routinesDir}/${change.path}.sql`;
+            const exists = await fs.pathExists(fullPath);
+
+            let newSql;
+            if (exists)
+                newSql = await fs.readFile(fullPath, 'utf8');
+            const oldSql = await engine.fetchRoutine(type, schema, name);
+            const oldSum = engine.getShaSum(type, schema, name);
+            const isEqual = newSql == oldSql;
+
+            let actionMsg;
+            if ((exists && isEqual) || (!exists && !oldSql))
+                actionMsg = '[I]'.blue;
+            else if (exists)
+                actionMsg = '[+]'.green;
+            else
+                actionMsg = '[-]'.red;
+
+            const typeMsg = `[${change.type.abbr}]`[change.type.color];
+            console.log('', actionMsg.bold, typeMsg.bold, change.fullName);
+
+            if (!isEqual) {
+                const scapedSchema = pushConn.escapeId(schema, true);
+
+                if (exists) {
+                    if (change.type.name === 'VIEW')
+                        await pushConn.query(`USE ${scapedSchema}`);
+
+                    await myt.multiQuery(pushConn, newSql);
+
+                    if (change.isRoutine) {
+                        await conn.query(
+                            `INSERT IGNORE INTO mysql.procs_priv
+                                SELECT * FROM tProcsPriv
+                                    WHERE Db = ?
+                                        AND Routine_name = ?
+                                        AND Routine_type = ?`,
+                            [schema, name, change.type.name]
+                        );
+                    }
+
+                    if (opts.sums || oldSum)
+                        await engine.fetchShaSum(type, schema, name);
+                } else {
+                    const escapedName =
+                        scapedSchema + '.' +
+                        pushConn.escapeId(name, true);
+
+                    const query = `DROP ${change.type.name} IF EXISTS ${escapedName}`;
+                    await pushConn.query(query);
+
+                    engine.deleteShaSum(type, schema, name);
+                }
+
+                nRoutines++;
+            }
+        } catch (err) {
+            try {
+                await finalize();
+            } catch (e) {
+                console.error(e);
+            }
+            throw err;
+        }
+
+        await finalize();
+
+        if (nRoutines > 0) {
+            console.log(` -> ${nRoutines} routines have changed.`);
+        } else
+            console.log(` -> No routines changed.`);
+
+        if (gitExists && opts.commit) {
+            const repo = await nodegit.Repository.open(this.opts.workspace);
+            const head = await repo.getHeadCommit();
+
+            if (head && version.gitCommit !== head.sha())
+                await this.updateVersion('gitCommit', head.sha());
+        }
+
+        // End
+
+        await pushConn.end();
+    }
+
+    parseChanges(changes) {
+        const routines = [];
+        if (changes)
+            for (const change of changes)
+                routines.push(new Routine(change));
+        return routines;
+    }
+
+    async updateVersion(column, value) {
+        column = this.conn.escapeId(column, true);
+        await this.conn.query(
+            `INSERT INTO version
+                SET code = ?,
+                    ${column} = ?,
+                    updated = NOW()
+                ON DUPLICATE KEY UPDATE 
+                    ${column} = VALUES(${column}),
+                    updated = VALUES(updated)`,
+            [
+                this.opts.code,
+                value
+            ]
+        );
+    }
+
+}
+
+const typeMap = {
+    events: {
+        name: 'EVENT',
+        abbr: 'EVNT',
+        color: 'cyan'
+    },
+    functions: {
+        name: 'FUNCTION',
+        abbr: 'FUNC',
+        color: 'cyan'
+    },
+    procedures: {
+        name: 'PROCEDURE',
+        abbr: 'PROC',
+        color: 'yellow'
+    },
+    triggers: {
+        name: 'TRIGGER',
+        abbr: 'TRIG',
+        color: 'blue'
+    },
+    views: {
+        name: 'VIEW',
+        abbr: 'VIEW',
+        color: 'magenta'
+    },
+};
+
+const routineTypes = new Set([
+    'FUNCTION',
+    'PROCEDURE'
+]);
+
+class Routine {
+    constructor(change) {
+        const path = change.path;
+        const split = path.split('/');
+
+        const schema = split[0];
+        const type = typeMap[split[1]];
+        const name = split[2];
+
+        if (split.length !== 3 || !type)
+            throw new Error(`Wrong routine path for '${path}', check that the sql file is located in the correct directory`);
+
+        Object.assign(this, {
+            path,
+            mark: change.mark,
+            type,
+            schema,
+            name,
+            fullName: `${schema}.${name}`,
+            isRoutine: routineTypes.has(type.name)
+        });
+    }
+}
+
+module.exports = Push;
+
+if (require.main === module)
+    new Myt().run(Push);