@verdnatura/myt 1.5.23 → 1.5.25

package/README.md

@@ -250,7 +250,7 @@ $ myt create [-t <type>] <schema>.<name>
 Cleans all already applied versions older than  *maxOldVersions*.
 
 ```text
-$ myt clean
+$ myt clean [-p|--purge]
 ```
 
 ## Local server commands
@@ -276,8 +276,7 @@ $ myt fixtures [<remote>]
 ### run
 
 Builds and starts local database server container. It only rebuilds the image 
-when fixtures have been modified or when the day on which the image was built 
-is different to today.
+dump has been modified.
 
 ```text
 $ myt run [-c|--ci] [-r|--random]

package/myt-clean.js

@@ -8,27 +8,40 @@ const path = require('path');
  */
 class Clean extends Command {
     static usage = {
-        description: 'Cleans old applied versions'
+        description: 'Cleans old applied versions',
+        params: {
+            purge: 'Wether to remove non-existent scripts from DB log'
+        }
     };
 
     static opts = {
+        alias: {
+            purge: 'p'
+        },
+        boolean: [
+            'purge'
+        ],
         default: {
             remote: 'production'
         }
     };
 
     async run(myt, opts) {
-        await myt.dbConnect();
-        const version = await myt.fetchDbVersion() || {};
-        const number = version.number;
+        const conn = await myt.dbConnect();
+        const versionDirs = await fs.readdir(opts.versionsDir);
+        const archiveDir = path.join(opts.versionsDir, '.archive');
+
+        const dbVersion = await myt.fetchDbVersion() || {};
+        const number = parseInt(dbVersion.number);
 
         const oldVersions = [];
-        const versionDirs = await fs.readdir(opts.versionsDir);
         for (const versionDir of versionDirs) {
-            const dirVersion = myt.parseVersionDir(versionDir);
-            if (!dirVersion) continue;
+            const version = await myt.loadVersion(versionDir);
+            const shouldArchive = version
+                && !version.apply
+                && parseInt(version.number) < number;
 
-            if (parseInt(dirVersion.number) < parseInt(number))
+            if (shouldArchive)
                 oldVersions.push(versionDir);
         }
 
@@ -36,7 +49,6 @@ class Clean extends Command {
         && oldVersions.length > opts.maxOldVersions) {
             oldVersions.splice(-opts.maxOldVersions);
 
-            const archiveDir = path.join(opts.versionsDir, '.archive');
             if (!await fs.pathExists(archiveDir))
                 await fs.mkdir(archiveDir);
 
@@ -46,9 +58,63 @@ class Clean extends Command {
                     path.join(archiveDir, oldVersion)
                 );
 
-            console.log(`Old versions deleted: ${oldVersions.length}`);
+            console.log(`Old versions archived: ${oldVersions.length}`);
         } else
-            console.log(`No versions to delete.`);
+            console.log(`No versions to archive.`);
+
+        if (opts.purge) {
+            const versionDb = new VersionDb(myt, opts.versionsDir);
+            versionDb.load();
+
+            const archiveDb = new VersionDb(myt, archiveDir);
+            archiveDb.load();
+
+            const [res] = await conn.query(
+                `SELECT number, file FROM versionLog
+                    WHERE code = ?
+                    ORDER BY number, file`,
+                [opts.code]
+            );
+
+            for (const script of res) {
+                const hasVersion = await versionDb.hasScript(script);
+                const hasArchive = await archiveDb.hasScript(script);
+    
+                if (!hasVersion && !hasArchive) {
+                    await conn.query(
+                        `DELETE FROM versionLog
+                            WHERE code = ? AND number = ? AND file = ?`,
+                        [opts.code, script.number, script.file]
+                    );
+                }
+            }
+        }
+    }
+}
+
+class VersionDb {
+    constructor(myt, baseDir) {
+        Object.assign(this, {myt, baseDir});
+    }
+
+    async load() {
+        const versionMap = this.versionMap = new Map();
+        if (await fs.pathExists(this.baseDir)) {
+            const dirs = await fs.readdir(this.baseDir);
+            for (const dir of dirs) {
+                const version = this.myt.parseVersionDir(dir);
+                if (!version) continue;
+                versionMap.set(version.number, dir);
+            }
+        }
+        return versionMap;
+    }
+
+    async hasScript(script) {
+        const dir = this.versionMap.get(script.number);
+        if (!dir) return false;
+        const scriptPath = path.join(this.baseDir, dir, script.file);
+        return await fs.pathExists(scriptPath);
     }
 }
 

package/myt-push.js

@@ -90,17 +90,17 @@ class Push extends Command {
 
         // Get database version
 
-        const version = await myt.fetchDbVersion() || {};
+        const dbVersion = await myt.fetchDbVersion() || {};
 
         console.log(
             `Database information:`
-            + `\n -> Version: ${version.number}`
-            + `\n -> Commit: ${version.gitCommit}`
+            + `\n -> Version: ${dbVersion.number}`
+            + `\n -> Commit: ${dbVersion.gitCommit}`
         );
 
-        if (!version.number)
-            version.number = String('0').padStart(opts.versionDigits, '0');
-        if (!/^[0-9]*$/.test(version.number))
+        if (!dbVersion.number)
+            dbVersion.number = String('0').padStart(opts.versionDigits, '0');
+        if (!/^[0-9]*$/.test(dbVersion.number))
             throw new Error('Wrong database version');
 
         // Prevent push to production by mistake
@@ -142,16 +142,25 @@ class Push extends Command {
         let silent = true;
         const versionsDir = opts.versionsDir;
 
-        function logVersion(version, name, error) {
-            console.log('', version.bold, name);
+        function logVersion(version, name, action, error) {
+            let actionMsg;
+            switch(action) {
+                case 'apply':
+                    actionMsg = '[A]'.green;
+                    break;
+                case 'ignore':
+                    actionMsg = '[I]'.blue;
+                    break;
+                default:
+                    actionMsg = '[W]'.yellow;
+            }
+
+            console.log('', (actionMsg + version).bold, name);
         }
         function logScript(type, message, error) {
             console.log(' ', type.bold, message);
         }
-        function isUndoScript(script) {
-            return /\.undo\.sql$/.test(script);
-        }
-       
+
         const skipFiles = new Set([
             'README.md',
             '.archive'
@@ -159,87 +168,49 @@ class Push extends Command {
 
         if (await fs.pathExists(versionsDir)) {
             const versionDirs = await fs.readdir(versionsDir);
-            const [[realm]] = await this.conn.query(
-                `SELECT realm
-                    FROM versionConfig`
-            );
-
             for (const versionDir of versionDirs) {
                 if (skipFiles.has(versionDir)) continue;
+                const version = await myt.loadVersion(versionDir);
 
-                const dirVersion = myt.parseVersionDir(versionDir);
-                if (!dirVersion) {
-                    logVersion('[?????]'.yellow, versionDir,
+                if (!version) {
+                    logVersion('[?????]'.yellow, versionDir, 'warn',
                         `Wrong directory name.`
                     );
                     continue;
                 }
-
-                const versionNumber = dirVersion.number;
-                const versionName = dirVersion.name;
-
-                if (versionNumber.length != version.number.length) {
-                    logVersion('[*****]'.gray, versionDir,
-                        `Bad version length, should have ${version.number.length} characters.`
+                if (version.number.length != dbVersion.number.length) {
+                    logVersion('[*****]'.gray, versionDir, 'warn'
+                        `Bad version length, should have ${dbVersion.number.length} characters.`
                     );
                     continue;
                 }
 
-                const scriptsDir = `${versionsDir}/${versionDir}`;
-                const scripts = await fs.readdir(scriptsDir);
-
-                const [versionLog] = await conn.query(
-                    `SELECT file FROM versionLog
-                        WHERE code = ?
-                            AND number = ?
-                            AND errorNumber IS NULL`,
-                    [opts.code, versionNumber]
-                );
-
-                for (const script of scripts)
-                if (!isUndoScript(script)
-                && versionLog.findIndex(x => x.file == script) === -1) {
-                    silent = false;
-                    break;
-                }
-
+                const {apply} = version;
+                if (apply) silent = false;
                 if (silent) continue;
-                logVersion(`[${versionNumber}]`.cyan, versionName);
 
-                for (const script of scripts) {
-                    const match = script.match(/^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(?:\.undo)?\.sql$/);
-                    
-                    if (!match) {
-                        logScript('[W]'.yellow, script, `Wrong file name.`);
-                        continue;
-                    }
+                const action = apply ? 'apply' : 'ignore';
+                logVersion(`[${version.number}]`.cyan, version.name, action);
 
-                    const skipRealm = match[1] && match[1] !== realm;
+                if (!apply) continue;
 
-                    if (isUndoScript(script) || skipRealm)
+                for (const script of version.scripts) {
+                    const scriptFile = script.file;
+                    
+                    if (!script.matchRegex) {
+                        logScript('[W]'.yellow, scriptFile, `Wrong file name.`);
                         continue;
+                    }
 
-                    const [[row]] = await conn.query(
-                        `SELECT errorNumber FROM versionLog
-                            WHERE code = ?
-                                AND number = ?
-                                AND file = ?`,
-                        [
-                            opts.code,
-                            versionNumber,
-                            script
-                        ]
-                    );
-                    const apply = !row || row.errorNumber;
-                    const actionMsg = apply ? '[+]'.green : '[I]'.blue;
-                    
-                    logScript(actionMsg, script);
-                    if (!apply) continue;
+                    const actionMsg = script.apply ? '[+]'.green : '[I]'.blue;
+                    logScript(actionMsg, scriptFile);
+    
+                    if (!script.apply) continue;
 
                     let err;
                     try {
                         await connExt.queryFromFile(pushConn,
-                            `${scriptsDir}/${script}`);
+                            `${versionsDir}/${versionDir}/${scriptFile}`);
                     } catch (e) {
                         err = e;
                     }
@@ -260,8 +231,8 @@ class Push extends Command {
                                 errorMessage = VALUES(errorMessage)`,
                         [
                             opts.code,
-                            versionNumber,
-                            script,
+                            version.number,
+                            scriptFile,
                             err && err.errno,
                             err && err.message
                         ]
@@ -271,7 +242,7 @@ class Push extends Command {
                     nChanges++;
                 }
 
-                await this.updateVersion('number', versionNumber);
+                await this.updateVersion('number', version.number);
             }
         }
 
@@ -282,7 +253,7 @@ class Push extends Command {
         const gitExists = await fs.pathExists(`${opts.workspace}/.git`);
 
         let nRoutines = 0;
-        const changes = await this.changedRoutines(version.gitCommit);
+        const changes = await this.changedRoutines(dbVersion.gitCommit);
 
         const routines = [];
         for (const change of changes)
@@ -359,7 +330,7 @@ class Push extends Command {
 
             const typeMsg = `[${change.type.abbr}]`[change.type.color];
             console.log('',
-                (statusMsg + actionMsg).bold,
+                (actionMsg + statusMsg).bold,
                 typeMsg.bold,
                 change.fullName
             );
@@ -419,7 +390,7 @@ class Push extends Command {
             const repo = await nodegit.Repository.open(this.opts.workspace);
             const head = await repo.getHeadCommit();
 
-            if (head && version.gitCommit !== head.sha())
+            if (head && dbVersion.gitCommit !== head.sha())
                 await this.updateVersion('gitCommit', head.sha());
         }
 

package/myt-run.js

@@ -9,10 +9,9 @@ const connExt = require('./lib/conn');
 const SqlString = require('sqlstring');
 
 /**
- * Builds the database image and runs a container. It only rebuilds the
- * image when fixtures have been modified or when the day on which the
- * image was built is different to today. Some workarounds have been used
- * to avoid a bug with OverlayFS driver on MacOS.
+ * Builds the database image and runs a container. It only rebuilds the image
+ * when dump has been modified. Some workarounds have been used to avoid a bug
+ * with OverlayFS driver on MacOS.
  */
 class Run extends Command {
     static usage = {
@@ -140,7 +139,7 @@ class Run extends Command {
         const hasTriggers = await fs.exists(`${dumpDataDir}/triggers.sql`);
 
         Object.assign(opts, {
-            triggers: hasTriggers,
+            triggers: !hasTriggers,
             commit: true,
             dbConfig
         });

package/myt-version.js

@@ -74,9 +74,9 @@ class Version extends Command {
             const versionNames = new Set();
             const versionDirs = await fs.readdir(opts.versionsDir);
             for (const versionDir of versionDirs) {
-                const dirVersion = myt.parseVersionDir(versionDir);
-                if (!dirVersion) continue;
-                versionNames.add(dirVersion.name);
+                const version = myt.parseVersionDir(versionDir);
+                if (!version) continue;
+                versionNames.add(version.name);
             }
 
             if (!versionName) {

package/myt.js

@@ -11,6 +11,8 @@ const mysql = require('mysql2/promise');
 const nodegit = require('nodegit');
 const camelToSnake = require('./lib/util').camelToSnake;
 
+const scriptRegex = /^[0-9]{2}-[a-zA-Z0-9_]+(?:\.(?!undo)([a-zA-Z0-9_]+))?(\.undo)?\.sql$/;
+
 class Myt {
     static usage = {
         description: 'Utility for database versioning',
@@ -305,6 +307,11 @@ class Myt {
                     `${__dirname}/assets/structure.sql`, 'utf8');
                 await conn.query(structure);
             }
+
+            const [[realm]] = await conn.query(
+                `SELECT realm FROM versionConfig`
+            );
+            this.realm = realm;
         }
 
         return this.conn;
@@ -332,6 +339,58 @@ class Myt {
         };
     }
 
+    async loadVersion(versionDir) {
+        const {opts} = this;
+
+        const info = this.parseVersionDir(versionDir);
+        if (!info) return null;
+
+        const versionsDir = opts.versionsDir;
+        const scriptsDir = `${versionsDir}/${versionDir}`;
+        const scriptList = await fs.readdir(scriptsDir);
+
+        const [res] = await this.conn.query(
+            `SELECT file, errorNumber IS NOT NULL hasError
+                FROM versionLog
+                WHERE code = ?
+                    AND number = ?`,
+            [opts.code, info.number]
+        );
+        const versionLog = new Map();
+        res.map(x => versionLog.set(x.file, x));
+
+        let applyVersion = false;
+        const scripts = [];
+
+        for (const file of scriptList) {
+            const match = file.match(scriptRegex);
+            if (match) {
+                const scriptRealm = match[1];
+                const isUndo = !!match[2];
+
+                if ((scriptRealm && scriptRealm !== this.realm) || isUndo)
+                    continue;
+            }
+
+            const logInfo = versionLog.get(file);
+            const apply = !logInfo || logInfo.hasError;
+            if (apply) applyVersion = true;
+
+            scripts.push({
+                file,
+                matchRegex: !!match,
+                apply
+            });
+        }
+
+        return {
+            number: info.number,
+            name: info.name,
+            scripts,
+            apply: applyVersion
+        };
+    }
+
     async openRepo() {
         const {opts} = this;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@verdnatura/myt",
-  "version": "1.5.23",
+  "version": "1.5.25",
   "author": "Verdnatura Levante SL",
   "description": "MySQL version control",
   "license": "GPL-3.0",