@verdnatura/myt 1.8.6 → 2.0.2

package/myt-apply.js

@@ -0,0 +1,203 @@
+const Myt = require('./myt');
+const Command = require('./lib/command');
+const Push = require('./myt-push');
+const fs = require('fs-extra');
+const path = require('path');
+const connExt = require('./lib/conn');
+const SqlString = require('sqlstring');
+const spawn = require('child_process').spawn;
+
+class Apply extends Command {
+    static usage = {
+        description: 'Initialize database',
+        params: {
+            structure: 'Apply only structure',
+            changes: 'Apply only changes',
+            realm: 'Name of fixture realm to use',
+            docker: 'Whether to load git changes from file',
+            load: 'Commit SHA to save'
+        },
+        operand: 'realm'
+    };
+
+    static args = {
+        alias: {
+            structure: 's',
+            changes: 'c',
+            realm: 'm',
+            docker: 'k',
+            load: 'l'
+        },
+        boolean: [
+            'structure',
+            'changes',
+            'docker'
+        ],
+        string: [
+            'realm',
+            'load'
+        ],
+        default: {
+            remote: 'socket'
+        }
+    };
+
+    static reporter = {
+        mockingDate: 'Mocking date functions.',
+        applyingFixtures: 'Applying fixtures.',
+        applyingRealm: 'Applying realm fixtures.',
+        creatingTriggers: 'Creating triggers.',
+    };
+
+    async _run(myt, ctx, cfg, opts) {
+        const {
+            structureDir,
+            dumpDir,
+            fixturesDir
+        } = ctx;
+
+        const {triggersImport} = cfg;
+
+        const conn = await myt.createConnection();
+        const applyAll = !(opts.structure || opts.changes);
+
+        if (ctx.isProtectedRemote)
+            throw new Error('Cannot apply to protected remote');
+
+        if (applyAll || opts.structure) {
+            await this.importFile(path.join(structureDir, 'before.sql'));
+
+            const importFiles = [
+                'structure.sql',
+                'data.sql',
+            ];
+
+            if (triggersImport == 'before')
+                importFiles.push('triggers.sql');
+
+            importFiles.push('privileges.sql');
+
+            for (const file of importFiles)
+                await this.importFile(path.join(dumpDir, file), true);
+
+            await this.importFile(path.join(structureDir, 'after.sql'));
+
+            // Mock date functions
+
+            this.emit('mockingDate');
+            const mockDateScript = path.join(structureDir, 'mock-date.sql');
+
+            if (cfg.mockDate) {
+                if (!await fs.pathExists(mockDateScript))
+                    throw new Error(`Date mock enabled but mock script does not exist: ${mockDateScript}`);
+
+                let sql = await fs.readFile(mockDateScript, 'utf8');
+                sql = sql.replace(/@mockDate/g, SqlString.escape(cfg.mockDate));
+                await connExt.multiQuery(conn, sql);
+            }
+        }
+
+        if (applyAll || opts.changes) {
+            // Apply changes
+
+            await myt.run(Push, {
+                triggers: triggersImport == 'after',
+                docker: true,
+                load: opts.load,
+                commit: true
+            });
+
+            // Apply fixtures
+
+            this.emit('applyingFixtures');
+            const fixturesFiles = [
+                ['before.sql'],
+                ['.dump.sql', true],
+                ['after.sql'],
+                ['local.sql'],
+            ];
+            for (const [file, force] of fixturesFiles)
+                await this.importFile(path.join(fixturesDir, file), force);
+
+            // Apply realms
+
+            if (opts.realm) {
+                this.emit('applyingRealm');
+                const realmDir = `realms/${opts.realm}`;
+                let realmFiles =  await fs.readdir(realmDir);
+                realmFiles = realmFiles.map(file => path.parse(file).name);
+                for (const file of realmFiles) {
+                    await this.importFile(path.join(realmDir, `${file}.sql`));
+                }
+            }
+
+            // Create triggers
+
+            if (triggersImport == 'after') {
+                this.emit('creatingTriggers');
+
+                for (const schema of cfg.schemas) {
+                    const triggersPath = `${ctx.routinesDir}/${schema}/triggers`;
+                    if (!await fs.pathExists(triggersPath))
+                        continue;
+
+                    const triggersDir = await fs.readdir(triggersPath);
+                    for (const triggerFile of triggersDir)
+                        await connExt.queryFromFile(conn, `${triggersPath}/${triggerFile}`);
+                }
+            }
+
+            await conn.end();
+        }
+    }
+
+    async importFile(file, force) {
+        const {cfg, ctx} = this;
+
+        if (!await fs.exists(file)) {
+            if (cfg.debug)
+                console.debug('Import:', `${file} does not exist, ignoring`);
+            return;
+        }
+
+        const iniPath = path.join(ctx.mytDir, 'remotes', ctx.iniFile);
+        const execArgs = [
+            `--defaults-file=${iniPath}`,
+            '--default-character-set=utf8',
+            '--comments',
+        ];
+
+        if (force)
+            execArgs.push('--force');
+
+        let stdio;
+        const mysqlBin = 'mariadb';
+        if (cfg.debug === true) {
+            const quotedArgs = execArgs
+                .map(x => /\s/g.test(x) ? `"${x}"` : x)
+                .join(' ');
+            console.debug('Command:', `${mysqlBin} ${quotedArgs} < ${file}`.yellow);
+
+            stdio = ['pipe', 'inherit', 'inherit'];
+        } else {
+            stdio = ['pipe', 'ignore', 'ignore'];
+        }
+        
+        const child = spawn(mysqlBin, execArgs, {stdio});
+        fs.createReadStream(file).pipe(child.stdin);
+
+        return await new Promise((resolve, reject) => {
+            child.on('exit', code => {
+                if (code !== 0)
+                    reject(new Error(`${mysqlBin} exit code ${code}`));
+                else
+                    resolve(code);
+            });
+        });
+    }
+}
+
+module.exports = Apply;
+
+if (require.main === module)
+    new Myt().cli(Apply);

package/myt-build.js

@@ -0,0 +1,283 @@
+const Myt = require('./myt');
+const Command = require('./lib/command');
+const docker = require('./lib/docker');
+const fs = require('fs-extra');
+const path = require('path');
+const {hashElement} = require('folder-hash');
+const {createHash} = require('crypto');
+
+class Build extends Command {
+    static usage = {
+        description: 'Build local database server container',
+        params: {
+            name: 'Built image name',
+            tag: 'Built image tag',
+            force: 'Whether to force build',
+            realm: 'Name of fixture realm to use'
+        },
+        operand: 'realm'
+    };
+
+    static args = {
+        alias: {
+            name: 'n',
+            tag: 't',
+            force: 'f',
+            realm: 'm'
+        },
+        string: [
+            'name',
+            'tag',
+            'realm'
+        ],
+        boolean: [
+            'force'
+        ]
+    };
+
+    static reporter = {
+        buildingBaseImages: 'Building base images.',
+        buildingServerImage: 'Building server image.',
+        serverImageTag: function(tag) {
+            console.log('Image:', tag);
+        }
+    };
+
+    async _run(myt, ctx, cfg, opts) {
+        const {
+            dumpDir,
+            dockerDir,
+            version,
+            subdir
+        } = ctx;
+
+        const {
+            realm,
+            force
+        } = opts;
+
+        const serverDir = path.join(__dirname, 'docker/server');
+
+        if (!await fs.pathExists(`${dumpDir}/structure.sql`))
+            throw new Error('Dump file not found');
+
+        // Initialize
+
+        const imageLabels = [`myt.version=${version}`];
+        const tagHash = createHash('sha1');
+        const repo = await myt.openRepo();
+
+        const commit = await repo.getHeadCommit();
+        const commitSha = commit?.sha() || '';
+        tagHash.update(commitSha);
+        imageLabels.push(`myt.commit-sha=${commitSha}`);
+
+        // Get changed files hash
+
+        const repoExt = require('./lib/repo');
+        const regex = subdir && new RegExp(`^${subdir}\/(.+)`);
+        const gitChanges = new Set();
+        const dirnames = new Set();
+        const {sep} = path;
+
+        async function pushChanges(diff) {
+            if (!diff) return;
+            const patches = await diff.patches();
+            for (const patch of patches) {
+                let filePath = patch.newFile().path();
+                if (regex) {
+                    const match = filePath.match(regex);
+                    if (!match) continue;
+                }
+                gitChanges.add(filePath);
+
+                const parts = path.dirname(filePath).split(sep);
+                for(let i = 0; i < parts.length; i++)
+                    dirnames.add(parts.slice(0, i + 1).join(sep));
+            }
+        }
+
+        await pushChanges(await repoExt.getUnstaged(repo));
+        await pushChanges(await repoExt.getStaged(repo));
+
+        if (cfg.debug)
+            console.debug('Uncommited:'.magenta, gitChanges);
+
+        let changesSha;
+        if (gitChanges.size) {
+            const {hash} = await hashElement(subdir, {
+                algo: 'sha1',
+                encoding: 'hex',
+                folders: {
+                    include: [...dirnames],
+                    matchBasename: false,
+                    matchPath: true
+                },
+                files: {
+                    include: [...gitChanges],
+                    matchBasename: false,
+                    matchPath: true
+                }
+            });
+
+            changesSha = hash;
+            tagHash.update(changesSha);
+            imageLabels.push(`myt.changes-sha=${changesSha}`);
+        }
+
+        if (realm)
+            imageLabels.push(`myt.realm=${realm}`);
+
+        // Get image tag
+
+        const tagSha = tagHash.digest('hex');
+        const shortSha = tagSha.substring(0, cfg.gitShortLen);
+
+        let imageTag = shortSha;
+        if (realm) imageTag += `-${realm}`;
+
+        const imageName = opts.name || cfg.code;
+        const tag = `${imageName}:${imageTag}`;
+
+        // Check for existent image
+
+        let labels;
+        try {
+            labels = await docker.inspect(tag, {
+                format: '{{json .Config.Labels}}'
+            });
+        } catch (err) {
+            if (err.code !== 1) throw err;
+            labels = {};
+        }
+
+        const useCache = !force
+            && labels['myt.version'] == version
+            && labels['myt.commit-sha'] == commitSha
+            && (
+                (!changesSha && !labels['myt.changes-sha'])
+                || (labels['myt.changes-sha'] == changesSha)
+            )
+            && (
+                (!realm && !labels['myt.realm'])
+                || (labels['myt.realm'] == realm)
+            );
+
+        if (!useCache) {
+            // Build base server image
+
+            const baseTag = `myt/base:${version}`;
+            const mytTag = `myt/app:${version}`;
+
+            let baseLabels;
+            try {
+                baseLabels = await docker.inspect(baseTag,
+                    {format: '{{json .Config.Labels}}'}
+                );
+            } catch (err) {
+                if (err.code !== 1) throw err;
+            }
+
+            let buildBase = !baseLabels;
+
+            const buildBaseLabels = [];
+            const dbDockerfile = path.join(dockerDir, 'Dockerfile');
+            const dbDockerExists = await fs.pathExists(dbDockerfile);
+
+            if (!buildBase && dbDockerExists) {
+                const {hash} = await hashElement(dockerDir, {
+                    algo: 'sha1',
+                    encoding: 'hex',
+                });
+
+                buildBase = baseLabels['myt.server-sha'] != hash;
+                buildBaseLabels.push(`myt.server-sha=${hash}`);
+            }
+
+            if (buildBase || force) {
+                this.emit('buildingBaseImages');
+
+                let dbTag;
+                if (dbDockerExists) {
+                    dbTag = `myt/db:${version}`;
+                    await docker.build(dockerDir, {
+                        tag: dbTag,
+                        file: dbDockerfile,
+                        label: buildBaseLabels,
+                    }, cfg.debug);
+                } else {
+                    dbTag = cfg.dbImageTag;
+                }
+
+                const buildArgs = [];
+                if (dbTag)
+                    buildArgs.push(`DB_TAG=${dbTag}`);
+
+                await docker.build(__dirname, {
+                    tag: baseTag,
+                    file: path.join(serverDir, 'Dockerfile.base'),
+                    buildArg: buildArgs,
+                    label: buildBaseLabels,
+                }, cfg.debug);
+
+                await docker.build(__dirname, {
+                    tag: mytTag,
+                    file: path.join(serverDir, 'Dockerfile.myt'),
+                    buildArg: `BASE_TAG=${baseTag}`,
+                    label: buildBaseLabels,
+                }, cfg.debug);
+            }
+
+            // Build dump image
+
+            this.emit('buildingServerImage');
+
+            let dbCommit;
+            const versionFile = path.join(dumpDir, 'version.json');
+            if (await fs.exists(versionFile)) {
+                const dbVersion = JSON.parse(await fs.readFile(versionFile, 'utf8'));
+                dbCommit = dbVersion.gitCommit;
+            }
+            const changes = await myt.getChanges(dbCommit);
+
+            const changesFile = path.join(ctx.routinesDir, '.changes.json');
+            await fs.writeFile(
+                changesFile,
+                JSON.stringify(changes, null, 1)
+            );
+
+            try {
+                const buildArgs = [
+                    `ROOT_PASS=${cfg.rootPassword}`,
+                    `BASE_TAG=${baseTag}`,
+                    `MYT_TAG=${mytTag}`,
+                    `MYT_COMMIT=${commitSha}`
+                ];
+
+                if (realm)
+                    buildArgs.push(`MYT_REALM=${realm}`);
+
+                const tags = [tag, `${imageName}:latest`];
+                if (opts.tag)
+                    tags.push(`${imageName}:${opts.tag}`);
+
+                await docker.build(ctx.mytDir, {
+                    tag: tags,
+                    file: path.join(serverDir, 'Dockerfile'),
+                    label: imageLabels,
+                    buildArg: buildArgs
+                }, cfg.debug);
+            } finally {
+                await fs.unlink(changesFile);
+            }
+        }
+
+        this.emit('serverImageTag', tag);
+        return tag;
+    }
+}
+
+module.exports = Build;
+
+if (require.main === module)
+    new Myt().cli(Build);

package/myt-clean.js

@@ -14,7 +14,7 @@ class Clean extends Command {
         }
     };
 
-    static opts = {
+    static args = {
         alias: {
             purge: 'p'
         },
@@ -41,15 +41,17 @@ class Clean extends Command {
         }
     };
 
-    async run(myt, opts) {
+    async _run(myt, ctx, cfg, opts) {
+        const {versionsDir} = ctx;
+
         const conn = await myt.dbConnect();
-        const archiveDir = path.join(opts.versionsDir, '.archive');
+        const archiveDir = path.join(versionsDir, '.archive');
 
         const dbVersion = await myt.fetchDbVersion() || {};
         const number = parseInt(dbVersion.number);
 
         const oldVersions = [];
-        const versionDirs = await fs.readdir(opts.versionsDir);
+        const versionDirs = await fs.readdir(versionsDir);
         for (const versionDir of versionDirs) {
             const version = await myt.loadVersion(versionDir);
             const shouldArchive = version
@@ -60,15 +62,17 @@ class Clean extends Command {
                 oldVersions.push(versionDir);
         }
 
-        if (opts.maxOldVersions
-        && oldVersions.length > opts.maxOldVersions) {
-            oldVersions.splice(-opts.maxOldVersions);
+        const {maxOldVersions} = cfg;
+
+        if (maxOldVersions
+        && oldVersions.length > maxOldVersions) {
+            oldVersions.splice(-maxOldVersions);
 
             if (!await fs.pathExists(archiveDir))
                 await fs.mkdir(archiveDir);
 
             for (const oldVersion of oldVersions) {
-                const srcDir = path.join(opts.versionsDir, oldVersion);
+                const srcDir = path.join(versionsDir, oldVersion);
                 const dstDir = path.join(archiveDir, oldVersion);
 
                 if (!await fs.pathExists(dstDir))
@@ -91,7 +95,7 @@ class Clean extends Command {
             this.emit('versionsArchived');
 
         if (opts.purge) {
-            const versionDb = new VersionDb(myt, opts.versionsDir);
+            const versionDb = new VersionDb(myt, versionsDir);
             versionDb.load();
 
             const archiveDb = new VersionDb(myt, archiveDir);
@@ -101,7 +105,7 @@ class Clean extends Command {
                 `SELECT number, file FROM versionLog
                     WHERE code = ?
                     ORDER BY number, file`,
-                [opts.code]
+                [cfg.code]
             );
 
             let nPurged = 0;
@@ -113,7 +117,7 @@ class Clean extends Command {
                     await conn.query(
                         `DELETE FROM versionLog
                             WHERE code = ? AND number = ? AND file = ?`,
-                        [opts.code, script.number, script.file]
+                        [cfg.code, script.number, script.file]
                     );
                     nPurged++;
                 }

package/myt-create.js

@@ -13,7 +13,7 @@ class Create extends Command {
         operand: 'name'
     };
 
-    static opts = {
+    static args = {
         alias: {
             type: 't',
             name: 'n'
@@ -27,12 +27,14 @@ class Create extends Command {
         }
     };
 
-    async cli(myt, opts) {
-        await super.cli(myt, opts);
+    async cli() {
+        await super.cli();
         console.log('Routine created.');
     }
 
-    async run(myt, opts) {
+    async _run(myt, ctx, cfg, opts) {
+        const {type} = opts;
+
         const match = opts.name.match(/^(\w+)\.(\w+)$/);
         if (!match)
             throw new Error('Invalid object name, should contain schema and routine name');
@@ -43,30 +45,36 @@ class Create extends Command {
         const params = {
             schema,
             name,
-            definer: opts.defaultDefiner
+            definer: cfg.defaultDefiner
         };
 
-        switch (opts.type) {
+        switch (type) {
             case 'event':
-            case 'function':
             case 'procedure':
             case 'trigger':
-                params.body = "BEGIN\n-- Your code goes here\nEND";
+                params.body = "BEGIN\n\t-- Your code goes here\nEND";
+                break;
+            case 'function':
+                params.body = "BEGIN\n\tRETURN 1;\nEND";
                 break;
             case 'view':
                 params.definition = "SELECT TRUE"
                 break;
         }
 
-        const exporter = new Exporter(opts.type, opts.replace);
+        const exporter = new Exporter(type, cfg.replace);
         await exporter.init();
         const sql = exporter.format(params);
 
-        const routineDir = `${opts.routinesDir}/${schema}/${opts.type}s`;
+        const routineDir = `${ctx.routinesDir}/${schema}/${type}s`;
         if (!await fs.pathExists(routineDir))
-            await fs.mkdir(routineDir);
+            await fs.mkdir(routineDir, {recursive: true});
 
         const routineFile = `${routineDir}/${name}.sql`;
+
+        if (await fs.exists(routineFile))
+            throw new Error('Routine already exists');
+        
         await fs.writeFile(routineFile, sql);
     }
 }