knex-migrator 4.0.5

package/lib/locking.js

@@ -0,0 +1,113 @@
+const moment = require('moment');
+const debug = require('debug')('knex-migrator:locking');
+const errors = require('./errors');
+const database = require('./database');
+
+/**
+ * @description Private helper to lock the migrations_lock table.
+ * @TODO Locks in Sqlite won't work, because Sqlite doesn't offer read locks (https://github.com/TryGhost/knex-migrator/issues/87)
+ * @returns {*}
+ */
+module.exports.lock = function (connection) {
+    debug('Lock.');
+
+    return database.createTransaction(connection, function (transacting) {
+        return transacting('migrations_lock')
+            .where({
+                lock_key: 'km01'
+            })
+            .forUpdate()
+            .then(function (data) {
+                if (!data || !data.length || data[0].locked) {
+                    throw new errors.MigrationsAreLockedError({
+                        message: 'Migrations are running at the moment. Please wait that the lock get`s released.',
+                        context: 'Either the release was never released because of a e.g. died process or a parallel process is migrating at the moment.',
+                        help: 'If your database looks okay, you can manually release the lock by running `UPDATE migrations_lock set locked=0 where lock_key=\'km01\';`.'
+                    });
+                }
+
+                return (transacting || connection)('migrations_lock')
+                    .where({
+                        lock_key: 'km01'
+                    })
+                    .update({
+                        locked: 1,
+                        acquired_at: moment().format('YYYY-MM-DD HH:mm:ss')
+                    });
+            })
+            .catch(function (err) {
+                if (errors.utils.isIgnitionError(err)) {
+                    throw err;
+                }
+
+                throw new errors.LockError({
+                    message: 'Error while acquire the migration lock.',
+                    err: err
+                });
+            });
+    });
+};
+
+/**
+ * @description Private helper to determine whether the database is locked or not.
+ * @returns {Bluebird<boolean>}
+ */
+module.exports.isLocked = function (connection) {
+    return connection('migrations_lock')
+        .where({
+            lock_key: 'km01'
+        })
+        .then(function (data) {
+            if (!data || !data.length || data[0].locked) {
+                throw new errors.MigrationsAreLockedError({
+                    message: 'Migration lock was never released or currently a migration is running.',
+                    help: 'If you are sure no migration is running, check your data and if your database is in a broken state, you could run `knex-migrator rollback`.'
+                });
+            }
+
+            return false;
+        });
+};
+
+/**
+ * @description Private helper to unlock the database table "migrations_lock".
+ * @TODO Locks in Sqlite won't work, because Sqlite doesn't offer read locks (https://github.com/TryGhost/knex-migrator/issues/87)
+ * @returns {*}
+ */
+module.exports.unlock = function (connection) {
+    debug('Unlock.');
+
+    return database.createTransaction(connection, function (transacting) {
+        return transacting('migrations_lock')
+            .where({
+                lock_key: 'km01'
+            })
+            .forUpdate()
+            .then(function (data) {
+                if (!data || !data.length || !data[0].locked) {
+                    throw new errors.MigrationsAreLockedError({
+                        message: 'Migration lock was already released?.'
+                    });
+                }
+
+                return transacting('migrations_lock')
+                    .where({
+                        lock_key: 'km01'
+                    })
+                    .update({
+                        locked: 0,
+                        released_at: moment().format('YYYY-MM-DD HH:mm:ss')
+                    });
+            })
+            .catch(function (err) {
+                if (errors.utils.isIgnitionError(err)) {
+                    throw err;
+                }
+
+                throw new errors.UnlockError({
+                    message: 'Error while releasing the migration lock.',
+                    err: err
+                });
+            });
+    });
+};

package/lib/utils.js

@@ -0,0 +1,213 @@
+const path = require('path'),
+    _ = require('lodash'),
+    fs = require('fs'),
+    compareVer = require('compare-ver'),
+    Promise = require('bluebird'),
+    resolve = Promise.promisify(require('resolve')),
+    debug = require('debug')('knex-migrator:utils'),
+    errors = require('./errors');
+
+/**
+ * @description This helper function offers two ways of loading the knex-migrator configuration.
+ *
+ * 1. via JS object
+ * 2. via file location
+ *
+ * The expected format is:
+ *
+ * {
+ *   database: Object,
+ *   migrationPath: String,
+ *   currentVersion: String
+ * }
+ *
+ * @param {Object} options
+ * @returns {*}
+ */
+module.exports.loadConfig = function loadConfig(options) {
+    if (options.knexMigratorConfig) {
+        return options.knexMigratorConfig;
+    }
+
+    const knexMigratorFilePath = options.knexMigratorFilePath || process.cwd();
+
+    try {
+        return require(path.join(path.resolve(knexMigratorFilePath), '/MigratorConfig.js'));
+    } catch (err) {
+        if (err.code === 'MODULE_NOT_FOUND') {
+            throw new errors.KnexMigrateError({
+                message: 'Please provide a file named MigratorConfig.js in your project root.',
+                help: 'Read through the README.md to see which values are expected.'
+            });
+        }
+
+        throw new errors.KnexMigrateError({err: err});
+    }
+};
+
+/**
+ * @description List all migration files from disk based on a path.
+ *
+ * @param absolutePath
+ * @returns {Array}
+ */
+exports.listFiles = function listFiles(absolutePath) {
+    let files = [];
+
+    try {
+        files = fs.readdirSync(absolutePath);
+    } catch (err) {
+        throw new errors.KnexMigrateError({
+            code: 'MIGRATION_PATH',
+            message: 'MigrationPath is wrong: ' + absolutePath
+        });
+    }
+
+    files = files.filter(function (file) {
+        // CASE: ignore dot files
+        return !file.match(/^\./);
+    });
+
+    debug(files);
+    return files;
+};
+
+/**
+ * @description Reads all migration files from disk based on a path.
+ * It returns an Array of migration files including it's up/down hooks, config and the name.
+ *
+ * @param absolutePath
+ * @returns {Array}
+ */
+exports.readTasks = function readTasks(absolutePath) {
+    let tasks = [];
+
+    const files = exports.listFiles(absolutePath);
+
+    _.each(files, function (file) {
+        let executeFn = require(path.join(absolutePath, file));
+
+        try {
+            tasks.push({
+                up: executeFn.up,
+                down: executeFn.down,
+                config: executeFn.config,
+                name: file
+            });
+        } catch (err) {
+            debug(err.message);
+
+            throw new errors.MigrationScript({
+                message: err.message,
+                help: 'Cannot load Migrationscript.',
+                context: file
+            });
+        }
+    });
+
+    debug(tasks);
+    return tasks;
+};
+
+/**
+ * @description Reads all version folders from disk in correct order.
+ *
+ * @param absolutePath
+ * @returns {*}
+ */
+exports.readVersionFolders = function readFolders(absolutePath) {
+    let folders = [],
+        toReturn = [];
+
+    try {
+        folders = fs.readdirSync(absolutePath);
+    } catch (err) {
+        throw new errors.KnexMigrateError({
+            message: 'MigrationPath is wrong: ' + absolutePath,
+            code: 'READ_FOLDERS'
+        });
+    }
+
+    if (!folders.length) {
+        return folders;
+    }
+
+    folders.forEach((folderToAdd) => {
+        let index = null;
+
+        // CASE: ignore dot files
+        if (folderToAdd.match(/^\./)) {
+            debug('Ignore Dotfile: ' + folderToAdd);
+            return;
+        }
+
+        toReturn.forEach((existingElement, _index) => {
+            if (index !== null) {
+                return;
+            }
+
+            // CASE: folder to add is smaller, push before this element
+            if (compareVer.gt(folderToAdd, existingElement) === -1) {
+                index = _index;
+            }
+        });
+
+        if (index === null) {
+            if (!toReturn.length) {
+                index = 0;
+            } else {
+                index = toReturn.length;
+            }
+        }
+
+        toReturn.splice(index, 0, folderToAdd);
+    });
+
+    debug(toReturn);
+    return toReturn;
+};
+
+/**
+ * @description Auto detect local installation to avoid version incompatible behaviour
+ */
+exports.getKnexMigrator = function getKnexMigrator(options) {
+    options = options || {};
+
+    return resolve('knex-migrator', {basedir: options.path})
+        .then(function (localCLIPath) {
+            return require(localCLIPath);
+        })
+        .catch(function () {
+            return require('./');
+        });
+};
+
+/**
+ * @description A helper function to figure out if a version is greater than another version.
+ *
+ * Valid versions are:
+ * - 1
+ * - 1.1
+ * - 1.1.0
+ *
+ * It's up to you which pattern you would like to use.
+ *
+ * @param options
+ * @returns {boolean}
+ */
+exports.isGreaterThanVersion = function isGreaterThanVersion(options) {
+    let greaterVersion = options.greaterVersion;
+    let smallerVersion = options.smallerVersion;
+
+    // CASE: are they semver like strings?
+    if (new RegExp(/\./g).test(greaterVersion) && new RegExp(/\./g).test(smallerVersion)) {
+        // -1 less than, 0 equal, 1 greater than
+        return compareVer.gt(greaterVersion, smallerVersion) === 1;
+    }
+
+    // CASE: must be numbers / number like strings
+    greaterVersion = Number(greaterVersion.toString());
+    smallerVersion = Number(smallerVersion.toString());
+
+    return greaterVersion > smallerVersion;
+};

package/logging.js

@@ -0,0 +1,8 @@
+var logging = require('ghost-ignition').logging;
+
+module.exports = logging({
+    env: process.env.NODE_ENV,
+    mode: 'long',
+    level: 'info',
+    transports: ['stdout', 'stderr']
+});

package/migrations/add-primary-key-to-lock-table.js

@@ -0,0 +1,58 @@
+const debug = require('debug')('knex-migrator:lock-table');
+
+/**
+ * Checks if primary key index exists in a table over the given columns.
+ */
+function hasPrimaryKeySQLite(tableName, knex) {
+    const client = knex.client.config.client;
+
+    if (client !== 'sqlite3') {
+        throw new Error('Must use hasPrimaryKeySQLite on an SQLite3 database');
+    }
+
+    return knex.raw(`PRAGMA index_list('${tableName}');`)
+        .then((rawConstraints) => {
+            const tablePrimaryKey = rawConstraints.find(c => c.origin === 'pk');
+            return tablePrimaryKey;
+        });
+}
+
+/**
+ * Adds an primary key index to a table over the given columns.
+ */
+function addPrimaryKey(tableName, columns, knex) {
+    const isSQLite = knex.client.config.client === 'sqlite3';
+    if (isSQLite) {
+        return hasPrimaryKeySQLite(tableName, knex)
+            .then((primaryKeyExists) => {
+                if (primaryKeyExists) {
+                    debug(`Primary key constraint for: ${columns} already exists for table: ${tableName}`);
+                    return;
+                }
+
+                return knex.schema.table(tableName, function (table) {
+                    table.primary(columns);
+                });
+            });
+    }
+
+    return knex.schema.table(tableName, function (table) {
+        table.primary(columns);
+    }).catch((err) => {
+        if (err.code === 'ER_MULTIPLE_PRI_KEY') {
+            debug(`Primary key constraint for: ${columns} already exists for table: ${tableName}`);
+            return;
+        }
+        throw err;
+    });
+}
+
+/**
+ * @description Private helper to create add a primary key to the migration lock table. The helper is called as part of `runDatabaseUpgrades`.
+ * @returns {*}
+ */
+module.exports.up = function (connection) {
+    debug('Add primary key to the lock table.');
+
+    return addPrimaryKey('migrations_lock', 'lock_key', connection);
+};

package/migrations/field-length.js

@@ -0,0 +1,22 @@
+const debug = require('debug')('knex-migrator:field-length');
+
+/**
+ * @description Private helper to migrate the migrations table. It will add missing constraints to existing fields.
+ * @returns {*}
+ */
+module.exports.up = function (connection) {
+    debug('Ensure Field Length.');
+
+    return connection.schema.hasTable('migrations')
+        .then(function (exists) {
+            if (exists) {
+                return connection.schema.alterTable('migrations', function (table) {
+                    table.string('name', 120).nullable(false).alter();
+                    table.string('version', 70).nullable(false).alter();
+                }).catch(function () {
+                    // ignore for now, it's not a urgent, required change
+                    return Promise.resolve();
+                });
+            }
+        });
+};

package/migrations/index.js

@@ -0,0 +1,17 @@
+const lockTable = require('./lock-table');
+const fieldLength = require('./field-length');
+const useIndex = require('./use-index');
+const addPKToLockTable = require('./add-primary-key-to-lock-table');
+
+/**
+ * @description Helper to run database migrations for the database tables, which knex-migrator is using.
+ *
+ * You can use this helper to execute more database migrations. It get's called as soon as you hit any knex-migrator command.
+ */
+module.exports.run = function (connection) {
+    return Promise.resolve()
+        .then(() => lockTable.up(connection))
+        .then(() => fieldLength.up(connection))
+        .then(() => useIndex.up(connection))
+        .then(() => addPKToLockTable.up(connection));
+};

package/migrations/lock-table.js

@@ -0,0 +1,41 @@
+const debug = require('debug')('knex-migrator:lock-table');
+const errors = require('../lib/errors');
+
+/**
+ * @description Private helper to create the migration lock table. The helper is called as part of `runDatabaseUpgrades`.
+ * @returns {*}
+ */
+module.exports.up = function (connection) {
+    debug('Ensure Lock Table.');
+
+    return connection.schema.hasTable('migrations_lock')
+        .then(function (table) {
+            if (table) {
+                return;
+            }
+
+            return connection.schema.createTable('migrations_lock', function (table) {
+                table.string('lock_key', 191).nullable(false).primary();
+                table.boolean('locked').default(0);
+                table.dateTime('acquired_at').nullable();
+                table.dateTime('released_at').nullable();
+            }).then(function () {
+                return connection('migrations_lock')
+                    .insert({
+                        lock_key: 'km01',
+                        locked: 0
+                    });
+            }).catch(function (err) {
+                // CASE: sqlite db is locked (e.g. concurrent migrations are running)
+                if (err.errno === 5) {
+                    throw new errors.MigrationsAreLockedError({
+                        message: 'Migrations are running at the moment. Please wait that the lock get`s released.',
+                        context: 'Either the release was never released because of a e.g. died process or a parallel process is migrating at the moment.',
+                        help: 'If you know what you are doing, you can manually release the lock by running `UPDATE migrations_lock set locked=0 where lock_key=\'km01\';`.'
+                    });
+                }
+
+                throw err;
+            });
+        });
+};

package/migrations/use-index.js

@@ -0,0 +1,24 @@
+const debug = require('debug')('knex-migrator:use-index');
+
+/**
+ * @description Private helper to migrate the migrations table. It will add missing indexes to existing fields.
+ * @returns {*}
+ */
+module.exports.up = function (connection) {
+    debug('Ensure Unique Index.');
+
+    return connection.schema.hasTable('migrations')
+        .then(function (exists) {
+            if (exists) {
+                return connection.schema.alterTable('migrations', function (table) {
+                    table.unique(['name', 'version']);
+                }).catch(function () {
+                    // @NOTE: ignore for now, it's not a urgent, required change
+                    // e.g. index exists already (1061,1)
+                    // e.g. can't index because of already existing duplicates
+                    return Promise.resolve();
+                });
+            }
+        });
+};
+

package/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "knex-migrator",
+  "version": "4.0.5",
+  "description": "Database migrations with knex.",
+  "keywords": [
+    "ghost",
+    "migration",
+    "knex",
+    "knex-migrations",
+    "knex migration",
+    "knex migrations",
+    "bookshelf migration",
+    "bookshelf"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:TryGhost/knex-migrator.git"
+  },
+  "bugs": {
+    "url": "https://github.com/TryGhost//knex-migrator/issues"
+  },
+  "author": {
+    "name": "Ghost Foundation",
+    "email": "hello@ghost.org",
+    "web": "https://ghost.org"
+  },
+  "main": "lib",
+  "license": "MIT",
+  "scripts": {
+    "lint": "eslint --ext .js --cache lib/** test/**",
+    "test": "LEVEL=fatal _mocha --require test/utils.js --report --exit lcovonly -- test/**/*_spec.js",
+    "test:unit": "yarn lint && LEVEL=fatal _mocha --require test/utils.js --report lcovonly -- test/unit/*_spec.js",
+    "posttest": "yarn lint",
+    "coverage": "nyc --reporter=lcov _mocha --require test/utils.js -- test/*_spec.js",
+    "preship": "yarn test",
+    "ship": "STATUS=$(git status --porcelain); echo $STATUS; if [ -z \"$STATUS\" ]; then yarn publish && git push ${GHOST_UPSTREAM:-upstream} master --follow-tags; fi"
+  },
+  "bin": {
+    "knex-migrator": "./bin/knex-migrator",
+    "knex-migrator-init": "./bin/knex-migrator-init",
+    "knex-migrator-health": "./bin/knex-migrator-health",
+    "knex-migrator-migrate": "./bin/knex-migrator-migrate",
+    "knex-migrator-reset": "./bin/knex-migrator-reset",
+    "knex-migrator-rollback": "./bin/knex-migrator-rollback"
+  },
+  "engines": {
+    "node": "^12.10.0 || ^14.14.0"
+  },
+  "dependencies": {
+    "bluebird": "3.7.2",
+    "commander": "5.1.0",
+    "compare-ver": "2.0.2",
+    "debug": "4.3.2",
+    "ghost-ignition": "4.6.3",
+    "knex": "0.21.19",
+    "lodash": "4.17.21",
+    "moment": "2.24.0",
+    "nconf": "0.11.3",
+    "resolve": "1.20.0"
+  },
+  "files": [
+    "bin",
+    "lib",
+    "migrations",
+    "logging.js"
+  ],
+  "devDependencies": {
+    "eslint": "6.8.0",
+    "eslint-plugin-ghost": "0.2.0",
+    "mocha": "7.2.0",
+    "nyc": "15.1.0",
+    "rimraf": "3.0.2",
+    "should": "13.2.3",
+    "sinon": "9.2.4"
+  },
+  "optionalDependencies": {
+    "mysql": "2.18.1",
+    "sqlite3": "5.0.2"
+  }
+}