knex-migrator 4.0.3 → 4.1.1

package/README.md

@@ -6,14 +6,14 @@ A database migration tool for [knex.js](https://github.com/tgriesser/knex), whic
 
 - [x] JS API
 - [x] CLI Tool
-- [x] Differentiation between database initialisation and migration (Support for a database schema, [like we use in Ghost](https://github.com/TryGhost/Ghost/blob/1.16.2/core/server/data/schema/schema.js))
+- [x] Differentiation between database initialization and migration (Support for a database schema, [like we use in Ghost](https://github.com/TryGhost/Ghost/blob/1.16.2/core/server/data/schema/schema.js))
 - [x] Support for database creation
 - [x] Hooks
 - [x] Rollback to latest version
 - [x] Auto-Rollback on error
 - [x] Database health check
 - [x] Supports transactions
-- [x] Full atomic, support for separate DML/DDL scripts (no autocommit) 
+- [x] Full atomic, support for separate DML/DDL scripts (no autocommit)
 - [x] Migration lock
 - [x] Full debug & pretty log support
 - [x] Custom migration folder structure
@@ -36,8 +36,8 @@ Add me to your globals:
 
 - Replicas are unsupported, because Knex.js [doesn't support them](https://github.com/tgriesser/knex/issues/2253).
 - Sqlite does **not** support read locks by default. Read [here](https://github.com/TryGhost/knex-migrator/issues/87) why.
-- [Comparision](https://github.com/TryGhost/knex-migrator/issues/119) with other available migration tools.
-- Don't mix DDL/DML statements in a migration script. In MySQL DDL statements use implicit commits. 
+- [Comparison](https://github.com/TryGhost/knex-migrator/issues/119) with other available migration tools.
+- Don't mix DDL/DML statements in a migration script. In MySQL DDL statements use implicit commits.
 - It's highly recommended to write both the `up` and the `down` function to ensure a full rollback.
 - If your process dies while migrations are running, knex-migrator won't be able to release the migration lock.
   To release to lock you can run `knex-migrator rollback`. **But** it's recommended to check your database first to see in which state it is.
@@ -73,7 +73,7 @@ Please take a look at [this real example](https://github.com/TryGhost/Ghost/blob
 
 ```
 project/
-    migrations/        
+    migrations/
         hooks/
             init/
                 index.js
@@ -88,9 +88,9 @@ project/
         versions/
             1.0/
                 1-add-events-table.js
-                2-normalise-settings.js                
+                2-normalise-settings.js
             2.0/
-                1-add-timestamps-columns.js                
+                1-add-timestamps-columns.js
             2.1/
                 1-remove-empty-strings.js
                 2-add-webhooks-table.js
@@ -199,7 +199,7 @@ Commands:
 
 #### knex-migrator init
 
-- Initialises your database based on your init scripts
+- Initializes your database based on your init scripts
 - Creates the database if it was not created yet
 
 ##### Options
@@ -223,10 +223,10 @@ Commands:
 ##### Options
 
 ```bash
-# The bersion you would like to migrate to
+# The version you would like to migrate to
 --v
 
-# Combo Feature to check whether the database was already initialised
+# Combo Feature to check whether the database was already initialized
 --init
 
 # Force the execution no matter which current version you are on
@@ -312,13 +312,13 @@ knexMigrator.reset
 knexMigrator.isDatabaseOK()
   .then(function() {
      // database is OK
-     // initialisation & migrations are not missing
+     // initialization & migrations are not missing
   })
   .catch(function(err) {
       if (err.code === 'DB_NOT_INITIALISED') {
           return knexMigrator.init();
       }
-      
+
       if (err.code === 'DB_NEEDS_MIGRATION') {
         return knexMigrator.migrate();
       }

package/lib/index.js

@@ -119,7 +119,7 @@ KnexMigrator.prototype.init = function init(options) {
                             }
                         })
                         .then(function () {
-                            const initTasks = utils.readTasks(path.join(self.migrationPath, 'init'));
+                            const initTasks = utils.listFiles(path.join(self.migrationPath, 'init'));
 
                             /**
                              * CASE 1: You can disable init completion manually
@@ -159,12 +159,12 @@ KnexMigrator.prototype.init = function init(options) {
                                 // CASE: Run over all migration scripts and add the file name to the database.
                                 return Promise.each(versionsToMigrateTo, function (versionToMigrateTo) {
                                     let versionPath = path.join(self.migrationPath, self.subfolder, versionToMigrateTo);
-                                    let filesToMigrateTo = utils.readTasks(versionPath) || [];
+                                    let filesToMigrateTo = utils.listFiles(versionPath) || [];
 
                                     return Promise.each(filesToMigrateTo, function (fileToMigrateTo) {
                                         // CASE: check if migration exists, do not insert twice
                                         return transacting('migrations')
-                                            .where('name', fileToMigrateTo.name)
+                                            .where('name', fileToMigrateTo)
                                             .then(function (migrationExists) {
                                                 if (migrationExists.length) {
                                                     return Promise.resolve();
@@ -172,7 +172,7 @@ KnexMigrator.prototype.init = function init(options) {
 
                                                 return transacting('migrations')
                                                     .insert({
-                                                        name: fileToMigrateTo.name,
+                                                        name: fileToMigrateTo,
                                                         version: versionToMigrateTo,
                                                         currentVersion: self.currentVersion
                                                     });
@@ -286,11 +286,10 @@ KnexMigrator.prototype.init = function init(options) {
  * @param {Object} options - Custom options you can pass in (version, force, init, only, skip)
  * @returns {Bluebird<any>}
  */
-KnexMigrator.prototype.migrate = function migrate(options) {
+KnexMigrator.prototype.migrate = async function migrate(options) {
     options = options || {};
 
-    let self = this,
-        onlyVersion = options.version,
+    let onlyVersion = options.version,
         force = options.force,
         init = options.init,
         onlyFile = options.only,
@@ -308,14 +307,11 @@ KnexMigrator.prototype.migrate = function migrate(options) {
 
     // CASE: `--init` flag is passed. Combo feature.
     if (init) {
-        return this.init()
-            .then(function () {
-                return self.migrate(_.omit(options, 'init'));
-            });
+        await this.init();
     }
 
     try {
-        hooks = require(path.join(self.migrationPath, '/hooks/migrate'));
+        hooks = require(path.join(this.migrationPath, '/hooks/migrate'));
     } catch (err) {
         debug('Hook Error: ' + err.message);
         debug('No hooks found, no problem.');
@@ -323,152 +319,130 @@ KnexMigrator.prototype.migrate = function migrate(options) {
 
     this.connection = database.connect(this.dbConfig);
 
-    return database.ensureConnectionWorks(this.connection)
-        .then(function () {
-            return migrations.run(self.connection);
-        })
-        .then(function () {
-            return locking.lock(self.connection);
-        })
-        .then(function () {
-            return self._integrityCheck({
-                force: force
-            });
-        })
-        .then(function (result) {
-            _.each(result, function (value, version) {
-                // CASE: Log which versions won't be executed based on the "only" flag
-                if (onlyVersion && version !== onlyVersion) {
-                    debug('Do not execute: ' + version);
-                    return;
-                }
-            });
+    try {
+        await database.ensureConnectionWorks(this.connection);
 
-            if (onlyVersion) {
-                // CASE: filter out versions which should not run
-                let containsVersion = _.find(result, function (obj, key) {
-                    return key === onlyVersion;
-                });
+        await migrations.run(this.connection);
 
-                if (!containsVersion) {
-                    logging.warn('Cannot find requested version: ' + onlyVersion);
-                }
-            }
+        await locking.lock(this.connection);
 
-            _.each(result, function (value, version) {
-                // CASE: compare files on disk with files in database
-                if (value.expected !== value.actual) {
-                    debug('Need to execute migrations for: ' + version);
-                    versionsToMigrate.push(version);
-                }
-            });
-        })
-        .then(function executeBeforeHook() {
-            if (!versionsToMigrate.length) {
-                return;
-            }
+        const result = await this._integrityCheck({force});
 
-            if (hooks.before) {
-                debug('Before hook');
-                return hooks.before({
-                    connection: self.connection
-                });
-            }
-        })
-        .then(function executeMigrations() {
-            if (!versionsToMigrate.length) {
+        _.each(result, function (_value, version) {
+            // CASE: Log which versions won't be executed based on the "only" flag
+            if (onlyVersion && version !== onlyVersion) {
+                debug('Do not execute: ' + version);
                 return;
             }
+        });
 
-            return Promise.each(versionsToMigrate, function (versionToMigrate) {
-                return self._migrateTo({
-                    version: versionToMigrate,
-                    only: onlyFile,
-                    hooks: hooks
-                });
+        if (onlyVersion) {
+            // CASE: filter out versions which should not run
+            let containsVersion = _.find(result, function (_obj, key) {
+                return key === onlyVersion;
             });
-        })
-        .then(function executeAfterHook() {
-            if (!versionsToMigrate.length) {
-                return;
-            }
 
-            if (hooks.after) {
-                debug('After hook');
-                return hooks.after({
-                    connection: self.connection
-                });
-            }
-        })
-        .then(function () {
-            return locking.unlock(self.connection);
-        })
-        .catch(function (err) {
-            // CASE: Do not rollback if migrations are locked
-            if (err instanceof errors.MigrationsAreLockedError) {
-                throw err;
+            if (!containsVersion) {
+                logging.warn('Cannot find requested version: ' + onlyVersion);
             }
+        }
 
-            // CASE: Do not rollback migration scripts, if lock error
-            if (err instanceof errors.LockError) {
-                throw err;
+        _.each(result, function (value, version) {
+            // CASE: compare files on disk with files in database
+            if (value.expected !== value.actual) {
+                debug('Need to execute migrations for: ' + version);
+                versionsToMigrate.push(version);
             }
+        });
 
-            // CASE: ETIMEDOUT, ENOTFOUND
-            if (err instanceof errors.DatabaseError) {
-                throw err;
+        if (versionsToMigrate.length) {
+            if (hooks.before) {
+                debug('Before hook');
+                await hooks.before({
+                    connection: this.connection
+                });
             }
 
-            if (err.context && err.context.name) {
-                debug(`Task failed: ${err.context.name}`);
-            }
+            logging.info('Running migrations.');
 
-            debug(`Rolling back: ${err.message}`);
+            for (const versionToMigrate of versionsToMigrate) {
+                try {
+                    await this._migrateTo({
+                        version: versionToMigrate,
+                        only: onlyFile,
+                        hooks: hooks
+                    });
+                } catch (err) {
+                    // CASE: Do not rollback if migrations are locked
+                    if (err instanceof errors.MigrationsAreLockedError) {
+                        throw err;
+                    }
 
-            versionsToMigrate.reverse();
+                    // CASE: Do not rollback migration scripts, if lock error
+                    if (err instanceof errors.LockError) {
+                        throw err;
+                    }
 
-            // CASE: rollback in reversed order
-            return Promise.each(versionsToMigrate, function (version) {
-                return self._rollback({version: version, task: err.context});
-            }).then(function () {
-                throw err;
-            }).catch(function (innerErr) {
-                if (errors.utils.isIgnitionError(innerErr)) {
-                    throw err;
-                }
+                    // CASE: ETIMEDOUT, ENOTFOUND
+                    if (err instanceof errors.DatabaseError) {
+                        throw err;
+                    }
 
-                throw new errors.RollbackError({
-                    message: innerErr.message,
-                    err: innerErr,
-                    context: `OuterError: ${err.message}`
-                });
-            }).finally(function () {
-                return locking.unlock(self.connection);
-            });
-        }).finally(function () {
-            let ops = [];
+                    if (err.context && err.context.name) {
+                        debug(`Task failed: ${err.context.name}`);
+                    }
 
-            if (hooks.shutdown) {
-                ops.push(function shutdownHook() {
-                    debug('Shutdown hook');
-                    return hooks.shutdown({
-                        executedFromShell: self.executedFromShell
-                    });
-                });
+                    logging.info(`Rolling back: ${err.message}.`);
+
+                    const versionsMigrated = versionsToMigrate.slice(
+                        0,
+                        versionsToMigrate.indexOf(versionToMigrate) + 1
+                    );
+
+                    versionsMigrated.reverse();
+
+                    try {
+                        for (const versionMigrated of versionsMigrated) {
+                            await this._rollback({version: versionMigrated, task: err.context});
+                        }
+                        logging.info(`Rollback was successful.`);
+                        throw err;
+                    } catch (innerErr) {
+                        if (errors.utils.isIgnitionError(innerErr)) {
+                            throw err;
+                        }
+
+                        throw new errors.RollbackError({
+                            message: innerErr.message,
+                            err: innerErr,
+                            context: `OuterError: ${err.message}`
+                        });
+                    } finally {
+                        await locking.unlock(this.connection);
+                    }
+                }
             }
 
-            ops.push(function destroyConnection() {
-                debug('Destroy connection');
-                return self.connection.destroy()
-                    .then(function () {
-                        debug('Destroyed connection');
-                    });
+            if (hooks.after) {
+                debug('After hook');
+                await hooks.after({
+                    connection: this.connection
+                });
+            }
+        }
+        await locking.unlock(this.connection);
+    } finally {
+        if (hooks.shutdown) {
+            debug('Shutdown hook');
+            await hooks.shutdown({
+                executedFromShell: this.executedFromShell
             });
+        }
 
-            return Promise.each(ops, function (op) {
-                return op.bind(self)();
-            });
-        });
+        debug('Destroy connection');
+        await this.connection.destroy();
+        debug('Destroyed connection');
+    }
 };
 
 /**
@@ -1045,8 +1019,8 @@ KnexMigrator.prototype._migrateTo = function _migrateTo(options) {
                 throw new errors.MigrationScriptError({
                     message: 'Field length of %field% in %table% is too long!'.replace('%field%', field).replace('%table%', table),
                     context: 'This usually happens if your database encoding is utf8mb4.\n' +
-                    'All unique fields and indexes must be lower than 191 characters.\n' +
-                    'Please correct your field length and reset your database with knex-migrator reset.\n',
+                        'All unique fields and indexes must be lower than 191 characters.\n' +
+                        'Please correct your field length and reset your database with knex-migrator reset.\n',
                     help: 'Read more here: https://github.com/TryGhost/knex-migrator/issues/51\n',
                     err: err
                 });
@@ -1132,7 +1106,6 @@ KnexMigrator.prototype._integrityCheck = function _integrityCheck(options) {
         subfolder = this.subfolder,
         force = options.force,
         folders = [],
-        operations = {},
         toReturn = {},
         futureVersions = [];
 
@@ -1148,29 +1121,69 @@ KnexMigrator.prototype._integrityCheck = function _integrityCheck(options) {
         // ignore
     }
 
-    _.each(folders, function (folder) {
-        // CASE: versions/1.1-members or versions/2.0-payments
-        if (folder !== 'init') {
-            try {
-                folder = folder.match(/([\d._]+)/)[0];
-            } catch (err) {
-                logging.warn('Cannot parse folder name.');
-                logging.warn('Ignore Folder: ' + folder);
-                return;
-            }
-        }
+    return this
+        .connection('migrations')
+        .select('version')
+        .count('version', {as: 'c'})
+        .groupBy('version')
+        .then((dbMigrations) => {
+            _.each(folders, function (folder) {
+                // CASE: versions/1.1-members or versions/2.0-payments
+                if (folder !== 'init') {
+                    try {
+                        folder = folder.match(/([\d._]+)/)[0];
+                    } catch (err) {
+                        logging.warn('Cannot parse folder name.');
+                        logging.warn('Ignore Folder: ' + folder);
+                        return;
+                    }
+                }
+
+                // CASE:
+                // if your current version is 1.0 and you add migration scripts for the next version 1.1
+                // we won't execute them until your current version changes to 1.1 or until you force KM to migrate to it
+                if (self.currentVersion && !force) {
+                    if (utils.isGreaterThanVersion({smallerVersion: self.currentVersion, greaterVersion: folder})) {
+                        futureVersions.push(folder);
+                    }
+                }
 
-        // CASE:
-        // if your current version is 1.0 and you add migration scripts for the next version 1.1
-        // we won't execute them until your current version changes to 1.1 or until you force KM to migrate to it
-        if (self.currentVersion && !force) {
-            if (utils.isGreaterThanVersion({smallerVersion: self.currentVersion, greaterVersion: folder})) {
-                futureVersions.push(folder);
+                let actual = 0;
+                let expected;
+
+                const migrationCount = dbMigrations.find(m => m.version === folder);
+                if (migrationCount) {
+                    actual = migrationCount.c;
+                }
+
+                if (folder !== 'init') {
+                    expected = utils.listFiles(path.join(self.migrationPath, subfolder, folder)).length;
+                } else {
+                    expected = utils.listFiles(path.join(self.migrationPath, folder)).length;
+                }
+
+                debug('Version ' + folder + ' expected: ' + expected);
+                debug('Version ' + folder + ' actual: ' + actual);
+
+                toReturn[folder] = {
+                    expected: expected,
+                    actual: actual
+                };
+            });
+
+            // CASE: ensure that either you have to run `migrate --force` or they ran already
+            if (futureVersions.length) {
+                _.each(futureVersions, function (futureVersion) {
+                    if (toReturn[futureVersion].actual !== toReturn[futureVersion].expected) {
+                        logging.warn('knex-migrator is skipping ' + futureVersion);
+                        logging.warn('Current version in MigratorConfig.js is smaller then requested version, use --force to proceed!');
+                        logging.warn('Please run `knex-migrator migrate --v ' + futureVersion + ' --force` to proceed!');
+                        delete toReturn[futureVersion];
+                    }
+                });
             }
-        }
 
-        operations[folder] = self.connection('migrations').where({
-            version: folder
+            return toReturn;
         }).catch(function onMigrationsLookupError(err) {
             // CASE: no database selected (database.connection.database="")
             if (err.errno === 1046) {
@@ -1199,43 +1212,6 @@ KnexMigrator.prototype._integrityCheck = function _integrityCheck(options) {
 
             throw err;
         });
-    });
-
-    return Promise.props(operations)
-        .then(function (result) {
-            _.each(result, function (value, version) {
-                let actual = value.length,
-                    expected = actual;
-
-                if (version !== 'init') {
-                    expected = utils.readTasks(path.join(self.migrationPath, subfolder, version)).length;
-                } else {
-                    expected = utils.readTasks(path.join(self.migrationPath, version)).length;
-                }
-
-                debug('Version ' + version + ' expected: ' + expected);
-                debug('Version ' + version + ' actual: ' + actual);
-
-                toReturn[version] = {
-                    expected: expected,
-                    actual: actual
-                };
-            });
-
-            // CASE: ensure that either you have to run `migrate --force` or they ran already
-            if (futureVersions.length) {
-                _.each(futureVersions, function (futureVersion) {
-                    if (toReturn[futureVersion].actual !== toReturn[futureVersion].expected) {
-                        logging.warn('knex-migrator is skipping ' + futureVersion);
-                        logging.warn('Current version in MigratorConfig.js is smaller then requested version, use --force to proceed!');
-                        logging.warn('Please run `knex-migrator migrate --v ' + futureVersion + ' --force` to proceed!');
-                        delete toReturn[futureVersion];
-                    }
-                });
-            }
-
-            return toReturn;
-        });
 };
 
 module.exports = KnexMigrator;

package/lib/utils.js

@@ -46,15 +46,13 @@ module.exports.loadConfig = function loadConfig(options) {
 };
 
 /**
- * @description Reads all migration files from disk based on a path.
- * It returns an Array of migration files including it's up/down hooks, config and the name.
+ * @description List all migration files from disk based on a path.
  *
  * @param absolutePath
  * @returns {Array}
  */
-exports.readTasks = function readTasks(absolutePath) {
-    let files = [],
-        tasks = [];
+exports.listFiles = function listFiles(absolutePath) {
+    let files = [];
 
     try {
         files = fs.readdirSync(absolutePath);
@@ -65,13 +63,28 @@ exports.readTasks = function readTasks(absolutePath) {
         });
     }
 
-    _.each(files, function (file) {
+    files = files.filter(function (file) {
         // CASE: ignore dot files
-        if (file.match(/^\./)) {
-            debug('Ignore Dotfile: ' + file);
-            return;
-        }
+        return !file.match(/^\./);
+    });
+
+    debug(files);
+    return files;
+};
+
+/**
+ * @description Reads all migration files from disk based on a path.
+ * It returns an Array of migration files including it's up/down hooks, config and the name.
+ *
+ * @param absolutePath
+ * @returns {Array}
+ */
+exports.readTasks = function readTasks(absolutePath) {
+    let tasks = [];
+
+    const files = exports.listFiles(absolutePath);
 
+    _.each(files, function (file) {
         let executeFn = require(path.join(absolutePath, file));
 
         try {
@@ -92,7 +105,7 @@ exports.readTasks = function readTasks(absolutePath) {
         }
     });
 
-    debug(files);
+    debug(tasks);
     return tasks;
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "knex-migrator",
-  "version": "4.0.3",
+  "version": "4.1.1",
   "description": "Database migrations with knex.",
   "keywords": [
     "ghost",
@@ -44,18 +44,18 @@
     "knex-migrator-rollback": "./bin/knex-migrator-rollback"
   },
   "engines": {
-    "node": "^10.13.0 || ^12.10.0 || ^14.14.0"
+    "node": "^12.22.1 || ^14.17.0 || ^16.13.0"
   },
   "dependencies": {
     "bluebird": "3.7.2",
     "commander": "5.1.0",
     "compare-ver": "2.0.2",
-    "debug": "4.3.1",
-    "ghost-ignition": "4.6.1",
+    "debug": "4.3.2",
+    "ghost-ignition": "4.6.3",
     "knex": "0.21.19",
     "lodash": "4.17.21",
     "moment": "2.24.0",
-    "nconf": "0.11.2",
+    "nconf": "0.11.3",
     "resolve": "1.20.0"
   },
   "files": [