db-migrate 1.0.0 → 1.2.0

package/README.md

@@ -20,6 +20,12 @@ Support this project by becoming a sponsor. Your logo will show up here with a l
 
 # Usage
 
+## Upgrading from 0.11
+
+db-migrate 0.11 and older are end of life. See the
+[changelog of 1.0.0](CHANGELOG.md) for what changed, v1 migrations keep
+working unchanged.
+
 ## Requirements
 
 Officially supported is Node.js 24 and newer. Older versions may work, but
@@ -61,6 +67,14 @@ db-migrate to use the `tunnel` setting of your database config:
 
     $ npm install db-migrate-plugin-tunnel-ssh
 
+## db-migrate for meta
+
+[meta-db-migrate](https://github.com/db-migrate/meta-db-migrate) is db-migrate
+rewritten in meta: a program ships with its migrations compiled in, as one
+binary. It uses the same migrations table and the same state as db-migrate,
+v2 migrations included, so a database migrated by one of them can be carried
+on by the other.
+
 ## Resources and usage instructions
 
 Please follow the link below, for usage instructions examples and the full

package/api.js

@@ -154,6 +154,14 @@ dbmigrate.prototype = {
    * @return boolean
    */
   addConfiguration: function (description, args, type) {
+    // the command line options are parsed already when the API is available
+    if (typeof this.internals.argv.describe !== 'function') {
+      log.warn(
+        'addConfiguration has no effect, the options are parsed already.'
+      );
+      return false;
+    }
+
     var name = args.shift();
     this.internals.argv.describe(name, description);
 
@@ -368,7 +376,8 @@ dbmigrate.prototype = {
    */
   createDatabase: function (dbname, callback) {
     var executeDB = load('db');
-    this.internals.argv._.push(dbname);
+    // in module mode there are no positional arguments yet
+    this.internals.argv._ = (this.internals.argv._ || []).concat(dbname);
     this.internals.mode = 'create';
     return Promise.resolve(executeDB(this.internals, this.config)).asCallback(
       callback
@@ -380,7 +389,8 @@ dbmigrate.prototype = {
    */
   dropDatabase: function (dbname, callback) {
     var executeDB = load('db');
-    this.internals.argv._.push(dbname);
+    // in module mode there are no positional arguments yet
+    this.internals.argv._ = (this.internals.argv._ || []).concat(dbname);
     this.internals.mode = 'drop';
     return Promise.resolve(executeDB(this.internals, this.config)).asCallback(
       callback

package/connect.js

@@ -2,15 +2,37 @@
  * This file is going to disappear.
  * Only still here for backwards compatibility.
  * */
-var recursive = require('final-fs').readdirRecursive;
 var fs = require('fs');
 var driver = require('./lib/driver');
 var path = require('path');
 var log = require('db-migrate-shared').log;
+var resolveEnv = require('./lib/config').resolveEnv;
 const Promise = require('bluebird');
 
 Promise.promisifyAll(driver);
 
+/**
+ * The config.json of a scope, with { "ENV": ... } resolved, or null.
+ */
+function scopeConfig (dirPath, scope) {
+  const file = path.resolve(dirPath, scope, 'config.json');
+
+  if (!fs.existsSync(file)) {
+    return null;
+  }
+
+  log.info('loaded extra config for migration subfolder: "' + scope + '/config.json"');
+  return resolveEnv(JSON.parse(fs.readFileSync(file, 'utf8')));
+}
+
+/**
+ * A scope config with more than database or schema connects on its own,
+ * inheriting the settings of the environment it does not set itself.
+ */
+const SWITCH_ONLY = ['database', 'schema'];
+const ownConnection = conf =>
+  Object.keys(conf).some(key => SWITCH_ONLY.indexOf(key) === -1);
+
 exports.connect = async function (config, PassedClass) {
   var internals = {};
   var prefix = 'migration';
@@ -20,123 +42,47 @@ exports.connect = async function (config, PassedClass) {
     config = config.config;
   }
 
-  const db2 = await driver.connectAsync(config, internals);
-
-  return Promise.fromCallback(callback => {
-    driver.connect(config, internals, function (err, db) {
-      if (err) {
-        callback(err);
-        return;
-      }
-
-      const realClose = db.close;
-      // close both lines with one disconnect action
-      db.close = function (cb) {
-        db2.close(function () {});
-        db.close = realClose;
-        db.close(cb);
-      };
-
-      var dirPath = path.resolve(
-        internals.argv['migrations-dir'] || 'migrations'
-      );
-      if (internals.migrationMode) {
-        if (internals.migrationMode !== 'all') {
-          var switched = false;
-          var newConf;
+  const dirPath = path.resolve(internals.argv['migrations-dir'] || 'migrations');
+  const scope = internals.migrationMode;
+  const conf = scope ? scopeConfig(dirPath, scope) : null;
+  let switchTo = null;
 
-          try {
-            newConf = require(path.resolve(
-              internals.argv['migrations-dir'] || 'migrations',
-              internals.migrationMode
-            ) + '/config.json');
-            log.info(
-              'loaded extra config for migration subfolder: "' +
-                internals.migrationMode +
-                '/config.json"'
-            );
-            switched = true;
-          } catch (e) {}
-
-          if (switched) {
-            db.switchDatabase(newConf, function (err) {
-              if (err) {
-                return callback(err);
-              }
-              internals.locTitle = internals.migrationMode;
-              callback(
-                null,
-                new PassedClass(
-                  db,
-                  dirPath,
-                  internals.mode !== 'static',
-                  internals,
-                  prefix,
-                  { db2 }
-                )
-              );
-            });
-          } else {
-            internals.locTitle = internals.migrationMode;
-            callback(
-              null,
-              new PassedClass(
-                db,
-                dirPath,
-                internals.mode !== 'static',
-                internals,
-                prefix,
-                { db2 }
-              )
-            );
-          }
-        } else {
-          recursive(
-            dirPath,
-            false,
-            internals.argv['migrations-dir'] || 'migrations'
-          ).then(function (files) {
-            var oldClose = db.close;
-
-            files = files.filter(function (file) {
-              return file !== 'migrations' && fs.statSync(file).isDirectory();
-            });
+  if (conf && ownConnection(conf)) {
+    // a database of its own, with its own migrations and state tables
+    config = Object.assign({}, config, conf);
+  } else if (conf && (conf.database || conf.schema)) {
+    switchTo = conf;
+  }
 
-            files.push('');
+  const db2 = await driver.connectAsync(config, internals);
+  const db = await driver.connectAsync(config, internals);
+
+  const realClose = db.close;
+  // close both lines with one disconnect action
+  db.close = function (cb) {
+    db2.close(function () {});
+    db.close = realClose;
+    db.close(cb);
+  };
+
+  if (switchTo) {
+    // both lines, so the state is kept in the database of the scope as well
+    await Promise.promisify(db.switchDatabase, { context: db })(switchTo);
+    await Promise.promisify(db2.switchDatabase, { context: db2 })(switchTo);
+  }
 
-            db.close = function (cb) {
-              migrationFiles(
-                files,
-                callback,
-                config,
-                internals,
-                PassedClass,
-                db,
-                oldClose,
-                prefix,
-                db2,
-                cb
-              );
-            };
+  if (scope) {
+    internals.locTitle = scope;
+  }
 
-            db.close();
-          });
-        }
-      } else {
-        callback(
-          null,
-          new PassedClass(
-            db,
-            dirPath,
-            internals.mode !== 'static',
-            internals,
-            prefix,
-            { db2 }
-          )
-        );
-      }
-    });
-  });
+  return new PassedClass(
+    db,
+    dirPath,
+    internals.mode !== 'static',
+    internals,
+    prefix,
+    { db2 }
+  );
 };
 
 exports.driver = function (config, callback) {
@@ -148,69 +94,3 @@ exports.driver = function (config, callback) {
 
   driver.connect(config, internals, callback);
 };
-
-function migrationFiles (
-  files,
-  callback,
-  config,
-  internals,
-  PassedClass,
-  db,
-  close,
-  prefix,
-  db2,
-  cb
-) {
-  var file;
-  var switched = false;
-  var newConf;
-
-  if (files.length === 1) {
-    db.close = close;
-  }
-
-  file = files.pop();
-  log.info('Enter scope "' + (file !== '' ? file : '/') + '"');
-
-  if (file !== '') {
-    try {
-      fs.statSync(path.resolve(file + '/config.json'));
-      newConf = require(path.resolve(file + '/config.json'));
-      log.info(
-        'loaded extra config for migration subfolder: "' +
-          file +
-          '/config.json"'
-      );
-      switched = true;
-    } catch (e) {}
-  }
-
-  db.switchDatabase(switched ? newConf : config.database, function () {
-    internals.matching = file.substr(
-      file.indexOf(internals.argv['migrations-dir'] || 'migrations') +
-        (internals.argv['migrations-dir'] || 'migrations').length +
-        1
-    );
-
-    if (internals.matching.length === 0) {
-      internals.matching = '';
-    }
-
-    internals.locTitle = internals.matching;
-    callback(
-      null,
-      new PassedClass(
-        db,
-        internals.argv['migrations-dir'],
-        internals.mode !== 'static',
-        internals,
-        prefix,
-        { db2 }
-      )
-    );
-
-    if (typeof cb === 'function') {
-      cb();
-    }
-  });
-}

package/lib/chain.js

@@ -32,6 +32,43 @@ const describe = (method, args) =>
     .map(a => JSON.stringify(a))
     .join(', ')})`;
 
+/**
+ * Instructions declaring an object, which can adopt objects created outside
+ * of v2 migrations into the schema, see adopt.
+ */
+const ADOPTABLE = /^(create|add)/;
+
+/**
+ * Adopting is for objects unknown to the schema only.
+ */
+const assertUnknown = (m, args, schema) => {
+  const [t, a, b] = args;
+  let known = false;
+
+  switch (m) {
+    case 'createTable':
+    case 'createCollection':
+      known = !!schema.c[t];
+      break;
+    case 'addColumn':
+      known = !!(schema.c[t] && (schema.c[t][a] || schema.c[t].columns?.[a]));
+      break;
+    case 'addIndex':
+      known = !!(schema.i[t] && schema.i[t][a]);
+      break;
+    case 'addForeignKey':
+      known = !!(schema.f[t] && schema.f[t][b]);
+      break;
+  }
+
+  if (known) {
+    throw new Error(
+      `${describe(m, args)} is known to the schema already, adopt is for ` +
+        'objects created outside of v2 migrations only.'
+    );
+  }
+};
+
 let rollbackCounter = 0;
 let rollbackSignal = null;
 let signal = null;
@@ -107,6 +144,13 @@ class Chain {
     this.op = 0;
     this.recovery = null;
 
+    /**
+     * adopt: declares objects created outside of v2 migrations, e.g. by v1
+     * migrations, so v2 migrations can work with them. Only learns them,
+     * nothing is executed, and reverting it only forgets them again.
+     */
+    this.adopt = {};
+
     DEFAULT.concat(Object.keys(this.context.learnable || [])).forEach(
       method => {
         this[method] = function (...args) {
@@ -116,6 +160,15 @@ class Chain {
 
           return this.exec.apply(this, [method].concat(args));
         };
+
+        if (ADOPTABLE.test(method)) {
+          this.adopt[method] = async (...args) => {
+            this.transferInt();
+            assertUnknown(method, args, this.internals.schema);
+
+            return this.run(method, args, 'adopt');
+          };
+        }
       }
     );
   }
@@ -153,6 +206,21 @@ class Chain {
   }
 
   async exec (m, ...args) {
+    return this.run(m, args, null);
+  }
+
+  /**
+   * Learn an instruction only, without executing it, to revert an adopt.
+   */
+  async learnOnly (m, ...args) {
+    return this.run(m, args, 'learn');
+  }
+
+  /**
+   * mode: null to execute the instruction, 'adopt' or 'learn' to only learn
+   * it, skipping the links executing it on the database
+   */
+  async run (m, args, mode) {
     let ret;
     const stat = [];
     const op = ++this.op;
@@ -183,9 +251,22 @@ class Chain {
         continue;
       }
 
+      if (mode && this._step.executes) {
+        continue;
+      }
+
+      const mod = this.internals.modSchema;
+      const recorded = mod ? mod.s.length : 0;
       ret = await this._interface[m].apply(this._interface, args);
       stat.push(ret);
 
+      // an adopted object is only forgotten again when reverting
+      if (mode === 'adopt' && this._step.learns && mod) {
+        mod.s.slice(recorded).forEach(entry => {
+          entry.t = 2;
+        });
+      }
+
       if (this._step.hasStateUsage) {
         this._step.useState(ret);
       }

package/lib/commands/check.js

@@ -1,6 +1,7 @@
 var path = require('path');
 var log = require('db-migrate-shared').log;
 var migrationHook = require('./helper/migration-hook.js');
+var eachScope = require('./helper/scopes.js');
 
 async function prepare (internals, config) {
   try {
@@ -40,7 +41,7 @@ async function prepare (internals, config) {
   }
 }
 
-module.exports = async function (internals, config) {
+async function execute (internals, config) {
   const migrator = await prepare(internals, config);
 
   try {
@@ -49,4 +50,8 @@ module.exports = async function (internals, config) {
   } catch (err) {
     return internals.onComplete(migrator, internals, err);
   }
+}
+
+module.exports = function (internals, config) {
+  return eachScope(internals, () => execute(internals, config));
 };

package/lib/commands/create-migration.js

@@ -199,12 +199,8 @@ function shouldCreateV2File (internals, config) {
 }
 
 async function createSqlFiles (internals, config) {
-  let migrationsDir = internals.argv['migrations-dir'];
-
-  if (internals.migrationMode && internals.migrationMode !== 'all') {
-    migrationsDir =
-      internals.argv['migrations-dir'] + '/' + internals.migrationMode;
-  }
+  // the directory of the migration, its scope included already
+  const migrationsDir = internals.argv['migrations-dir'];
 
   const sqlDir = migrationsDir + '/sqls';
   try {

package/lib/commands/db.js

@@ -1,63 +1,60 @@
 'use strict';
 
 var log = require('db-migrate-shared').log;
-var assert = require('assert');
+var Promise = require('bluebird');
 
+/**
+ * Creates or drops a database, resolving once it is done.
+ */
 function executeDB (internals, config, callback) {
   var index = require('../../connect');
 
-  if (internals.argv._.length > 0) {
-    internals.argv.dbname = internals.argv._.shift().toString();
-  } else {
-    log.info('Error: You must enter a database name!');
-    return;
+  if (internals.argv._.length === 0) {
+    return Promise.reject(
+      new Error('You must enter a database name!')
+    ).asCallback(callback);
   }
 
+  internals.argv.dbname = internals.argv._.shift().toString();
   delete config.getCurrent().settings.database;
 
-  index.driver(config.getCurrent().settings, function (err, db) {
-    assert.ifError(err);
-
-    if (internals.mode === 'create') {
-      db.createDatabase(
-        internals.argv.dbname,
-        {
-          ifNotExists: true
-        },
-        function (err) {
-          if (err) {
-            if (err.error) err = err.error;
-            log.error('Error: Failed to create database!', err);
-          } else {
-            log.info('Created database "' + internals.argv.dbname + '"');
-          }
+  var create = internals.mode === 'create';
+  if (!create && internals.mode !== 'drop') {
+    return Promise.reject(
+      new Error(
+        (internals.mode ? 'Unknown db command "db:' + internals.mode + '"' : 'Missing db command') +
+          ', use db:create or db:drop'
+      )
+    ).asCallback(callback);
+  }
 
-          db.close();
-          if (typeof callback === 'function') callback(err);
-          else process.exit(err ? 1 : 0);
+  return Promise.fromCallback(function (cb) {
+    index.driver(config.getCurrent().settings, cb);
+  })
+    .then(function (db) {
+      return Promise.fromCallback(function (cb) {
+        if (create) {
+          db.createDatabase(internals.argv.dbname, { ifNotExists: true }, cb);
+        } else {
+          db.dropDatabase(internals.argv.dbname, { ifExists: true }, cb);
         }
-      );
-    } else if (internals.mode === 'drop') {
-      db.dropDatabase(
-        internals.argv.dbname,
-        {
-          ifExists: true
-        },
-        function (err) {
-          if (err) {
-            if (err.error) err = err.error;
-            log.error('Error: Failed to drop database!', err);
-          } else {
-            log.info('Deleted database "' + internals.argv.dbname + '"');
-          }
-
+      })
+        .catch(function (err) {
+          throw err && err.error ? err.error : err;
+        })
+        .finally(function () {
           db.close();
-          if (typeof callback === 'function') callback(err);
-          else process.exit(err ? 1 : 0);
-        }
+        });
+    })
+    .then(function () {
+      log.info(
+        (create ? 'Created' : 'Deleted') +
+          ' database "' +
+          internals.argv.dbname +
+          '"'
       );
-    }
-  });
+    })
+    .asCallback(callback);
 }
 
 module.exports = executeDB;

package/lib/commands/down.js

@@ -1,6 +1,7 @@
 var path = require('path');
 var log = require('db-migrate-shared').log;
 var migrationHook = require('./helper/migration-hook.js');
+var eachScope = require('./helper/scopes.js');
 
 async function prepare (internals, config) {
   try {
@@ -33,7 +34,7 @@ async function prepare (internals, config) {
   }
 }
 
-module.exports = async function (internals, config) {
+async function execute (internals, config) {
   const migrator = await prepare(internals, config);
 
   try {
@@ -42,4 +43,8 @@ module.exports = async function (internals, config) {
   } catch (err) {
     return internals.onComplete(migrator, internals, err);
   }
+}
+
+module.exports = function (internals, config) {
+  return eachScope(internals, () => execute(internals, config));
 };

package/lib/commands/fix.js

@@ -1,6 +1,7 @@
 var path = require('path');
 var log = require('db-migrate-shared').log;
 var migrationHook = require('./helper/migration-hook.js');
+var eachScope = require('./helper/scopes.js');
 
 async function prepare (internals, config) {
   try {
@@ -41,7 +42,7 @@ async function prepare (internals, config) {
   }
 }
 
-module.exports = async function (internals, config) {
+async function execute (internals, config) {
   const migrator = await prepare(internals, config);
 
   try {
@@ -50,4 +51,8 @@ module.exports = async function (internals, config) {
   } catch (err) {
     return internals.onComplete(migrator, internals, err);
   }
+}
+
+module.exports = function (internals, config) {
+  return eachScope(internals, () => execute(internals, config));
 };

package/lib/commands/fn/plugin.js

@@ -13,7 +13,7 @@ function registerPluginLoader (plugins) {
             'Attention, multiple overwrites registered for %s, we are ' +
               'only loading the first plugin %s!',
             name,
-            plugin.name
+            plugin[0].name
           );
         }
 

package/lib/commands/generated.js

@@ -16,6 +16,7 @@ const files = {
   "helper/load-config": require('./helper/load-config.js'),
   "helper/migration-hook": require('./helper/migration-hook.js'),
   "helper/register-events": require('./helper/register-events.js'),
+  "helper/scopes": require('./helper/scopes.js'),
   "index": require('./index.js'),
   "on-complete": require('./on-complete.js'),
   "run": require('./run.js'),

package/lib/commands/helper/scopes.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const log = require('db-migrate-shared').log;
+
+/**
+ * The scopes below a migrations directory, nested ones as "a/b". Folders
+ * named sqls hold the files of sql-file migrations, they are no scopes.
+ */
+function listScopes (dir, prefix = '') {
+  return fs
+    .readdirSync(dir, { withFileTypes: true })
+    .filter(entry => entry.isDirectory() && entry.name !== 'sqls')
+    .sort((a, b) => a.name.localeCompare(b.name))
+    .reduce((scopes, entry) => {
+      const scope = prefix + entry.name;
+      return scopes.concat(
+        scope,
+        listScopes(path.join(dir, entry.name), scope + '/')
+      );
+    }, []);
+}
+
+/**
+ * Runs a command for its scope, or with the scope "all" once for the
+ * migrations directory itself and once for every scope, one after another.
+ * The results of all scopes are returned together.
+ */
+module.exports = async function eachScope (internals, run) {
+  if (internals.migrationMode !== 'all') {
+    return run();
+  }
+
+  const dir = path.resolve(internals.argv['migrations-dir'] || 'migrations');
+  const scopes = [''].concat(listScopes(dir));
+  const forceExit = internals.argv['force-exit'];
+  let results = [];
+
+  try {
+    for (let i = 0; i < scopes.length; ++i) {
+      const scope = scopes[i];
+      log.info('Enter scope "' + (scope || '/') + '"');
+
+      internals.migrationMode = scope || undefined;
+      internals.matching = scope;
+      internals.locTitle = undefined;
+      // exiting is only up to the last scope
+      internals.argv['force-exit'] = forceExit && i === scopes.length - 1;
+
+      const result = await run();
+      results = results.concat(result === undefined ? [] : result);
+    }
+  } finally {
+    internals.migrationMode = 'all';
+    internals.argv['force-exit'] = forceExit;
+  }
+
+  return results;
+};

package/lib/commands/set-default-argv.js

@@ -79,8 +79,9 @@ module.exports = function (internals, isModule) {
       .boolean('coffee-file')
       .describe(
         'ignore-on-init',
-        'Create files that will run only if ignore-on-init in the env is set ' +
-          'to false (currently works only with SQL)'
+        'With create, create sql-file migrations whose up is skipped when ' +
+          'running with --ignore-on-init, e.g. to initialize a database ' +
+          'from a dump (sql-file migrations only)'
       )
       .boolean('ignore-on-init')
       .describe(
@@ -146,7 +147,25 @@ module.exports = function (internals, isModule) {
     });
   }
 
-  internals.argv = deepExtend(internals.argv, rc('db-migrate', {}));
+  // in module mode the arguments of the process belong to the application
+  var rcConfig = rc('db-migrate', {}, isModule ? {} : undefined);
+
+  // the long names of options are aliases on the command line only
+  [
+    ['state-table', 'state'],
+    ['migration-table', 'table']
+  ].forEach(function ([alias, name]) {
+    if (rcConfig[alias] !== undefined && rcConfig[name] === undefined) {
+      rcConfig[name] = rcConfig[alias];
+    }
+  });
+
+  internals.argv = deepExtend(internals.argv, rcConfig);
+
+  // positional arguments, the API adds its own ones in module mode
+  if (!Array.isArray(internals.argv._)) {
+    internals.argv._ = [];
+  }
   internals.argv.rcconfig = internals.argv.config;
   internals.argv.config = internals.argv.configFile || _config;
 

package/lib/commands/sync.js

@@ -1,6 +1,7 @@
 var path = require('path');
 var log = require('db-migrate-shared').log;
 var migrationHook = require('./helper/migration-hook.js');
+var eachScope = require('./helper/scopes.js');
 
 async function prepare (internals, config) {
   try {
@@ -40,7 +41,7 @@ async function prepare (internals, config) {
   }
 }
 
-module.exports = async function (internals, config) {
+async function execute (internals, config) {
   const migrator = await prepare(internals, config);
 
   try {
@@ -49,4 +50,8 @@ module.exports = async function (internals, config) {
   } catch (err) {
     return internals.onComplete(migrator, internals, err);
   }
+}
+
+module.exports = function (internals, config) {
+  return eachScope(internals, () => execute(internals, config));
 };

package/lib/commands/up.js

@@ -1,6 +1,7 @@
 var path = require('path');
 var log = require('db-migrate-shared').log;
 var migrationHook = require('./helper/migration-hook.js');
+var eachScope = require('./helper/scopes.js');
 
 async function prepare (internals, config) {
   try {
@@ -40,7 +41,7 @@ async function prepare (internals, config) {
   }
 }
 
-module.exports = async function (internals, config) {
+async function execute (internals, config) {
   const migrator = await prepare(internals, config);
 
   try {
@@ -49,4 +50,8 @@ module.exports = async function (internals, config) {
   } catch (err) {
     return internals.onComplete(migrator, internals, err);
   }
+}
+
+module.exports = function (internals, config) {
+  return eachScope(internals, () => execute(internals, config));
 };

package/lib/config.js

@@ -94,6 +94,9 @@ exports.loadFile = function (fileName, currentEnv, plugins) {
   return exports.loadObject(config, currentEnv);
 };
 
+// resolves { "ENV": ... } entries of a config, in place
+exports.resolveEnv = walkConfig;
+
 function walkConfig (level) {
   for (var configEntry in level) {
     if (level[configEntry] && level[configEntry].ENV) {

package/lib/executors/versioned/v1.js

@@ -2,11 +2,33 @@
 
 const Promise = require('bluebird');
 const maybePromised = require('../../temputils.js').maybePromised;
+const log = require('db-migrate-shared').log;
+
+/**
+ * A migration runs inside a transaction of the driver, unless it opts out
+ * with _meta.transactions = false, e.g. for statements which can not run
+ * inside one, like CREATE INDEX CONCURRENTLY of PostgreSQL.
+ */
+const transactional = file => {
+  const _meta = file.get()._meta || {};
+
+  if (_meta.transactions === false) {
+    log.verbose(`[migration] ${file.name} runs without a transaction`);
+    return false;
+  }
+
+  return true;
+};
+
+const start = (driver, file) =>
+  transactional(file) ? driver.startMigration() : Promise.resolve();
+
+const end = (driver, file) =>
+  transactional(file) ? driver.endMigration() : Promise.resolve();
 
 const execUnit = {
   up: function (context, driver, file) {
-    return context.driver
-      .startMigration()
+    return start(context.driver, file)
       .then(() => {
         const _file = file.get();
 
@@ -21,12 +43,11 @@ const execUnit = {
           file
         );
       })
-      .then(context.driver.endMigration.bind(context.driver));
+      .then(() => end(context.driver, file));
   },
 
   down: function (context, driver, file) {
-    return driver
-      .startMigration()
+    return start(driver, file)
       .then(() => {
         const _file = file.get();
 
@@ -41,7 +62,7 @@ const execUnit = {
           file
         );
       })
-      .then(context.driver.endMigration.bind(context.driver));
+      .then(() => end(context.driver, file));
   }
 };
 

package/lib/executors/versioned/v2.js

@@ -181,6 +181,15 @@ const execUnit = {
       );
       describeFailure(err, chain, context.internals);
 
+      if (context.internals.modSchema.s.some(entry => entry.t === 3)) {
+        log.error(
+          `Migration "${file.name}" failed and can not be rolled back, it ` +
+            'ran a step with { irreversible: true }. The steps executed ' +
+            'stay, the next run continues after them.'
+        );
+        throw err;
+      }
+
       log.error(
         `Migration "${file.name}" failed, rolling back: ${
           (err && err.message) || err
@@ -246,7 +255,16 @@ const execUnit = {
       });
     }
     // start migration, see up comments
-    await TranslateState(context._driver, file, driver, context.internals, context._pdriver);
+    try {
+      await TranslateState(context._driver, file, driver, context.internals, context._pdriver);
+    } catch (err) {
+      // refused before changing anything, the state is left as it was
+      if (err.irreversible && abort !== true) {
+        await State.endMigration(context._pdriver, file, context.internals);
+      }
+
+      throw err;
+    }
     await State.endMigration(context._pdriver, file, context.internals);
     return Promise.promisify(context.deleteMigrationRecord.bind(context))(file);
     // end migration, see up comments

package/lib/file.js

@@ -200,6 +200,11 @@ File.loadFromFileystem = function (dir, prefix, internals) {
 };
 
 File.loadFromDatabase = function (dir, prefix, driver, internals) {
+  // --ignore-completed-migrations starts at the first migration
+  if (internals.ignoreCompleted) {
+    return Promise.resolve([]);
+  }
+
   log.verbose(`[${prefix}] loading from database`);
   return driver[LOADER[prefix]]()
     .catch(function (err) {

package/lib/learn.js

@@ -1,15 +1,52 @@
 const Promise = require('bluebird');
 const Shadow = require('./driver/shadow');
 
+/**
+ * Objects unknown to the schema were not created by a v2 migration, but e.g.
+ * by a v1 migration or by hand. The errors point to the ways of handling them:
+ * adopting them into the schema first, or dropping them irreversibly.
+ */
+const IRREVERSIBLE = ', or pass { irreversible: true } to drop it without being able to revert it';
+
+const unknown = (what, adopt, drop) =>
+  new Error(
+    `${what} is unknown to the schema of db-migrate, it was not created by a ` +
+      `v2 migration. Declare it first with db.adopt.${adopt}` +
+      (drop ? IRREVERSIBLE : '') +
+      '.'
+  );
+
+const unknownTable = (t, drop) =>
+  unknown(`The table "${t}"`, `createTable("${t}", columns)`, drop);
+
+const unknownColumn = (t, c, drop) =>
+  unknown(`The column "${c}" of "${t}"`, `addColumn("${t}", "${c}", spec)`, drop);
+
+const unknownIndex = (t, i) =>
+  unknown(`The index "${i}" of "${t}"`, `addIndex("${t}", "${i}", columns)`);
+
+const unknownForeignKey = (t, k, drop) =>
+  unknown(
+    `The foreign key "${k}" of "${t}"`,
+    `addForeignKey("${t}", referencedTable, "${k}", mapping)`,
+    drop
+  );
+
+/**
+ * A step dropping an object unknown to the schema, asked for explicitly.
+ * It can not be reverted, a rollback refuses to start because of it.
+ */
+const irreversible = (action, args) => ({ t: 3, a: action, c: args });
+
 class STD {
   constructor ({ schema, modSchema: mod, unlearn }, driver) {
-    this.checkColumn = function (t, c) {
+    this.checkColumn = function (t, c, drop) {
       if (!this.schema[t]) {
-        throw new Error(`There is no ${t} table in schema!`);
+        throw unknownTable(t, drop);
       }
 
       if (!this.schema[t][c] && !this.schema[t].columns?.[c]) {
-        throw new Error(`There is no ${c} column in schema!`);
+        throw unknownColumn(t, c, drop);
       }
     };
     this.unlearn = unlearn;
@@ -32,10 +69,19 @@ class STD {
     this.modF = mod.f;
   }
 
-  dropTable (t) {
+  dropTable (t, o = {}) {
     let alter = {};
     alter = { c: {}, i: {}, f: {} };
 
+    if (!this.schema[t] && !this.unlearn) {
+      if (!o || o.irreversible !== true) {
+        throw unknownTable(t, true);
+      }
+
+      this.modC.push(irreversible('dropTable', [t]));
+      return Promise.resolve(alter);
+    }
+
     if (this.schema[t]) {
       alter.c[t] = this.schema[t];
       this.modS[t] = this.schema[t];
@@ -125,7 +171,19 @@ class STD {
 
   removeColumn (t, c, o = {}) {
     const alter = {};
-    this.checkColumn(t, c);
+
+    if (
+      o &&
+      o.irreversible === true &&
+      !this.unlearn &&
+      (!this.schema[t] ||
+        (!this.schema[t][c] && !this.schema[t].columns?.[c]))
+    ) {
+      this.modC.push(irreversible('removeColumn', [t, c]));
+      return Promise.resolve(alter);
+    }
+
+    this.checkColumn(t, c, true);
     const hasColumns = this.schema[t].columns !== undefined;
     const columns = hasColumns ? this.schema[t].columns : this.schema[t];
 
@@ -230,7 +288,7 @@ class STD {
 
   addColumn (t, c, s) {
     if (!this.schema[t]) {
-      throw new Error(`There is no ${t} table in schema!`);
+      throw unknownTable(t);
     }
     this.schema[t] = this.schema[t] || {};
     const hasColumns = this.schema[t].columns !== undefined;
@@ -292,11 +350,11 @@ class STD {
     }
 
     if (!this.schema[t]) {
-      throw new Error(`There is no ${t} table in schema!`);
+      throw unknownTable(t);
     }
 
     if (!this.indizies[t] || !this.indizies[t][i]) {
-      throw new Error(`There is no index ${i} in ${t} table!`);
+      throw unknownIndex(t, i);
     }
 
     alter.i[t] = {};
@@ -312,11 +370,11 @@ class STD {
 
   addForeignKey (t, rt, k, m, r) {
     if (!this.schema[t]) {
-      throw new Error(`There is no ${t} table in schema!`);
+      throw unknownTable(t);
     }
 
     if (!this.schema[rt]) {
-      throw new Error(`There is no ${rt} table in schema!`);
+      throw unknownTable(rt);
     }
 
     if (!this.foreign[t]) this.foreign[t] = {};
@@ -335,13 +393,19 @@ class STD {
   removeForeignKey (t, k, o) {
     let alter = {};
     alter = { c: {}, i: {}, f: {} };
+    const known = this.schema[t] && this.foreign[t] && this.foreign[t][k];
+
+    if (!known && o && o.irreversible === true && !this.unlearn) {
+      this.modC.push(irreversible('removeForeignKey', [t, k]));
+      return Promise.resolve(alter);
+    }
 
     if (!this.schema[t]) {
-      throw new Error(`There is no ${t} table in schema!`);
+      throw unknownTable(t, true);
     }
 
     if (!this.foreign[t] || !this.foreign[t][k]) {
-      throw new Error(`There is no foreign key ${k} in ${t} table!`);
+      throw unknownForeignKey(t, k, true);
     }
 
     alter.f[t] = {};
@@ -377,6 +441,9 @@ module.exports = {
   // learns the schema and records the reverse operations
   learns: true,
 
+  unknownTable,
+  unknownColumn,
+
   getInterface: (context, file, driver, internals) => {
     if (context.learnable) {
       const _std = new STD(internals, context);

package/lib/methods/v2/conventions.js

@@ -28,6 +28,9 @@ const noAction = prop => {
 };
 
 module.exports = {
+  // adds the default column to tables created on the database
+  executes: true,
+
   getInterface: (context, file, driver, internals) => {
     if (context.conventions) {
       const st = new AddConventions(internals, file, context);

package/lib/methods/v2/migrate.js

@@ -1,4 +1,7 @@
 module.exports = {
+  // executes the instructions on the database
+  executes: true,
+
   getInterface (context, file, driver) {
     return driver;
   }

package/lib/methods/v2/translatestate.js

@@ -100,6 +100,10 @@ async function processEntry (
       case 1:
         await f[action](driver, args, internals);
         break;
+      case 2:
+        // an adopted object, only forgotten again
+        await driver.learnOnly.apply(driver, [action].concat(args));
+        break;
 
       default:
         throw new Error(`Invalid state record, of type ${type}`);
@@ -110,8 +114,28 @@ async function processEntry (
   await State.update(pdriver, file, internals.modSchema, internals);
 }
 
+/**
+ * Steps run with { irreversible: true } can not be reverted, the migration
+ * is not reverted at all then, instead of only partially.
+ */
+const irreversibleSteps = internals =>
+  internals.modSchema.s
+    .filter(entry => entry.t === 3)
+    .map(entry => `step ${entry.n} ${entry.a}(${entry.c.map(x => JSON.stringify(x)).join(', ')})`);
+
 module.exports = async (context, file, driver, internals, pdriver) => {
   const mod = internals.modSchema;
+  const irreversible = irreversibleSteps(internals);
+
+  if (irreversible.length > 0) {
+    const err = new Error(
+      `Migration "${file.name}" can not be reverted, it ran ` +
+        `${irreversible.join(', ')} with { irreversible: true }.`
+    );
+    // nothing was changed
+    err.irreversible = true;
+    throw err;
+  }
 
   const chain = new Chain(context, file, driver, internals, pdriver);
   internals.unlearn = true;

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.0.0",
+  "version": "1.2.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },
@@ -57,9 +57,9 @@
     "@hapi/code": "9.x",
     "@hapi/eslint-plugin": "^6.0.0",
     "@hapi/lab": "25.x",
-    "db-migrate-base": "^2.1.1",
-    "db-migrate-mysql": "^2.2.0",
-    "db-migrate-sqlite3": "^0.5.0",
+    "db-migrate-base": "^2.4.1",
+    "db-migrate-mysql": "^3.1.1",
+    "db-migrate-sqlite3": "^1.1.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^14.1.1",
     "eslint-plugin-import": "^2.26.0",
@@ -79,5 +79,9 @@
     "pretest": "eslint *.js lib/*.js",
     "test": "lab",
     "test:coverage": "lab -r html -o coverage.html"
+  },
+  "overrides": {
+    "tar": "^7.5.21",
+    "node-gyp": "^12.0.0"
   }
 }