db-migrate 1.0.0-beta.9 → 1.0.0

package/lib/commands/sync.js

@@ -21,15 +21,15 @@ async function prepare (internals, config) {
     );
 
     if (internals.locTitle) {
-      migrator.migrationsDir = path.resolve(
+      migrator.directory = path.resolve(
         internals.argv['migrations-dir'],
         internals.locTitle
       );
     } else {
-      migrator.migrationsDir = path.resolve(internals.argv['migrations-dir']);
+      migrator.directory = path.resolve(internals.argv['migrations-dir']);
     }
 
-    internals.migrationsDir = migrator.migrationsDir;
+    internals.migrationsDir = migrator.directory;
 
     await migrator.createMigrationsTable();
     log.verbose('migration table created');

package/lib/commands/undo-seed.js

@@ -1,54 +1,14 @@
 'use strict';
 
-var assert = require('assert');
-var path = require('path');
-var log = require('db-migrate-shared').log;
-var _assert = require('./helper/assert');
-
-function executeUndoSeed (internals, config, callback) {
-  var index = require('./connect');
-  var Seeder = require('./lib/walker.js');
-
-  if (!internals.argv.count) {
-    log.info('Defaulting to running 1 down seed.');
-    internals.argv.count = 1;
-  }
-
-  if (internals.argv._.length > 0) {
-    internals.argv.destination = internals.argv._.shift().toString();
-  }
-
-  index.connect(
-    {
-      config: config.getCurrent().settings,
-      internals: internals,
-      prefix: 'seed'
-    },
-    Seeder,
-    function (err, seeder) {
-      assert.ifError(err);
-
-      var seedDir =
-        internals.mode !== 'static' ? 'vcseeder-dir' : 'staticseeder-dir';
-
-      seeder.seedDir = path.resolve(internals.argv[seedDir]);
-
-      if (internals.mode === 'static') {
-        internals.onComplete(seeder, callback, {
-          stack: "Static seeders can't be undone. Use VC Seeders instead!"
-        });
-      } else {
-        seeder.createSeedsTable(function (err) {
-          if (_assert(err, callback)) {
-            seeder.down(
-              internals.argv,
-              internals.onComplete.bind(this, seeder, internals, callback)
-            );
-          }
-        });
-      }
-    }
+/**
+ * Seeders are not part of db-migrate 1.0, their implementation was never
+ * finished. A new concept follows separately, see #687.
+ */
+module.exports = function () {
+  return Promise.reject(
+    new Error(
+      'Seeders are not supported by db-migrate 1.0. The unfinished seeders ' +
+        'of db-migrate 0.11 are still available there.'
+    )
   );
-}
-
-module.exports = executeUndoSeed;
+};

package/lib/commands/up.js

@@ -21,15 +21,15 @@ async function prepare (internals, config) {
     );
 
     if (internals.locTitle) {
-      migrator.migrationsDir = path.resolve(
+      migrator.directory = path.resolve(
         internals.argv['migrations-dir'],
         internals.locTitle
       );
     } else {
-      migrator.migrationsDir = path.resolve(internals.argv['migrations-dir']);
+      migrator.directory = path.resolve(internals.argv['migrations-dir']);
     }
 
-    internals.migrationsDir = migrator.migrationsDir;
+    internals.migrationsDir = migrator.directory;
 
     await migrator.createMigrationsTable();
     log.verbose('migration table created');

package/lib/driver/index.js

@@ -16,6 +16,64 @@ var ShadowProto = {
   }
 };
 
+/**
+ * Attach the failed statement to errors of the driver, so the error output
+ * can show what failed. Drivers already setting sql (mysql2) keep theirs.
+ */
+const annotateSql = (err, sql, hasParams) => {
+  if (err && typeof err === 'object' && err.sql === undefined) {
+    try {
+      err.sql = sql;
+      // the values are not kept, they might be sensitive
+      Object.defineProperty(err, 'sqlParams', { value: hasParams });
+    } catch (e) {}
+  }
+
+  return err;
+};
+
+function trackSql (db) {
+  ['runSql', 'all'].forEach(method => {
+    const original = db[method];
+
+    if (typeof original !== 'function') {
+      return;
+    }
+
+    db[method] = function (sql, ...args) {
+      const callback =
+        typeof args[args.length - 1] === 'function' ? args.pop() : null;
+      const hasParams = args.length > 0;
+
+      if (callback) {
+        // the returned promise is left alone, the callback handles errors
+        return original.call(this, sql, ...args, function (err, ...res) {
+          return callback.call(
+            this,
+            err ? annotateSql(err, sql, hasParams) : err,
+            ...res
+          );
+        });
+      }
+
+      const ret = original.call(this, sql, ...args);
+      if (ret && typeof ret.catch === 'function') {
+        return ret.catch(err => {
+          throw annotateSql(err, sql, hasParams);
+        });
+      }
+
+      return ret;
+    };
+  });
+}
+
+/**
+ * Established tunnels by their config. db-migrate opens more than one
+ * connection, which all share the tunnel listening on the local port.
+ */
+const tunnels = {};
+
 exports.connect = function (config, intern, callback) {
   var driver, req;
   let plugin = false;
@@ -37,7 +95,7 @@ exports.connect = function (config, intern, callback) {
 
   if (config.driver === undefined) {
     throw new Error(
-      'config must include a driver key specifing which driver to use'
+      'config must include a driver key specifying which driver to use'
     );
   }
 
@@ -92,64 +150,75 @@ exports.connect = function (config, intern, callback) {
   log.verbose('connecting');
 
   var connect = function (config) {
-    driver.connect(config, intern, function (err, db) {
-      if (err) {
-        callback(err);
-        return;
-      }
-      log.verbose('connected');
+    driver.connect(
+      // safe disconnect of our config object into userspace
+      JSON.parse(JSON.stringify(config)),
+      intern,
+      function (err, db) {
+        if (err) {
+          callback(err);
+          return;
+        }
+        log.verbose('connected');
 
-      if (!global.immunity) {
-        db = Shadow.infect(db, internals, ShadowProto);
-      }
+        if (!global.immunity) {
+          db = Shadow.infect(db, internals, ShadowProto);
+        }
 
-      callback(null, db);
-    });
+        trackSql(db);
+        callback(null, db);
+      }
+    );
   };
 
   if (config.tunnel) {
-    var tunnel = require('tunnel-ssh');
+    // tunnels are provided by plugins, e.g. db-migrate-plugin-tunnel-ssh
     var tunnelConfig = JSON.parse(JSON.stringify(config.tunnel));
     const { tunnelType } = tunnelConfig;
+    const type = tunnelType && tunnelType !== 'ssh' ? tunnelType : 'ssh';
+    const hook = `connection:tunnel:${type}`;
 
     if (plugins) {
-      plugin = plugins.overwrite(
-        `connection:tunnel:${
-          tunnelType && tunnelType !== 'ssh' ? tunnelType : 'ssh'
-        }`
+      plugin = plugins.overwrite(hook);
+    }
+
+    if (!plugin) {
+      return callback(
+        new Error(
+          `A ${type} tunnel is configured, but no plugin provides it. ` +
+            `Install db-migrate-plugin-tunnel-${type}, e.g. ` +
+            `npm install db-migrate-plugin-tunnel-${type}`
+        )
       );
     }
 
     tunnelConfig.dstHost = config.host;
     tunnelConfig.dstPort = config.port;
 
-    if (plugin) {
-      tunnel = plugin[
-        `connection:tunnel:${
-          tunnelType && tunnelType !== 'ssh' ? tunnelType : 'ssh'
-        }`
-      ](tunnelConfig);
-    }
+    // Point the db host/port to our local tunnel, on a copy as the config
+    // is used again for further connections
+    config = Object.assign({}, config, {
+      host: '127.0.0.1',
+      port: tunnelConfig.localPort
+    });
 
-    if (tunnelConfig.privateKeyPath) {
-      tunnelConfig.privateKey = require('fs').readFileSync(
-        tunnelConfig.privateKeyPath
-      );
+    const key = JSON.stringify([type, tunnelConfig]);
+    if (!tunnels[key]) {
+      tunnels[key] = Promise.resolve().then(() => plugin[hook](tunnelConfig));
+      // a failed tunnel may be retried
+      tunnels[key].catch(() => delete tunnels[key]);
     }
 
-    // Reassign the db host/port to point to our local ssh tunnel
-    config.host = '127.0.0.1';
-    config.port = tunnelConfig.localPort;
-
-    tunnel(tunnelConfig, function (err) {
-      if (err) {
-        callback(err);
-        return;
-      }
-      log.verbose('SSH tunnel connected on port ', tunnelConfig.localPort);
-
-      connect(config);
-    });
+    tunnels[key]
+      .then(
+        function () {
+          log.verbose(`${type} tunnel connected on port`, tunnelConfig.localPort);
+          connect(config);
+        },
+        function (err) {
+          callback(err);
+        }
+      );
   } else {
     connect(config);
   }

package/lib/executors/versioned/v2.js

@@ -5,10 +5,108 @@ const State = require('../../state');
 const log = require('db-migrate-shared').log;
 const Learn = require('../../learn');
 const Chain = require('../../chain');
-const StateTravel = require('../../methods/v2/statetravel');
+const StateTravel = require('../../methods/v2/statetravel')();
+const StateNoStepTravel = require('../../methods/v2/statetravel')(false);
 const Migrate = require('../../methods/v2/migrate');
+const StateDone = require('../../methods/v2/statedone');
 const TranslateState = require('../../methods/v2/translatestate');
 const AddConventions = require('../../methods/v2/conventions');
+const util = require('util');
+
+/**
+ * How to recover from an interrupted run of a migration, set per migration
+ * by _meta.recovery:
+ *
+ * skip: skip the steps already executed and continue with the rest
+ * rollback: revert the steps already executed and run the migration again
+ */
+const RECOVERY_MODES = ['skip', 'rollback'];
+const RECOVERY_DEFAULT = 'skip';
+
+const recoveryMode = (file, interrupted) => {
+  const name = file.name;
+  const mode = file.get()._meta.recovery || RECOVERY_DEFAULT;
+
+  if (RECOVERY_MODES.indexOf(mode) === -1) {
+    throw new Error(
+      `Invalid recovery mode "${mode}" in migration "${name}", use one of ` +
+        RECOVERY_MODES.join(', ')
+    );
+  }
+
+  if (interrupted.rollback) {
+    if (mode !== 'rollback') {
+      log.warn(
+        `[recovery] ${name}: the previous run was interrupted while rolling ` +
+          'back, continuing the rollback'
+      );
+    }
+
+    return 'rollback';
+  }
+
+  if (mode === 'skip' && interrupted.changed) {
+    throw new Error(
+      `Migration "${name}" was interrupted at step ${interrupted.step} and ` +
+        'changed since, so the executed steps can not be skipped safely. ' +
+        "Set _meta.recovery to 'rollback' to revert them, or repair the " +
+        'state manually.'
+    );
+  }
+
+  return mode;
+};
+
+/**
+ * Revert the steps executed by an interrupted run. A step started but not
+ * done did not change the database, so its reverse operation is dropped.
+ */
+/**
+ * Keep the reverse operations of the steps executed on the database. The
+ * failed step only changed the database, if its main statement went through
+ * before failing, signaled by the driver (e.g. a foreign key of createTable
+ * failed after the table was created).
+ */
+const executedSteps = (internals, failed, signaled) => {
+  const done = internals.migrationDone;
+
+  internals.modSchema.s = internals.modSchema.s.filter(
+    entry =>
+      entry.n <= done || (signaled && failed > done && entry.n === failed)
+  );
+};
+
+/**
+ * Name the instruction a migration failed at, or after if the instruction
+ * itself finished and the migration code failed afterwards.
+ */
+const describeFailure = (err, chain, internals) => {
+  if (!err || typeof err !== 'object' || err.instruction || chain.op === 0) {
+    return;
+  }
+
+  const after = internals.migrationDone >= chain.op;
+  err.instruction = `${after ? 'after' : 'at'} step ${chain.op} ${
+    chain.current
+  }`;
+};
+
+const recoverByRollback = async (context, driver, file, interrupted) => {
+  const internals = context.internals;
+  const pdriver = context._pdriver;
+
+  await State.rollingBack(pdriver, internals);
+  internals.modSchema.s = internals.modSchema.s.filter(
+    entry => !(entry.n > interrupted.done)
+  );
+  await TranslateState(context._driver, file, driver, internals, pdriver);
+
+  internals.unlearn = false;
+  internals.rollback = false;
+  internals.rollbackContinue = false;
+  internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
+  await State.endMigration(pdriver, file, internals);
+};
 
 const execUnit = {
   _extend: (context, type) => {
@@ -28,15 +126,87 @@ const execUnit = {
 
   up: async function (context, driver, file) {
     const _file = file.get();
-    const chain = new Chain(context._driver, file, driver, context.internals);
+    const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
       chain.addChain(AddConventions);
     }
-    chain.addChain(Learn);
     chain.addChain(StateTravel);
+    chain.addChain(Learn);
+    chain.addChain(StateNoStepTravel);
     chain.addChain(Migrate);
+    chain.addChain(StateDone);
+
+    const interrupted = await State.startMigration(
+      context._pdriver,
+      file,
+      context.internals,
+      { recover: true }
+    );
+
+    if (interrupted) {
+      const mode = recoveryMode(file, interrupted);
+      log.warn(
+        `[recovery] ${file.name}: the previous run was interrupted at step ` +
+          `${interrupted.step}, ${interrupted.done} steps were executed, ` +
+          `recovering by ${mode}`
+      );
+
+      if (mode === 'rollback') {
+        await recoverByRollback(context, driver, file, interrupted);
+        await State.startMigration(context._pdriver, file, context.internals);
+      } else {
+        chain.recovery = interrupted;
+      }
+    }
+
+    context.internals.migrationDone = chain.recovery ? chain.recovery.done : 0;
 
-    await State.startMigration(context._driver, file, context.internals);
+    // startMigration - needs secondary instance since we can not afford to
+    // loose state and the transaction start will include these for roll back
+    // we will disable them probably at all from DDL when the driver does not
+    // explicitly signal DDL transaction support (like crdb)
+    try {
+      await _file.migrate(chain, {
+        options: context.internals.safeOptions,
+        seedLink: context.seedLink,
+        dbm: context.internals.safeOptions.dbmigrate
+      });
+    } catch (err) {
+      // transfer last state
+      chain.transferInt();
+      executedSteps(
+        context.internals,
+        chain.op,
+        chain.udriver._counter.previousSignal() === true
+      );
+      describeFailure(err, chain, context.internals);
+
+      log.error(
+        `Migration "${file.name}" failed, rolling back: ${
+          (err && err.message) || err
+        }`
+      );
+      await State.rollingBack(context._pdriver, context.internals);
+      await execUnit.down(context, driver, file, { abort: true });
+      throw err;
+    }
+    await Promise.promisify(context.writeMigrationRecord.bind(context))(file);
+    return State.endMigration(context._pdriver, file, context.internals);
+    // end migration, same as start migration
+  },
+
+  fix: async function (context, driver, file) {
+    const _file = file.get();
+    const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
+    if (!_file._meta.noDefaultColumn) {
+      chain.addChain(AddConventions);
+    }
+    chain.addChain(Learn);
+    chain.addChain(StateTravel);
+
+    await State.startMigration(context._pdriver, file, context.internals, {
+      op: 'fix'
+    });
     // startMigration - needs secondary instance since we can not afford to
     // loose state and the transaction start will include these for roll back
     // we will disable them probably at all from DDL when the driver does not
@@ -53,23 +223,31 @@ const execUnit = {
       // transfer last state
       chain.transferInt();
 
+      describeFailure(err, chain, context.internals);
       log.error(
-        'An error occured. No alternative failure strategy defined. Rolling back!',
-        err
+        `Migration "${file.name}" failed, rolling back: ${
+          (err && err.message) || err
+        }`
       );
       await execUnit.down(context, driver, file);
       throw err;
     }
-    await Promise.promisify(context.writeMigrationRecord.bind(context))(file);
-    return State.endMigration(context._driver, file, context.internals);
+    await State.endMigration(context._pdriver, file, context.internals);
+    log.verbose(`[fix] current schema`, util.inspect(context.internals.schema, false, null, true));
     // end migration, same as start migration
   },
 
-  down: async function (context, driver, file) {
+  down: async function (context, driver, file, { abort } = {}) {
+    // if we get the abort signal this means we are in a rollback routine
+    // which means in turn we want to avoid reloading the state
+    if (abort !== true) {
+      await State.startMigration(context._pdriver, file, context.internals, {
+        op: 'down'
+      });
+    }
     // start migration, see up comments
-    await State.startMigration(context._driver, file, context.internals);
-    await TranslateState(context._driver, file, driver, context.internals);
-    await State.endMigration(context._driver, file, context.internals);
+    await TranslateState(context._driver, file, driver, context.internals, context._pdriver);
+    await State.endMigration(context._pdriver, file, context.internals);
     return Promise.promisify(context.deleteMigrationRecord.bind(context))(file);
     // end migration, see up comments
   }

package/lib/file.js

@@ -56,6 +56,12 @@ function parseTitle (name) {
 
 const filesRegEx = /\.js$/;
 
+/**
+ * Loaders of plugins for their file extensions, files without one are
+ * required.
+ */
+let loaders = {};
+
 const File = function () {
   if (arguments.length >= 3) {
     this.title = arguments[0];
@@ -81,7 +87,27 @@ File.prototype = {
   },
 
   get: function () {
-    return this._required || (this._required = require(this.path));
+    if (!this._required) {
+      let file = this.path;
+      let loader = loaders[path.extname(file).slice(1)];
+
+      // migrations recorded in the database carry no extension, require
+      // resolves its own ones, files of plugins have to be looked up
+      if (!loader) {
+        const extension = Object.keys(loaders).find(ext =>
+          fs.existsSync(`${file}.${ext}`)
+        );
+
+        if (extension) {
+          file = `${file}.${extension}`;
+          loader = loaders[extension];
+        }
+      }
+
+      this._required = loader ? loader(file) : require(file);
+    }
+
+    return this._required;
   },
 
   write: function (data) {
@@ -104,6 +130,7 @@ File.registerHook = function (Plugin, internals) {
     filesRegEx: filesRegEx,
     extensions: 'js'
   };
+  loaders = {};
 
   if (!plugin) {
     return Promise.resolve(null);
@@ -137,6 +164,14 @@ File.registerHook = function (Plugin, internals) {
       if (parser && parser.extensions) {
         internals.parser.extensions =
           internals.parser.extensions + '|' + parser.extensions;
+
+        // a parser may load its files itself, returning the migration
+        // module, instead of registering them for require
+        if (typeof parser.load === 'function') {
+          parser.extensions.split('|').forEach(extension => {
+            loaders[extension] = parser.load;
+          });
+        }
       }
     })
     .then(function () {