db-migrate 1.0.0-beta.16

package/lib/executors/versioned/v2.js

@@ -0,0 +1,78 @@
+'use strict';
+
+const Promise = require('bluebird');
+const State = require('../../state');
+const log = require('db-migrate-shared').log;
+const Learn = require('../../learn');
+const Chain = require('../../chain');
+const StateTravel = require('../../methods/v2/statetravel');
+const Migrate = require('../../methods/v2/migrate');
+const TranslateState = require('../../methods/v2/translatestate');
+const AddConventions = require('../../methods/v2/conventions');
+
+const execUnit = {
+  _extend: (context, type) => {
+    return {
+      atomic: function (actions) {
+        const action = actions[type];
+        const reverse = actions[type === 'up' ? 'up' : 'down'];
+
+        if (!action || !reverse) {
+          return Promise.reject(new Error('invalid operation'));
+        }
+
+        return action().catch(() => reverse());
+      }
+    };
+  },
+
+  up: async function (context, driver, file) {
+    const _file = file.get();
+    const chain = new Chain(context._driver, file, driver, context.internals);
+    if (!_file._meta.noDefaultColumn) {
+      chain.addChain(AddConventions);
+    }
+    chain.addChain(Learn);
+    chain.addChain(StateTravel);
+    chain.addChain(Migrate);
+
+    await State.startMigration(context._driver, file, context.internals);
+    // startMigration - needs secondary instance since we can not afford to
+    // loose state and the transaction start will include these for roll back
+    // we will disable them probably at all from DDL when the driver does not
+    // explicitly signal DDL transaction support (like crdb)
+    try {
+      await _file.migrate(chain, {
+        options: context.internals.safeOptions,
+        seedLink: context.seedLink,
+        dbm: context.internals.safeOptions.dbmigrate
+      });
+    } catch (err) {
+      context.internals.rollback = true;
+
+      // transfer last state
+      chain.transferInt();
+
+      log.error(
+        'An error occured. No alternative failure strategy defined. Rolling back!',
+        err
+      );
+      await execUnit.down(context, driver, file);
+      throw err;
+    }
+    await Promise.promisify(context.writeMigrationRecord.bind(context))(file);
+    return State.endMigration(context._driver, file, context.internals);
+    // end migration, same as start migration
+  },
+
+  down: async function (context, driver, file) {
+    // start migration, see up comments
+    await State.startMigration(context._driver, file, context.internals);
+    await TranslateState(context._driver, file, driver, context.internals);
+    await State.endMigration(context._driver, file, context.internals);
+    return Promise.promisify(context.deleteMigrationRecord.bind(context))(file);
+    // end migration, see up comments
+  }
+};
+
+module.exports = execUnit;

package/lib/file.js

@@ -0,0 +1,190 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const log = require('db-migrate-shared').log;
+const inflection = require('inflection');
+const Promise = require('bluebird');
+const lpad = require('db-migrate-shared').util.lpad;
+
+Promise.promisifyAll(fs);
+
+const LOADER = {
+  seeder: 'allLoadedSeedsAsync',
+  migration: 'allLoadedMigrationsAsync'
+};
+
+function formatPath (dir, name) {
+  return path.join(dir, name);
+}
+
+function formatName (title, date) {
+  return formatDate(date) + '-' + formatTitle(title);
+}
+
+function formatDate (date) {
+  return [
+    date.getUTCFullYear(),
+    lpad(date.getUTCMonth() + 1, '0', 2),
+    lpad(date.getUTCDate(), '0', 2),
+    lpad(date.getUTCHours(), '0', 2),
+    lpad(date.getUTCMinutes(), '0', 2),
+    lpad(date.getUTCSeconds(), '0', 2)
+  ].join('');
+}
+
+function formatTitle (title) {
+  return inflection.dasherize(title);
+}
+
+function parseDate (name) {
+  const date = new Date();
+  const match = name.match(/(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})-[^.]+/);
+  date.setUTCFullYear(match[1]);
+  date.setUTCDate(match[3]);
+  date.setUTCMonth(match[2] - 1);
+  date.setUTCHours(match[4]);
+  date.setUTCMinutes(match[5]);
+  date.setUTCSeconds(match[6]);
+  return date;
+}
+
+function parseTitle (name) {
+  const match = name.match(/\d{14}-([^.]+)/);
+  return inflection.humanize(match[1], true);
+}
+
+const filesRegEx = /\.js$/;
+
+const File = function () {
+  if (arguments.length >= 3) {
+    this.title = arguments[0];
+    this.date = arguments[2];
+    this.name = this.formatName(this.title, this.date);
+    this.path = this.formatPath(arguments[1], this.name);
+    this.internals = arguments[4];
+  } else if (arguments.length === 2) {
+    this.path = arguments[0];
+    this.name = this.parseName(this.path);
+    this.date = this.parseDate(this.name);
+    this.title = this.parseTitle(this.name);
+    this.internals = arguments[1];
+  }
+
+  if (this.internals) this.internals = this.internals.safeOptions;
+};
+
+File.prototype = {
+  parseName: function (path) {
+    const match = path.match(/(\d{14}-[^.]+)(?:\.*?)?/);
+    return match[1];
+  },
+
+  get: function () {
+    return this._required || (this._required = require(this.path));
+  },
+
+  write: function (data) {
+    return fs.writeFileAsync(this.path, data);
+  },
+
+  parseTitle: parseTitle,
+  parseDate: parseDate,
+  formatTitle: formatTitle,
+  formatPath: formatPath,
+  formatName: formatName
+};
+
+File.registerHook = function (Plugin, internals) {
+  const plugin = [].concat(Plugin.hook('file:hook:require') || []).concat(
+    Plugin.hook('migrator:migration:hook:require') || [] // backwards compatible
+  );
+
+  internals.parser = internals.parser || {
+    filesRegEx: filesRegEx,
+    extensions: 'js'
+  };
+
+  if (!plugin) {
+    return Promise.resolve(null);
+  }
+
+  return Promise.resolve(plugin)
+    .map(function (plugin) {
+      // Backwards compatibility and notice for everyone so they
+      // can bug the plugin maintainer
+      if (plugin['migrator:migration:hook:require']) {
+        log.warn(
+          `The plugin '${plugin.name}' is outdated! The hook` +
+            `migrator:migration:hook:require was deprecated!`
+        );
+        log.warn(
+          `Report the maintainer of '${plugin.name}' to patch the ` +
+            `hook instead with file:hook:require.`
+        );
+
+        if (plugin.maintainer && plugin.maintainer.repository) {
+          log.verbose(
+            `The repo of the ${plugin.name} plugin is ${plugin.maintainer.repository}!`
+          );
+        }
+        return plugin['migrator:migration:hook:require']();
+      }
+
+      return plugin['file:hook:require']();
+    })
+    .each(function (parser) {
+      if (parser && parser.extensions) {
+        internals.parser.extensions =
+          internals.parser.extensions + '|' + parser.extensions;
+      }
+    })
+    .then(function () {
+      internals.parser.filesRegEx = new RegExp(
+        '\\.(' + internals.parser.extensions + ')$'
+      );
+
+      return internals.parser;
+    });
+};
+
+File.loadFromFileystem = function (dir, prefix, internals) {
+  log.verbose(`[${prefix}] loading from dir`, dir);
+
+  return fs
+    .readdirAsync(dir)
+    .filter(function (files) {
+      return internals.parser.filesRegEx.test(files);
+    })
+    .then(function (files) {
+      return files.sort();
+    })
+    .map(function (file) {
+      return new File(path.join(dir, file), internals);
+    });
+};
+
+File.loadFromDatabase = function (dir, prefix, driver, internals) {
+  log.verbose(`[${prefix}] loading from database`);
+  return driver[LOADER[prefix]]()
+    .catch(function (err) {
+      if (internals.dryRun) {
+        return [];
+      } else {
+        return Promise.reject(err);
+      }
+    })
+    .filter(function (result) {
+      return (
+        result.name.substr(0, result.name.lastIndexOf('/')) ===
+        internals.matching
+      );
+    })
+    .map(function (result) {
+      return new File(path.join(dir, result.name), internals);
+    });
+};
+
+Promise.promisifyAll(fs);
+
+module.exports = File;

package/lib/interface/migratorInterface.js

@@ -0,0 +1,127 @@
+/**
+ * The migrator interface provides the ability to handle all abilities which
+ * are needed to successfully modify the DD of your tables.
+ * This includes all DDL methods provided by SQL naturally.
+ *
+ * Considering the zero downtime deployment we are handling migrations like
+ * the following:
+ *
+ * Zero Downtime notes
+ *
+ *  *) The very first step is always a full dump, for the sake of security
+ *
+ * - all operations that are deleting a currently existing column, e.g. renaming a column:
+ *      - 1. duplicate the column
+ *   - if renamed:
+ *      -  1. create a trigger to update both columns
+ *      -  2. disable/delete trigger after successful deployment
+ *   - if deleted:
+ *      - 1. apply steps from the renaming process
+ *      - 2. create a backup of this column (+PK)/table and delete it afterwards (after the successful deployment)
+ * - all operations that modify content
+ *   - 1. Retrieve data to be applied
+ *   - 2. Extract PKs of to be updated rows
+ *   - 3. Create backup of to be updated rows
+ *   - 4. Apply changes
+ */
+
+function dummy () {
+  arguments[arguments.length - 1]('not implemented');
+}
+
+function MigratorInterface () {}
+
+MigratorInterface.prototype = {
+  renameCollection: dummy,
+
+  dropCollection: dummy,
+
+  createCollection: dummy,
+
+  checkDBMS: dummy,
+
+  createDatabase: dummy,
+
+  switchDatabase: dummy,
+
+  dropDatabase: dummy,
+
+  bindForeignKey: dummy,
+
+  createTable: dummy,
+
+  dropTable: dummy,
+
+  renameTable: dummy,
+
+  addColumn: dummy,
+
+  removeColumn: dummy,
+
+  renameColumn: dummy,
+
+  changeColumn: dummy,
+
+  addIndex: dummy,
+
+  removeIndex: dummy,
+
+  addForeignKey: dummy,
+
+  removeForeignKey: dummy,
+
+  createMigrationsTable: dummy,
+
+  createSeedsTable: dummy,
+
+  normalizeColumnSpec: dummy,
+
+  runSql: dummy,
+
+  createColumnDef: dummy,
+
+  mapDataType: dummy,
+
+  createColumnConstraint: dummy,
+
+  recurseCallbackArray: dummy,
+
+  allLoadedMigrations: dummy,
+
+  allLoadedSeeds: dummy,
+
+  addSeedRecord: dummy,
+
+  startMigration: dummy,
+
+  endMigration: dummy,
+
+  addMigrationRecord: dummy,
+
+  deleteMigration: dummy,
+
+  deleteSeed: dummy,
+
+  quoteDDLArr: dummy,
+
+  quoteArr: dummy,
+
+  escapeString: dummy,
+
+  escape: dummy,
+
+  escapeDDL: dummy,
+
+  all: dummy,
+
+  close: dummy,
+
+  insert: dummy
+};
+
+module.exports = MigratorInterface;
+module.exports.deprecated = {};
+
+module.exports.extending = {};
+
+module.exports.exclude = {};

package/lib/interface/seederInterface.js

@@ -0,0 +1,145 @@
+/**
+  * The seeder interface provides the ability to handle all operations, that
+  * are not DDL specific and thus not a migration.
+  *
+  * These operations are currently, but not limited to:
+  *
+  * inserting data
+  * removing data
+  * searching data
+  * truncating whole tables
+  *
+  * This functionality is provided in two ways to the user. First there are
+  * traditional seeder. You can call them whenever you want, and how often
+  * you want. This might be handy when initializing a new development
+  * environment, but not if you want to use it for the production environment.
+  *
+  * The second seeder type, the version controlled seeders, short VC Seeds,
+  * is targeting the production environment and development environment.
+  * There is technically no big difference between them, except the following
+  * details:
+  *
+  * A VC Seed can be called from a migration, via seed.execute and seed.link. A
+  * normal seeder can not. Also A VC Seeder has a down and up function, like
+  * the way the migrations work, the static has a truncate function instead,
+  * which gets called before the seed function. This is because a seeder is
+  * mostly used to initialize something, a VC Seeder instead is used to
+  * populate changes made to the dataset and to have a clear separation of
+  * DDL and Data Maniupulations.
+  * And last but not least, a VC Seed can not be executed if it was already
+  * executed. You will need to roll back it first, it just acts pretty similar
+  * to migrations. A normal seed can be executed just as often as you want
+  * without the need to rollback your data.
+  *
+  * To note: If you rollback a migration, linked to a seeder, db-migrate will
+  * also rollback the seed. This is also a reason why you can't rollback a
+  * specific migration, you would going to break that much, you probably loose
+  * a bunch of valueable time.
+  */
+
+var Promise = require('bluebird');
+
+function insert (table, options, callback) {
+  var foreignLinkage = this.foreignLinkage[table];
+
+  for (var o = 0; o < options.length; ++o) {
+    var option = Object.keys(options[o]);
+
+    for (var i = 0; i < option.length; ++i) {
+      if (typeof options[option[i]] === 'object') {
+        if (options[option[i]].type === 'lookup') {
+          if (!options[option[i]].table) {
+            if (foreignLinkage[option[i]]) {
+              options[option[i]].table = foreignLinkage[option[i]].table;
+              options[option[i]].field = foreignLinkage[option[i]].field;
+            } else {
+              return Promise.reject(
+                new Error('missing foreign key linkage!')
+              ).nodeify(callback);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return lookup(options)
+    .then(function () {
+      return this.driver.insert(options);
+    })
+    .catch(function (e) {
+      throw e;
+    })
+    .nodeify(callback);
+}
+
+function lookup (options) {
+  var lookups = [];
+  var i = 0;
+
+  for (var o = 0; o < options.length; ++o) {
+    var option = Object.keys(options[o]);
+
+    for (; i < option.length; ++i) {
+      if (typeof options[option] === 'object') {
+        if (options[option].type === 'lookup') {
+          lookups.push(
+            this.driver.lookup(options[option]).catch(function (err) {
+              throw err;
+            })
+          );
+        }
+      }
+    }
+  }
+
+  return Promise.settle(lookups);
+}
+
+function dummy () {
+  arguments[arguments.length - 1]('not implemented');
+}
+
+var SeederInterface = {
+  lookup: dummy,
+
+  insert: dummy,
+
+  delete: dummy,
+
+  runSql: dummy,
+
+  buildWhereClause: dummy,
+
+  quoteDDLArr: dummy,
+
+  quoteArr: dummy,
+
+  escapeString: dummy,
+
+  escape: dummy,
+
+  escapeDDL: dummy,
+
+  checkDBMS: dummy,
+
+  update: dummy,
+
+  truncate: dummy,
+
+  switchDatabase: dummy,
+
+  remove: dummy,
+
+  close: dummy
+};
+
+module.exports = SeederInterface;
+module.exports.deprecated = {};
+module.exports.extending = {
+  _l: function (field) {
+    return { type: 'lookup', field: field };
+  },
+  __Test__insert: insert,
+  __Test__lookup: lookup
+};