db-migrate 1.0.0-beta.3 → 1.0.0-beta.31

package/lib/driver/index.js

@@ -18,6 +18,8 @@ var ShadowProto = {
 
 exports.connect = function (config, intern, callback) {
   var driver, req;
+  let plugin = false;
+  const { plugins } = intern;
 
   var mod = internals.mod;
   internals = intern;
@@ -90,27 +92,51 @@ exports.connect = function (config, intern, callback) {
   log.verbose('connecting');
 
   var connect = function (config) {
-    driver.connect(config, intern, function (err, db) {
-      if (err) {
-        callback(err);
-        return;
-      }
-      log.verbose('connected');
+    driver.connect(
+      // safe disconnect of our config object into userspace
+      JSON.parse(JSON.stringify(config)),
+      intern,
+      function (err, db) {
+        if (err) {
+          callback(err);
+          return;
+        }
+        log.verbose('connected');
 
-      if (!global.immunity) {
-        db = Shadow.infect(db, internals, ShadowProto);
-      }
+        if (!global.immunity) {
+          db = Shadow.infect(db, internals, ShadowProto);
+        }
 
-      callback(null, db);
-    });
+        callback(null, db);
+      }
+    );
   };
 
   if (config.tunnel) {
     var tunnel = require('tunnel-ssh');
     var tunnelConfig = JSON.parse(JSON.stringify(config.tunnel));
+    const { tunnelType } = tunnelConfig;
+
+    if (plugins) {
+      plugin = plugins.overwrite(
+        `connection:tunnel:${
+          tunnelType && tunnelType !== 'ssh' ? tunnelType : 'ssh'
+        }`
+      );
+    }
+
     tunnelConfig.dstHost = config.host;
     tunnelConfig.dstPort = config.port;
 
+    if (plugin) {
+      tunnel =
+        plugin[
+          `connection:tunnel:${
+            tunnelType && tunnelType !== 'ssh' ? tunnelType : 'ssh'
+          }`
+        ](tunnelConfig);
+    }
+
     if (tunnelConfig.privateKeyPath) {
       tunnelConfig.privateKey = require('fs').readFileSync(
         tunnelConfig.privateKeyPath

package/lib/executors/versioned/v2.js

@@ -9,6 +9,7 @@ const StateTravel = require('../../methods/v2/statetravel');
 const Migrate = require('../../methods/v2/migrate');
 const TranslateState = require('../../methods/v2/translatestate');
 const AddConventions = require('../../methods/v2/conventions');
+const util = require('util');
 
 const execUnit = {
   _extend: (context, type) => {
@@ -28,7 +29,7 @@ const execUnit = {
 
   up: async function (context, driver, file) {
     const _file = file.get();
-    const chain = new Chain(context._driver, file, driver, context.internals);
+    const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
       chain.addChain(AddConventions);
     }
@@ -36,7 +37,7 @@ const execUnit = {
     chain.addChain(StateTravel);
     chain.addChain(Migrate);
 
-    await State.startMigration(context._driver, file, context.internals);
+    await State.startMigration(context._pdriver, file, context.internals);
     // startMigration - needs secondary instance since we can not afford to
     // loose state and the transaction start will include these for roll back
     // we will disable them probably at all from DDL when the driver does not
@@ -48,6 +49,11 @@ const execUnit = {
         dbm: context.internals.safeOptions.dbmigrate
       });
     } catch (err) {
+      context.internals.rollback = true;
+
+      // transfer last state
+      chain.transferInt();
+
       log.error(
         'An error occured. No alternative failure strategy defined. Rolling back!',
         err
@@ -56,15 +62,53 @@ const execUnit = {
       throw err;
     }
     await Promise.promisify(context.writeMigrationRecord.bind(context))(file);
-    return State.endMigration(context._driver, file, context.internals);
+    return State.endMigration(context._pdriver, file, context.internals);
+    // end migration, same as start migration
+  },
+
+  fix: async function (context, driver, file) {
+    const _file = file.get();
+    const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
+    if (!_file._meta.noDefaultColumn) {
+      chain.addChain(AddConventions);
+    }
+    chain.addChain(Learn);
+    chain.addChain(StateTravel);
+
+    await State.startMigration(context._pdriver, file, context.internals);
+    // startMigration - needs secondary instance since we can not afford to
+    // loose state and the transaction start will include these for roll back
+    // we will disable them probably at all from DDL when the driver does not
+    // explicitly signal DDL transaction support (like crdb)
+    try {
+      await _file.migrate(chain, {
+        options: context.internals.safeOptions,
+        seedLink: context.seedLink,
+        dbm: context.internals.safeOptions.dbmigrate
+      });
+    } catch (err) {
+      context.internals.rollback = true;
+
+      // transfer last state
+      chain.transferInt();
+
+      log.error(
+        'An error occured. No alternative failure strategy defined. Rolling back!',
+        err
+      );
+      await execUnit.down(context, driver, file);
+      throw err;
+    }
+    await State.endMigration(context._pdriver, file, context.internals);
+    log.verbose(`[fix] current schema`, util.inspect(context.internals.schema, false, null, true));
     // end migration, same as start migration
   },
 
   down: async function (context, driver, file) {
     // start migration, see up comments
-    await State.startMigration(context._driver, file, context.internals);
-    await TranslateState(context._driver, file, driver, context.internals);
-    await State.endMigration(context._driver, file, context.internals);
+    await State.startMigration(context._pdriver, file, context.internals);
+    await TranslateState(context._driver, file, driver, context.internals, context._pdriver);
+    await State.endMigration(context._pdriver, file, context.internals);
     return Promise.promisify(context.deleteMigrationRecord.bind(context))(file);
     // end migration, see up comments
   }

package/lib/learn.js

@@ -2,25 +2,34 @@ const Promise = require('bluebird');
 const Shadow = require('./driver/shadow');
 
 class STD {
-  constructor ({ schema, modSchema: mod }, driver) {
+  constructor ({ schema, modSchema: mod, unlearn }, driver) {
     this.checkColumn = function (t, c) {
       if (!this.schema[t]) {
         throw new Error(`There is no ${t} table in schema!`);
       }
 
-      if (!this.schema[t][c]) {
+      if (!this.schema[t][c] && !this.schema[t].columns?.[c]) {
         throw new Error(`There is no ${c} column in schema!`);
       }
     };
+    this.unlearn = unlearn;
     this.validations = {};
     this.driver = driver;
     this.indizies = schema.i;
     this.schema = schema.c;
     this.foreign = schema.f;
+    if (!schema.e) {
+      schema.e = {};
+    }
+    this.extra = schema.e;
+    if (unlearn) {
+      this.modC = [];
+    } else {
+      this.modC = mod.s;
+    }
     this.modS = mod.c;
     this.modI = mod.i;
     this.modF = mod.f;
-    this.modC = mod.s;
   }
 
   dropTable (t) {
@@ -87,6 +96,16 @@ class STD {
       delete this.schema[t];
     }
 
+    if (this.foreign[t]) {
+      this.modF[n] = this.foreign[n];
+      delete this.foreign[t];
+    }
+
+    if (this.indizies[t]) {
+      this.modI[n] = this.indizies[t];
+      delete this.indizies[t];
+    }
+
     this.modC.push({ t: 0, a: 'renameTable', c: [n, t] });
 
     return Promise.resolve();
@@ -107,8 +126,10 @@ class STD {
   removeColumn (t, c, o = {}) {
     const alter = {};
     this.checkColumn(t, c);
+    const hasColumns = this.schema[t].columns !== undefined;
+    const columns = hasColumns ? this.schema[t].columns : this.schema[t];
 
-    if (this.schema[t][c].notNull === true) {
+    if (columns[c].notNull === true && !this.unlearn) {
       if (this.validations.columnStrategies !== true) {
         if (
           this.driver._meta &&
@@ -156,10 +177,10 @@ class STD {
 
     this.modS[t] = {};
 
-    if (this.schema[t][c].notNull === true) {
+    if (columns[c].notNull === true && !this.unlearn) {
       switch (o.columnStrategy) {
         case 'delay':
-          this.modS[t][c] = this.schema[t][c];
+          this.modS[t][c] = columns[c];
 
           o.passthrough = o.passthrough || {};
           o.passthrough.column =
@@ -174,19 +195,23 @@ class STD {
 
           break;
         case 'defaultValue':
-          this.modS[t][c] = this.schema[t][c];
+          this.modS[t][c] = columns[c];
           this.modS[t][c].defaultValue = o.passthrough.defaultValue;
 
           this.modC.push({ t: 1, a: 'addColumn', c: [t, c, o] });
           break;
       }
-    } else {
-      this.modS[t][c] = this.schema[t][c];
+    } else if (!this.unlearn) {
+      this.modS[t][c] = columns[c];
 
       this.modC.push({ t: 1, a: 'addColumn', c: [t, c, o] });
     }
 
-    delete this.schema[t][c];
+    if (hasColumns) {
+      delete this.schema[t].columns[c];
+    } else {
+      delete this.schema[t][c];
+    }
 
     return Promise.resolve(alter);
   }
@@ -207,7 +232,10 @@ class STD {
       throw new Error(`There is no ${t} table in schema!`);
     }
     this.schema[t] = this.schema[t] || {};
-    this.schema[t][c] = s;
+    const hasColumns = this.schema[t].columns !== undefined;
+    const columns = hasColumns ? this.schema[t].columns : this.schema[t];
+
+    columns[c] = s;
 
     this.modC.push({ t: 0, a: 'removeColumn', c: [t, c] });
 
@@ -217,8 +245,13 @@ class STD {
   changeColumn (t, c, s) {
     this.checkColumn(t, c);
 
-    this.modS[t][c] = this.schema[t][c];
-    this.schema[t][c] = Object.assign(this.schema[t][c], s);
+    if (!this.modS[t]) this.modS[t] = {};
+
+    const hasColumns = this.schema[t].columns !== undefined;
+    const columns = hasColumns ? this.schema[t].columns : this.schema[t];
+
+    this.modS[t][c] = columns[c];
+    columns[c] = Object.assign(columns[c], s);
 
     this.modC.push({ t: 1, a: 'changeColumn', c: [t, c] });
 
@@ -227,7 +260,7 @@ class STD {
 
   addIndex (t, i, c, u) {
     if (Array.isArray(c)) {
-      c.forEach(x => this.checkColumn(t, x));
+      c.forEach(x => this.checkColumn(t, typeof x  === 'object' ? x.name : x));
     } else {
       this.checkColumn(t, c);
     }

package/lib/methods/v2/statetravel.js

@@ -18,25 +18,28 @@ const DEFAULT = {
 };
 
 class StateTravel {
-  constructor (internals, file, driver) {
+  constructor (internals, file, driver, pdriver) {
     this.file = file;
     this.internals = internals;
     this.driver = driver;
+    this.pdriver = pdriver;
     this._counter = 0;
+    this._default = async function _default () {
+      await State.step(this.pdriver, ++this._counter, this.internals);
+
+      return State.update(
+        this.pdriver,
+        this.file,
+        this.internals.modSchema,
+        this.internals
+      );
+    };
   }
 }
 
 Object.keys(DEFAULT).forEach(m => {
   StateTravel.prototype[m] = async function (...args) {
-    await State.step(this.driver, ++this._counter, this.internals);
-    const res = await State.update(
-      this.driver,
-      this.file,
-      this.internals.modSchema,
-      this.internals
-    );
-
-    return res;
+    return this._default();
   };
 });
 
@@ -47,9 +50,9 @@ const noTravelError = prop => {
 };
 
 module.exports = {
-  getInterface: (context, file, driver, internals) => {
+  getInterface: (context, file, driver, internals, pdriver) => {
     if (context.learnable) {
-      const st = new StateTravel(internals, file, context);
+      const st = new StateTravel(internals, file, context, pdriver);
       return Shadow.overshadow(
         driver,
         Object.assign(st, context.statechanger),
@@ -59,7 +62,7 @@ module.exports = {
 
     return Shadow.overshadow(
       driver,
-      new StateTravel(internals, file, context),
+      new StateTravel(internals, file, context, pdriver),
       noTravelError
     );
   }

package/lib/methods/v2/translatestate.js

@@ -3,6 +3,7 @@ const State = require('../../state');
 const Chain = require('../../chain');
 const Migrate = require('./migrate');
 const Learn = require('../../learn');
+const log = require('db-migrate-shared').log;
 
 const methods = {
   createTable: async (driver, [t], internals) => {
@@ -43,6 +44,26 @@ const methods = {
 
   changeColumn: async (driver, [t, c], internals) => {
     return driver.changeColumn(t, c, internals.modSchema.c[t][c]);
+  },
+
+  addIndex: async (driver, [t, i], internals) => {
+    if (!internals.modSchema.i[t][i]) {
+      log.warn(
+        `There was an attempt to create the index "${i}" in the table "${t}".`
+      );
+      log.warn(
+        `This index did not exist at the time of deleting it. ` +
+          `This can happen if an index was created by hand and then deleted ` +
+          `by db-migrate. Make sure to recreate it by hand to avoid any problems.`
+      );
+      return Promise.resolve();
+    }
+    return driver.addIndex(
+      t,
+      i,
+      internals.modSchema.i[t][i].c,
+      internals.modSchema.i[t][i].u ? true : undefined
+    );
   }
 };
 
@@ -51,34 +72,50 @@ async function processEntry (
   file,
   driver,
   internals,
+  pdriver,
   { t: type, a: action, c: args }
 ) {
-  const f = Object.assign(methods, context.reverse);
-  switch (type) {
-    case 0:
-      await driver[action].apply(driver, args);
-      break;
-    case 1:
-      await f[action](driver, args, internals);
-      break;
-
-    default:
-      throw new Error(`Invalid state record, of type ${type}`);
+  let skip = false;
+
+  if (
+    internals.rollback === true &&
+    internals.rollbackContinue !== true &&
+    driver.udriver._meta?.signalColumns?.indexOf(action) !== -1
+  ) {
+    internals.rollbackContinue = true;
+    skip = driver.udriver._counter.previousSignal() === null;
+  }
+
+  if (!skip) {
+    const f = Object.assign(methods, context.reverse);
+    switch (type) {
+      case 0:
+        await driver[action].apply(driver, args);
+        break;
+      case 1:
+        await f[action](driver, args, internals);
+        break;
+
+      default:
+        throw new Error(`Invalid state record, of type ${type}`);
+    }
   }
 
   internals.modSchema.s.shift();
-  await State.update(context, file, internals.modSchema, internals);
+  await State.update(pdriver, file, internals.modSchema, internals);
 }
 
-module.exports = async (context, file, driver, internals) => {
+module.exports = async (context, file, driver, internals, pdriver) => {
   const mod = internals.modSchema;
 
-  const chain = new Chain(context, file, driver, internals);
+  const chain = new Chain(context, file, driver, internals, pdriver);
+  internals.unlearn = true;
+
   chain.addChain(Learn);
   chain.addChain(Migrate);
   await Promise.resolve(mod.s.reverse()).each(args =>
-    processEntry(context, file, chain, internals, args)
+    processEntry(context, file, chain, internals, pdriver, args)
   );
 
-  await State.deleteState(context, file, internals);
+  await State.deleteState(pdriver, file, internals);
 };

package/lib/state.js

@@ -1,10 +1,25 @@
 const MSTATE = '__dbmigrate_state__';
 const SSTATE = '__dbmigrate_schema__';
+const RUNON = 'run_on';
+
 const log = require('db-migrate-shared').log;
+const fs = require('fs').promises;
+const path = require('path');
+const crypto = require('crypto');
+
+const ID = crypto.randomBytes(32).toString('base64');
+let owner = false;
 
 module.exports = {
+  /**
+   * Lock state will add the basic state controller an ID
+   * and a process controlled date.
+   * It is important that all systems are in sync. We will use the
+   * database clock instead if possible.
+   */
   lockState: async function (driver, state, internals) {
     if (!state) {
+      owner = true;
       return driver._insertKV(
         internals.migrationState,
         MSTATE,
@@ -12,29 +27,72 @@ module.exports = {
           s: {
             step: 0,
             fin: 0,
+            ID,
             date: new Date()
           }
         })
       );
     } else {
+      let runOn = state.run_on;
       state = state.value;
       state = JSON.parse(state);
-      state.s.date = new Date();
-      return driver._updateKV(
+
+      if (!state.s.ID || state.s.ID !== ID) {
+        owner = true;
+        state.s.ID = ID;
+      }
+
+      const d = new Date();
+
+      state.s.date = d;
+      await driver._updateKVC(
         internals.migrationState,
         MSTATE,
-
-        JSON.stringify(state)
+        JSON.stringify(state),
+        RUNON,
+        runOn
       );
+
+      state = await driver._getKV(internals.migrationState, MSTATE);
+
+      runOn = state.run_on;
+      state = state.value;
+      state = JSON.parse(state);
+
+      if (state.s.ID !== ID) {
+        owner = false;
+      }
     }
   },
 
-  init: async function (driver, internals) {
+  isOwner: function () {
+    return owner;
+  },
+
+  init: async function (driver, internals, { emptyState, backupState }) {
     await driver._createKV(internals.migrationState);
-    const schema = await driver._getKV(internals.migrationState, SSTATE);
+    const _schema = await driver._getKV(internals.migrationState, SSTATE);
+
+    if (_schema && backupState) {
+      const newName = `${internals.migrationState}_b_${Math.floor((new Date() - 0) / 1000)}`;
+      log.info(`[state] Created a backup of ${internals.migrationState} by writing to file ${newName}.dbmigrate`);
+
+      await fs.writeFile(path.resolve(`${newName}.dbmigrate`), JSON.stringify(_schema), 'utf8');
+
+      if (emptyState) {
+        await driver.renameTable(internals.migrationState,
+          newName);
+
+        log.info(`[state] Created a backup of ${internals.migrationState} by renaming table to ${newName}`);
+        await driver._createKV(internals.migrationState);
+        await driver._insertKV(internals.migrationState, SSTATE, '{}');
+      }
+    }
+
+    const schema = emptyState !== true ? _schema : null;
     if (schema) {
       internals.schema = JSON.parse(schema.value);
-    } else {
+    } else if (!emptyState) {
       await driver._insertKV(internals.migrationState, SSTATE, '{}');
     }
 
@@ -50,7 +108,7 @@ module.exports = {
     const state = await driver._getKV(internals.migrationState, MSTATE);
     const mig = await driver._getKV(internals.migrationState, file.name);
 
-    if (mig) {
+    if (mig && mig.value !== '{}') {
       internals.modSchema = JSON.parse(mig.value);
     }
 
@@ -77,7 +135,7 @@ module.exports = {
       return Promise.resolve();
     }
 
-    await driver._updateKV(internals.migrationState, SSTATE, internals.schema);
+    await driver._updateKV(internals.migrationState, SSTATE, JSON.stringify(internals.schema));
 
     return driver._updateKV(
       internals.migrationState,
@@ -95,17 +153,40 @@ module.exports = {
       return Promise.resolve();
     }
 
-    let { value: state } = await driver._getKV(
+    let state = await driver._getKV(
       internals.migrationState,
       MSTATE
     );
+
+    let runOn = state.run_on;
+    state = state.value;
     state = JSON.parse(state);
-    state.s.date = new Date();
-    return driver._updateKV(
+
+    if (!state.s.ID || state.s.ID !== ID) {
+      owner = true;
+      state.s.ID = ID;
+    }
+
+    const d = new Date();
+
+    state.s.date = d;
+    await driver._updateKVC(
       internals.migrationState,
       MSTATE,
-      JSON.stringify(state)
+      JSON.stringify(state),
+      RUNON,
+      runOn
     );
+
+    state = await driver._getKV(internals.migrationState, MSTATE);
+
+    runOn = state.run_on;
+    state = state.value;
+    state = JSON.parse(state);
+
+    if (state.s.ID !== ID) {
+      owner = false;
+    }
   },
 
   step: async function (driver, step, internals) {
@@ -145,6 +226,9 @@ module.exports = {
     state.s.date = new Date();
     state.s.fin = 1;
 
+    // unlock from current process
+    state.s.ID = 0;
+
     return driver._updateKV(
       internals.migrationState,
       MSTATE,

package/lib/template.js

@@ -258,7 +258,8 @@ Template.TemplateType = {
   SQL_FILE_LOADER: 2,
   DEFAULT_COFFEE: 3,
   COFFEE_SQL_FILE_LOADER: 4,
-  SQL_FILE_LOADER_IGNORE_ON_INIT: 5
+  SQL_FILE_LOADER_IGNORE_ON_INIT: 5,
+  V2_DEFAULT: 6
 };
 
 Template.prototype.getTemplate = function () {
@@ -277,6 +278,8 @@ Template.prototype.getTemplate = function () {
       return this.sqlFileLoaderIgnoreOnInitTemplate();
     case Template.TemplateType.DEFAULT_JS:
       return this.defaultJsTemplate();
+    case Template.TemplateType.V2_DEFAULT:
+      return require('./templates/v2.js')();
     default:
       if (this.plugins) {
         plugin = this.plugins.overwrite(

package/lib/templates/v2.js

@@ -0,0 +1,12 @@
+'use strict';
+
+module.exports = () => `'use strict';
+
+exports.migrate = async (db, opt) => {
+  const type = opt.dbm.dataType;
+  return null;
+};
+
+exports._meta = {
+  version: 2
+};`;