db-migrate 1.2.0 → 1.4.0

package/lib/driver/index.js

@@ -166,6 +166,8 @@ exports.connect = function (config, intern, callback) {
         }
 
         trackSql(db);
+        // the dialect of parameters in all, $1 instead of ? for pg
+        db._dbmDriver = typeof config.driver === 'string' ? config.driver : null;
         callback(null, db);
       }
     );

package/lib/executors/versioned/v2.js

@@ -12,84 +12,8 @@ const StateDone = require('../../methods/v2/statedone');
 const TranslateState = require('../../methods/v2/translatestate');
 const AddConventions = require('../../methods/v2/conventions');
 const util = require('util');
-
-/**
- * How to recover from an interrupted run of a migration, set per migration
- * by _meta.recovery:
- *
- * skip: skip the steps already executed and continue with the rest
- * rollback: revert the steps already executed and run the migration again
- */
-const RECOVERY_MODES = ['skip', 'rollback'];
-const RECOVERY_DEFAULT = 'skip';
-
-const recoveryMode = (file, interrupted) => {
-  const name = file.name;
-  const mode = file.get()._meta.recovery || RECOVERY_DEFAULT;
-
-  if (RECOVERY_MODES.indexOf(mode) === -1) {
-    throw new Error(
-      `Invalid recovery mode "${mode}" in migration "${name}", use one of ` +
-        RECOVERY_MODES.join(', ')
-    );
-  }
-
-  if (interrupted.rollback) {
-    if (mode !== 'rollback') {
-      log.warn(
-        `[recovery] ${name}: the previous run was interrupted while rolling ` +
-          'back, continuing the rollback'
-      );
-    }
-
-    return 'rollback';
-  }
-
-  if (mode === 'skip' && interrupted.changed) {
-    throw new Error(
-      `Migration "${name}" was interrupted at step ${interrupted.step} and ` +
-        'changed since, so the executed steps can not be skipped safely. ' +
-        "Set _meta.recovery to 'rollback' to revert them, or repair the " +
-        'state manually.'
-    );
-  }
-
-  return mode;
-};
-
-/**
- * Revert the steps executed by an interrupted run. A step started but not
- * done did not change the database, so its reverse operation is dropped.
- */
-/**
- * Keep the reverse operations of the steps executed on the database. The
- * failed step only changed the database, if its main statement went through
- * before failing, signaled by the driver (e.g. a foreign key of createTable
- * failed after the table was created).
- */
-const executedSteps = (internals, failed, signaled) => {
-  const done = internals.migrationDone;
-
-  internals.modSchema.s = internals.modSchema.s.filter(
-    entry =>
-      entry.n <= done || (signaled && failed > done && entry.n === failed)
-  );
-};
-
-/**
- * Name the instruction a migration failed at, or after if the instruction
- * itself finished and the migration code failed afterwards.
- */
-const describeFailure = (err, chain, internals) => {
-  if (!err || typeof err !== 'object' || err.instruction || chain.op === 0) {
-    return;
-  }
-
-  const after = internals.migrationDone >= chain.op;
-  err.instruction = `${after ? 'after' : 'at'} step ${chain.op} ${
-    chain.current
-  }`;
-};
+const { recoveryMode, executedSteps, describeFailure } = require('../../recovery');
+const Dml = require('../../dml');
 
 const recoverByRollback = async (context, driver, file, interrupted) => {
   const internals = context.internals;
@@ -125,6 +49,11 @@ const execUnit = {
   },
 
   up: async function (context, driver, file) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      return Dml.up(context, driver, file);
+    }
+
     const _file = file.get();
     const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
@@ -205,6 +134,12 @@ const execUnit = {
   },
 
   fix: async function (context, driver, file) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      log.info(`[fix] skipping "${file.name}", dml migrations change no schema`);
+      return;
+    }
+
     const _file = file.get();
     const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
@@ -247,6 +182,11 @@ const execUnit = {
   },
 
   down: async function (context, driver, file, { abort } = {}) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      return Dml.down(context, driver, file, { abort });
+    }
+
     // if we get the abort signal this means we are in a rollback routine
     // which means in turn we want to avoid reloading the state
     if (abort !== true) {

package/lib/jobs.js

@@ -0,0 +1,177 @@
+'use strict';
+
+const crypto = require('crypto');
+const log = require('db-migrate-shared').log;
+
+/**
+ * Background migrations, dml migrations with _meta.background, are not run
+ * by up, but registered as jobs. Workers started by executeWork take the
+ * jobs one by one and run them. A migration counts as run once its job is
+ * done, until then it is running.
+ *
+ * All jobs live in one row of the state table, changed by compare and swap
+ * only, so any number of workers can work on them at once:
+ *
+ * s: queued, running or failed
+ * blocking: later jobs wait until this one is done
+ * ID: the worker running it, n: renewed by every write of the worker, a
+ *     job whose n does not change for the timeout is taken over
+ * step, learned, done, rb: the progress of the run, like for migrations run
+ *     in the foreground, see State
+ * h: hash of the migration file when it started
+ * err: why it failed
+ */
+
+const JOBS = '__dbmigrate_jobs__';
+
+const nonce = () => crypto.randomBytes(8).toString('hex');
+
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const read = async (pdriver, internals) => {
+  const row = await pdriver._getKV(internals.migrationState, JOBS);
+  return { row, jobs: row ? JSON.parse(row.value).jobs : {} };
+};
+
+const list = async (pdriver, internals) => (await read(pdriver, internals)).jobs;
+
+/**
+ * Changes the jobs by mutate, retried until no other worker wrote in
+ * between. mutate returns false to leave them as they are.
+ */
+const change = async (pdriver, internals, mutate) => {
+  for (let attempt = 0; ; attempt++) {
+    const { row, jobs } = await read(pdriver, internals);
+
+    if (mutate(jobs) === false) {
+      return jobs;
+    }
+
+    const value = JSON.stringify({ jobs, n: nonce() });
+
+    if (!row) {
+      try {
+        await pdriver._insertKV(internals.migrationState, JOBS, value);
+      } catch (err) {
+        // inserted by another worker first, try again with theirs
+        log.verbose('[jobs] insert failed', err.message);
+      }
+    } else {
+      await pdriver._updateKVC(
+        internals.migrationState,
+        JOBS,
+        value,
+        'value',
+        row.value
+      );
+    }
+
+    const after = await pdriver._getKV(internals.migrationState, JOBS);
+    if (after && after.value === value) {
+      return jobs;
+    }
+
+    if (attempt > 50) {
+      throw new Error('[jobs] could not change the jobs, too much contention');
+    }
+
+    await delay(Math.floor(Math.random() * 20 * Math.min(attempt + 1, 10)));
+  }
+};
+
+/**
+ * Registers a migration as job, or a failed one again. A job which failed
+ * with irreversible steps keeps its progress, so it continues after them.
+ */
+const register = async (pdriver, internals, file) => {
+  const blocking = (file.get()._meta || {}).blocking === true;
+
+  await change(pdriver, internals, jobs => {
+    const job = jobs[file.name];
+
+    if (job && job.s !== 'failed') {
+      return false;
+    }
+
+    jobs[file.name] = Object.assign(
+      { step: 0, learned: 0, done: 0, rb: 0 },
+      job,
+      { s: 'queued', blocking, ID: 0, n: nonce(), err: undefined }
+    );
+  });
+
+  const mig = await pdriver._getKV(internals.migrationState, file.name);
+  if (!mig) {
+    await pdriver._insertKV(internals.migrationState, file.name, '{}');
+  }
+
+  log.info(`[jobs] ${file.name} runs in the background`);
+};
+
+/**
+ * Takes the first job to run, in the order of their names. A running job is
+ * taken over if it did not change for timeout ms, measured on our clock in
+ * seen. Jobs after a blocking job wait until it is done.
+ *
+ * Resolves to the name of the job taken, or null.
+ */
+const claim = async (pdriver, internals, id, seen, timeout) => {
+  const now = Date.now();
+  let taken = null;
+
+  await change(pdriver, internals, jobs => {
+    taken = null;
+
+    for (const name of Object.keys(jobs).sort()) {
+      const job = jobs[name];
+      let free = job.s === 'queued';
+
+      if (job.s === 'running' && job.ID !== id) {
+        const last = seen[name];
+
+        if (!last || last.n !== job.n) {
+          seen[name] = { n: job.n, since: now };
+        } else if (now - last.since >= timeout) {
+          log.warn(
+            `[jobs] taking over ${name}, its worker did not respond for ` +
+              `${timeout} ms`
+          );
+          free = true;
+        }
+      }
+
+      if (free) {
+        Object.assign(job, { s: 'running', ID: id, n: nonce() });
+        taken = name;
+        return;
+      }
+
+      if (job.blocking) {
+        return false;
+      }
+    }
+
+    return false;
+  });
+
+  return taken;
+};
+
+/**
+ * Changes a job we run. If another worker took it over in between, lost is
+ * called and nothing is changed.
+ */
+const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
+  change(pdriver, internals, jobs => {
+    const job = jobs[name];
+
+    if (!job || job.ID !== id) {
+      lost();
+      return false;
+    }
+
+    mutate(job, jobs);
+    job.n = nonce();
+  });
+
+module.exports = { JOBS, list, register, claim, update };

package/lib/recovery.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const log = require('db-migrate-shared').log;
+
+/**
+ * How to recover from an interrupted run of a migration, set per migration
+ * by _meta.recovery:
+ *
+ * skip: skip the steps already executed and continue with the rest
+ * rollback: revert the steps already executed and run the migration again
+ */
+const RECOVERY_MODES = ['skip', 'rollback'];
+const RECOVERY_DEFAULT = 'skip';
+
+const recoveryMode = (file, interrupted) => {
+  const name = file.name;
+  const mode = file.get()._meta.recovery || RECOVERY_DEFAULT;
+
+  if (RECOVERY_MODES.indexOf(mode) === -1) {
+    throw new Error(
+      `Invalid recovery mode "${mode}" in migration "${name}", use one of ` +
+        RECOVERY_MODES.join(', ')
+    );
+  }
+
+  if (interrupted.rollback) {
+    if (mode !== 'rollback') {
+      log.warn(
+        `[recovery] ${name}: the previous run was interrupted while rolling ` +
+          'back, continuing the rollback'
+      );
+    }
+
+    return 'rollback';
+  }
+
+  if (mode === 'skip' && interrupted.changed) {
+    throw new Error(
+      `Migration "${name}" was interrupted at step ${interrupted.step} and ` +
+        'changed since, so the executed steps can not be skipped safely. ' +
+        "Set _meta.recovery to 'rollback' to revert them, or repair the " +
+        'state manually.'
+    );
+  }
+
+  return mode;
+};
+
+/**
+ * Revert the steps executed by an interrupted run. A step started but not
+ * done did not change the database, so its reverse operation is dropped.
+ */
+/**
+ * Keep the reverse operations of the steps executed on the database. The
+ * failed step only changed the database, if its main statement went through
+ * before failing, signaled by the driver (e.g. a foreign key of createTable
+ * failed after the table was created).
+ */
+const executedSteps = (internals, failed, signaled) => {
+  const done = internals.migrationDone;
+
+  internals.modSchema.s = internals.modSchema.s.filter(
+    entry =>
+      entry.n <= done || (signaled && failed > done && entry.n === failed)
+  );
+};
+
+/**
+ * Name the instruction a migration failed at, or after if the instruction
+ * itself finished and the migration code failed afterwards.
+ */
+const describeFailure = (err, chain, internals) => {
+  if (!err || typeof err !== 'object' || err.instruction || chain.op === 0) {
+    return;
+  }
+
+  const after = internals.migrationDone >= chain.op;
+  err.instruction = `${after ? 'after' : 'at'} step ${chain.op} ${
+    chain.current
+  }`;
+};
+
+module.exports = { RECOVERY_MODES, recoveryMode, executedSteps, describeFailure };

package/lib/seed.js

@@ -0,0 +1,199 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const Promise = require('bluebird');
+const log = require('db-migrate-shared').log;
+const { FLAG, insertRows } = require('./dml');
+
+/**
+ * Static seeds insert data for development and tests, seeds/<name>.js:
+ *
+ *   exports.seed = async db => {
+ *     await db.insert('pets', [{ name: 'Rex' }, { name: 'Tom' }]);
+ *   };
+ *
+ * The rows are flagged with seed:<name> in their __dbmigrate__flag column.
+ * Seeding again removes the rows of the previous run first, so seeds can be
+ * changed and run any time. The tables each seed inserted into are kept in
+ * the state, under SEEDS, in the order the seeds ran.
+ */
+
+const SEEDS = '__dbmigrate_seeds__';
+
+const flag = name => `seed:${name}`;
+
+const loadState = async (pdriver, internals) => {
+  const row = await pdriver._getKV(internals.migrationState, SEEDS);
+  return { exists: !!row, seeds: row ? JSON.parse(row.value) : {} };
+};
+
+const saveState = (pdriver, internals, state) => {
+  if (internals.dryRun) {
+    return Promise.resolve();
+  }
+
+  const value = JSON.stringify(state.seeds);
+
+  if (state.exists) {
+    return pdriver._updateKV(internals.migrationState, SEEDS, value);
+  }
+
+  state.exists = true;
+  return pdriver._insertKV(internals.migrationState, SEEDS, value);
+};
+
+const columns = (internals, table) => {
+  const t = internals.schema.c[table];
+  return t ? t.columns || t : null;
+};
+
+class SeedDb {
+  constructor (driver, internals, name, touch) {
+    this.driver = driver;
+    this.internals = internals;
+    this.name = name;
+    this.touch = touch;
+  }
+
+  async insert (table, ...args) {
+    const { rows } = insertRows(args);
+    const known = columns(this.internals, table);
+
+    if (!known || !known[FLAG]) {
+      throw new Error(
+        `Seed "${this.name}" can not insert into "${table}", seeds insert ` +
+          `into tables created by v2 migrations only, by their ${FLAG} ` +
+          'column, so the rows can be removed again.'
+      );
+    }
+
+    if (!rows.length) {
+      return;
+    }
+
+    await this.touch(table);
+    return this.driver.insert(
+      table,
+      rows.map(row => Object.assign({}, row, { [FLAG]: flag(this.name) }))
+    );
+  }
+
+  all (...args) {
+    return this.driver.all(...args);
+  }
+}
+
+/**
+ * Removes the rows of a seed, the tables in reverse order, for rows
+ * referencing rows of the same seed. Tables dropped since are skipped.
+ */
+const remove = async (driver, internals, state, name) => {
+  const tables = (state.seeds[name] || []).slice().reverse();
+
+  await Promise.each(tables, table => {
+    if (!columns(internals, table)) {
+      return;
+    }
+
+    return driver.runSql(
+      `DELETE FROM ${driver.escapeDDL(table)} WHERE ${driver.escapeDDL(FLAG)} = ?`,
+      [flag(name)]
+    );
+  });
+
+  delete state.seeds[name];
+};
+
+const files = dir => {
+  if (!fs.existsSync(dir)) {
+    return [];
+  }
+
+  return fs
+    .readdirSync(dir)
+    .filter(file => /\.js$/.test(file))
+    .sort()
+    .map(file => ({
+      name: file.replace(/\.js$/, ''),
+      path: path.join(dir, file)
+    }));
+};
+
+const select = (list, name, what) => {
+  if (!name) {
+    return list;
+  }
+
+  const found = list.filter(seed => seed.name === name);
+  if (!found.length) {
+    throw new Error(`There is no ${what} "${name}"`);
+  }
+
+  return found;
+};
+
+/**
+ * Seeds all seeds of the directory, or the one named. The rows of the
+ * seeds run before are removed first, of all seeds the last run first,
+ * including seeds whose files were removed since.
+ */
+const run = async (walker, name) => {
+  const internals = walker.internals;
+  const driver = walker._driver;
+  const pdriver = walker._pdriver;
+  const dir = path.resolve(internals.argv['seeds-dir']);
+  const seeds = select(files(dir), name, 'seed');
+  const state = await loadState(pdriver, internals);
+
+  const previous = name ? [name] : Object.keys(state.seeds).reverse();
+  await Promise.each(previous, seed => remove(driver, internals, state, seed));
+  await saveState(pdriver, internals, state);
+
+  await Promise.each(seeds, async seed => {
+    log.info(`[seed] ${seed.name}`);
+    delete require.cache[require.resolve(seed.path)];
+    const mod = require(seed.path);
+
+    if (typeof mod.seed !== 'function') {
+      throw new Error(`The seed "${seed.name}" exports no seed function`);
+    }
+
+    state.seeds[seed.name] = [];
+    const touch = async table => {
+      const tables = state.seeds[seed.name];
+      if (tables.indexOf(table) === -1) {
+        // recorded before inserting, so a failing seed is removed as well
+        tables.push(table);
+        await saveState(pdriver, internals, state);
+      }
+    };
+
+    await mod.seed(new SeedDb(driver, internals, seed.name, touch), {
+      options: internals.safeOptions,
+      dbm: internals.safeOptions.dbmigrate
+    });
+    await saveState(pdriver, internals, state);
+  });
+};
+
+/**
+ * Removes the rows of all seeds, or of the one named.
+ */
+const undo = async (walker, name) => {
+  const internals = walker.internals;
+  const state = await loadState(walker._pdriver, internals);
+  const seeds = name
+    ? select(Object.keys(state.seeds).map(n => ({ name: n })), name, 'seeded seed')
+    : Object.keys(state.seeds)
+      .reverse()
+      .map(n => ({ name: n }));
+
+  await Promise.each(seeds, seed => {
+    log.info(`[seed] removing ${seed.name}`);
+    return remove(walker._driver, internals, state, seed.name);
+  });
+  await saveState(walker._pdriver, internals, state);
+};
+
+module.exports = { run, undo, SEEDS };

package/lib/walker.js

@@ -5,6 +5,8 @@ const State = require('./state');
 const log = require('db-migrate-shared').log;
 const Promise = require('bluebird');
 const File = require('./file.js');
+const Dml = require('./dml');
+const Jobs = require('./jobs');
 
 const INTERFACES = {
   migration: require('./interface/migratorInterface.js'),
@@ -133,6 +135,46 @@ Walker.prototype = {
     }
   },
 
+  /**
+   * Migrations running in the background are left out, until their job is
+   * done. Failed ones are run, which registers them again.
+   */
+  _withoutJobs: async function (toRun) {
+    if (this.prefix !== 'migration' || !this._pdriver || typeof this._pdriver._getKV !== 'function') {
+      return toRun;
+    }
+
+    const jobs = await Jobs.list(this._pdriver, this.internals);
+
+    return toRun.filter(file => {
+      const job = jobs[file.name];
+
+      if (job && job.s !== 'failed') {
+        log.info(`${this.title}${file.name} is running in the background`);
+        return false;
+      }
+
+      return true;
+    });
+  },
+
+  _registerJob: async function (file) {
+    Dml.checkType(file);
+
+    if (this.internals.dryRun) {
+      log.info(`${this.title}${file.name} would run in the background`);
+      return;
+    }
+
+    if (!State.supportsLock(this._pdriver)) {
+      throw new Error(
+        'Background migrations need a driver supporting the migration lock'
+      );
+    }
+
+    return Jobs.register(this._pdriver, this.internals, file);
+  },
+
   writeMigrationRecord: function (migration, callback) {
     const onComplete = err => {
       if (err) {
@@ -201,9 +243,11 @@ Walker.prototype = {
           this._driver,
           this.internals
         )
-      ]).spread((allFiles, completedFiles) =>
-        dbmUtil.filterUp(allFiles, completedFiles, partialName, count)
-      );
+      ])
+        .spread((allFiles, completedFiles) =>
+          dbmUtil.filterUp(allFiles, completedFiles, partialName, count)
+        )
+        .then(toRun => this._withoutJobs(toRun));
 
     if (this.internals.check) {
       return determine()
@@ -219,6 +263,10 @@ Walker.prototype = {
 
         return Promise.each(toRun, file => {
           log.verbose(this.title + 'preparing to run up:', file.name);
+          if (Dml.isBackground(file)) {
+            return this._registerJob(file).catch(failedIn(file));
+          }
+
           const _meta = file.get()._meta || {};
           const version = _meta.version || 1;
           this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };