db-migrate 1.4.0 → 1.5.0

package/lib/chain.js

@@ -1,4 +1,6 @@
 const log = require('db-migrate-shared').log;
+const Release = require('./release');
+const { unknownTable, unknownColumn } = require('./learn');
 
 const DEFAULT = [
   'renameCollection',
@@ -151,6 +153,8 @@ class Chain {
      */
     this.adopt = {};
 
+    this.deprecations();
+
     DEFAULT.concat(Object.keys(this.context.learnable || [])).forEach(
       method => {
         this[method] = function (...args) {
@@ -173,6 +177,86 @@ class Chain {
     );
   }
 
+  /**
+   * deprecateTable, deprecateColumn and dropDeprecated, see Release. The
+   * marks are learned only, relaxing notNull of a column is executed.
+   */
+  deprecations () {
+    const internals = this.internals;
+
+    this.deprecateTable = async (t, options) => {
+      this.transferInt();
+      if (!internals.schema.c[t]) {
+        throw unknownTable(t);
+      }
+
+      const entry = Release.entry(internals, t, options);
+      return this.run('setDeprecated', ['tables', t, null, entry], 'learn');
+    };
+
+    this.deprecateColumn = async (t, c, options) => {
+      this.transferInt();
+      const columns = Release.columnsOf(internals.schema, t);
+      if (!columns) {
+        throw unknownTable(t);
+      }
+
+      if (!columns[c]) {
+        throw unknownColumn(t, c);
+      }
+
+      const entry = Release.entry(internals, c, options);
+      if (columns[c].notNull) {
+        // the application stops writing it. The whole spec, MySQL changes
+        // a column by its full definition
+        await this.changeColumn(
+          t,
+          c,
+          Object.assign({}, columns[c], { notNull: false })
+        );
+      }
+
+      return this.run('setDeprecated', ['columns', t, c, entry], 'learn');
+    };
+
+    /**
+     * dropDeprecated() drops what is due, dropDeprecated(table) or
+     * dropDeprecated(table, column) the one named, due or not.
+     */
+    this.dropDeprecated = async (t, c) => {
+      this.transferInt();
+      const items = Release.list(
+        internals,
+        internals.releaseCurrent || 0,
+        internals.releaseIndex || {}
+      );
+      let targets;
+
+      if (t === undefined) {
+        targets = items.filter(
+          item => item.age >= Release.settings(internals, item.entry).releases
+        );
+      } else {
+        targets = items.filter(item =>
+          c === undefined
+            ? item.kind === 'tables' && item.t === t
+            : item.kind === 'columns' && item.t === t && item.c === c
+        );
+
+        if (!targets.length) {
+          throw new Error(
+            `${c === undefined ? `The table "${t}"` : `The column "${c}" of "${t}"`} ` +
+              'is not deprecated, deprecate it first'
+          );
+        }
+      }
+
+      for (const item of targets) {
+        await Release.drop(this, item);
+      }
+    };
+  }
+
   transferInt () {
     rollbackCounter = counter;
     counter = 0;

package/lib/dml.js

@@ -282,6 +282,11 @@ class Dml {
       throw new Stopped();
     }
 
+    // migrations paused the jobs
+    if (this.work.yielding && (await this.work.yielding())) {
+      throw new Stopped();
+    }
+
     if (this.work.pause) {
       await delay(this.work.pause);
     }
@@ -658,12 +663,20 @@ class Dml {
     const condition = this.where(where);
     const key = this.key(table, options, 'delete');
     const time = options.value === undefined;
+    const purge = options.purge;
+
+    if (purge !== undefined && purge !== true && !isObject(purge)) {
+      throw new Error(
+        `delete("${table}") takes purge: true or { releases, drop }, to purge ` +
+          'the rows with a later release'
+      );
+    }
 
     return this.step(
       'delete',
       [table],
       op => ({ t: 4, a: 'soft', c: [table, column, key, deleted(this.file, op)] }),
-      entry => {
+      async entry => {
         const mark = entry.c[3];
         const set =
           `${this.ddl(column)} = ${time ? 'CURRENT_TIMESTAMP' : '?'}, ` +
@@ -682,6 +695,17 @@ class Dml {
           );
         }
 
+        if (purge) {
+          await require('./release').schedulePurge(
+            this.pdriver,
+            this.internals,
+            mark,
+            table,
+            key,
+            purge
+          );
+        }
+
         return this.untilDone(table, key, active, options, match =>
           this.driver.runSql(
             `UPDATE ${this.ddl(table)} SET ${set} WHERE ${match.sql}`,
@@ -710,7 +734,15 @@ class Dml {
       'purge',
       [table].concat(migration ? [migration] : []),
       () => ({ t: 3, a: 'purge', c: [table] }),
-      () => {
+      async () => {
+        // nothing left to purge with a later release
+        await require('./release').forgetPurges(
+          this.pdriver,
+          this.internals,
+          table,
+          mark
+        );
+
         const condition = {
           sql: `${this.ddl(FLAG)} LIKE ? ESCAPE '!'`,
           params: [`%${like(mark)}%`]
@@ -902,8 +934,9 @@ const reverts = {
     );
   },
 
-  soft: (d, [table, column, key, mark]) =>
-    d.untilDone(
+  soft: async (d, [table, column, key, mark]) => {
+    await require('./release').forgetPurges(d.pdriver, d.internals, table, mark);
+    return d.untilDone(
       table,
       key,
       {
@@ -918,7 +951,8 @@ const reverts = {
             `WHERE ${match.sql}`,
           [mark].concat(match.params)
         )
-    )
+    );
+  }
 };
 
 const irreversibleSteps = internals =>

package/lib/driver/shadow.js

@@ -70,5 +70,20 @@ exports.overshadow = function (db, ShadowProto, ShadowError) {
     }
   }
 
+  // instructions of db-migrate itself, which no driver has, e.g. setDeprecated
+  let proto = Object.getPrototypeOf(ShadowProto);
+  while (proto && proto !== Object.prototype) {
+    Object.getOwnPropertyNames(proto).forEach(prop => {
+      if (
+        prop !== 'constructor' &&
+        newDb[prop] === undefined &&
+        typeof ShadowProto[prop] === 'function'
+      ) {
+        newDb[prop] = ShadowProto[prop];
+      }
+    });
+    proto = Object.getPrototypeOf(proto);
+  }
+
   return newDb;
 };

package/lib/jobs.js

@@ -2,6 +2,7 @@
 
 const crypto = require('crypto');
 const log = require('db-migrate-shared').log;
+const State = require('./state');
 
 /**
  * Background migrations, dml migrations with _meta.background, are not run
@@ -20,6 +21,12 @@ const log = require('db-migrate-shared').log;
  *     in the foreground, see State
  * h: hash of the migration file when it started
  * err: why it failed
+ *
+ * Migrations take precedence over jobs. Whoever holds the migration lock
+ * pauses the jobs before migrating: no job is taken anymore, the running
+ * ones stop after their current batch, and are continued once the lock is
+ * released. The pause is stored with the jobs, so taking a job and pausing
+ * never miss each other. It holds as long as its holder holds the lock.
  */
 
 const JOBS = '__dbmigrate_jobs__';
@@ -28,26 +35,30 @@ const nonce = () => crypto.randomBytes(8).toString('hex');
 
 const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 
+const WAIT = { wait: true };
+
 const read = async (pdriver, internals) => {
   const row = await pdriver._getKV(internals.migrationState, JOBS);
-  return { row, jobs: row ? JSON.parse(row.value).jobs : {} };
+  const state = row ? JSON.parse(row.value) : { jobs: {} };
+  return { row, state, jobs: state.jobs };
 };
 
 const list = async (pdriver, internals) => (await read(pdriver, internals)).jobs;
 
 /**
- * Changes the jobs by mutate, retried until no other worker wrote in
- * between. mutate returns false to leave them as they are.
+ * Changes the jobs by mutate(jobs, state), retried until no other worker
+ * wrote in between. mutate returns false to leave them as they are.
  */
 const change = async (pdriver, internals, mutate) => {
   for (let attempt = 0; ; attempt++) {
-    const { row, jobs } = await read(pdriver, internals);
+    const { row, state, jobs } = await read(pdriver, internals);
 
-    if (mutate(jobs) === false) {
+    if (mutate(jobs, state) === false) {
       return jobs;
     }
 
-    const value = JSON.stringify({ jobs, n: nonce() });
+    state.n = nonce();
+    const value = JSON.stringify(state);
 
     if (!row) {
       try {
@@ -108,19 +119,61 @@ const register = async (pdriver, internals, file) => {
   log.info(`[jobs] ${file.name} runs in the background`);
 };
 
+/**
+ * Whether the jobs are paused by a migration. A pause holds as long as its
+ * holder holds the migration lock and keeps it alive, the lock row did not
+ * stay the same for the lock timeout, measured on our clock in seen.
+ *
+ * Resolves to the pause, or null.
+ */
+const paused = async (pdriver, internals, seen = {}) => {
+  const { state } = await read(pdriver, internals);
+
+  if (!state.pause) {
+    return null;
+  }
+
+  const lock = await State.readLock(pdriver, internals);
+  if (!lock || State.holder(lock) !== state.pause.ID) {
+    return null;
+  }
+
+  const now = Date.now();
+  const last = seen.lock;
+  if (!last || last.value !== lock.value || String(last.run_on) !== String(lock.run_on)) {
+    seen.lock = { value: lock.value, run_on: lock.run_on, since: now };
+  } else if (now - last.since >= State.lockOptions(internals).timeout) {
+    return null;
+  }
+
+  return state.pause;
+};
+
 /**
  * Takes the first job to run, in the order of their names. A running job is
  * taken over if it did not change for timeout ms, measured on our clock in
  * seen. Jobs after a blocking job wait until it is done.
  *
- * Resolves to the name of the job taken, or null.
+ * Resolves to the name of the job taken, null if there is none, or WAIT
+ * while the jobs are paused.
  */
 const claim = async (pdriver, internals, id, seen, timeout) => {
   const now = Date.now();
+  const valid = await paused(pdriver, internals, seen);
   let taken = null;
+  let wait = false;
 
-  await change(pdriver, internals, jobs => {
+  await change(pdriver, internals, (jobs, state) => {
     taken = null;
+    // a pause found invalid before is ignored, one set in between holds
+    wait = !!state.pause && (!!valid || state.pause.n !== seen.invalid);
+    if (state.pause && !valid) {
+      seen.invalid = seen.invalid || state.pause.n;
+    }
+
+    if (wait) {
+      return false;
+    }
 
     for (const name of Object.keys(jobs).sort()) {
       const job = jobs[name];
@@ -154,9 +207,81 @@ const claim = async (pdriver, internals, id, seen, timeout) => {
     return false;
   });
 
+  if (wait) {
+    return WAIT;
+  }
+
   return taken;
 };
 
+const isRunning = (job, id) => job.s === 'running' && job.ID !== id;
+
+/**
+ * Pauses the jobs for the migrations of the lock holder id, and waits until
+ * the running jobs stopped. A job whose worker does not respond for the lock
+ * timeout is not waited for, it is continued by another worker later.
+ */
+const pause = async (pdriver, internals, id) => {
+  const { timeout, interval } = State.lockOptions(internals);
+  const seen = {};
+
+  await change(pdriver, internals, (jobs, state) => {
+    state.pause = { ID: id, n: nonce() };
+  });
+
+  for (;;) {
+    const now = Date.now();
+    const jobs = await list(pdriver, internals);
+    const running = Object.keys(jobs).filter(name => {
+      const job = jobs[name];
+
+      if (!isRunning(job, id)) {
+        return false;
+      }
+
+      const last = seen[name];
+      if (!last || last.n !== job.n) {
+        seen[name] = { n: job.n, since: now };
+        return true;
+      }
+
+      return now - last.since < timeout;
+    });
+
+    if (!running.length) {
+      return;
+    }
+
+    log.info(
+      `[jobs] waiting for ${running.length} background job(s) to pause: ` +
+        running.join(', ')
+    );
+    // they stop after their current batch, usually soon
+    await delay(Math.min(interval, 200));
+  }
+};
+
+/**
+ * Forgets a job, once its migration is reverted.
+ */
+const remove = (pdriver, internals, name) =>
+  change(pdriver, internals, jobs => {
+    if (!jobs[name]) {
+      return false;
+    }
+
+    delete jobs[name];
+  });
+
+const resume = (pdriver, internals, id) =>
+  change(pdriver, internals, (jobs, state) => {
+    if (!state.pause || state.pause.ID !== id) {
+      return false;
+    }
+
+    delete state.pause;
+  });
+
 /**
  * Changes a job we run. If another worker took it over in between, lost is
  * called and nothing is changed.
@@ -174,4 +299,4 @@ const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
     job.n = nonce();
   });
 
-module.exports = { JOBS, list, register, claim, update };
+module.exports = { JOBS, WAIT, list, register, claim, update, paused, pause, resume, remove };

package/lib/learn.js

@@ -59,6 +59,14 @@ class STD {
       schema.e = {};
     }
     this.extra = schema.e;
+    if (!schema.d) {
+      schema.d = {};
+    }
+    /**
+     * deprecated tables and columns, d.tables[table] and
+     * d.columns[table][column], see Release
+     */
+    this.deprecated = schema.d;
     if (unlearn) {
       this.modC = [];
     } else {
@@ -137,6 +145,15 @@ class STD {
   }
 
   renameTable (t, n) {
+    // deprecations move with the table
+    ['tables', 'columns'].forEach(kind => {
+      const d = this.deprecated[kind];
+      if (d && d[t] !== undefined) {
+        d[n] = d[t];
+        delete d[t];
+      }
+    });
+
     if (this.schema[t]) {
       this.schema[n] = this.schema[t];
       delete this.schema[t];
@@ -310,7 +327,9 @@ class STD {
     const columns = hasColumns ? this.schema[t].columns : this.schema[t];
 
     this.modS[t][c] = columns[c];
-    columns[c] = Object.assign(columns[c], s);
+    // a copy, the previous spec is kept to revert it. Reverting restores
+    // the previous spec as a whole.
+    columns[c] = Object.assign({}, this.unlearn ? {} : columns[c], s);
 
     this.modC.push({ t: 1, a: 'changeColumn', c: [t, c] });
 
@@ -419,6 +438,38 @@ class STD {
     return Promise.resolve(alter);
   }
 
+  /**
+   * Marks a table or column as deprecated, or removes the mark with entry
+   * null. Only learned, nothing is executed.
+   */
+  setDeprecated (kind, t, c, entry) {
+    const d = this.deprecated;
+    if (!d[kind]) d[kind] = {};
+
+    let previous;
+    if (kind === 'tables') {
+      previous = d.tables[t] || null;
+      if (entry) {
+        d.tables[t] = entry;
+      } else {
+        delete d.tables[t];
+      }
+    } else {
+      if (!d.columns[t]) d.columns[t] = {};
+      previous = d.columns[t][c] || null;
+      if (entry) {
+        d.columns[t][c] = entry;
+      } else {
+        delete d.columns[t][c];
+        if (!Object.keys(d.columns[t]).length) delete d.columns[t];
+      }
+    }
+
+    this.modC.push({ t: 2, a: 'setDeprecated', c: [kind, t, c, previous] });
+
+    return Promise.resolve();
+  }
+
   //
   //  checkDBMS: dummy,
   //

package/lib/methods/v2/statedone.js

@@ -13,6 +13,7 @@ module.exports = {
         return State.done(pdriver, internals.migrationOp, internals);
       };
 
-    return Shadow.overshadow(driver, {}, done);
+    // learned only, no method of the driver
+    return Shadow.overshadow(driver, { setDeprecated: done() }, done);
   }
 };

package/lib/methods/v2/statetravel.js

@@ -14,7 +14,8 @@ const DEFAULT = {
   addIndex: 'removeIndex',
   removeIndex: 'addIndex',
   addForeignKey: 'removeForeignKey',
-  removeForeignKey: 'addForeignKey'
+  removeForeignKey: 'addForeignKey',
+  setDeprecated: 'setDeprecated'
 };
 
 /**

package/lib/release.js

@@ -0,0 +1,502 @@
+'use strict';
+
+const crypto = require('crypto');
+const log = require('db-migrate-shared').log;
+
+/**
+ * Releases and the deprecation of tables and columns.
+ *
+ * A migration starts a release with _meta.release, any label. The releases
+ * are counted in the order of the migrations, a migration without a label
+ * belongs to the release before it, those before the first label to release
+ * 0. The labels themselves are not interpreted.
+ *
+ * db.deprecateTable and db.deprecateColumn mark a table or column in the
+ * release of their migration R. Once a migration starts a later release,
+ * db-migrate runs the steps due before it, as a migration of its own named
+ * __dbmigrate_release__:<label>:
+ *
+ * R+1: the table or column is renamed to __dbm_deprecated_<name>_<time>, so
+ *      whatever still uses it fails, while the data is still there
+ * R+N: it is dropped, N = releases (4 by default), with drop 'auto' only.
+ *      With drop 'manual' (default) db-migrate warns it is due, until
+ *      db.dropDeprecated drops it.
+ *
+ * Reverting the first migration of a release reverts the steps run before
+ * it as well.
+ */
+
+const PREFIX = '__dbmigrate_release__:';
+const PURGES = '__dbmigrate_purges__';
+const DEFAULTS = { releases: 4, drop: 'manual' };
+const DROPS = ['auto', 'manual'];
+
+/**
+ * The release of every migration: { label, index }, by name.
+ */
+const releases = files => {
+  const map = {};
+  const index = {};
+  let current = { label: null, index: 0 };
+
+  files
+    .slice()
+    .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+    .forEach(file => {
+      let label;
+      try {
+        label = (file.get()._meta || {}).release;
+      } catch (err) {
+        // an old migration which can not be loaded anymore
+        log.verbose(`[release] can not load ${file.name}: ${err.message}`);
+      }
+
+      if (label !== undefined && label !== null && String(label) !== current.label) {
+        const key = String(label);
+        if (index[key] === undefined) {
+          index[key] = Object.keys(index).length + 1;
+        }
+
+        current = { label: key, index: index[key] };
+      }
+
+      map[file.name] = current;
+    });
+
+  return { map, index };
+};
+
+/**
+ * The name a deprecated table or column is renamed to.
+ */
+const hiddenName = name => {
+  const time = Math.floor(Date.now() / 1000);
+  const plain = `__dbm_deprecated_${name}_${time}`;
+
+  if (plain.length <= 60) {
+    return plain;
+  }
+
+  const hash = crypto.createHash('sha256').update(name).digest('hex');
+  return `__dbm_deprecated_${hash.slice(0, 16)}_${time}`;
+};
+
+const checkOptions = (options = {}) => {
+  const out = {};
+
+  if (options.releases !== undefined) {
+    const n = Number(options.releases);
+    if (!Number.isInteger(n) || n < 1) {
+      throw new Error(
+        `releases is the number of releases until dropping, at least 1, not ${options.releases}`
+      );
+    }
+
+    out.releases = n;
+  }
+
+  if (options.drop !== undefined) {
+    if (DROPS.indexOf(options.drop) === -1) {
+      throw new Error(`drop is 'auto' or 'manual', not ${options.drop}`);
+    }
+
+    out.drop = options.drop;
+  }
+
+  return out;
+};
+
+/**
+ * The options of an entry, given for it, else of the project, else the
+ * defaults. The project sets them as deprecation: { releases, drop }, in
+ * the rc file or the options of the programmable API.
+ */
+const settings = (internals, entry) => {
+  const project = checkOptions((internals.argv && internals.argv.deprecation) || {});
+  return Object.assign({}, DEFAULTS, project, entry.o || {});
+};
+
+const entry = (internals, name, options) => {
+  const e = { r: internals.releaseLabel || null, to: hiddenName(name) };
+  const o = checkOptions(options);
+
+  if (Object.keys(o).length) {
+    e.o = o;
+  }
+
+  return e;
+};
+
+const columnsOf = (schema, table) => {
+  const t = schema.c[table];
+  return t ? t.columns || t : null;
+};
+
+/**
+ * All deprecations with their state:
+ *
+ * name: the current name, renamed or not
+ * renamed: whether it was renamed
+ * age: releases since deprecated, up to index
+ */
+const list = (internals, index, releaseIndex) => {
+  const schema = internals.schema;
+  const d = schema.d || {};
+  const out = [];
+  const age = e => index - (e.r === null ? 0 : releaseIndex[e.r] || 0);
+
+  Object.keys(d.tables || {}).forEach(t => {
+    const e = d.tables[t];
+    const renamed = !!schema.c[e.to];
+    if (!renamed && !schema.c[t]) {
+      return;
+    }
+
+    out.push({ kind: 'tables', t, entry: e, renamed, name: renamed ? e.to : t, age: age(e) });
+  });
+
+  Object.keys(d.columns || {}).forEach(t => {
+    const columns = columnsOf(schema, t) || {};
+    Object.keys(d.columns[t]).forEach(c => {
+      const e = d.columns[t][c];
+      const renamed = !!columns[e.to];
+      if (!renamed && !columns[c]) {
+        return;
+      }
+
+      out.push({ kind: 'columns', t, c, entry: e, renamed, name: renamed ? e.to : c, age: age(e) });
+    });
+  });
+
+  return out;
+};
+
+const describe = item =>
+  item.kind === 'tables'
+    ? `table "${item.t}"`
+    : `column "${item.c}" of "${item.t}"`;
+
+/**
+ * Drops a deprecated table or column and forgets the deprecation, within a
+ * v2 migration.
+ */
+const drop = async (db, item) => {
+  if (item.kind === 'tables') {
+    await db.dropTable(item.name);
+  } else {
+    await db.removeColumn(item.t, item.name);
+  }
+
+  await db.run('setDeprecated', [item.kind, item.t, item.c || null, null], 'learn');
+};
+
+/**
+ * The steps due when the release with index starts.
+ */
+const due = (internals, index, releaseIndex) => {
+  const renames = [];
+  const drops = [];
+
+  list(internals, index, releaseIndex).forEach(item => {
+    const { releases, drop } = settings(internals, item.entry);
+
+    if (item.age >= releases && drop === 'auto') {
+      drops.push(item);
+    } else if (item.age >= 1 && !item.renamed) {
+      renames.push(item);
+    }
+  });
+
+  return { renames, drops };
+};
+
+/**
+ * Rows deleted in soft mode with { purge }, to delete for good once enough
+ * releases passed: { [mark]: { t, key, r, o } }, stored in the state.
+ * Changed by compare and swap, background jobs delete in soft mode as well.
+ */
+const purges = async (pdriver, internals) => {
+  const row = await pdriver._getKV(internals.migrationState, PURGES);
+  return { row, value: row ? JSON.parse(row.value) : {} };
+};
+
+const changePurges = async (pdriver, internals, mutate) => {
+  if (internals.dryRun) {
+    return;
+  }
+
+  for (let attempt = 0; attempt < 50; attempt++) {
+    const { row, value } = await purges(pdriver, internals);
+    if (mutate(value) === false) {
+      return;
+    }
+
+    const next = JSON.stringify(value);
+    if (!row) {
+      try {
+        await pdriver._insertKV(internals.migrationState, PURGES, next);
+      } catch (err) {
+        log.verbose('[release] purges inserted concurrently', err.message);
+      }
+    } else {
+      await pdriver._updateKVC(internals.migrationState, PURGES, next, 'value', row.value);
+    }
+
+    const after = await pdriver._getKV(internals.migrationState, PURGES);
+    if (after && after.value === next) {
+      return;
+    }
+  }
+
+  throw new Error('[release] could not change the purges, too much contention');
+};
+
+/**
+ * Schedules purging the rows of a soft delete, { purge: true } takes the
+ * options of the project.
+ */
+const schedulePurge = (pdriver, internals, mark, t, key, purge) => {
+  const o = purge === true ? {} : checkOptions(purge);
+  const entry = { t, key, r: internals.releaseLabel || null };
+  if (Object.keys(o).length) {
+    entry.o = o;
+  }
+
+  return changePurges(pdriver, internals, value => {
+    value[mark] = entry;
+  });
+};
+
+/**
+ * Forgets the purges done, of a soft delete or by db.purge, matching
+ * the table and the prefix of the mark.
+ */
+const forgetPurges = (pdriver, internals, t, prefix) =>
+  changePurges(pdriver, internals, value => {
+    const marks = Object.keys(value).filter(
+      mark => value[mark].t === t && mark.indexOf(prefix) === 0
+    );
+
+    if (!marks.length) {
+      return false;
+    }
+
+    marks.forEach(mark => delete value[mark]);
+  });
+
+const duePurges = async (pdriver, internals, index, releaseIndex) => {
+  const { value } = await purges(pdriver, internals);
+
+  return Object.keys(value).map(mark => {
+    const e = value[mark];
+    const age = index - (e.r === null ? 0 : releaseIndex[e.r] || 0);
+    return Object.assign({ mark, age }, e, settings(internals, e));
+  }).filter(e => e.age >= e.releases);
+};
+
+const releaseFile = (label, migrate) => ({
+  name: PREFIX + label,
+  path: null,
+  get: () => ({ _meta: { version: 2 }, migrate })
+});
+
+/**
+ * The walker, without recording the release steps as migrations.
+ */
+const unrecorded = walker =>
+  Object.create(walker, {
+    writeMigrationRecord: { value: (file, cb) => cb() },
+    deleteMigrationRecord: { value: (file, cb) => cb() }
+  });
+
+/**
+ * Runs the steps due before the first migration of the release label.
+ */
+const start = async (walker, label, index, releaseIndex) => {
+  const internals = walker.internals;
+  const { renames, drops } = due(internals, index, releaseIndex);
+  const purged = (await duePurges(walker._pdriver, internals, index, releaseIndex)).filter(
+    e => e.drop === 'auto'
+  );
+
+  if (purged.length) {
+    await purge(walker, label, purged);
+  }
+
+  if (!renames.length && !drops.length) {
+    return;
+  }
+
+  log.info(`[release] starting release ${label}`);
+  renames.forEach(item =>
+    log.info(`[release] renaming the deprecated ${describe(item)} to ${item.entry.to}`)
+  );
+  drops.forEach(item => log.info(`[release] dropping the deprecated ${describe(item)}`));
+
+  const file = releaseFile(label, async db => {
+    for (const item of renames) {
+      if (item.kind === 'tables') {
+        await db.renameTable(item.t, item.entry.to);
+      } else {
+        await db.renameColumn(item.t, item.c, item.entry.to);
+      }
+    }
+
+    for (const item of drops) {
+      await drop(db, item);
+    }
+  });
+
+  internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
+  if (internals.dryRun) {
+    return;
+  }
+
+  await require('./executors/versioned/v2').up(unrecorded(walker), walker.driver, file);
+};
+
+/**
+ * Deletes the rows of the soft deletes due for good, before the first
+ * migration of the release label. This can not be reverted, the steps of
+ * the release are recorded as irreversible.
+ */
+const purge = async (walker, label, due) => {
+  const internals = walker.internals;
+  const pdriver = walker._pdriver;
+  const name = PREFIX + label;
+  const { Dml, FLAG } = require('./dml');
+  const file = releaseFile(label, async () => {});
+  const d = new Dml(walker, file, internals, pdriver, null, {
+    step: async () => {},
+    learned: async () => {},
+    done: async () => {}
+  });
+
+  for (const e of due) {
+    log.info(`[release] purging the rows of "${e.t}" deleted in soft mode by ${e.mark.slice(5)}`);
+    if (!internals.dryRun) {
+      await d.untilDone(
+        e.t,
+        e.key,
+        { sql: `${d.ddl(FLAG)} LIKE ? ESCAPE '!'`, params: [`%${e.mark.replace(/[!%_]/g, c => '!' + c)}%`] },
+        {},
+        match => walker._driver.runSql(`DELETE FROM ${d.ddl(e.t)} WHERE ${match.sql}`, match.params)
+      );
+    }
+  }
+
+  if (internals.dryRun) {
+    return;
+  }
+
+  await changePurges(pdriver, internals, value => {
+    due.forEach(e => delete value[e.mark]);
+  });
+
+  // the release can not be reverted anymore
+  const row = await pdriver._getKV(internals.migrationState, name);
+  const state = row ? JSON.parse(row.value) : { i: {}, c: {}, f: {}, s: [] };
+  state.s = (state.s || []).concat(
+    due.map(e => ({ t: 3, a: 'purge', c: [e.t], n: 0 }))
+  );
+
+  if (row) {
+    await pdriver._updateKV(internals.migrationState, name, JSON.stringify(state));
+  } else {
+    await pdriver._insertKV(internals.migrationState, name, JSON.stringify(state));
+  }
+};
+
+/**
+ * Reverts the steps run before the first migration of the release label,
+ * once its last migration is reverted.
+ */
+const revert = async (walker, label) => {
+  const internals = walker.internals;
+  const name = PREFIX + label;
+  const state = await walker._pdriver._getKV(internals.migrationState, name);
+
+  if (!state || internals.dryRun) {
+    return;
+  }
+
+  log.info(`[release] reverting the steps of release ${label}`);
+  internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
+  await require('./executors/versioned/v2').down(
+    unrecorded(walker),
+    walker.driver,
+    releaseFile(label, async () => {})
+  );
+};
+
+/**
+ * Refuses reverting a release whose steps can not be reverted, rows purged.
+ */
+const assertRevertible = async (walker, label) => {
+  const row = await walker._pdriver._getKV(
+    walker.internals.migrationState,
+    PREFIX + label
+  );
+  const steps = row ? JSON.parse(row.value).s || [] : [];
+  const irreversible = steps.filter(entry => entry.t === 3);
+
+  if (irreversible.length) {
+    const err = new Error(
+      `Release ${label} can not be reverted, it ${irreversible
+        .map(entry => `purged "${entry.c[0]}"`)
+        .join(', ')} before its first migration.`
+    );
+    err.irreversible = true;
+    throw err;
+  }
+};
+
+/**
+ * Warns about what is due for dropping with drop 'manual'.
+ */
+const warnDuePurges = async (pdriver, internals, index, releaseIndex) => {
+  (await duePurges(pdriver, internals, index, releaseIndex))
+    .filter(e => e.drop === 'manual')
+    .forEach(e =>
+      log.warn(
+        `[release] the rows of "${e.t}" deleted in soft mode by ` +
+          `${e.mark.slice(5)} are due for purging, deleted ${e.age} releases ` +
+          `ago. Purge them in a dml migration with db.purge(${JSON.stringify(e.t)}, ` +
+          `${JSON.stringify(e.mark.slice(5, e.mark.lastIndexOf('#')))}).`
+      )
+    );
+};
+
+const warnDue = (internals, index, releaseIndex) => {
+  list(internals, index, releaseIndex).forEach(item => {
+    const { releases, drop } = settings(internals, item.entry);
+
+    if (drop === 'manual' && item.age >= releases) {
+      log.warn(
+        `[release] the deprecated ${describe(item)} is due for dropping, ` +
+          `deprecated ${item.age} releases ago. Drop it in a migration with ` +
+          `db.dropDeprecated(${JSON.stringify(item.t)}${
+            item.c ? `, ${JSON.stringify(item.c)}` : ''
+          }).`
+      );
+    }
+  });
+};
+
+module.exports = {
+  PREFIX,
+  PURGES,
+  schedulePurge,
+  forgetPurges,
+  warnDuePurges,
+  releases,
+  entry,
+  list,
+  settings,
+  drop,
+  start,
+  revert,
+  assertRevertible,
+  warnDue,
+  columnsOf
+};

package/lib/state.js

@@ -97,7 +97,7 @@ const createConcurrently = async create => {
  * The stored schema starts as '{}', keep the structure the walker expects.
  */
 const parseSchema = value =>
-  Object.assign({ i: {}, c: {}, f: {}, e: {} }, JSON.parse(value));
+  Object.assign({ i: {}, c: {}, f: {}, e: {}, d: {} }, JSON.parse(value));
 
 const stopHeartbeat = () => {
   if (heartbeat) {
@@ -155,6 +155,12 @@ const claim = (row, driver, internals) => {
 module.exports = {
   createConcurrently,
 
+  // the id of this process in the lock, the holder of a lock row, see Jobs
+  ID,
+  holder,
+  lockOptions,
+  readLock: read,
+
   // for tests, simulates a process which died while holding the lock
   __stopHeartbeat: stopHeartbeat,
 

package/lib/walker.js

@@ -7,6 +7,7 @@ const Promise = require('bluebird');
 const File = require('./file.js');
 const Dml = require('./dml');
 const Jobs = require('./jobs');
+const Release = require('./release');
 
 const INTERFACES = {
   migration: require('./interface/migratorInterface.js'),
@@ -39,7 +40,7 @@ const Walker = function (driver, directory, mode, intern, prefix, opts = {}) {
    * f: foreignKey
    * e: extra items for the schema (like ENUM types)
    */
-  this.internals.schema = { i: {}, c: {}, f: {}, e: {} };
+  this.internals.schema = { i: {}, c: {}, f: {}, e: {}, d: {} };
   /**
    * s: commands
    * i: index
@@ -127,10 +128,20 @@ Walker.prototype = {
       return run(toRun);
     }
 
+    let paused = false;
     try {
+      // migrations take precedence over the jobs of background migrations
+      await Jobs.pause(this._pdriver, this.internals, State.ID);
+      paused = true;
       await State.reloadSchema(this._pdriver, this.internals);
       return await run(await determine());
     } finally {
+      if (paused) {
+        await Jobs.resume(this._pdriver, this.internals, State.ID).catch(err =>
+          log.warn(`[jobs] could not resume the jobs: ${err.message}`)
+        );
+      }
+
       await State.release(this._pdriver, this.internals);
     }
   },
@@ -158,6 +169,52 @@ Walker.prototype = {
     });
   },
 
+  /**
+   * The migrations of the jobs, newest first.
+   */
+  _jobFiles: async function () {
+    if (this.prefix !== 'migration' || !this._pdriver || typeof this._pdriver._getKV !== 'function') {
+      return [];
+    }
+
+    const jobs = await Jobs.list(this._pdriver, this.internals);
+    const names = Object.keys(jobs);
+    if (!names.length) {
+      return [];
+    }
+
+    const files = await File.loadFromFileystem(this.directory, this.prefix, this.internals);
+    return names
+      .sort()
+      .reverse()
+      .map(name => {
+        const file = files.find(f => f.name === name);
+        if (!file) {
+          throw new Error(
+            `The migration of the background job "${name}" is missing, it ` +
+              'can not be reverted'
+          );
+        }
+
+        file.job = true;
+        return file;
+      });
+  },
+
+  /**
+   * Reverts what the job of a background migration did so far, it was
+   * paused by taking the lock, and forgets the job.
+   */
+  _revertJob: async function (file) {
+    log.info(`${this.title}reverting the background job of ${file.name}`);
+    this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
+
+    if (!this.internals.dryRun) {
+      await Dml.down(this, this.driver, file);
+      await Jobs.remove(this._pdriver, this.internals, file.name);
+    }
+  },
+
   _registerJob: async function (file) {
     Dml.checkType(file);
 
@@ -244,14 +301,18 @@ Walker.prototype = {
           this.internals
         )
       ])
-        .spread((allFiles, completedFiles) =>
-          dbmUtil.filterUp(allFiles, completedFiles, partialName, count)
-        )
+        .spread((allFiles, completedFiles) => {
+          this._releases(allFiles, completedFiles);
+          return dbmUtil.filterUp(allFiles, completedFiles, partialName, count);
+        })
         .then(toRun => this._withoutJobs(toRun));
 
     if (this.internals.check) {
       return determine()
-        .then(toRun => this._checkResult(toRun))
+        .then(async toRun => {
+          await this._warnDue();
+          return this._checkResult(toRun);
+        })
         .nodeify(callback);
     }
 
@@ -261,8 +322,10 @@ Walker.prototype = {
           log.info(this.title + 'Nothing to run');
         }
 
-        return Promise.each(toRun, file => {
+        return Promise.each(toRun, async file => {
           log.verbose(this.title + 'preparing to run up:', file.name);
+          await this._enterRelease(file).catch(failedIn(file));
+
           if (Dml.isBackground(file)) {
             return this._registerJob(file).catch(failedIn(file));
           }
@@ -277,11 +340,94 @@ Walker.prototype = {
               file
             )
           ).catch(failedIn(file));
+        }).then(async res => {
+          await this._warnDue();
+          return res;
         });
       })
     ).nodeify(callback);
   },
 
+  /**
+   * The releases of the migrations, and the latest release migrated.
+   */
+  _releases: function (allFiles, completedFiles) {
+    const release = Release.releases(allFiles);
+    this._release = release;
+    this._applied = completedFiles.reduce(
+      (max, file) => Math.max(max, (release.map[file.name] || { index: 0 }).index),
+      0
+    );
+  },
+
+  /**
+   * Runs the steps due, before the first migration of a new release.
+   */
+  _enterRelease: async function (file) {
+    const release = this._release || { map: {}, index: {} };
+    const r = release.map[file.name] || { label: null, index: 0 };
+
+    if (r.index > this._applied) {
+      this.internals.releaseIndex = release.index;
+      await Release.start(this, r.label, r.index, release.index);
+      this._applied = r.index;
+    }
+
+    this.internals.releaseLabel = r.label;
+    this.internals.releaseIndex = release.index;
+    this.internals.releaseCurrent = this._applied;
+  },
+
+  /**
+   * The label of the release whose steps are reverted after reverting file,
+   * its last migration. Refuses before reverting anything, if the steps of
+   * the release can not be reverted.
+   */
+  _leavingRelease: async function (file) {
+    const release = this._release;
+    const r = release && release.map[file.name];
+
+    if (!r || !r.label) {
+      return null;
+    }
+
+    const completed = await File.loadFromDatabase(
+      this.directory,
+      this.prefix,
+      this._driver,
+      this.internals
+    );
+    const remaining = completed
+      .filter(f => f.name !== file.name)
+      .reduce(
+        (max, f) => Math.max(max, (release.map[f.name] || { index: 0 }).index),
+        0
+      );
+
+    if (remaining >= r.index) {
+      return null;
+    }
+
+    await Release.assertRevertible(this, r.label);
+    return r.label;
+  },
+
+  _warnDue: async function () {
+    if (!this._release) {
+      return;
+    }
+
+    Release.warnDue(this.internals, this._applied, this._release.index);
+    if (this._pdriver && typeof this._pdriver._getKV === 'function') {
+      await Release.warnDuePurges(
+        this._pdriver,
+        this.internals,
+        this._applied,
+        this._release.index
+      );
+    }
+  },
+
   down: function (options, callback) {
     const partialName = options.destination;
     const count = options.count;
@@ -291,9 +437,28 @@ Walker.prototype = {
         this.prefix,
         this._driver,
         this.internals
-      ).then(completedFiles =>
-        dbmUtil.filterDown(completedFiles, partialName, count)
-      );
+      ).then(async completedFiles => {
+        // the releases, from the migrations still there
+        const allFiles = await File.loadFromFileystem(
+          this.directory,
+          this.prefix,
+          this.internals
+        ).catch(err => {
+          if (err.code === 'ENOENT') {
+            return [];
+          }
+
+          throw err;
+        });
+        this._releases(allFiles, completedFiles);
+
+        return dbmUtil.filterDown(
+          // jobs of background migrations are the latest, still running
+          (await this._jobFiles()).concat(completedFiles),
+          partialName,
+          count
+        );
+      });
 
     if (this.internals.check) {
       return determine()
@@ -309,16 +474,20 @@ Walker.prototype = {
 
         return Promise.each(toRun, file => {
           log.verbose(this.title + 'preparing to run down:', file.name);
+          if (file.job) {
+            return this._revertJob(file).catch(failedIn(file));
+          }
+
           const _meta = file.get()._meta || {};
           const version = _meta.version || 1;
           this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
-          return Promise.resolve(
-            require(`./executors/versioned/v${version}`).down(
-              this,
-              this.driver,
-              file
+          return this._leavingRelease(file)
+            .then(leaving =>
+              require(`./executors/versioned/v${version}`)
+                .down(this, this.driver, file)
+                .then(() => leaving && Release.revert(this, leaving))
             )
-          ).catch(failedIn(file));
+            .catch(failedIn(file));
         });
       })
     ).nodeify(callback);

package/lib/work.js

@@ -8,6 +8,7 @@ const State = require('./state');
 const File = require('./file');
 const Jobs = require('./jobs');
 const Dml = require('./dml');
+const Release = require('./release');
 
 /**
  * Runs the jobs of background migrations, see Jobs.
@@ -85,7 +86,7 @@ const hashFile = file =>
     .update(require('fs').readFileSync(file.path))
     .digest('hex');
 
-const runJob = async (walker, name, id, options, stopping, result) => {
+const runJob = async (walker, name, id, options, stopping, result, seen) => {
   const internals = walker.internals;
   const pdriver = walker._pdriver;
   let lost = false;
@@ -100,6 +101,8 @@ const runJob = async (walker, name, id, options, stopping, result) => {
     internals
   );
   const file = files.find(f => f.name === name);
+  // the release of the migration, a soft delete schedules its purge by it
+  internals.releaseLabel = (Release.releases(files).map[name] || {}).label || null;
 
   if (!file) {
     await update(job => {
@@ -153,7 +156,15 @@ const runJob = async (walker, name, id, options, stopping, result) => {
       {
         batch: options.batch,
         pause: options.pause,
-        stopping: () => lost || stopping()
+        stopping: () => lost || stopping(),
+        yielding: async () => {
+          if (await Jobs.paused(pdriver, internals, seen)) {
+            log.info(`[jobs] ${name} pauses for migrations`);
+            return true;
+          }
+
+          return false;
+        }
       },
       {
         rollingBack: () => update(job => { job.rb = 1; }),
@@ -227,8 +238,14 @@ module.exports = function executeWork (internals, config, opts = {}) {
           options.timeout
         );
 
+        if (name === Jobs.WAIT) {
+          // paused by migrations, the jobs continue afterwards
+          await delay(options.interval);
+          continue;
+        }
+
         if (name) {
-          await runJob(walker, name, id, options, stopping, result);
+          await runJob(walker, name, id, options, stopping, result, seen);
           continue;
         }
 

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.4.0",
+  "version": "1.5.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },