db-migrate 1.6.0 → 1.7.0

package/api.js

@@ -421,6 +421,27 @@ dbmigrate.prototype = {
     this.internals.onComplete = callback;
   },
 
+  /**
+   * What db-migrate knows about the database: pending migrations, the lock,
+   * the jobs of background migrations, the release and what is deprecated
+   * or due, see lib/status.js.
+   */
+  status: function (scope, callback) {
+    if (typeof scope === 'function') {
+      callback = scope;
+      scope = undefined;
+    }
+
+    if (scope) {
+      this.internals.migrationMode = scope;
+      this.internals.matching = scope;
+    }
+
+    return Promise.resolve(load('status')(this.internals, this.config)).asCallback(
+      callback
+    );
+  },
+
   /**
    * Runs the jobs of background migrations, see lib/work.js for the
    * options. Returns { done, stop }, done resolves with the jobs done and

package/lib/commands/generated.js

@@ -23,6 +23,7 @@ const files = {
   "run": require('./run.js'),
   "seed": require('./seed.js'),
   "set-default-argv": require('./set-default-argv.js'),
+  "status": require('./status.js'),
   "sync": require('./sync.js'),
   "undo-seed": require('./undo-seed.js'),
   "up": require('./up.js'),

package/lib/commands/run.js

@@ -97,6 +97,19 @@ function run (internals, config) {
         report(load('db')(internals, config));
       }
       break;
+    case 'status':
+      if (folder[1]) {
+        internals.matching = folder[1];
+        internals.migrationMode = folder[1];
+      }
+
+      report(
+        load('status')(internals, config).then(result => {
+          console.log(require('../status').format(result));
+        })
+      );
+      break;
+
     case 'work': {
       // runs the jobs of background migrations, until none is left or, with
       // --watch, until stopped
@@ -166,7 +179,7 @@ function run (internals, config) {
         );
       } else {
         log.error(
-          'Invalid Action: Must be [up|down|check|create|reset|sync|db|seed|work].'
+          'Invalid Action: Must be [up|down|check|create|reset|sync|db|seed|work|status].'
         );
         yargs.showHelp();
         process.exit(1);

package/lib/commands/set-default-argv.js

@@ -27,7 +27,7 @@ module.exports = function (internals, isModule) {
     internals.argv = yargs
       .default(defaultConfig)
       .usage(
-        'Usage: db-migrate [up|down|check|reset|sync|create|db|seed|work] ' +
+        'Usage: db-migrate [up|down|check|reset|sync|create|db|seed|work|status] ' +
           '[[dbname/]migrationName|all] [options]'
       )
       .describe(

package/lib/commands/status.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const path = require('path');
+const migrationHook = require('./helper/migration-hook.js');
+const status = require('../status');
+
+/**
+ * The status of the database, see lib/status.js.
+ */
+module.exports = async function (internals, config) {
+  await migrationHook(internals);
+  const Migrator = require('../walker.js');
+  const index = require('../../connect');
+
+  const migrator = await index.connect(
+    {
+      config: config.getCurrent().settings,
+      internals: internals,
+      prefix: 'migration'
+    },
+    Migrator
+  );
+
+  migrator.directory = path.resolve(
+    internals.argv['migrations-dir'],
+    internals.locTitle || ''
+  );
+
+  try {
+    await migrator.createMigrationsTable();
+    const result = await status(migrator);
+    return internals.onComplete(migrator, internals, null, result);
+  } catch (err) {
+    return internals.onComplete(migrator, internals, err);
+  }
+};

package/lib/dml.js

@@ -798,6 +798,35 @@ class Dml {
     );
   }
 
+  /**
+   * dropBackups([migration]), drops the backups of update and delete of the
+   * migrations due, or of the one named, due or not. They can not be
+   * reverted afterwards, nor this migration.
+   */
+  dropBackups (migration) {
+    return this.step(
+      'dropBackups',
+      migration ? [migration] : [],
+      () => ({ t: 3, a: 'dropBackups', c: [migration || null] }),
+      async () => {
+        const Release = require('./release');
+        const groups = await Release.backupsOf(
+          this.pdriver,
+          this.internals,
+          this.internals.releaseCurrent || 0,
+          this.internals.releaseIndex || {},
+          !!migration
+        );
+
+        for (const group of groups) {
+          if (!migration || group.m === migration) {
+            await Release.dropBackups(this.driver, this.pdriver, this.internals, group);
+          }
+        }
+      }
+    );
+  }
+
   /**
    * Reading is no step, it does not change anything.
    */
@@ -835,6 +864,14 @@ class Dml {
     entry.b = 1;
     entry.o = 0;
     await this.save();
+    // dropped once the migration is final, after enough releases
+    await require('./release').registerBackup(
+      this.pdriver,
+      this.internals,
+      entry.c[1],
+      this.file.name,
+      table
+    );
   }
 
   /**
@@ -991,6 +1028,7 @@ const revert = async (context, file, internals, pdriver, d) => {
         entry.b = 2;
         await d.save();
         await d.driver.runSql(`DROP TABLE IF EXISTS ${d.ddl(entry.c[1])}`);
+        await require('./release').forgetBackups(pdriver, internals, [entry.c[1]]);
       }
     }
 

package/lib/learn.js

@@ -145,10 +145,12 @@ class STD {
   }
 
   renameTable (t, n) {
-    // deprecations move with the table
+    // deprecations move with the table, unless it is renamed for its own
+    // deprecation, which stays with its name
     ['tables', 'columns'].forEach(kind => {
       const d = this.deprecated[kind];
-      if (d && d[t] !== undefined) {
+      const own = kind === 'tables' && d && d[t] && d[t].to === n;
+      if (d && d[t] !== undefined && !own) {
         d[n] = d[t];
         delete d[t];
       }

package/lib/release.js

@@ -28,6 +28,7 @@ const log = require('db-migrate-shared').log;
 
 const PREFIX = '__dbmigrate_release__:';
 const PURGES = '__dbmigrate_purges__';
+const BACKUPS = '__dbmigrate_backups__';
 const DEFAULTS = { releases: 4, drop: 'manual' };
 const DROPS = ['auto', 'manual'];
 
@@ -221,13 +222,18 @@ const purges = async (pdriver, internals) => {
   return { row, value: row ? JSON.parse(row.value) : {} };
 };
 
-const changePurges = async (pdriver, internals, mutate) => {
+/**
+ * Changes a row of the state by compare and swap, retried until nobody
+ * wrote in between. mutate returns false to leave it as it is.
+ */
+const changeState = async (pdriver, internals, key, mutate) => {
   if (internals.dryRun) {
     return;
   }
 
   for (let attempt = 0; attempt < 50; attempt++) {
-    const { row, value } = await purges(pdriver, internals);
+    const row = await pdriver._getKV(internals.migrationState, key);
+    const value = row ? JSON.parse(row.value) : {};
     if (mutate(value) === false) {
       return;
     }
@@ -235,23 +241,26 @@ const changePurges = async (pdriver, internals, mutate) => {
     const next = JSON.stringify(value);
     if (!row) {
       try {
-        await pdriver._insertKV(internals.migrationState, PURGES, next);
+        await pdriver._insertKV(internals.migrationState, key, next);
       } catch (err) {
-        log.verbose('[release] purges inserted concurrently', err.message);
+        log.verbose(`[release] ${key} inserted concurrently`, err.message);
       }
     } else {
-      await pdriver._updateKVC(internals.migrationState, PURGES, next, 'value', row.value);
+      await pdriver._updateKVC(internals.migrationState, key, next, 'value', row.value);
     }
 
-    const after = await pdriver._getKV(internals.migrationState, PURGES);
+    const after = await pdriver._getKV(internals.migrationState, key);
     if (after && after.value === next) {
       return;
     }
   }
 
-  throw new Error('[release] could not change the purges, too much contention');
+  throw new Error(`[release] could not change ${key}, too much contention`);
 };
 
+const changePurges = (pdriver, internals, mutate) =>
+  changeState(pdriver, internals, PURGES, mutate);
+
 /**
  * Schedules purging the rows of a soft delete, { purge: true } takes the
  * options of the project.
@@ -285,6 +294,77 @@ const forgetPurges = (pdriver, internals, t, prefix) =>
     marks.forEach(mark => delete value[mark]);
   });
 
+/**
+ * The backup tables of update and delete in copy mode, kept to revert their
+ * migration: { [backup]: { m: migration, t: table, r: release } }. Once
+ * enough releases passed, the migration is final, its backups are dropped.
+ */
+const registerBackup = (pdriver, internals, backup, migration, t) =>
+  changeState(pdriver, internals, BACKUPS, value => {
+    value[backup] = { m: migration, t, r: internals.releaseLabel || null };
+  });
+
+const forgetBackups = (pdriver, internals, backups) =>
+  changeState(pdriver, internals, BACKUPS, value => {
+    if (!backups.some(b => value[b])) {
+      return false;
+    }
+
+    backups.forEach(b => delete value[b]);
+  });
+
+/**
+ * The migrations with backups, with their age and options, those due only
+ * unless all.
+ */
+const backupsOf = async (pdriver, internals, index, releaseIndex, all = false) => {
+  const row = await pdriver._getKV(internals.migrationState, BACKUPS);
+  const value = row ? JSON.parse(row.value) : {};
+  const migrations = {};
+
+  Object.keys(value).forEach(backup => {
+    const e = value[backup];
+    if (!migrations[e.m]) {
+      const age = index - (e.r === null ? 0 : releaseIndex[e.r] || 0);
+      migrations[e.m] = Object.assign({ m: e.m, age, backups: [] }, settings(internals, e));
+    }
+
+    migrations[e.m].backups.push(backup);
+  });
+
+  return Object.keys(migrations)
+    .map(m => migrations[m])
+    .filter(g => all || g.age >= g.releases);
+};
+
+/**
+ * Drops the backups of a migration, which can not be reverted afterwards:
+ * its steps are recorded as irreversible first.
+ */
+const dropBackups = async (driver, pdriver, internals, group) => {
+  log.info(`[release] dropping the backups of ${group.m}, it can not be reverted anymore`);
+  if (internals.dryRun) {
+    return;
+  }
+
+  const row = await pdriver._getKV(internals.migrationState, group.m);
+  if (row) {
+    const state = JSON.parse(row.value);
+    state.s = (state.s || []).map(e =>
+      e.t === 4 && group.backups.indexOf(e.c[1]) !== -1
+        ? { t: 3, a: e.a, c: [e.c[0]], n: e.n }
+        : e
+    );
+    await pdriver._updateKV(internals.migrationState, group.m, JSON.stringify(state));
+  }
+
+  for (const backup of group.backups) {
+    await driver.runSql(`DROP TABLE IF EXISTS ${driver.escapeDDL(backup)}`);
+  }
+
+  await forgetBackups(pdriver, internals, group.backups);
+};
+
 const duePurges = async (pdriver, internals, index, releaseIndex) => {
   const { value } = await purges(pdriver, internals);
 
@@ -326,6 +406,14 @@ const start = async (walker, label, index, releaseIndex, mode = 'up') => {
     if (purged.length) {
       await purge(walker, label, purged);
     }
+
+    const final = (
+      await backupsOf(walker._pdriver, internals, index, releaseIndex)
+    ).filter(g => g.drop === 'auto');
+
+    for (const group of final) {
+      await dropBackups(walker._driver, walker._pdriver, internals, group);
+    }
   }
 
   if (!renames.length && !drops.length) {
@@ -463,6 +551,19 @@ const assertRevertible = async (walker, label) => {
 /**
  * Warns about what is due for dropping with drop 'manual'.
  */
+const warnDueBackups = async (pdriver, internals, index, releaseIndex) => {
+  (await backupsOf(pdriver, internals, index, releaseIndex))
+    .filter(g => g.drop === 'manual')
+    .forEach(g =>
+      log.warn(
+        `[release] the backups of ${g.m} are due for dropping, it ran ` +
+          `${g.age} releases ago. Drop them in a dml migration with ` +
+          `db.dropBackups(${JSON.stringify(g.m)}), it can not be reverted ` +
+          'afterwards.'
+      )
+    );
+};
+
 const warnDuePurges = async (pdriver, internals, index, releaseIndex) => {
   (await duePurges(pdriver, internals, index, releaseIndex))
     .filter(e => e.drop === 'manual')
@@ -495,6 +596,12 @@ const warnDue = (internals, index, releaseIndex) => {
 module.exports = {
   PREFIX,
   PURGES,
+  BACKUPS,
+  registerBackup,
+  forgetBackups,
+  backupsOf,
+  dropBackups,
+  warnDueBackups,
   schedulePurge,
   forgetPurges,
   warnDuePurges,

package/lib/status.js

@@ -0,0 +1,179 @@
+'use strict';
+
+const dbmUtil = require('db-migrate-shared').util;
+const File = require('./file');
+const State = require('./state');
+const Jobs = require('./jobs');
+const Release = require('./release');
+
+/**
+ * What db-migrate knows about the database, without changing anything:
+ *
+ * pending: migrations not run yet, background migrations running excluded
+ * lock: the process holding the migration lock, and the migration it was
+ *       interrupted in, if any
+ * jobs: the jobs of background migrations, paused by whom if so
+ * release: the latest release migrated
+ * deprecated: tables and columns deprecated, renamed or not, and when due
+ * purges, backups: soft deletes and backups due
+ */
+module.exports = async function status (walker) {
+  const internals = walker.internals;
+  const pdriver = walker._pdriver;
+  const supports = State.supportsLock(pdriver);
+
+  const [allFiles, completed] = await Promise.all([
+    File.loadFromFileystem(walker.directory, walker.prefix, internals).catch(err => {
+      if (err.code === 'ENOENT') {
+        return [];
+      }
+
+      throw err;
+    }),
+    File.loadFromDatabase(walker.directory, walker.prefix, walker._driver, internals)
+  ]);
+
+  walker._releases(allFiles, completed);
+  const release = walker._release;
+  const index = walker._applied;
+
+  const jobs = supports ? await Jobs.list(pdriver, internals) : {};
+  const pending = dbmUtil
+    .filterUp(allFiles, completed, undefined, Number.MAX_VALUE)
+    .filter(file => !jobs[file.name] || jobs[file.name].s === 'failed')
+    .map(file => file.name);
+
+  const result = {
+    pending,
+    lock: null,
+    jobs: [],
+    paused: null,
+    release: null,
+    deprecated: [],
+    purges: [],
+    backups: []
+  };
+
+  const labels = Object.keys(release.index);
+  result.release = labels.find(label => release.index[label] === index) || null;
+
+  if (supports) {
+    const lock = await State.readLock(pdriver, internals);
+    if (lock) {
+      const s = JSON.parse(lock.value).s;
+      result.lock = {
+        held: !!s.ID,
+        since: s.date,
+        interrupted: s.fin === 0 && s.f ? { migration: s.f, op: s.o, step: s.step, done: s.done } : null
+      };
+    }
+
+    const paused = await Jobs.paused(pdriver, internals);
+    result.paused = paused ? paused.ID : null;
+    result.jobs = Object.keys(jobs)
+      .sort()
+      .map(name => {
+        const job = jobs[name];
+        return {
+          name,
+          state: job.s,
+          blocking: !!job.blocking,
+          step: job.step,
+          done: job.done,
+          error: job.err || null
+        };
+      });
+
+    const due = (e, age) => {
+      const options = Release.settings(internals, e);
+      return Object.assign({ age, due: age >= options.releases }, options);
+    };
+
+    result.deprecated = Release.list(internals, index, release.index).map(item =>
+      Object.assign(
+        {
+          table: item.t,
+          column: item.c || null,
+          name: item.name,
+          renamed: item.renamed,
+          release: item.entry.r
+        },
+        due(item.entry, item.age)
+      )
+    );
+
+    const purges = await pdriver._getKV(internals.migrationState, Release.PURGES);
+    const scheduled = purges ? JSON.parse(purges.value) : {};
+    result.purges = Object.keys(scheduled).map(mark => {
+      const e = scheduled[mark];
+      const age = index - (e.r === null ? 0 : release.index[e.r] || 0);
+      return Object.assign({ table: e.t, by: mark.slice(5), release: e.r }, due(e, age));
+    });
+
+    result.backups = (
+      await Release.backupsOf(pdriver, internals, index, release.index, true)
+    ).map(g => ({
+      migration: g.m,
+      tables: g.backups,
+      age: g.age,
+      due: g.age >= g.releases,
+      releases: g.releases,
+      drop: g.drop
+    }));
+  }
+
+  return result;
+};
+
+/**
+ * The status as text, for the command line.
+ */
+module.exports.format = s => {
+  const lines = [];
+  const section = (title, items, line) => {
+    lines.push(`${title}:${items.length ? '' : ' none'}`);
+    items.forEach(item => lines.push(`  ${line(item)}`));
+  };
+
+  section('Pending migrations', s.pending, name => name);
+  lines.push(`Release: ${s.release === null ? 'none' : s.release}`);
+
+  if (s.lock) {
+    lines.push(
+      `Migration lock: ${s.lock.held ? `held since ${s.lock.since}` : 'free'}` +
+        (s.lock.interrupted
+          ? `, ${s.lock.interrupted.op} of ${s.lock.interrupted.migration} ` +
+            `interrupted at step ${s.lock.interrupted.step}`
+          : '')
+    );
+  }
+
+  section(
+    `Background jobs${s.paused ? ' (paused for migrations)' : ''}`,
+    s.jobs,
+    job =>
+      `${job.name}: ${job.state}${job.blocking ? ', blocking' : ''}` +
+      (job.step ? `, step ${job.step}, ${job.done} done` : '') +
+      (job.error ? `, ${job.error}` : '')
+  );
+  section(
+    'Deprecated',
+    s.deprecated,
+    d =>
+      `${d.column ? `column "${d.column}" of "${d.table}"` : `table "${d.table}"`}` +
+      `${d.renamed ? ` renamed to ${d.name}` : ''}, ${d.age} of ${d.releases} ` +
+      `releases${d.due ? `, due to drop (${d.drop})` : ''}`
+  );
+  section(
+    'Purges',
+    s.purges,
+    p => `"${p.table}" by ${p.by}, ${p.age} of ${p.releases} releases${p.due ? `, due (${p.drop})` : ''}`
+  );
+  section(
+    'Backups',
+    s.backups,
+    b => `${b.migration}: ${b.tables.length} table(s), ${b.age} of ${b.releases} releases${b.due ? `, due (${b.drop})` : ''}`
+  );
+
+  return lines.join('\n');
+};

package/lib/walker.js

@@ -425,6 +425,12 @@ Walker.prototype = {
         this._applied,
         this._release.index
       );
+      await Release.warnDueBackups(
+        this._pdriver,
+        this.internals,
+        this._applied,
+        this._release.index
+      );
     }
   },
 

package/lib/work.js

@@ -102,7 +102,10 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
   );
   const file = files.find(f => f.name === name);
   // the release of the migration, a soft delete schedules its purge by it
-  internals.releaseLabel = (Release.releases(files).map[name] || {}).label || null;
+  const release = Release.releases(files);
+  internals.releaseLabel = (release.map[name] || {}).label || null;
+  internals.releaseIndex = release.index;
+  internals.releaseCurrent = (release.map[name] || {}).index || 0;
 
   if (!file) {
     await update(job => {

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.6.0",
+  "version": "1.7.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },