db-migrate 1.5.0 → 1.6.0

package/lib/chain.js

@@ -190,7 +190,7 @@ class Chain {
         throw unknownTable(t);
       }
 
-      const entry = Release.entry(internals, t, options);
+      const entry = Release.entry(internals, t, options, this.file);
       return this.run('setDeprecated', ['tables', t, null, entry], 'learn');
     };
 
@@ -205,7 +205,7 @@ class Chain {
         throw unknownColumn(t, c);
       }
 
-      const entry = Release.entry(internals, c, options);
+      const entry = Release.entry(internals, c, options, this.file);
       if (columns[c].notNull) {
         // the application stops writing it. The whole spec, MySQL changes
         // a column by its full definition

package/lib/commands/run.js

@@ -97,6 +97,49 @@ function run (internals, config) {
         report(load('db')(internals, config));
       }
       break;
+    case 'work': {
+      // runs the jobs of background migrations, until none is left or, with
+      // --watch, until stopped
+      const argv = internals.argv;
+      const options = {};
+      [
+        ['parallel', 'parallel'],
+        ['pause', 'pause'],
+        ['batch', 'batch'],
+        ['interval', 'interval'],
+        ['job-timeout', 'timeout']
+      ].forEach(([arg, option]) => {
+        if (argv[arg] !== undefined) {
+          options[option] = Number(argv[arg]);
+        }
+      });
+      options.watch = argv.watch === true;
+
+      const worker = load('work')(internals, config, options);
+      const stop = () => {
+        log.info('[jobs] stopping after the current batches');
+        worker.stop();
+      };
+      process.once('SIGINT', stop);
+      process.once('SIGTERM', stop);
+
+      report(
+        worker.done.then(result => {
+          log.info(`[jobs] ${result.done.length} job(s) done`);
+
+          if (result.failed.length) {
+            throw new Error(
+              `${result.failed.length} background job(s) failed: ` +
+                result.failed.map(f => f.name).join(', ')
+            );
+          }
+
+          process.exit(0);
+        })
+      );
+      break;
+    }
+
     case 'seed': {
       // seed [name], seed down [name], seed reset
       const args = internals.argv._.map(String);
@@ -123,7 +166,7 @@ function run (internals, config) {
         );
       } else {
         log.error(
-          'Invalid Action: Must be [up|down|check|create|reset|sync|db|seed].'
+          'Invalid Action: Must be [up|down|check|create|reset|sync|db|seed|work].'
         );
         yargs.showHelp();
         process.exit(1);

package/lib/commands/set-default-argv.js

@@ -27,7 +27,7 @@ module.exports = function (internals, isModule) {
     internals.argv = yargs
       .default(defaultConfig)
       .usage(
-        'Usage: db-migrate [up|down|check|reset|sync|create|db|seed] ' +
+        'Usage: db-migrate [up|down|check|reset|sync|create|db|seed|work] ' +
           '[[dbname/]migrationName|all] [options]'
       )
       .describe(
@@ -128,6 +128,22 @@ module.exports = function (internals, isModule) {
           ' held by another process is considered stale and taken over.'
       )
       .number('lock-timeout')
+      .describe('parallel', 'work: jobs of background migrations run at once.')
+      .number('parallel')
+      .describe('pause', 'work: milliseconds to wait between two batches of a job.')
+      .number('pause')
+      .describe('batch', 'work: rows per batch of a job.')
+      .number('batch')
+      .describe('watch', 'work: keep looking for new jobs, until stopped.')
+      .boolean('watch')
+      .describe('interval', 'work: milliseconds between looking for new jobs.')
+      .number('interval')
+      .describe(
+        'job-timeout',
+        'work: milliseconds after which the job of a worker which stopped ' +
+          'responding is taken over.'
+      )
+      .number('job-timeout')
       .describe(
         'lock-interval',
         'Milliseconds between checks while waiting for the migration lock.'

package/lib/release.js

@@ -67,10 +67,11 @@ const releases = files => {
 };
 
 /**
- * The name a deprecated table or column is renamed to.
+ * The name a deprecated table or column is renamed to, with the time of the
+ * migration deprecating it, so fix learns the same name again.
  */
-const hiddenName = name => {
-  const time = Math.floor(Date.now() / 1000);
+const hiddenName = (name, file) => {
+  const time = String(file.name).split('-')[0];
   const plain = `__dbm_deprecated_${name}_${time}`;
 
   if (plain.length <= 60) {
@@ -116,8 +117,8 @@ const settings = (internals, entry) => {
   return Object.assign({}, DEFAULTS, project, entry.o || {});
 };
 
-const entry = (internals, name, options) => {
-  const e = { r: internals.releaseLabel || null, to: hiddenName(name) };
+const entry = (internals, name, options, file) => {
+  const e = { r: internals.releaseLabel || null, to: hiddenName(name, file) };
   const o = checkOptions(options);
 
   if (Object.keys(o).length) {
@@ -310,24 +311,28 @@ const unrecorded = walker =>
   });
 
 /**
- * Runs the steps due before the first migration of the release label.
+ * Runs the steps due before the first migration of the release label. fix
+ * only learns them again, like the migrations it learns.
  */
-const start = async (walker, label, index, releaseIndex) => {
+const start = async (walker, label, index, releaseIndex, mode = 'up') => {
   const internals = walker.internals;
   const { renames, drops } = due(internals, index, releaseIndex);
-  const purged = (await duePurges(walker._pdriver, internals, index, releaseIndex)).filter(
-    e => e.drop === 'auto'
-  );
 
-  if (purged.length) {
-    await purge(walker, label, purged);
+  if (mode === 'up') {
+    const purged = (
+      await duePurges(walker._pdriver, internals, index, releaseIndex)
+    ).filter(e => e.drop === 'auto');
+
+    if (purged.length) {
+      await purge(walker, label, purged);
+    }
   }
 
   if (!renames.length && !drops.length) {
     return;
   }
 
-  log.info(`[release] starting release ${label}`);
+  log.info(`[release] ${mode === 'fix' ? 'learning' : 'starting'} release ${label}`);
   renames.forEach(item =>
     log.info(`[release] renaming the deprecated ${describe(item)} to ${item.entry.to}`)
   );
@@ -352,7 +357,11 @@ const start = async (walker, label, index, releaseIndex) => {
     return;
   }
 
-  await require('./executors/versioned/v2').up(unrecorded(walker), walker.driver, file);
+  await require('./executors/versioned/v2')[mode](
+    unrecorded(walker),
+    walker.driver,
+    file
+  );
 };
 
 /**

package/lib/state.js

@@ -328,6 +328,9 @@ module.exports = {
       }
     }
 
+    // fix learns the schema from scratch, it must not be loaded again
+    internals.learnFromScratch = emptyState === true;
+
     const schema = emptyState !== true ? _schema : null;
     if (schema) {
       internals.schema = parseSchema(schema.value);
@@ -351,6 +354,10 @@ module.exports = {
    * so the schema loaded during init is outdated.
    */
   reloadSchema: async function (driver, internals) {
+    if (internals.learnFromScratch) {
+      return;
+    }
+
     const schema = await driver._getKV(internals.migrationState, SSTATE);
     if (schema) {
       internals.schema = parseSchema(schema.value);
@@ -376,7 +383,8 @@ module.exports = {
   ) {
     const mig = await driver._getKV(internals.migrationState, file.name);
 
-    if (mig && mig.value !== '{}') {
+    // fix learns the records again from scratch
+    if (mig && mig.value !== '{}' && op !== 'fix') {
       internals.modSchema = JSON.parse(mig.value);
       // records of older runs, they belong to no step of this run
       (internals.modSchema.s || []).forEach(entry => {

package/lib/walker.js

@@ -363,13 +363,13 @@ Walker.prototype = {
   /**
    * Runs the steps due, before the first migration of a new release.
    */
-  _enterRelease: async function (file) {
+  _enterRelease: async function (file, mode = 'up') {
     const release = this._release || { map: {}, index: {} };
     const r = release.map[file.name] || { label: null, index: 0 };
 
     if (r.index > this._applied) {
       this.internals.releaseIndex = release.index;
-      await Release.start(this, r.label, r.index, release.index);
+      await Release.start(this, r.label, r.index, release.index, mode);
       this._applied = r.index;
     }
 
@@ -513,9 +513,23 @@ Walker.prototype = {
         this.prefix,
         this._driver,
         this.internals
-      ).then(completedFiles =>
-        dbmUtil.filterDown(completedFiles, partialName, count).sort(sortFn)
-      );
+      ).then(async completedFiles => {
+        // the releases, learned again from the start
+        const allFiles = await File.loadFromFileystem(
+          this.directory,
+          this.prefix,
+          this.internals
+        ).catch(err => {
+          if (err.code === 'ENOENT') {
+            return [];
+          }
+
+          throw err;
+        });
+        this._releases(allFiles, []);
+
+        return dbmUtil.filterDown(completedFiles, partialName, count).sort(sortFn);
+      });
 
     if (this.internals.check) {
       return determine()
@@ -529,8 +543,9 @@ Walker.prototype = {
           log.info(this.title + 'Nothing to run');
         }
 
-        return Promise.each(toRun, file => {
+        return Promise.each(toRun, async file => {
           log.verbose(this.title + 'preparing to run fix:', file.name);
+          await this._enterRelease(file, 'fix').catch(failedIn(file));
           const _meta = file.get()._meta || {};
           const version = _meta.version || 1;
           if (version < 2) {

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.5.0",
+  "version": "1.6.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },