db-migrate 1.4.0 → 1.4.1

package/lib/dml.js

@@ -282,6 +282,11 @@ class Dml {
       throw new Stopped();
     }
 
+    // migrations paused the jobs
+    if (this.work.yielding && (await this.work.yielding())) {
+      throw new Stopped();
+    }
+
     if (this.work.pause) {
       await delay(this.work.pause);
     }

package/lib/jobs.js

@@ -2,6 +2,7 @@
 
 const crypto = require('crypto');
 const log = require('db-migrate-shared').log;
+const State = require('./state');
 
 /**
  * Background migrations, dml migrations with _meta.background, are not run
@@ -20,6 +21,12 @@ const log = require('db-migrate-shared').log;
  *     in the foreground, see State
  * h: hash of the migration file when it started
  * err: why it failed
+ *
+ * Migrations take precedence over jobs. Whoever holds the migration lock
+ * pauses the jobs before migrating: no job is taken anymore, the running
+ * ones stop after their current batch, and are continued once the lock is
+ * released. The pause is stored with the jobs, so taking a job and pausing
+ * never miss each other. It holds as long as its holder holds the lock.
  */
 
 const JOBS = '__dbmigrate_jobs__';
@@ -28,26 +35,30 @@ const nonce = () => crypto.randomBytes(8).toString('hex');
 
 const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 
+const WAIT = { wait: true };
+
 const read = async (pdriver, internals) => {
   const row = await pdriver._getKV(internals.migrationState, JOBS);
-  return { row, jobs: row ? JSON.parse(row.value).jobs : {} };
+  const state = row ? JSON.parse(row.value) : { jobs: {} };
+  return { row, state, jobs: state.jobs };
 };
 
 const list = async (pdriver, internals) => (await read(pdriver, internals)).jobs;
 
 /**
- * Changes the jobs by mutate, retried until no other worker wrote in
- * between. mutate returns false to leave them as they are.
+ * Changes the jobs by mutate(jobs, state), retried until no other worker
+ * wrote in between. mutate returns false to leave them as they are.
  */
 const change = async (pdriver, internals, mutate) => {
   for (let attempt = 0; ; attempt++) {
-    const { row, jobs } = await read(pdriver, internals);
+    const { row, state, jobs } = await read(pdriver, internals);
 
-    if (mutate(jobs) === false) {
+    if (mutate(jobs, state) === false) {
       return jobs;
     }
 
-    const value = JSON.stringify({ jobs, n: nonce() });
+    state.n = nonce();
+    const value = JSON.stringify(state);
 
     if (!row) {
       try {
@@ -108,19 +119,61 @@ const register = async (pdriver, internals, file) => {
   log.info(`[jobs] ${file.name} runs in the background`);
 };
 
+/**
+ * Whether the jobs are paused by a migration. A pause holds as long as its
+ * holder holds the migration lock and keeps it alive, the lock row did not
+ * stay the same for the lock timeout, measured on our clock in seen.
+ *
+ * Resolves to the pause, or null.
+ */
+const paused = async (pdriver, internals, seen = {}) => {
+  const { state } = await read(pdriver, internals);
+
+  if (!state.pause) {
+    return null;
+  }
+
+  const lock = await State.readLock(pdriver, internals);
+  if (!lock || State.holder(lock) !== state.pause.ID) {
+    return null;
+  }
+
+  const now = Date.now();
+  const last = seen.lock;
+  if (!last || last.value !== lock.value || String(last.run_on) !== String(lock.run_on)) {
+    seen.lock = { value: lock.value, run_on: lock.run_on, since: now };
+  } else if (now - last.since >= State.lockOptions(internals).timeout) {
+    return null;
+  }
+
+  return state.pause;
+};
+
 /**
  * Takes the first job to run, in the order of their names. A running job is
  * taken over if it did not change for timeout ms, measured on our clock in
  * seen. Jobs after a blocking job wait until it is done.
  *
- * Resolves to the name of the job taken, or null.
+ * Resolves to the name of the job taken, null if there is none, or WAIT
+ * while the jobs are paused.
  */
 const claim = async (pdriver, internals, id, seen, timeout) => {
   const now = Date.now();
+  const valid = await paused(pdriver, internals, seen);
   let taken = null;
+  let wait = false;
 
-  await change(pdriver, internals, jobs => {
+  await change(pdriver, internals, (jobs, state) => {
     taken = null;
+    // a pause found invalid before is ignored, one set in between holds
+    wait = !!state.pause && (!!valid || state.pause.n !== seen.invalid);
+    if (state.pause && !valid) {
+      seen.invalid = seen.invalid || state.pause.n;
+    }
+
+    if (wait) {
+      return false;
+    }
 
     for (const name of Object.keys(jobs).sort()) {
       const job = jobs[name];
@@ -154,9 +207,81 @@ const claim = async (pdriver, internals, id, seen, timeout) => {
     return false;
   });
 
+  if (wait) {
+    return WAIT;
+  }
+
   return taken;
 };
 
+const isRunning = (job, id) => job.s === 'running' && job.ID !== id;
+
+/**
+ * Pauses the jobs for the migrations of the lock holder id, and waits until
+ * the running jobs stopped. A job whose worker does not respond for the lock
+ * timeout is not waited for, it is continued by another worker later.
+ */
+const pause = async (pdriver, internals, id) => {
+  const { timeout, interval } = State.lockOptions(internals);
+  const seen = {};
+
+  await change(pdriver, internals, (jobs, state) => {
+    state.pause = { ID: id, n: nonce() };
+  });
+
+  for (;;) {
+    const now = Date.now();
+    const jobs = await list(pdriver, internals);
+    const running = Object.keys(jobs).filter(name => {
+      const job = jobs[name];
+
+      if (!isRunning(job, id)) {
+        return false;
+      }
+
+      const last = seen[name];
+      if (!last || last.n !== job.n) {
+        seen[name] = { n: job.n, since: now };
+        return true;
+      }
+
+      return now - last.since < timeout;
+    });
+
+    if (!running.length) {
+      return;
+    }
+
+    log.info(
+      `[jobs] waiting for ${running.length} background job(s) to pause: ` +
+        running.join(', ')
+    );
+    // they stop after their current batch, usually soon
+    await delay(Math.min(interval, 200));
+  }
+};
+
+/**
+ * Forgets a job, once its migration is reverted.
+ */
+const remove = (pdriver, internals, name) =>
+  change(pdriver, internals, jobs => {
+    if (!jobs[name]) {
+      return false;
+    }
+
+    delete jobs[name];
+  });
+
+const resume = (pdriver, internals, id) =>
+  change(pdriver, internals, (jobs, state) => {
+    if (!state.pause || state.pause.ID !== id) {
+      return false;
+    }
+
+    delete state.pause;
+  });
+
 /**
  * Changes a job we run. If another worker took it over in between, lost is
  * called and nothing is changed.
@@ -174,4 +299,4 @@ const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
     job.n = nonce();
   });
 
-module.exports = { JOBS, list, register, claim, update };
+module.exports = { JOBS, WAIT, list, register, claim, update, paused, pause, resume, remove };

package/lib/state.js

@@ -155,6 +155,12 @@ const claim = (row, driver, internals) => {
 module.exports = {
   createConcurrently,
 
+  // the id of this process in the lock, the holder of a lock row, see Jobs
+  ID,
+  holder,
+  lockOptions,
+  readLock: read,
+
   // for tests, simulates a process which died while holding the lock
   __stopHeartbeat: stopHeartbeat,
 

package/lib/walker.js

@@ -127,10 +127,20 @@ Walker.prototype = {
       return run(toRun);
     }
 
+    let paused = false;
     try {
+      // migrations take precedence over the jobs of background migrations
+      await Jobs.pause(this._pdriver, this.internals, State.ID);
+      paused = true;
       await State.reloadSchema(this._pdriver, this.internals);
       return await run(await determine());
     } finally {
+      if (paused) {
+        await Jobs.resume(this._pdriver, this.internals, State.ID).catch(err =>
+          log.warn(`[jobs] could not resume the jobs: ${err.message}`)
+        );
+      }
+
       await State.release(this._pdriver, this.internals);
     }
   },
@@ -158,6 +168,52 @@ Walker.prototype = {
     });
   },
 
+  /**
+   * The migrations of the jobs, newest first.
+   */
+  _jobFiles: async function () {
+    if (this.prefix !== 'migration' || !this._pdriver || typeof this._pdriver._getKV !== 'function') {
+      return [];
+    }
+
+    const jobs = await Jobs.list(this._pdriver, this.internals);
+    const names = Object.keys(jobs);
+    if (!names.length) {
+      return [];
+    }
+
+    const files = await File.loadFromFileystem(this.directory, this.prefix, this.internals);
+    return names
+      .sort()
+      .reverse()
+      .map(name => {
+        const file = files.find(f => f.name === name);
+        if (!file) {
+          throw new Error(
+            `The migration of the background job "${name}" is missing, it ` +
+              'can not be reverted'
+          );
+        }
+
+        file.job = true;
+        return file;
+      });
+  },
+
+  /**
+   * Reverts what the job of a background migration did so far, it was
+   * paused by taking the lock, and forgets the job.
+   */
+  _revertJob: async function (file) {
+    log.info(`${this.title}reverting the background job of ${file.name}`);
+    this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
+
+    if (!this.internals.dryRun) {
+      await Dml.down(this, this.driver, file);
+      await Jobs.remove(this._pdriver, this.internals, file.name);
+    }
+  },
+
   _registerJob: async function (file) {
     Dml.checkType(file);
 
@@ -291,8 +347,13 @@ Walker.prototype = {
         this.prefix,
         this._driver,
         this.internals
-      ).then(completedFiles =>
-        dbmUtil.filterDown(completedFiles, partialName, count)
+      ).then(async completedFiles =>
+        dbmUtil.filterDown(
+          // jobs of background migrations are the latest, still running
+          (await this._jobFiles()).concat(completedFiles),
+          partialName,
+          count
+        )
       );
 
     if (this.internals.check) {
@@ -309,6 +370,10 @@ Walker.prototype = {
 
         return Promise.each(toRun, file => {
           log.verbose(this.title + 'preparing to run down:', file.name);
+          if (file.job) {
+            return this._revertJob(file).catch(failedIn(file));
+          }
+
           const _meta = file.get()._meta || {};
           const version = _meta.version || 1;
           this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };

package/lib/work.js

@@ -85,7 +85,7 @@ const hashFile = file =>
     .update(require('fs').readFileSync(file.path))
     .digest('hex');
 
-const runJob = async (walker, name, id, options, stopping, result) => {
+const runJob = async (walker, name, id, options, stopping, result, seen) => {
   const internals = walker.internals;
   const pdriver = walker._pdriver;
   let lost = false;
@@ -153,7 +153,15 @@ const runJob = async (walker, name, id, options, stopping, result) => {
       {
         batch: options.batch,
         pause: options.pause,
-        stopping: () => lost || stopping()
+        stopping: () => lost || stopping(),
+        yielding: async () => {
+          if (await Jobs.paused(pdriver, internals, seen)) {
+            log.info(`[jobs] ${name} pauses for migrations`);
+            return true;
+          }
+
+          return false;
+        }
       },
       {
         rollingBack: () => update(job => { job.rb = 1; }),
@@ -227,8 +235,14 @@ module.exports = function executeWork (internals, config, opts = {}) {
           options.timeout
         );
 
+        if (name === Jobs.WAIT) {
+          // paused by migrations, the jobs continue afterwards
+          await delay(options.interval);
+          continue;
+        }
+
         if (name) {
-          await runJob(walker, name, id, options, stopping, result);
+          await runJob(walker, name, id, options, stopping, result, seen);
           continue;
         }
 

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.4.0",
+  "version": "1.4.1",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },