db-migrate 1.3.0 → 1.4.0

package/lib/driver/index.js

@@ -166,6 +166,8 @@ exports.connect = function (config, intern, callback) {
         }
 
         trackSql(db);
+        // the dialect of parameters in all, $1 instead of ? for pg
+        db._dbmDriver = typeof config.driver === 'string' ? config.driver : null;
         callback(null, db);
       }
     );

package/lib/jobs.js

@@ -0,0 +1,177 @@
+'use strict';
+
+const crypto = require('crypto');
+const log = require('db-migrate-shared').log;
+
+/**
+ * Background migrations, dml migrations with _meta.background, are not run
+ * by up, but registered as jobs. Workers started by executeWork take the
+ * jobs one by one and run them. A migration counts as run once its job is
+ * done, until then it is running.
+ *
+ * All jobs live in one row of the state table, changed by compare and swap
+ * only, so any number of workers can work on them at once:
+ *
+ * s: queued, running or failed
+ * blocking: later jobs wait until this one is done
+ * ID: the worker running it, n: renewed by every write of the worker, a
+ *     job whose n does not change for the timeout is taken over
+ * step, learned, done, rb: the progress of the run, like for migrations run
+ *     in the foreground, see State
+ * h: hash of the migration file when it started
+ * err: why it failed
+ */
+
+const JOBS = '__dbmigrate_jobs__';
+
+const nonce = () => crypto.randomBytes(8).toString('hex');
+
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const read = async (pdriver, internals) => {
+  const row = await pdriver._getKV(internals.migrationState, JOBS);
+  return { row, jobs: row ? JSON.parse(row.value).jobs : {} };
+};
+
+const list = async (pdriver, internals) => (await read(pdriver, internals)).jobs;
+
+/**
+ * Changes the jobs by mutate, retried until no other worker wrote in
+ * between. mutate returns false to leave them as they are.
+ */
+const change = async (pdriver, internals, mutate) => {
+  for (let attempt = 0; ; attempt++) {
+    const { row, jobs } = await read(pdriver, internals);
+
+    if (mutate(jobs) === false) {
+      return jobs;
+    }
+
+    const value = JSON.stringify({ jobs, n: nonce() });
+
+    if (!row) {
+      try {
+        await pdriver._insertKV(internals.migrationState, JOBS, value);
+      } catch (err) {
+        // inserted by another worker first, try again with theirs
+        log.verbose('[jobs] insert failed', err.message);
+      }
+    } else {
+      await pdriver._updateKVC(
+        internals.migrationState,
+        JOBS,
+        value,
+        'value',
+        row.value
+      );
+    }
+
+    const after = await pdriver._getKV(internals.migrationState, JOBS);
+    if (after && after.value === value) {
+      return jobs;
+    }
+
+    if (attempt > 50) {
+      throw new Error('[jobs] could not change the jobs, too much contention');
+    }
+
+    await delay(Math.floor(Math.random() * 20 * Math.min(attempt + 1, 10)));
+  }
+};
+
+/**
+ * Registers a migration as job, or a failed one again. A job which failed
+ * with irreversible steps keeps its progress, so it continues after them.
+ */
+const register = async (pdriver, internals, file) => {
+  const blocking = (file.get()._meta || {}).blocking === true;
+
+  await change(pdriver, internals, jobs => {
+    const job = jobs[file.name];
+
+    if (job && job.s !== 'failed') {
+      return false;
+    }
+
+    jobs[file.name] = Object.assign(
+      { step: 0, learned: 0, done: 0, rb: 0 },
+      job,
+      { s: 'queued', blocking, ID: 0, n: nonce(), err: undefined }
+    );
+  });
+
+  const mig = await pdriver._getKV(internals.migrationState, file.name);
+  if (!mig) {
+    await pdriver._insertKV(internals.migrationState, file.name, '{}');
+  }
+
+  log.info(`[jobs] ${file.name} runs in the background`);
+};
+
+/**
+ * Takes the first job to run, in the order of their names. A running job is
+ * taken over if it did not change for timeout ms, measured on our clock in
+ * seen. Jobs after a blocking job wait until it is done.
+ *
+ * Resolves to the name of the job taken, or null.
+ */
+const claim = async (pdriver, internals, id, seen, timeout) => {
+  const now = Date.now();
+  let taken = null;
+
+  await change(pdriver, internals, jobs => {
+    taken = null;
+
+    for (const name of Object.keys(jobs).sort()) {
+      const job = jobs[name];
+      let free = job.s === 'queued';
+
+      if (job.s === 'running' && job.ID !== id) {
+        const last = seen[name];
+
+        if (!last || last.n !== job.n) {
+          seen[name] = { n: job.n, since: now };
+        } else if (now - last.since >= timeout) {
+          log.warn(
+            `[jobs] taking over ${name}, its worker did not respond for ` +
+              `${timeout} ms`
+          );
+          free = true;
+        }
+      }
+
+      if (free) {
+        Object.assign(job, { s: 'running', ID: id, n: nonce() });
+        taken = name;
+        return;
+      }
+
+      if (job.blocking) {
+        return false;
+      }
+    }
+
+    return false;
+  });
+
+  return taken;
+};
+
+/**
+ * Changes a job we run. If another worker took it over in between, lost is
+ * called and nothing is changed.
+ */
+const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
+  change(pdriver, internals, jobs => {
+    const job = jobs[name];
+
+    if (!job || job.ID !== id) {
+      lost();
+      return false;
+    }
+
+    mutate(job, jobs);
+    job.n = nonce();
+  });
+
+module.exports = { JOBS, list, register, claim, update };

package/lib/walker.js

@@ -5,6 +5,8 @@ const State = require('./state');
 const log = require('db-migrate-shared').log;
 const Promise = require('bluebird');
 const File = require('./file.js');
+const Dml = require('./dml');
+const Jobs = require('./jobs');
 
 const INTERFACES = {
   migration: require('./interface/migratorInterface.js'),
@@ -133,6 +135,46 @@ Walker.prototype = {
     }
   },
 
+  /**
+   * Migrations running in the background are left out, until their job is
+   * done. Failed ones are run, which registers them again.
+   */
+  _withoutJobs: async function (toRun) {
+    if (this.prefix !== 'migration' || !this._pdriver || typeof this._pdriver._getKV !== 'function') {
+      return toRun;
+    }
+
+    const jobs = await Jobs.list(this._pdriver, this.internals);
+
+    return toRun.filter(file => {
+      const job = jobs[file.name];
+
+      if (job && job.s !== 'failed') {
+        log.info(`${this.title}${file.name} is running in the background`);
+        return false;
+      }
+
+      return true;
+    });
+  },
+
+  _registerJob: async function (file) {
+    Dml.checkType(file);
+
+    if (this.internals.dryRun) {
+      log.info(`${this.title}${file.name} would run in the background`);
+      return;
+    }
+
+    if (!State.supportsLock(this._pdriver)) {
+      throw new Error(
+        'Background migrations need a driver supporting the migration lock'
+      );
+    }
+
+    return Jobs.register(this._pdriver, this.internals, file);
+  },
+
   writeMigrationRecord: function (migration, callback) {
     const onComplete = err => {
       if (err) {
@@ -201,9 +243,11 @@ Walker.prototype = {
           this._driver,
           this.internals
         )
-      ]).spread((allFiles, completedFiles) =>
-        dbmUtil.filterUp(allFiles, completedFiles, partialName, count)
-      );
+      ])
+        .spread((allFiles, completedFiles) =>
+          dbmUtil.filterUp(allFiles, completedFiles, partialName, count)
+        )
+        .then(toRun => this._withoutJobs(toRun));
 
     if (this.internals.check) {
       return determine()
@@ -219,6 +263,10 @@ Walker.prototype = {
 
         return Promise.each(toRun, file => {
           log.verbose(this.title + 'preparing to run up:', file.name);
+          if (Dml.isBackground(file)) {
+            return this._registerJob(file).catch(failedIn(file));
+          }
+
           const _meta = file.get()._meta || {};
           const version = _meta.version || 1;
           this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };

package/lib/work.js

@@ -0,0 +1,260 @@
+'use strict';
+
+const crypto = require('crypto');
+const path = require('path');
+const Promise = require('bluebird');
+const log = require('db-migrate-shared').log;
+const State = require('./state');
+const File = require('./file');
+const Jobs = require('./jobs');
+const Dml = require('./dml');
+
+/**
+ * Runs the jobs of background migrations, see Jobs.
+ *
+ * parallel: jobs run at once by this worker, each on connections of its own
+ * pause: ms to wait between two batches of a job
+ * batch: rows per batch, unless the instruction sets its own
+ * interval: ms between looking for new jobs
+ * timeout: ms after which a job of a worker which stopped responding is
+ *          taken over, the heartbeat renews it every third of it
+ * watch: keep looking for new jobs, instead of ending once none is left
+ *
+ * Returns { done, stop }. done resolves with the jobs done and failed, once
+ * no job is left, or after stop() was called. stop() lets the running jobs
+ * stop after their current batch, they are continued later.
+ */
+
+const DEFAULTS = {
+  parallel: 1,
+  pause: 0,
+  batch: undefined,
+  interval: 5000,
+  timeout: 60000,
+  watch: false
+};
+
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const connect = async (internals, config) => {
+  const Migrator = require('./walker');
+  const index = require('../connect');
+
+  const walker = await index.connect(
+    {
+      config: config.getCurrent().settings,
+      internals,
+      prefix: 'migration'
+    },
+    Migrator
+  );
+
+  walker.directory = path.resolve(internals.argv['migrations-dir']);
+  await walker.createMigrationsTable();
+
+  if (!State.supportsLock(walker._pdriver)) {
+    walker.driver.close(() => {});
+    throw new Error(
+      'Background migrations need a driver supporting the migration lock'
+    );
+  }
+
+  return walker;
+};
+
+const close = walker =>
+  new Promise(resolve => walker.driver.close(() => resolve()));
+
+/**
+ * The interrupted run of a job, like State.startMigration reports it.
+ */
+const interruption = (job, hash) =>
+  job.step > 0
+    ? {
+      step: job.step,
+      learned: job.learned,
+      done: job.done,
+      rollback: job.rb === 1,
+      changed: !!job.h && !!hash && job.h !== hash
+    }
+    : null;
+
+const hashFile = file =>
+  crypto
+    .createHash('sha256')
+    .update(require('fs').readFileSync(file.path))
+    .digest('hex');
+
+const runJob = async (walker, name, id, options, stopping, result) => {
+  const internals = walker.internals;
+  const pdriver = walker._pdriver;
+  let lost = false;
+  const update = mutate =>
+    Jobs.update(pdriver, internals, name, id, mutate, () => {
+      lost = true;
+    });
+
+  const files = await File.loadFromFileystem(
+    walker.directory,
+    walker.prefix,
+    internals
+  );
+  const file = files.find(f => f.name === name);
+
+  if (!file) {
+    await update(job => {
+      job.s = 'failed';
+      job.ID = 0;
+      job.err = 'the migration file is missing';
+    });
+    throw new Error(`The migration file of the job ${name} is missing`);
+  }
+
+  const heartbeat = setInterval(() => {
+    update(() => {}).catch(err => log.error(err.message));
+  }, Math.max(Math.floor(options.timeout / 3), 1));
+  heartbeat.unref();
+
+  try {
+    const jobs = await Jobs.list(pdriver, internals);
+    const hash = hashFile(file);
+    const interrupted = interruption(jobs[name], hash);
+
+    await State.reloadSchema(pdriver, internals);
+    const mig = await pdriver._getKV(internals.migrationState, name);
+    internals.modSchema = Object.assign(
+      { i: {}, c: {}, f: {}, s: [] },
+      mig ? JSON.parse(mig.value) : {}
+    );
+
+    if (!interrupted) {
+      await update(job => {
+        job.h = hash;
+      });
+    }
+
+    log.info(
+      `[jobs] ${interrupted ? 'continuing' : 'running'} ${name}`
+    );
+
+    const progress = {
+      step: op => update(job => { job.step = op; }),
+      learned: op => update(job => { job.learned = op; }),
+      done: op => update(job => { job.done = op; })
+    };
+
+    await Dml.execute(
+      walker,
+      file,
+      internals,
+      pdriver,
+      interrupted,
+      progress,
+      {
+        batch: options.batch,
+        pause: options.pause,
+        stopping: () => lost || stopping()
+      },
+      {
+        rollingBack: () => update(job => { job.rb = 1; }),
+        restart: () =>
+          update(job => {
+            Object.assign(job, { step: 0, learned: 0, done: 0, rb: 0, h: hash });
+          })
+      }
+    );
+
+    await Promise.promisify(walker.writeMigrationRecord.bind(walker))(file);
+    await update((job, jobs) => {
+      delete jobs[name];
+    });
+    log.info(`[jobs] ${name} is done`);
+    result.done.push(name);
+  } catch (err) {
+    if (lost) {
+      log.warn(`[jobs] ${name} was taken over by another worker`);
+      return;
+    }
+
+    if (err && err.stopped) {
+      await update(job => {
+        job.s = 'queued';
+        job.ID = 0;
+      });
+      log.info(`[jobs] ${name} stopped, it continues with the next run`);
+      return;
+    }
+
+    // a job rolled back starts over, else it continues after the steps
+    // which can not be reverted
+    await update(job => {
+      job.s = 'failed';
+      job.ID = 0;
+      job.err = (err && err.message) || String(err);
+      if (err && err.rolledBack) {
+        Object.assign(job, { step: 0, learned: 0, done: 0, rb: 0 });
+      }
+    });
+    log.error(`[jobs] ${name} failed: ${(err && err.message) || err}`);
+    result.failed.push({ name, error: err });
+  } finally {
+    clearInterval(heartbeat);
+  }
+};
+
+module.exports = function executeWork (internals, config, opts = {}) {
+  const options = Object.assign({}, DEFAULTS, opts);
+  const id = crypto.randomBytes(16).toString('base64');
+  const seen = {};
+  const result = { done: [], failed: [] };
+  let stopped = false;
+  const stopping = () => stopped;
+
+  const slot = async () => {
+    // internals of its own, each slot migrates on its own connections
+    const own = Object.assign({}, internals, {
+      argv: Object.assign({}, internals.argv)
+    });
+    const walker = await connect(own, config);
+
+    try {
+      while (!stopping()) {
+        const name = await Jobs.claim(
+          walker._pdriver,
+          own,
+          id,
+          seen,
+          options.timeout
+        );
+
+        if (name) {
+          await runJob(walker, name, id, options, stopping, result);
+          continue;
+        }
+
+        if (!options.watch) {
+          return;
+        }
+
+        await delay(options.interval);
+      }
+    } finally {
+      await close(walker);
+    }
+  };
+
+  const slots = [];
+  for (let i = 0; i < Math.max(Number(options.parallel) || 1, 1); i++) {
+    slots.push(slot());
+  }
+
+  const done = Promise.all(slots).then(() => result);
+
+  return {
+    done,
+    stop: () => {
+      stopped = true;
+      return done;
+    }
+  };
+};

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.3.0",
+  "version": "1.4.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },
@@ -59,7 +59,7 @@
     "@hapi/lab": "25.x",
     "db-migrate-base": "^2.5.0",
     "db-migrate-mysql": "^3.2.0",
-    "db-migrate-sqlite3": "^1.2.0",
+    "db-migrate-sqlite3": "^1.2.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^14.1.1",
     "eslint-plugin-import": "^2.26.0",