db-migrate 1.8.0 → 1.8.1

package/lib/commands/run.js

@@ -127,6 +127,10 @@ function run (internals, config) {
         }
       });
       options.watch = argv.watch === true;
+      // work:<scope>, the jobs of a scope with a database of its own
+      if (folder[1]) {
+        options.scope = folder[1];
+      }
 
       const worker = load('work')(internals, config, options);
       const stop = () => {

package/lib/commands/set-default-argv.js

@@ -179,7 +179,23 @@ module.exports = function (internals, isModule) {
     }
   });
 
-  internals.argv = deepExtend(internals.argv, rcConfig);
+  // the options given on the command line or to the API win over the rc
+  // files, those of the command line with all their aliases
+  var given = {};
+  if (isModule) {
+    given = Object.assign({}, internals.cmdOptions);
+  } else {
+    var defaulted = yargs.parsed.defaulted || {};
+    var aliases = yargs.parsed.aliases || {};
+    Object.keys(internals.argv).forEach(function (key) {
+      var names = [key].concat(aliases[key] || []);
+      if (key !== '_' && key !== '$0' && !names.some(n => defaulted[n])) {
+        given[key] = internals.argv[key];
+      }
+    });
+  }
+
+  internals.argv = deepExtend(internals.argv, rcConfig, given);
 
   // positional arguments, the API adds its own ones in module mode
   if (!Array.isArray(internals.argv._)) {

package/lib/dml.js

@@ -101,6 +101,15 @@ const deleted = (file, op) => `|del:${file.name}#${op}`;
 // a LIKE pattern matching text literally, with ! as escape character
 const like = text => text.replace(/[!%_]/g, c => '!' + c);
 
+/**
+ * The rows carrying the mark of a soft delete as a whole, followed by the
+ * end of the flag or by the next mark: |del:m#1 is no part of |del:m#10.
+ */
+const markedBy = (ddl, mark) => ({
+  sql: `(${ddl(FLAG)} LIKE ? ESCAPE '!' OR ${ddl(FLAG)} LIKE ? ESCAPE '!')`,
+  params: [`%${like(mark)}`, `%${like(mark)}|%`]
+});
+
 const isObject = x =>
   x !== null &&
   typeof x === 'object' &&
@@ -196,7 +205,9 @@ class Dml {
     this.recovery = recovery;
     this.progress = progress || foreground(pdriver, internals);
     this.work = work;
-    this.transactional = meta(file).transactions !== false;
+    // --non-transactional turns them off for all migrations
+    this.transactional =
+      meta(file).transactions !== false && !internals.notransactions;
     this.op = 0;
     this.current = null;
 
@@ -247,6 +258,19 @@ class Dml {
       : `(${sql} || CAST(? AS TEXT))`;
   }
 
+  /**
+   * The flag without a mark, followed by the parameters '|' and the mark
+   * with a '|' after it, so only the mark as a whole is taken out. NULL
+   * once nothing is left.
+   */
+  unmarked () {
+    const replaced = `REPLACE(${this.concat(this.ddl(FLAG))}, ?, '|')`;
+
+    return this.driver._dbmDriver === 'mysql'
+      ? `NULLIF(TRIM(TRAILING '|' FROM ${replaced}), '')`
+      : `NULLIF(RTRIM(${replaced}, '|'), '')`;
+  }
+
   /**
    * Runs fn inside a transaction, rolled back if it fails.
    */
@@ -695,6 +719,24 @@ class Dml {
           );
         }
 
+        // MySQL keeps 255 characters of a flag, a mark cut off is never
+        // found again
+        if (this.driver._dbmDriver === 'mysql') {
+          const [{ n }] = await this.select(
+            `SELECT COUNT(*) AS n FROM ${this.ddl(table)} WHERE ${active.sql} ` +
+              `AND CHAR_LENGTH(COALESCE(${this.ddl(FLAG)}, '')) + ? > 255`,
+            active.params.concat([mark.length])
+          );
+
+          if (Number(n) > 0) {
+            throw new Error(
+              `delete("${table}") in soft mode can not mark ${n} row(s), ` +
+                `their ${FLAG} would be longer than the 255 characters ` +
+                'MySQL keeps of it'
+            );
+          }
+        }
+
         if (purge) {
           await require('./release').schedulePurge(
             this.pdriver,
@@ -972,21 +1014,17 @@ const reverts = {
   },
 
   soft: async (d, [table, column, key, mark]) => {
-    await require('./release').forgetPurges(d.pdriver, d.internals, table, mark);
+    await require('./release').forgetPurges(d.pdriver, d.internals, table, mark, true);
     return d.untilDone(
       table,
       key,
-      {
-        sql: `${d.ddl(FLAG)} LIKE ? ESCAPE '!'`,
-        params: [`%${like(mark)}%`]
-      },
+      markedBy(d.ddl.bind(d), mark),
       {},
       match =>
         d.driver.runSql(
           `UPDATE ${d.ddl(table)} SET ${d.ddl(column)} = NULL, ` +
-            `${d.ddl(FLAG)} = NULLIF(REPLACE(${d.ddl(FLAG)}, ?, ''), '') ` +
-            `WHERE ${match.sql}`,
-          [mark].concat(match.params)
+            `${d.ddl(FLAG)} = ${d.unmarked()} WHERE ${match.sql}`,
+          ['|', `${mark}|`].concat(match.params)
         )
     );
   }
@@ -1183,5 +1221,6 @@ module.exports = {
   Dml,
   Stopped,
   FLAG,
+  markedBy,
   insertRows
 };

package/lib/jobs.js

@@ -21,6 +21,7 @@ const State = require('./state');
  *     in the foreground, see State
  * h: hash of the migration file when it started
  * err: why it failed
+ * sc: the scope the migration belongs to, missing for the top level
  *
  * Migrations take precedence over jobs. Whoever holds the migration lock
  * pauses the jobs before migrating: no job is taken anymore, the running
@@ -107,7 +108,8 @@ const register = async (pdriver, internals, file) => {
     jobs[file.name] = Object.assign(
       { step: 0, learned: 0, done: 0, rb: 0 },
       job,
-      { s: 'queued', blocking, ID: 0, n: nonce(), err: undefined }
+      { s: 'queued', blocking, ID: 0, n: nonce(), err: undefined },
+      internals.matching ? { sc: internals.matching } : {}
     );
   });
 
@@ -119,6 +121,13 @@ const register = async (pdriver, internals, file) => {
   log.info(`[jobs] ${file.name} runs in the background`);
 };
 
+/**
+ * The scope of a job, the one of internals for jobs registered before the
+ * scope was kept.
+ */
+const scopeOf = (job, internals) =>
+  job && typeof job.sc === 'string' ? job.sc : internals.matching || '';
+
 /**
  * Whether the jobs are paused by a migration. A pause holds as long as its
  * holder holds the migration lock and keeps it alive, the lock row did not
@@ -299,4 +308,4 @@ const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
     job.n = nonce();
   });
 
-module.exports = { JOBS, WAIT, list, register, claim, update, paused, pause, resume, remove };
+module.exports = { JOBS, WAIT, list, scopeOf, register, claim, update, paused, pause, resume, remove };

package/lib/release.js

@@ -279,12 +279,14 @@ const schedulePurge = (pdriver, internals, mark, t, key, purge) => {
 
 /**
  * Forgets the purges done, of a soft delete or by db.purge, matching
- * the table and the prefix of the mark.
+ * the table and the prefix of the mark, or the mark itself when exact.
  */
-const forgetPurges = (pdriver, internals, t, prefix) =>
+const forgetPurges = (pdriver, internals, t, prefix, exact = false) =>
   changePurges(pdriver, internals, value => {
     const marks = Object.keys(value).filter(
-      mark => value[mark].t === t && mark.indexOf(prefix) === 0
+      mark =>
+        value[mark].t === t &&
+        (exact ? mark === prefix : mark.indexOf(prefix) === 0)
     );
 
     if (!marks.length) {
@@ -461,7 +463,7 @@ const purge = async (walker, label, due) => {
   const internals = walker.internals;
   const pdriver = walker._pdriver;
   const name = PREFIX + label;
-  const { Dml, FLAG } = require('./dml');
+  const { Dml, markedBy } = require('./dml');
   const file = releaseFile(label, async () => {});
   const d = new Dml(walker, file, internals, pdriver, null, {
     step: async () => {},
@@ -475,7 +477,7 @@ const purge = async (walker, label, due) => {
       await d.untilDone(
         e.t,
         e.key,
-        { sql: `${d.ddl(FLAG)} LIKE ? ESCAPE '!'`, params: [`%${e.mark.replace(/[!%_]/g, c => '!' + c)}%`] },
+        markedBy(d.ddl.bind(d), e.mark),
         {},
         match => walker._driver.runSql(`DELETE FROM ${d.ddl(e.t)} WHERE ${match.sql}`, match.params)
       );

package/lib/state.js

@@ -324,6 +324,16 @@ module.exports = {
 
         log.info(`[state] Created a backup of ${internals.migrationState} by renaming table to ${newName}`);
         await driver._createKV(internals.migrationState);
+        // all but the schema, as without a backup: fix learns the schema and
+        // the records of v2 migrations, not the jobs, seeds, purges, backups
+        // and the records of dml migrations
+        const key = driver.escapeDDL('key');
+        const columns = `${key}, ${driver.escapeDDL('value')}, ${driver.escapeDDL('run_on')}`;
+        await driver.runSql(
+          `INSERT INTO ${driver.escapeDDL(internals.migrationState)} (${columns}) ` +
+            `SELECT ${columns} FROM ${driver.escapeDDL(newName)} WHERE ${key} <> ?`,
+          [SSTATE]
+        );
         await driver._insertKV(internals.migrationState, SSTATE, '{}');
       }
     }

package/lib/walker.js

@@ -184,7 +184,12 @@ Walker.prototype = {
     }
 
     const jobs = await Jobs.list(this._pdriver, this.internals);
-    const names = Object.keys(jobs);
+    // the jobs of this scope, scopes without a database of their own share
+    // the jobs with the top level
+    const scope = this.internals.matching || '';
+    const names = Object.keys(jobs).filter(
+      name => Jobs.scopeOf(jobs[name], this.internals) === scope
+    );
     if (!names.length) {
       return [];
     }

package/lib/work.js

@@ -20,6 +20,8 @@ const Release = require('./release');
  * timeout: ms after which a job of a worker which stopped responding is
  *          taken over, the heartbeat renews it every third of it
  * watch: keep looking for new jobs, instead of ending once none is left
+ * scope: the scope whose database the jobs are kept in, for a scope with a
+ *        config.json of its own; the jobs of other scopes run in their scope
  *
  * Returns { done, stop }. done resolves with the jobs done and failed, once
  * no job is left, or after stop() was called. stop() lets the running jobs
@@ -50,7 +52,10 @@ const connect = async (internals, config) => {
     Migrator
   );
 
-  walker.directory = path.resolve(internals.argv['migrations-dir']);
+  walker.directory = path.resolve(
+    internals.argv['migrations-dir'],
+    internals.locTitle || ''
+  );
   await walker.createMigrationsTable();
 
   if (!State.supportsLock(walker._pdriver)) {
@@ -86,6 +91,19 @@ const hashFile = file =>
     .update(require('fs').readFileSync(file.path))
     .digest('hex');
 
+/**
+ * The record of the migration, reverting it deletes it.
+ */
+const keepRecord = async (pdriver, internals, name) => {
+  const mig = await pdriver._getKV(internals.migrationState, name);
+
+  if (!mig) {
+    await pdriver._insertKV(internals.migrationState, name, '{}');
+  }
+
+  return mig;
+};
+
 const runJob = async (walker, name, id, options, stopping, result, seen) => {
   const internals = walker.internals;
   const pdriver = walker._pdriver;
@@ -95,6 +113,12 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
       lost = true;
     });
 
+  // the scope the job was registered in, the one of the worker for jobs
+  // registered before the scope was kept
+  const scope = Jobs.scopeOf((await Jobs.list(pdriver, internals))[name], internals);
+  internals.matching = scope;
+  walker.directory = path.resolve(internals.argv['migrations-dir'], scope);
+
   const files = await File.loadFromFileystem(
     walker.directory,
     walker.prefix,
@@ -116,6 +140,16 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
     throw new Error(`The migration file of the job ${name} is missing`);
   }
 
+  // its worker died between recording the migration and forgetting the job
+  const recorded = await walker._driver.allLoadedMigrationsAsync();
+  if (recorded.some(r => r.name === `${scope}/${name}`)) {
+    await update((job, jobs) => {
+      delete jobs[name];
+    });
+    log.info(`[jobs] ${name} is done`);
+    return;
+  }
+
   const heartbeat = setInterval(() => {
     update(() => {}).catch(err => log.error(err.message));
   }, Math.max(Math.floor(options.timeout / 3), 1));
@@ -127,7 +161,7 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
     const interrupted = interruption(jobs[name], hash);
 
     await State.reloadSchema(pdriver, internals);
-    const mig = await pdriver._getKV(internals.migrationState, name);
+    const mig = await keepRecord(pdriver, internals, name);
     internals.modSchema = Object.assign(
       { i: {}, c: {}, f: {}, s: [] },
       mig ? JSON.parse(mig.value) : {}
@@ -171,10 +205,13 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
       },
       {
         rollingBack: () => update(job => { job.rb = 1; }),
-        restart: () =>
-          update(job => {
+        restart: async () => {
+          // reverting deleted the record, the run starting over needs one
+          await keepRecord(pdriver, internals, name);
+          await update(job => {
             Object.assign(job, { step: 0, learned: 0, done: 0, rb: 0, h: hash });
-          })
+          });
+        }
       }
     );
 
@@ -216,8 +253,51 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
   }
 };
 
+/**
+ * A dry run, which jobs would run.
+ */
+const dryRun = async (internals, config) => {
+  const walker = await connect(internals, config);
+
+  try {
+    const jobs = await Jobs.list(walker._pdriver, internals);
+    const names = Object.keys(jobs)
+      .sort()
+      .filter(name => jobs[name].s !== 'failed');
+
+    names.forEach(name => log.info(`[jobs] ${name} would run`));
+  } finally {
+    await close(walker);
+  }
+
+  return { done: [], failed: [] };
+};
+
 module.exports = function executeWork (internals, config, opts = {}) {
   const options = Object.assign({}, DEFAULTS, opts);
+
+  // the jobs of all scopes are kept with the top level, but those of the
+  // scopes with a database of their own
+  if (options.scope === 'all') {
+    const done = Promise.reject(
+      new Error(
+        'work runs the jobs of one database, the top level with the scopes ' +
+          'sharing it, or work:<scope> for a scope with a database of its own'
+      )
+    );
+    return { done, stop: () => done };
+  }
+
+  if (options.scope) {
+    internals.migrationMode = options.scope;
+    internals.matching = options.scope;
+  }
+
+  if (internals.dryRun) {
+    const done = dryRun(internals, config);
+    return { done, stop: () => done };
+  }
+
   const id = crypto.randomBytes(16).toString('base64');
   const seen = {};
   const result = { done: [], failed: [] };

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.8.0",
+  "version": "1.8.1",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },