db-migrate 1.2.0 → 1.3.0

package/lib/executors/versioned/v2.js

@@ -12,84 +12,8 @@ const StateDone = require('../../methods/v2/statedone');
 const TranslateState = require('../../methods/v2/translatestate');
 const AddConventions = require('../../methods/v2/conventions');
 const util = require('util');
-
-/**
- * How to recover from an interrupted run of a migration, set per migration
- * by _meta.recovery:
- *
- * skip: skip the steps already executed and continue with the rest
- * rollback: revert the steps already executed and run the migration again
- */
-const RECOVERY_MODES = ['skip', 'rollback'];
-const RECOVERY_DEFAULT = 'skip';
-
-const recoveryMode = (file, interrupted) => {
-  const name = file.name;
-  const mode = file.get()._meta.recovery || RECOVERY_DEFAULT;
-
-  if (RECOVERY_MODES.indexOf(mode) === -1) {
-    throw new Error(
-      `Invalid recovery mode "${mode}" in migration "${name}", use one of ` +
-        RECOVERY_MODES.join(', ')
-    );
-  }
-
-  if (interrupted.rollback) {
-    if (mode !== 'rollback') {
-      log.warn(
-        `[recovery] ${name}: the previous run was interrupted while rolling ` +
-          'back, continuing the rollback'
-      );
-    }
-
-    return 'rollback';
-  }
-
-  if (mode === 'skip' && interrupted.changed) {
-    throw new Error(
-      `Migration "${name}" was interrupted at step ${interrupted.step} and ` +
-        'changed since, so the executed steps can not be skipped safely. ' +
-        "Set _meta.recovery to 'rollback' to revert them, or repair the " +
-        'state manually.'
-    );
-  }
-
-  return mode;
-};
-
-/**
- * Revert the steps executed by an interrupted run. A step started but not
- * done did not change the database, so its reverse operation is dropped.
- */
-/**
- * Keep the reverse operations of the steps executed on the database. The
- * failed step only changed the database, if its main statement went through
- * before failing, signaled by the driver (e.g. a foreign key of createTable
- * failed after the table was created).
- */
-const executedSteps = (internals, failed, signaled) => {
-  const done = internals.migrationDone;
-
-  internals.modSchema.s = internals.modSchema.s.filter(
-    entry =>
-      entry.n <= done || (signaled && failed > done && entry.n === failed)
-  );
-};
-
-/**
- * Name the instruction a migration failed at, or after if the instruction
- * itself finished and the migration code failed afterwards.
- */
-const describeFailure = (err, chain, internals) => {
-  if (!err || typeof err !== 'object' || err.instruction || chain.op === 0) {
-    return;
-  }
-
-  const after = internals.migrationDone >= chain.op;
-  err.instruction = `${after ? 'after' : 'at'} step ${chain.op} ${
-    chain.current
-  }`;
-};
+const { recoveryMode, executedSteps, describeFailure } = require('../../recovery');
+const Dml = require('../../dml');
 
 const recoverByRollback = async (context, driver, file, interrupted) => {
   const internals = context.internals;
@@ -125,6 +49,11 @@ const execUnit = {
   },
 
   up: async function (context, driver, file) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      return Dml.up(context, driver, file);
+    }
+
     const _file = file.get();
     const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
@@ -205,6 +134,12 @@ const execUnit = {
   },
 
   fix: async function (context, driver, file) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      log.info(`[fix] skipping "${file.name}", dml migrations change no schema`);
+      return;
+    }
+
     const _file = file.get();
     const chain = new Chain(context._driver, file, driver, context.internals, context._pdriver);
     if (!_file._meta.noDefaultColumn) {
@@ -247,6 +182,11 @@ const execUnit = {
   },
 
   down: async function (context, driver, file, { abort } = {}) {
+    Dml.checkType(file);
+    if (Dml.isDml(file)) {
+      return Dml.down(context, driver, file, { abort });
+    }
+
     // if we get the abort signal this means we are in a rollback routine
     // which means in turn we want to avoid reloading the state
     if (abort !== true) {

package/lib/recovery.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const log = require('db-migrate-shared').log;
+
+/**
+ * How to recover from an interrupted run of a migration, set per migration
+ * by _meta.recovery:
+ *
+ * skip: skip the steps already executed and continue with the rest
+ * rollback: revert the steps already executed and run the migration again
+ */
+const RECOVERY_MODES = ['skip', 'rollback'];
+const RECOVERY_DEFAULT = 'skip';
+
+const recoveryMode = (file, interrupted) => {
+  const name = file.name;
+  const mode = file.get()._meta.recovery || RECOVERY_DEFAULT;
+
+  if (RECOVERY_MODES.indexOf(mode) === -1) {
+    throw new Error(
+      `Invalid recovery mode "${mode}" in migration "${name}", use one of ` +
+        RECOVERY_MODES.join(', ')
+    );
+  }
+
+  if (interrupted.rollback) {
+    if (mode !== 'rollback') {
+      log.warn(
+        `[recovery] ${name}: the previous run was interrupted while rolling ` +
+          'back, continuing the rollback'
+      );
+    }
+
+    return 'rollback';
+  }
+
+  if (mode === 'skip' && interrupted.changed) {
+    throw new Error(
+      `Migration "${name}" was interrupted at step ${interrupted.step} and ` +
+        'changed since, so the executed steps can not be skipped safely. ' +
+        "Set _meta.recovery to 'rollback' to revert them, or repair the " +
+        'state manually.'
+    );
+  }
+
+  return mode;
+};
+
+/**
+ * Revert the steps executed by an interrupted run. A step started but not
+ * done did not change the database, so its reverse operation is dropped.
+ */
+/**
+ * Keep the reverse operations of the steps executed on the database. The
+ * failed step only changed the database, if its main statement went through
+ * before failing, signaled by the driver (e.g. a foreign key of createTable
+ * failed after the table was created).
+ */
+const executedSteps = (internals, failed, signaled) => {
+  const done = internals.migrationDone;
+
+  internals.modSchema.s = internals.modSchema.s.filter(
+    entry =>
+      entry.n <= done || (signaled && failed > done && entry.n === failed)
+  );
+};
+
+/**
+ * Name the instruction a migration failed at, or after if the instruction
+ * itself finished and the migration code failed afterwards.
+ */
+const describeFailure = (err, chain, internals) => {
+  if (!err || typeof err !== 'object' || err.instruction || chain.op === 0) {
+    return;
+  }
+
+  const after = internals.migrationDone >= chain.op;
+  err.instruction = `${after ? 'after' : 'at'} step ${chain.op} ${
+    chain.current
+  }`;
+};
+
+module.exports = { RECOVERY_MODES, recoveryMode, executedSteps, describeFailure };

package/lib/seed.js

@@ -0,0 +1,199 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const Promise = require('bluebird');
+const log = require('db-migrate-shared').log;
+const { FLAG, insertRows } = require('./dml');
+
+/**
+ * Static seeds insert data for development and tests, seeds/<name>.js:
+ *
+ *   exports.seed = async db => {
+ *     await db.insert('pets', [{ name: 'Rex' }, { name: 'Tom' }]);
+ *   };
+ *
+ * The rows are flagged with seed:<name> in their __dbmigrate__flag column.
+ * Seeding again removes the rows of the previous run first, so seeds can be
+ * changed and run any time. The tables each seed inserted into are kept in
+ * the state, under SEEDS, in the order the seeds ran.
+ */
+
+const SEEDS = '__dbmigrate_seeds__';
+
+const flag = name => `seed:${name}`;
+
+const loadState = async (pdriver, internals) => {
+  const row = await pdriver._getKV(internals.migrationState, SEEDS);
+  return { exists: !!row, seeds: row ? JSON.parse(row.value) : {} };
+};
+
+const saveState = (pdriver, internals, state) => {
+  if (internals.dryRun) {
+    return Promise.resolve();
+  }
+
+  const value = JSON.stringify(state.seeds);
+
+  if (state.exists) {
+    return pdriver._updateKV(internals.migrationState, SEEDS, value);
+  }
+
+  state.exists = true;
+  return pdriver._insertKV(internals.migrationState, SEEDS, value);
+};
+
+const columns = (internals, table) => {
+  const t = internals.schema.c[table];
+  return t ? t.columns || t : null;
+};
+
+class SeedDb {
+  constructor (driver, internals, name, touch) {
+    this.driver = driver;
+    this.internals = internals;
+    this.name = name;
+    this.touch = touch;
+  }
+
+  async insert (table, ...args) {
+    const { rows } = insertRows(args);
+    const known = columns(this.internals, table);
+
+    if (!known || !known[FLAG]) {
+      throw new Error(
+        `Seed "${this.name}" can not insert into "${table}", seeds insert ` +
+          `into tables created by v2 migrations only, by their ${FLAG} ` +
+          'column, so the rows can be removed again.'
+      );
+    }
+
+    if (!rows.length) {
+      return;
+    }
+
+    await this.touch(table);
+    return this.driver.insert(
+      table,
+      rows.map(row => Object.assign({}, row, { [FLAG]: flag(this.name) }))
+    );
+  }
+
+  all (...args) {
+    return this.driver.all(...args);
+  }
+}
+
+/**
+ * Removes the rows of a seed, the tables in reverse order, for rows
+ * referencing rows of the same seed. Tables dropped since are skipped.
+ */
+const remove = async (driver, internals, state, name) => {
+  const tables = (state.seeds[name] || []).slice().reverse();
+
+  await Promise.each(tables, table => {
+    if (!columns(internals, table)) {
+      return;
+    }
+
+    return driver.runSql(
+      `DELETE FROM ${driver.escapeDDL(table)} WHERE ${driver.escapeDDL(FLAG)} = ?`,
+      [flag(name)]
+    );
+  });
+
+  delete state.seeds[name];
+};
+
+const files = dir => {
+  if (!fs.existsSync(dir)) {
+    return [];
+  }
+
+  return fs
+    .readdirSync(dir)
+    .filter(file => /\.js$/.test(file))
+    .sort()
+    .map(file => ({
+      name: file.replace(/\.js$/, ''),
+      path: path.join(dir, file)
+    }));
+};
+
+const select = (list, name, what) => {
+  if (!name) {
+    return list;
+  }
+
+  const found = list.filter(seed => seed.name === name);
+  if (!found.length) {
+    throw new Error(`There is no ${what} "${name}"`);
+  }
+
+  return found;
+};
+
+/**
+ * Seeds all seeds of the directory, or the one named. The rows of the
+ * seeds run before are removed first, of all seeds the last run first,
+ * including seeds whose files were removed since.
+ */
+const run = async (walker, name) => {
+  const internals = walker.internals;
+  const driver = walker._driver;
+  const pdriver = walker._pdriver;
+  const dir = path.resolve(internals.argv['seeds-dir']);
+  const seeds = select(files(dir), name, 'seed');
+  const state = await loadState(pdriver, internals);
+
+  const previous = name ? [name] : Object.keys(state.seeds).reverse();
+  await Promise.each(previous, seed => remove(driver, internals, state, seed));
+  await saveState(pdriver, internals, state);
+
+  await Promise.each(seeds, async seed => {
+    log.info(`[seed] ${seed.name}`);
+    delete require.cache[require.resolve(seed.path)];
+    const mod = require(seed.path);
+
+    if (typeof mod.seed !== 'function') {
+      throw new Error(`The seed "${seed.name}" exports no seed function`);
+    }
+
+    state.seeds[seed.name] = [];
+    const touch = async table => {
+      const tables = state.seeds[seed.name];
+      if (tables.indexOf(table) === -1) {
+        // recorded before inserting, so a failing seed is removed as well
+        tables.push(table);
+        await saveState(pdriver, internals, state);
+      }
+    };
+
+    await mod.seed(new SeedDb(driver, internals, seed.name, touch), {
+      options: internals.safeOptions,
+      dbm: internals.safeOptions.dbmigrate
+    });
+    await saveState(pdriver, internals, state);
+  });
+};
+
+/**
+ * Removes the rows of all seeds, or of the one named.
+ */
+const undo = async (walker, name) => {
+  const internals = walker.internals;
+  const state = await loadState(walker._pdriver, internals);
+  const seeds = name
+    ? select(Object.keys(state.seeds).map(n => ({ name: n })), name, 'seeded seed')
+    : Object.keys(state.seeds)
+      .reverse()
+      .map(n => ({ name: n }));
+
+  await Promise.each(seeds, seed => {
+    log.info(`[seed] removing ${seed.name}`);
+    return remove(walker._driver, internals, state, seed.name);
+  });
+  await saveState(walker._pdriver, internals, state);
+};
+
+module.exports = { run, undo, SEEDS };

package/package.json

@@ -31,7 +31,7 @@
     "programmable",
     "api"
   ],
-  "version": "1.2.0",
+  "version": "1.3.0",
   "bugs": {
     "url": "https://github.com/db-migrate/node-db-migrate/issues"
   },
@@ -57,9 +57,9 @@
     "@hapi/code": "9.x",
     "@hapi/eslint-plugin": "^6.0.0",
     "@hapi/lab": "25.x",
-    "db-migrate-base": "^2.4.1",
-    "db-migrate-mysql": "^3.1.1",
-    "db-migrate-sqlite3": "^1.1.1",
+    "db-migrate-base": "^2.5.0",
+    "db-migrate-mysql": "^3.2.0",
+    "db-migrate-sqlite3": "^1.2.0",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^14.1.1",
     "eslint-plugin-import": "^2.26.0",