db-migrate 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api.js +19 -0
- package/connect.js +2 -3
- package/index.js +10 -1
- package/lib/commands/down.js +4 -1
- package/lib/commands/run.js +4 -0
- package/lib/commands/set-default-argv.js +17 -1
- package/lib/config.js +33 -12
- package/lib/dml.js +48 -9
- package/lib/file.js +16 -1
- package/lib/jobs.js +11 -2
- package/lib/release.js +7 -5
- package/lib/state.js +10 -0
- package/lib/template.js +0 -9
- package/lib/walker.js +12 -1
- package/lib/work.js +85 -5
- package/package.json +1 -1
package/api.js
CHANGED
|
@@ -192,6 +192,18 @@ dbmigrate.prototype = {
|
|
|
192
192
|
this.internals.argv = argv;
|
|
193
193
|
},
|
|
194
194
|
|
|
195
|
+
/**
|
|
196
|
+
* Every call starts from the options of the instance: the count,
|
|
197
|
+
* destination and scope of an earlier call do not carry over.
|
|
198
|
+
*/
|
|
199
|
+
_begin: function () {
|
|
200
|
+
delete this.internals.argv.count;
|
|
201
|
+
delete this.internals.argv.destination;
|
|
202
|
+
this.internals.migrationMode = undefined;
|
|
203
|
+
this.internals.matching = '';
|
|
204
|
+
this.internals.locTitle = undefined;
|
|
205
|
+
},
|
|
206
|
+
|
|
195
207
|
/**
|
|
196
208
|
* Executes up a given number of migrations or a specific one.
|
|
197
209
|
*
|
|
@@ -199,6 +211,7 @@ dbmigrate.prototype = {
|
|
|
199
211
|
*/
|
|
200
212
|
up: function (specification, opts, callback) {
|
|
201
213
|
var executeUp = load('up');
|
|
214
|
+
this._begin();
|
|
202
215
|
|
|
203
216
|
if (arguments.length > 0) {
|
|
204
217
|
if (typeof specification === 'string') {
|
|
@@ -229,6 +242,7 @@ dbmigrate.prototype = {
|
|
|
229
242
|
*/
|
|
230
243
|
down: function (specification, opts, callback) {
|
|
231
244
|
var executeDown = load('down');
|
|
245
|
+
this._begin();
|
|
232
246
|
|
|
233
247
|
if (arguments.length > 0) {
|
|
234
248
|
if (typeof specification === 'number') {
|
|
@@ -255,6 +269,7 @@ dbmigrate.prototype = {
|
|
|
255
269
|
|
|
256
270
|
fix: function (specification, opts, callback) {
|
|
257
271
|
var executeFix = load('fix');
|
|
272
|
+
this._begin();
|
|
258
273
|
|
|
259
274
|
if (arguments.length > 0) {
|
|
260
275
|
if (typeof specification === 'string') {
|
|
@@ -280,6 +295,7 @@ dbmigrate.prototype = {
|
|
|
280
295
|
|
|
281
296
|
check: function (specification, opts, callback) {
|
|
282
297
|
var executeCheck = load('check');
|
|
298
|
+
this._begin();
|
|
283
299
|
|
|
284
300
|
if (arguments.length > 0) {
|
|
285
301
|
if (typeof specification === 'number') {
|
|
@@ -308,6 +324,7 @@ dbmigrate.prototype = {
|
|
|
308
324
|
*/
|
|
309
325
|
sync: function (specification, opts, callback) {
|
|
310
326
|
var executeSync = load('sync');
|
|
327
|
+
this._begin();
|
|
311
328
|
|
|
312
329
|
if (arguments.length > 0) {
|
|
313
330
|
if (typeof specification === 'string') {
|
|
@@ -332,6 +349,7 @@ dbmigrate.prototype = {
|
|
|
332
349
|
*/
|
|
333
350
|
reset: function (scope, callback) {
|
|
334
351
|
var executeDown = load('down');
|
|
352
|
+
this._begin();
|
|
335
353
|
|
|
336
354
|
if (typeof scope === 'string') {
|
|
337
355
|
this.internals.migrationMode = scope;
|
|
@@ -427,6 +445,7 @@ dbmigrate.prototype = {
|
|
|
427
445
|
* or due, see lib/status.js.
|
|
428
446
|
*/
|
|
429
447
|
status: function (scope, callback) {
|
|
448
|
+
this._begin();
|
|
430
449
|
if (typeof scope === 'function') {
|
|
431
450
|
callback = scope;
|
|
432
451
|
scope = undefined;
|
package/connect.js
CHANGED
|
@@ -71,9 +71,8 @@ exports.connect = async function (config, PassedClass) {
|
|
|
71
71
|
await Promise.promisify(db2.switchDatabase, { context: db2 })(switchTo);
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
74
|
+
// of this run only, an instance of the API runs several
|
|
75
|
+
internals.locTitle = scope || undefined;
|
|
77
76
|
|
|
78
77
|
return new PassedClass(
|
|
79
78
|
db,
|
package/index.js
CHANGED
|
@@ -45,7 +45,16 @@ function loadPlugins (options) {
|
|
|
45
45
|
var hooks = {};
|
|
46
46
|
|
|
47
47
|
for (; i < length; ++i) {
|
|
48
|
-
var plugin
|
|
48
|
+
var plugin;
|
|
49
|
+
|
|
50
|
+
try {
|
|
51
|
+
// resolved like node does from the project, so plugins hoisted in a
|
|
52
|
+
// monorepo are found as well
|
|
53
|
+
plugin = require(require.resolve(plugins[i], { paths: [options.cwd] }));
|
|
54
|
+
} catch (err) {
|
|
55
|
+
log.warn(`Could not load the plugin ${plugins[i]}: ${err.message}`);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
49
58
|
|
|
50
59
|
if (!plugin.hooks || !plugin.loadPlugin) {
|
|
51
60
|
continue;
|
package/lib/commands/down.js
CHANGED
|
@@ -23,7 +23,10 @@ async function prepare (internals, config) {
|
|
|
23
23
|
Migrator
|
|
24
24
|
);
|
|
25
25
|
|
|
26
|
-
migrator.directory = path.resolve(
|
|
26
|
+
migrator.directory = path.resolve(
|
|
27
|
+
internals.argv['migrations-dir'],
|
|
28
|
+
internals.locTitle || ''
|
|
29
|
+
);
|
|
27
30
|
|
|
28
31
|
await migrator.createMigrationsTable();
|
|
29
32
|
log.verbose('migration table created');
|
package/lib/commands/run.js
CHANGED
|
@@ -127,6 +127,10 @@ function run (internals, config) {
|
|
|
127
127
|
}
|
|
128
128
|
});
|
|
129
129
|
options.watch = argv.watch === true;
|
|
130
|
+
// work:<scope>, the jobs of a scope with a database of its own
|
|
131
|
+
if (folder[1]) {
|
|
132
|
+
options.scope = folder[1];
|
|
133
|
+
}
|
|
130
134
|
|
|
131
135
|
const worker = load('work')(internals, config, options);
|
|
132
136
|
const stop = () => {
|
|
@@ -179,7 +179,23 @@ module.exports = function (internals, isModule) {
|
|
|
179
179
|
}
|
|
180
180
|
});
|
|
181
181
|
|
|
182
|
-
|
|
182
|
+
// the options given on the command line or to the API win over the rc
|
|
183
|
+
// files, those of the command line with all their aliases
|
|
184
|
+
var given = {};
|
|
185
|
+
if (isModule) {
|
|
186
|
+
given = Object.assign({}, internals.cmdOptions);
|
|
187
|
+
} else {
|
|
188
|
+
var defaulted = yargs.parsed.defaulted || {};
|
|
189
|
+
var aliases = yargs.parsed.aliases || {};
|
|
190
|
+
Object.keys(internals.argv).forEach(function (key) {
|
|
191
|
+
var names = [key].concat(aliases[key] || []);
|
|
192
|
+
if (key !== '_' && key !== '$0' && !names.some(n => defaulted[n])) {
|
|
193
|
+
given[key] = internals.argv[key];
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
internals.argv = deepExtend(internals.argv, rcConfig, given);
|
|
183
199
|
|
|
184
200
|
// positional arguments, the API adds its own ones in module mode
|
|
185
201
|
if (!Array.isArray(internals.argv._)) {
|
package/lib/config.js
CHANGED
|
@@ -97,16 +97,28 @@ exports.loadFile = function (fileName, currentEnv, plugins) {
|
|
|
97
97
|
// resolves { "ENV": ... } entries of a config, in place
|
|
98
98
|
exports.resolveEnv = walkConfig;
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* The value of { "ENV": "NAME", "default": value }, the default if the
|
|
102
|
+
* variable is not set or empty.
|
|
103
|
+
*/
|
|
104
|
+
function fromEnv (entry) {
|
|
105
|
+
var value = process.env[entry.ENV];
|
|
106
|
+
|
|
107
|
+
if (value === undefined || value === '') {
|
|
108
|
+
if (Object.prototype.hasOwnProperty.call(entry, 'default')) {
|
|
109
|
+
return entry.default;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
log.verbose('Environment variable ' + entry.ENV + ' is empty!');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return value;
|
|
116
|
+
}
|
|
117
|
+
|
|
100
118
|
function walkConfig (level) {
|
|
101
119
|
for (var configEntry in level) {
|
|
102
120
|
if (level[configEntry] && level[configEntry].ENV) {
|
|
103
|
-
|
|
104
|
-
log.verbose(
|
|
105
|
-
'Environment variable ' + level[configEntry].ENV + ' is empty!'
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
level[configEntry] = process.env[level[configEntry].ENV];
|
|
121
|
+
level[configEntry] = fromEnv(level[configEntry]);
|
|
110
122
|
} else if (level[configEntry] && typeof level[configEntry] === 'object') {
|
|
111
123
|
level[configEntry] = walkConfig(level[configEntry]);
|
|
112
124
|
}
|
|
@@ -121,11 +133,19 @@ exports.loadObject = function (_config, currentEnv) {
|
|
|
121
133
|
var config = JSON.parse(JSON.stringify(_config));
|
|
122
134
|
|
|
123
135
|
for (var env in config) {
|
|
136
|
+
if (env === 'defaultEnv' || env === 'default') {
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
|
|
124
140
|
if (config[env].ENV) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
141
|
+
// a database url from the environment, the other keys next to it
|
|
142
|
+
// are added to it
|
|
143
|
+
var url = fromEnv(config[env]);
|
|
144
|
+
if (url) {
|
|
145
|
+
var rest = Object.assign({}, config[env]);
|
|
146
|
+
delete rest.ENV;
|
|
147
|
+
delete rest.default;
|
|
148
|
+
out[env] = Object.assign(parseDatabaseUrl(url), walkConfig(rest));
|
|
129
149
|
}
|
|
130
150
|
} else if (typeof config[env] === 'string') {
|
|
131
151
|
out[env] = parseDatabaseUrl(config[env]);
|
|
@@ -183,7 +203,8 @@ exports.loadObject = function (_config, currentEnv) {
|
|
|
183
203
|
out.setCurrent(config.default);
|
|
184
204
|
} else if (config.defaultEnv) {
|
|
185
205
|
if (config.defaultEnv.ENV) {
|
|
186
|
-
|
|
206
|
+
// without the variable and a default, the default environments
|
|
207
|
+
out.setCurrent(fromEnv(config.defaultEnv) || ['dev', 'development']);
|
|
187
208
|
} else {
|
|
188
209
|
out.setCurrent(config.defaultEnv);
|
|
189
210
|
}
|
package/lib/dml.js
CHANGED
|
@@ -101,6 +101,15 @@ const deleted = (file, op) => `|del:${file.name}#${op}`;
|
|
|
101
101
|
// a LIKE pattern matching text literally, with ! as escape character
|
|
102
102
|
const like = text => text.replace(/[!%_]/g, c => '!' + c);
|
|
103
103
|
|
|
104
|
+
/**
|
|
105
|
+
* The rows carrying the mark of a soft delete as a whole, followed by the
|
|
106
|
+
* end of the flag or by the next mark: |del:m#1 is no part of |del:m#10.
|
|
107
|
+
*/
|
|
108
|
+
const markedBy = (ddl, mark) => ({
|
|
109
|
+
sql: `(${ddl(FLAG)} LIKE ? ESCAPE '!' OR ${ddl(FLAG)} LIKE ? ESCAPE '!')`,
|
|
110
|
+
params: [`%${like(mark)}`, `%${like(mark)}|%`]
|
|
111
|
+
});
|
|
112
|
+
|
|
104
113
|
const isObject = x =>
|
|
105
114
|
x !== null &&
|
|
106
115
|
typeof x === 'object' &&
|
|
@@ -196,7 +205,9 @@ class Dml {
|
|
|
196
205
|
this.recovery = recovery;
|
|
197
206
|
this.progress = progress || foreground(pdriver, internals);
|
|
198
207
|
this.work = work;
|
|
199
|
-
|
|
208
|
+
// --non-transactional turns them off for all migrations
|
|
209
|
+
this.transactional =
|
|
210
|
+
meta(file).transactions !== false && !internals.notransactions;
|
|
200
211
|
this.op = 0;
|
|
201
212
|
this.current = null;
|
|
202
213
|
|
|
@@ -247,6 +258,19 @@ class Dml {
|
|
|
247
258
|
: `(${sql} || CAST(? AS TEXT))`;
|
|
248
259
|
}
|
|
249
260
|
|
|
261
|
+
/**
|
|
262
|
+
* The flag without a mark, followed by the parameters '|' and the mark
|
|
263
|
+
* with a '|' after it, so only the mark as a whole is taken out. NULL
|
|
264
|
+
* once nothing is left.
|
|
265
|
+
*/
|
|
266
|
+
unmarked () {
|
|
267
|
+
const replaced = `REPLACE(${this.concat(this.ddl(FLAG))}, ?, '|')`;
|
|
268
|
+
|
|
269
|
+
return this.driver._dbmDriver === 'mysql'
|
|
270
|
+
? `NULLIF(TRIM(TRAILING '|' FROM ${replaced}), '')`
|
|
271
|
+
: `NULLIF(RTRIM(${replaced}, '|'), '')`;
|
|
272
|
+
}
|
|
273
|
+
|
|
250
274
|
/**
|
|
251
275
|
* Runs fn inside a transaction, rolled back if it fails.
|
|
252
276
|
*/
|
|
@@ -695,6 +719,24 @@ class Dml {
|
|
|
695
719
|
);
|
|
696
720
|
}
|
|
697
721
|
|
|
722
|
+
// MySQL keeps 255 characters of a flag, a mark cut off is never
|
|
723
|
+
// found again
|
|
724
|
+
if (this.driver._dbmDriver === 'mysql') {
|
|
725
|
+
const [{ n }] = await this.select(
|
|
726
|
+
`SELECT COUNT(*) AS n FROM ${this.ddl(table)} WHERE ${active.sql} ` +
|
|
727
|
+
`AND CHAR_LENGTH(COALESCE(${this.ddl(FLAG)}, '')) + ? > 255`,
|
|
728
|
+
active.params.concat([mark.length])
|
|
729
|
+
);
|
|
730
|
+
|
|
731
|
+
if (Number(n) > 0) {
|
|
732
|
+
throw new Error(
|
|
733
|
+
`delete("${table}") in soft mode can not mark ${n} row(s), ` +
|
|
734
|
+
`their ${FLAG} would be longer than the 255 characters ` +
|
|
735
|
+
'MySQL keeps of it'
|
|
736
|
+
);
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
|
|
698
740
|
if (purge) {
|
|
699
741
|
await require('./release').schedulePurge(
|
|
700
742
|
this.pdriver,
|
|
@@ -972,21 +1014,17 @@ const reverts = {
|
|
|
972
1014
|
},
|
|
973
1015
|
|
|
974
1016
|
soft: async (d, [table, column, key, mark]) => {
|
|
975
|
-
await require('./release').forgetPurges(d.pdriver, d.internals, table, mark);
|
|
1017
|
+
await require('./release').forgetPurges(d.pdriver, d.internals, table, mark, true);
|
|
976
1018
|
return d.untilDone(
|
|
977
1019
|
table,
|
|
978
1020
|
key,
|
|
979
|
-
|
|
980
|
-
sql: `${d.ddl(FLAG)} LIKE ? ESCAPE '!'`,
|
|
981
|
-
params: [`%${like(mark)}%`]
|
|
982
|
-
},
|
|
1021
|
+
markedBy(d.ddl.bind(d), mark),
|
|
983
1022
|
{},
|
|
984
1023
|
match =>
|
|
985
1024
|
d.driver.runSql(
|
|
986
1025
|
`UPDATE ${d.ddl(table)} SET ${d.ddl(column)} = NULL, ` +
|
|
987
|
-
`${d.ddl(FLAG)} =
|
|
988
|
-
|
|
989
|
-
[mark].concat(match.params)
|
|
1026
|
+
`${d.ddl(FLAG)} = ${d.unmarked()} WHERE ${match.sql}`,
|
|
1027
|
+
['|', `${mark}|`].concat(match.params)
|
|
990
1028
|
)
|
|
991
1029
|
);
|
|
992
1030
|
}
|
|
@@ -1183,5 +1221,6 @@ module.exports = {
|
|
|
1183
1221
|
Dml,
|
|
1184
1222
|
Stopped,
|
|
1185
1223
|
FLAG,
|
|
1224
|
+
markedBy,
|
|
1186
1225
|
insertRows
|
|
1187
1226
|
};
|
package/lib/file.js
CHANGED
|
@@ -221,10 +221,25 @@ File.loadFromDatabase = function (dir, prefix, driver, internals) {
|
|
|
221
221
|
);
|
|
222
222
|
})
|
|
223
223
|
.map(function (result) {
|
|
224
|
-
|
|
224
|
+
// the names are recorded with their scope, dir is the directory of the
|
|
225
|
+
// scope already
|
|
226
|
+
return new File(path.join(rootOf(dir, internals.matching), result.name), internals);
|
|
225
227
|
});
|
|
226
228
|
};
|
|
227
229
|
|
|
230
|
+
/**
|
|
231
|
+
* The migrations directory of a scope directory.
|
|
232
|
+
*/
|
|
233
|
+
function rootOf (dir, scope) {
|
|
234
|
+
const resolved = path.resolve(dir);
|
|
235
|
+
|
|
236
|
+
if (scope && resolved.endsWith(path.sep + path.normalize(scope))) {
|
|
237
|
+
return resolved.slice(0, -(path.normalize(scope).length + 1));
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
return dir;
|
|
241
|
+
}
|
|
242
|
+
|
|
228
243
|
Promise.promisifyAll(fs);
|
|
229
244
|
|
|
230
245
|
module.exports = File;
|
package/lib/jobs.js
CHANGED
|
@@ -21,6 +21,7 @@ const State = require('./state');
|
|
|
21
21
|
* in the foreground, see State
|
|
22
22
|
* h: hash of the migration file when it started
|
|
23
23
|
* err: why it failed
|
|
24
|
+
* sc: the scope the migration belongs to, missing for the top level
|
|
24
25
|
*
|
|
25
26
|
* Migrations take precedence over jobs. Whoever holds the migration lock
|
|
26
27
|
* pauses the jobs before migrating: no job is taken anymore, the running
|
|
@@ -107,7 +108,8 @@ const register = async (pdriver, internals, file) => {
|
|
|
107
108
|
jobs[file.name] = Object.assign(
|
|
108
109
|
{ step: 0, learned: 0, done: 0, rb: 0 },
|
|
109
110
|
job,
|
|
110
|
-
{ s: 'queued', blocking, ID: 0, n: nonce(), err: undefined }
|
|
111
|
+
{ s: 'queued', blocking, ID: 0, n: nonce(), err: undefined },
|
|
112
|
+
internals.matching ? { sc: internals.matching } : {}
|
|
111
113
|
);
|
|
112
114
|
});
|
|
113
115
|
|
|
@@ -119,6 +121,13 @@ const register = async (pdriver, internals, file) => {
|
|
|
119
121
|
log.info(`[jobs] ${file.name} runs in the background`);
|
|
120
122
|
};
|
|
121
123
|
|
|
124
|
+
/**
|
|
125
|
+
* The scope of a job, the one of internals for jobs registered before the
|
|
126
|
+
* scope was kept.
|
|
127
|
+
*/
|
|
128
|
+
const scopeOf = (job, internals) =>
|
|
129
|
+
job && typeof job.sc === 'string' ? job.sc : internals.matching || '';
|
|
130
|
+
|
|
122
131
|
/**
|
|
123
132
|
* Whether the jobs are paused by a migration. A pause holds as long as its
|
|
124
133
|
* holder holds the migration lock and keeps it alive, the lock row did not
|
|
@@ -299,4 +308,4 @@ const update = (pdriver, internals, name, id, mutate, lost = () => {}) =>
|
|
|
299
308
|
job.n = nonce();
|
|
300
309
|
});
|
|
301
310
|
|
|
302
|
-
module.exports = { JOBS, WAIT, list, register, claim, update, paused, pause, resume, remove };
|
|
311
|
+
module.exports = { JOBS, WAIT, list, scopeOf, register, claim, update, paused, pause, resume, remove };
|
package/lib/release.js
CHANGED
|
@@ -279,12 +279,14 @@ const schedulePurge = (pdriver, internals, mark, t, key, purge) => {
|
|
|
279
279
|
|
|
280
280
|
/**
|
|
281
281
|
* Forgets the purges done, of a soft delete or by db.purge, matching
|
|
282
|
-
* the table and the prefix of the mark.
|
|
282
|
+
* the table and the prefix of the mark, or the mark itself when exact.
|
|
283
283
|
*/
|
|
284
|
-
const forgetPurges = (pdriver, internals, t, prefix) =>
|
|
284
|
+
const forgetPurges = (pdriver, internals, t, prefix, exact = false) =>
|
|
285
285
|
changePurges(pdriver, internals, value => {
|
|
286
286
|
const marks = Object.keys(value).filter(
|
|
287
|
-
mark =>
|
|
287
|
+
mark =>
|
|
288
|
+
value[mark].t === t &&
|
|
289
|
+
(exact ? mark === prefix : mark.indexOf(prefix) === 0)
|
|
288
290
|
);
|
|
289
291
|
|
|
290
292
|
if (!marks.length) {
|
|
@@ -461,7 +463,7 @@ const purge = async (walker, label, due) => {
|
|
|
461
463
|
const internals = walker.internals;
|
|
462
464
|
const pdriver = walker._pdriver;
|
|
463
465
|
const name = PREFIX + label;
|
|
464
|
-
const { Dml,
|
|
466
|
+
const { Dml, markedBy } = require('./dml');
|
|
465
467
|
const file = releaseFile(label, async () => {});
|
|
466
468
|
const d = new Dml(walker, file, internals, pdriver, null, {
|
|
467
469
|
step: async () => {},
|
|
@@ -475,7 +477,7 @@ const purge = async (walker, label, due) => {
|
|
|
475
477
|
await d.untilDone(
|
|
476
478
|
e.t,
|
|
477
479
|
e.key,
|
|
478
|
-
|
|
480
|
+
markedBy(d.ddl.bind(d), e.mark),
|
|
479
481
|
{},
|
|
480
482
|
match => walker._driver.runSql(`DELETE FROM ${d.ddl(e.t)} WHERE ${match.sql}`, match.params)
|
|
481
483
|
);
|
package/lib/state.js
CHANGED
|
@@ -324,6 +324,16 @@ module.exports = {
|
|
|
324
324
|
|
|
325
325
|
log.info(`[state] Created a backup of ${internals.migrationState} by renaming table to ${newName}`);
|
|
326
326
|
await driver._createKV(internals.migrationState);
|
|
327
|
+
// all but the schema, as without a backup: fix learns the schema and
|
|
328
|
+
// the records of v2 migrations, not the jobs, seeds, purges, backups
|
|
329
|
+
// and the records of dml migrations
|
|
330
|
+
const key = driver.escapeDDL('key');
|
|
331
|
+
const columns = `${key}, ${driver.escapeDDL('value')}, ${driver.escapeDDL('run_on')}`;
|
|
332
|
+
await driver.runSql(
|
|
333
|
+
`INSERT INTO ${driver.escapeDDL(internals.migrationState)} (${columns}) ` +
|
|
334
|
+
`SELECT ${columns} FROM ${driver.escapeDDL(newName)} WHERE ${key} <> ?`,
|
|
335
|
+
[SSTATE]
|
|
336
|
+
);
|
|
327
337
|
await driver._insertKV(internals.migrationState, SSTATE, '{}');
|
|
328
338
|
}
|
|
329
339
|
}
|
package/lib/template.js
CHANGED
|
@@ -105,8 +105,6 @@ Template.prototype.sqlFileLoaderTemplate = function () {
|
|
|
105
105
|
' return new Promise( function( resolve, reject ) {',
|
|
106
106
|
" fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){",
|
|
107
107
|
' if (err) return reject(err);',
|
|
108
|
-
" console.log('received data: ' + data);",
|
|
109
|
-
'',
|
|
110
108
|
' resolve(data);',
|
|
111
109
|
' });',
|
|
112
110
|
' })',
|
|
@@ -122,8 +120,6 @@ Template.prototype.sqlFileLoaderTemplate = function () {
|
|
|
122
120
|
' return new Promise( function( resolve, reject ) {',
|
|
123
121
|
" fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){",
|
|
124
122
|
' if (err) return reject(err);',
|
|
125
|
-
" console.log('received data: ' + data);",
|
|
126
|
-
'',
|
|
127
123
|
' resolve(data);',
|
|
128
124
|
' });',
|
|
129
125
|
' })',
|
|
@@ -171,8 +167,6 @@ Template.prototype.sqlFileLoaderIgnoreOnInitTemplate = function () {
|
|
|
171
167
|
' return new Promise( function( resolve, reject ) {',
|
|
172
168
|
" fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){",
|
|
173
169
|
' if (err) return reject(err);',
|
|
174
|
-
" console.log('received data: ' + data);",
|
|
175
|
-
'',
|
|
176
170
|
' resolve(data);',
|
|
177
171
|
' });',
|
|
178
172
|
' })',
|
|
@@ -181,7 +175,6 @@ Template.prototype.sqlFileLoaderIgnoreOnInitTemplate = function () {
|
|
|
181
175
|
' });',
|
|
182
176
|
' }',
|
|
183
177
|
' else {',
|
|
184
|
-
" console.log('ignoring on init: ' + filePath)",
|
|
185
178
|
' return null;',
|
|
186
179
|
' }',
|
|
187
180
|
'};',
|
|
@@ -193,8 +186,6 @@ Template.prototype.sqlFileLoaderIgnoreOnInitTemplate = function () {
|
|
|
193
186
|
' return new Promise( function( resolve, reject ) {',
|
|
194
187
|
" fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){",
|
|
195
188
|
' if (err) return reject(err);',
|
|
196
|
-
" console.log('received data: ' + data);",
|
|
197
|
-
'',
|
|
198
189
|
' resolve(data);',
|
|
199
190
|
' });',
|
|
200
191
|
' })',
|
package/lib/walker.js
CHANGED
|
@@ -48,6 +48,12 @@ const Walker = function (driver, directory, mode, intern, prefix, opts = {}) {
|
|
|
48
48
|
* c: schema
|
|
49
49
|
*/
|
|
50
50
|
this.internals.modSchema = { i: {}, c: {}, f: {}, s: [] };
|
|
51
|
+
// the state of an earlier run of the same instance of the API, e.g. a
|
|
52
|
+
// down leaves unlearn set, which would keep up from recording anything
|
|
53
|
+
this.internals.unlearn = false;
|
|
54
|
+
this.internals.rollback = false;
|
|
55
|
+
this.internals.rollbackContinue = false;
|
|
56
|
+
this.internals.learnFromScratch = false;
|
|
51
57
|
this.mode = mode;
|
|
52
58
|
|
|
53
59
|
if (!this.mode) this.prefix = `static-${prefix}`;
|
|
@@ -178,7 +184,12 @@ Walker.prototype = {
|
|
|
178
184
|
}
|
|
179
185
|
|
|
180
186
|
const jobs = await Jobs.list(this._pdriver, this.internals);
|
|
181
|
-
|
|
187
|
+
// the jobs of this scope, scopes without a database of their own share
|
|
188
|
+
// the jobs with the top level
|
|
189
|
+
const scope = this.internals.matching || '';
|
|
190
|
+
const names = Object.keys(jobs).filter(
|
|
191
|
+
name => Jobs.scopeOf(jobs[name], this.internals) === scope
|
|
192
|
+
);
|
|
182
193
|
if (!names.length) {
|
|
183
194
|
return [];
|
|
184
195
|
}
|
package/lib/work.js
CHANGED
|
@@ -20,6 +20,8 @@ const Release = require('./release');
|
|
|
20
20
|
* timeout: ms after which a job of a worker which stopped responding is
|
|
21
21
|
* taken over, the heartbeat renews it every third of it
|
|
22
22
|
* watch: keep looking for new jobs, instead of ending once none is left
|
|
23
|
+
* scope: the scope whose database the jobs are kept in, for a scope with a
|
|
24
|
+
* config.json of its own; the jobs of other scopes run in their scope
|
|
23
25
|
*
|
|
24
26
|
* Returns { done, stop }. done resolves with the jobs done and failed, once
|
|
25
27
|
* no job is left, or after stop() was called. stop() lets the running jobs
|
|
@@ -50,7 +52,10 @@ const connect = async (internals, config) => {
|
|
|
50
52
|
Migrator
|
|
51
53
|
);
|
|
52
54
|
|
|
53
|
-
walker.directory = path.resolve(
|
|
55
|
+
walker.directory = path.resolve(
|
|
56
|
+
internals.argv['migrations-dir'],
|
|
57
|
+
internals.locTitle || ''
|
|
58
|
+
);
|
|
54
59
|
await walker.createMigrationsTable();
|
|
55
60
|
|
|
56
61
|
if (!State.supportsLock(walker._pdriver)) {
|
|
@@ -86,6 +91,19 @@ const hashFile = file =>
|
|
|
86
91
|
.update(require('fs').readFileSync(file.path))
|
|
87
92
|
.digest('hex');
|
|
88
93
|
|
|
94
|
+
/**
|
|
95
|
+
* The record of the migration, reverting it deletes it.
|
|
96
|
+
*/
|
|
97
|
+
const keepRecord = async (pdriver, internals, name) => {
|
|
98
|
+
const mig = await pdriver._getKV(internals.migrationState, name);
|
|
99
|
+
|
|
100
|
+
if (!mig) {
|
|
101
|
+
await pdriver._insertKV(internals.migrationState, name, '{}');
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return mig;
|
|
105
|
+
};
|
|
106
|
+
|
|
89
107
|
const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
90
108
|
const internals = walker.internals;
|
|
91
109
|
const pdriver = walker._pdriver;
|
|
@@ -95,6 +113,12 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
|
95
113
|
lost = true;
|
|
96
114
|
});
|
|
97
115
|
|
|
116
|
+
// the scope the job was registered in, the one of the worker for jobs
|
|
117
|
+
// registered before the scope was kept
|
|
118
|
+
const scope = Jobs.scopeOf((await Jobs.list(pdriver, internals))[name], internals);
|
|
119
|
+
internals.matching = scope;
|
|
120
|
+
walker.directory = path.resolve(internals.argv['migrations-dir'], scope);
|
|
121
|
+
|
|
98
122
|
const files = await File.loadFromFileystem(
|
|
99
123
|
walker.directory,
|
|
100
124
|
walker.prefix,
|
|
@@ -116,6 +140,16 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
|
116
140
|
throw new Error(`The migration file of the job ${name} is missing`);
|
|
117
141
|
}
|
|
118
142
|
|
|
143
|
+
// its worker died between recording the migration and forgetting the job
|
|
144
|
+
const recorded = await walker._driver.allLoadedMigrationsAsync();
|
|
145
|
+
if (recorded.some(r => r.name === `${scope}/${name}`)) {
|
|
146
|
+
await update((job, jobs) => {
|
|
147
|
+
delete jobs[name];
|
|
148
|
+
});
|
|
149
|
+
log.info(`[jobs] ${name} is done`);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
119
153
|
const heartbeat = setInterval(() => {
|
|
120
154
|
update(() => {}).catch(err => log.error(err.message));
|
|
121
155
|
}, Math.max(Math.floor(options.timeout / 3), 1));
|
|
@@ -127,7 +161,7 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
|
127
161
|
const interrupted = interruption(jobs[name], hash);
|
|
128
162
|
|
|
129
163
|
await State.reloadSchema(pdriver, internals);
|
|
130
|
-
const mig = await pdriver
|
|
164
|
+
const mig = await keepRecord(pdriver, internals, name);
|
|
131
165
|
internals.modSchema = Object.assign(
|
|
132
166
|
{ i: {}, c: {}, f: {}, s: [] },
|
|
133
167
|
mig ? JSON.parse(mig.value) : {}
|
|
@@ -171,10 +205,13 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
|
171
205
|
},
|
|
172
206
|
{
|
|
173
207
|
rollingBack: () => update(job => { job.rb = 1; }),
|
|
174
|
-
restart: () =>
|
|
175
|
-
|
|
208
|
+
restart: async () => {
|
|
209
|
+
// reverting deleted the record, the run starting over needs one
|
|
210
|
+
await keepRecord(pdriver, internals, name);
|
|
211
|
+
await update(job => {
|
|
176
212
|
Object.assign(job, { step: 0, learned: 0, done: 0, rb: 0, h: hash });
|
|
177
|
-
})
|
|
213
|
+
});
|
|
214
|
+
}
|
|
178
215
|
}
|
|
179
216
|
);
|
|
180
217
|
|
|
@@ -216,8 +253,51 @@ const runJob = async (walker, name, id, options, stopping, result, seen) => {
|
|
|
216
253
|
}
|
|
217
254
|
};
|
|
218
255
|
|
|
256
|
+
/**
|
|
257
|
+
* A dry run, which jobs would run.
|
|
258
|
+
*/
|
|
259
|
+
const dryRun = async (internals, config) => {
|
|
260
|
+
const walker = await connect(internals, config);
|
|
261
|
+
|
|
262
|
+
try {
|
|
263
|
+
const jobs = await Jobs.list(walker._pdriver, internals);
|
|
264
|
+
const names = Object.keys(jobs)
|
|
265
|
+
.sort()
|
|
266
|
+
.filter(name => jobs[name].s !== 'failed');
|
|
267
|
+
|
|
268
|
+
names.forEach(name => log.info(`[jobs] ${name} would run`));
|
|
269
|
+
} finally {
|
|
270
|
+
await close(walker);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return { done: [], failed: [] };
|
|
274
|
+
};
|
|
275
|
+
|
|
219
276
|
module.exports = function executeWork (internals, config, opts = {}) {
|
|
220
277
|
const options = Object.assign({}, DEFAULTS, opts);
|
|
278
|
+
|
|
279
|
+
// the jobs of all scopes are kept with the top level, but those of the
|
|
280
|
+
// scopes with a database of their own
|
|
281
|
+
if (options.scope === 'all') {
|
|
282
|
+
const done = Promise.reject(
|
|
283
|
+
new Error(
|
|
284
|
+
'work runs the jobs of one database, the top level with the scopes ' +
|
|
285
|
+
'sharing it, or work:<scope> for a scope with a database of its own'
|
|
286
|
+
)
|
|
287
|
+
);
|
|
288
|
+
return { done, stop: () => done };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (options.scope) {
|
|
292
|
+
internals.migrationMode = options.scope;
|
|
293
|
+
internals.matching = options.scope;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
if (internals.dryRun) {
|
|
297
|
+
const done = dryRun(internals, config);
|
|
298
|
+
return { done, stop: () => done };
|
|
299
|
+
}
|
|
300
|
+
|
|
221
301
|
const id = crypto.randomBytes(16).toString('base64');
|
|
222
302
|
const seen = {};
|
|
223
303
|
const result = { done: [], failed: [] };
|