@alexify/migronaut 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +33 -2
- package/bullmq.d.ts +449 -6
- package/index.d.ts +1010 -9
- package/migronaut.schema.json +93 -1
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +128 -14
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migrator.js +904 -12
- package/src/core/options.js +16 -0
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +107 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The driver's BSON, from the `mongodb` peer — loaded on first use, never at
|
|
3
|
+
* startup, and from this one module only (pinned by a test), so nothing else
|
|
4
|
+
* under src/ grows its own dependency on the driver's internals.
|
|
5
|
+
*
|
|
6
|
+
* What it is for: measuring a document the way the server will (a step's
|
|
7
|
+
* checkpoint must stay small) and writing one as relaxed EJSON for a dry
|
|
8
|
+
* run's report — both things only BSON itself can do exactly.
|
|
9
|
+
*/
|
|
10
|
+
let bson;
|
|
11
|
+
|
|
12
|
+
function loadBson() {
|
|
13
|
+
bson ??= require('mongodb').BSON;
|
|
14
|
+
return bson;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The size of `value` as a BSON document, in bytes */
|
|
18
|
+
const bsonSize = (value) => loadBson().calculateObjectSize(value);
|
|
19
|
+
|
|
20
|
+
/** `value` as relaxed EJSON — plain JSON a person can read, types tagged where JSON has none */
|
|
21
|
+
const toRelaxedEjson = (value) => loadBson().EJSON.serialize(value, { relaxed: true });
|
|
22
|
+
|
|
23
|
+
module.exports = { bsonSize, loadBson, toRelaxedEjson };
|
package/src/core/changelog.js
CHANGED
|
@@ -109,6 +109,26 @@ class Changelog {
|
|
|
109
109
|
return failed;
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Which of `names` are applied from a history that predates migronaut — a
|
|
114
|
+
* `baseline` or an import from migrate-mongo: never run here, so a
|
|
115
|
+
* background migration among them has no state, and counts as done.
|
|
116
|
+
*/
|
|
117
|
+
async getAdoptedNames(db, names) {
|
|
118
|
+
if (names.length === 0) return [];
|
|
119
|
+
const docs = await this.#coll(db)
|
|
120
|
+
.find({
|
|
121
|
+
status: 'applied',
|
|
122
|
+
origin: { $in: ['baseline', 'migrate-mongo'] },
|
|
123
|
+
name: { $in: names },
|
|
124
|
+
})
|
|
125
|
+
.project({ name: 1, _id: 0 })
|
|
126
|
+
.toArray();
|
|
127
|
+
const adopted = [];
|
|
128
|
+
for (const doc of docs) adopted.push(doc.name);
|
|
129
|
+
return adopted;
|
|
130
|
+
}
|
|
131
|
+
|
|
112
132
|
/** Return a single record by migration name, or null */
|
|
113
133
|
async getByName(db, name) {
|
|
114
134
|
return this.#coll(db).findOne({ name });
|
|
@@ -292,6 +312,18 @@ class Changelog {
|
|
|
292
312
|
await this.#coll(db).bulkWrite(ops, { ordered: false });
|
|
293
313
|
}
|
|
294
314
|
|
|
315
|
+
/**
|
|
316
|
+
* Re-pin an applied record to the file now on disk — `background repin`,
|
|
317
|
+
* so the strict drift check agrees with what is actually running.
|
|
318
|
+
*/
|
|
319
|
+
async setChecksum(db, name, checksum) {
|
|
320
|
+
const result = await this.#coll(db).updateOne(
|
|
321
|
+
{ name, status: 'applied' },
|
|
322
|
+
{ $set: { checksum } },
|
|
323
|
+
);
|
|
324
|
+
return result.matchedCount === 1;
|
|
325
|
+
}
|
|
326
|
+
|
|
295
327
|
/**
|
|
296
328
|
* Mark a migration as reverted. Sets `status='reverted'` and `revertedAt=now`.
|
|
297
329
|
* Never deletes the record — preserves the full audit history.
|
package/src/core/collections.js
CHANGED
|
@@ -8,6 +8,13 @@ const { errorText } = require('../utils/error.js');
|
|
|
8
8
|
const { importUserFile, tsLoadMessageOrNull } = require('../utils/loader.js');
|
|
9
9
|
const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('./index-spec.js');
|
|
10
10
|
const { normalizeDeclaredSearchIndex, searchIndexIssues } = require('./search-index-spec.js');
|
|
11
|
+
const { resolveVersioning, versioningIssues } = require('../versioning/config.js');
|
|
12
|
+
const {
|
|
13
|
+
isVersioningIndexKey,
|
|
14
|
+
mergeVersioningValidator,
|
|
15
|
+
validatorVersioningIssues,
|
|
16
|
+
versioningIndex,
|
|
17
|
+
} = require('./versioning-spec.js');
|
|
11
18
|
|
|
12
19
|
/**
|
|
13
20
|
* Declared collections: validating a definition, normalizing it for the
|
|
@@ -29,6 +36,7 @@ const DEFINITION_KEYS = [
|
|
|
29
36
|
'validationLevel',
|
|
30
37
|
'validationAction',
|
|
31
38
|
'prune',
|
|
39
|
+
'versioning',
|
|
32
40
|
];
|
|
33
41
|
const DEFINITION_KEY_SET = new Set(DEFINITION_KEYS);
|
|
34
42
|
const VALIDATION_LEVELS = ['off', 'strict', 'moderate'];
|
|
@@ -108,6 +116,40 @@ function searchIndexListIssues(searchIndexes, base, issues) {
|
|
|
108
116
|
}
|
|
109
117
|
}
|
|
110
118
|
|
|
119
|
+
/**
|
|
120
|
+
* Validate the `versioning` key and how it fits the rest of the definition:
|
|
121
|
+
* a declared validator must leave the managed fields to it, and a declared
|
|
122
|
+
* index must not duplicate the version index. Returns whether the definition
|
|
123
|
+
* is versioned (valid or not), so the validator checks below know a
|
|
124
|
+
* validator is coming.
|
|
125
|
+
*/
|
|
126
|
+
function versioningDefinitionIssues(definition, base, issues) {
|
|
127
|
+
if (definition.versioning === undefined) return false;
|
|
128
|
+
const path = join(base, 'versioning');
|
|
129
|
+
const found = versioningIssues(definition.versioning, path);
|
|
130
|
+
issues.push(...found);
|
|
131
|
+
if (found.length > 0) return true;
|
|
132
|
+
const versioning = resolveVersioning(definition.versioning);
|
|
133
|
+
if (isPlainObject(definition.validator) || definition.validator === null) {
|
|
134
|
+
for (const message of validatorVersioningIssues(definition.validator, versioning)) {
|
|
135
|
+
issues.push({ path: join(base, 'validator'), message });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if (versioning.index && Array.isArray(definition.indexes)) {
|
|
139
|
+
for (const [position, index] of definition.indexes.entries()) {
|
|
140
|
+
if (isPlainObject(index) && isVersioningIndexKey(index.key, versioning)) {
|
|
141
|
+
issues.push({
|
|
142
|
+
path: `${join(base, 'indexes')}[${position}]`,
|
|
143
|
+
message:
|
|
144
|
+
'is the version index, which versioning declares itself — remove it, or set ' +
|
|
145
|
+
'versioning.index: false',
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
|
|
111
153
|
/**
|
|
112
154
|
* Validate one collection definition, returning `{ path, message }` issues
|
|
113
155
|
* (empty when valid). `path` prefixes every issue (`collections[2]`, or
|
|
@@ -149,17 +191,19 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
|
|
|
149
191
|
if (
|
|
150
192
|
definition.indexes === undefined &&
|
|
151
193
|
definition.searchIndexes === undefined &&
|
|
152
|
-
definition.validator === undefined
|
|
194
|
+
definition.validator === undefined &&
|
|
195
|
+
definition.versioning === undefined
|
|
153
196
|
) {
|
|
154
197
|
issues.push({
|
|
155
198
|
path: self,
|
|
156
|
-
message: 'declares no indexes, searchIndexes or
|
|
199
|
+
message: 'declares no indexes, searchIndexes, validator or versioning — nothing to manage',
|
|
157
200
|
});
|
|
158
201
|
}
|
|
159
202
|
if (definition.indexes !== undefined) indexListIssues(definition.indexes, base, issues);
|
|
160
203
|
if (definition.searchIndexes !== undefined) {
|
|
161
204
|
searchIndexListIssues(definition.searchIndexes, base, issues);
|
|
162
205
|
}
|
|
206
|
+
const versioning = versioningDefinitionIssues(definition, base, issues);
|
|
163
207
|
|
|
164
208
|
const { validator } = definition;
|
|
165
209
|
if (validator !== undefined && validator !== null) {
|
|
@@ -170,7 +214,8 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
|
|
|
170
214
|
if (reason) report('validator', reason);
|
|
171
215
|
}
|
|
172
216
|
}
|
|
173
|
-
|
|
217
|
+
// The versioning rules are a validator of their own.
|
|
218
|
+
const hasValidator = (isPlainObject(validator) && !isEmptyObject(validator)) || versioning;
|
|
174
219
|
for (const [key, allowed] of [
|
|
175
220
|
['validationLevel', VALIDATION_LEVELS],
|
|
176
221
|
['validationAction', VALIDATION_ACTIONS],
|
|
@@ -222,13 +267,39 @@ function collectionsIssues(list, { reserved = [] } = {}) {
|
|
|
222
267
|
* names, server-form keys, the spec to send), search indexes with their
|
|
223
268
|
* default name and type, the validator cleaned for the wire.
|
|
224
269
|
* `indexes`/`searchIndexes`/`validator` stay `undefined` when not managed.
|
|
270
|
+
* `versioning` is resolved and folded in: its rules merged into the
|
|
271
|
+
* validator, its index appended to `indexes` — with `indexesPartial` when it
|
|
272
|
+
* is the only index declared, so the others stay unmanaged.
|
|
225
273
|
*/
|
|
226
274
|
function normalizeDefinition(definition, { name, source } = {}) {
|
|
227
275
|
const { validator } = definition;
|
|
276
|
+
let wireValidator = validator === undefined || validator === null ? validator : toWire(validator);
|
|
277
|
+
let indexes = definition.indexes?.map((index) => normalizeDeclaredIndex(index));
|
|
278
|
+
let validationLevel = definition.validationLevel;
|
|
279
|
+
let versioning;
|
|
280
|
+
let indexesPartial = false;
|
|
281
|
+
if (definition.versioning !== undefined) {
|
|
282
|
+
// Folded into an ordinary validator and an ordinary index, so the planner
|
|
283
|
+
// needs to know nothing about versioning.
|
|
284
|
+
versioning = resolveVersioning(definition.versioning);
|
|
285
|
+
// A validator synthesized for versioning alone applies `moderate`: an
|
|
286
|
+
// update to a legacy document that predates the rules stays possible.
|
|
287
|
+
if (wireValidator === undefined || isEmptyObject(wireValidator)) validationLevel ??= 'moderate';
|
|
288
|
+
wireValidator = mergeVersioningValidator(wireValidator, versioning);
|
|
289
|
+
if (versioning.index) {
|
|
290
|
+
// Marked: on a sharded collection the planner swaps in the shard-key-prefixed form.
|
|
291
|
+
const index = { ...normalizeDeclaredIndex(versioningIndex(versioning)), versionIndex: true };
|
|
292
|
+
// Declaring the version index alone must not make every other index of
|
|
293
|
+
// the collection "undeclared" — and so a candidate for prune.
|
|
294
|
+
indexesPartial = indexes === undefined;
|
|
295
|
+
indexes = [...(indexes ?? []), index];
|
|
296
|
+
}
|
|
297
|
+
}
|
|
228
298
|
return {
|
|
229
299
|
name: definition.name ?? name,
|
|
230
300
|
source,
|
|
231
|
-
indexes
|
|
301
|
+
indexes,
|
|
302
|
+
...(indexesPartial ? { indexesPartial } : {}),
|
|
232
303
|
...(definition.searchIndexes !== undefined
|
|
233
304
|
? {
|
|
234
305
|
searchIndexes: definition.searchIndexes.map((index) =>
|
|
@@ -236,14 +307,13 @@ function normalizeDefinition(definition, { name, source } = {}) {
|
|
|
236
307
|
),
|
|
237
308
|
}
|
|
238
309
|
: {}),
|
|
239
|
-
validator:
|
|
240
|
-
...(
|
|
241
|
-
? { validationLevel: definition.validationLevel }
|
|
242
|
-
: {}),
|
|
310
|
+
validator: wireValidator,
|
|
311
|
+
...(validationLevel !== undefined ? { validationLevel } : {}),
|
|
243
312
|
...(definition.validationAction !== undefined
|
|
244
313
|
? { validationAction: definition.validationAction }
|
|
245
314
|
: {}),
|
|
246
315
|
...(definition.prune !== undefined ? { prune: definition.prune } : {}),
|
|
316
|
+
...(versioning !== undefined ? { versioning } : {}),
|
|
247
317
|
};
|
|
248
318
|
}
|
|
249
319
|
|
package/src/core/config.js
CHANGED
|
@@ -38,6 +38,11 @@ const DEFAULT_CONFIG = {
|
|
|
38
38
|
onSearchUnavailable: 'fail',
|
|
39
39
|
waitForSearchIndexes: false,
|
|
40
40
|
searchIndexWaitTimeoutMs: 600_000,
|
|
41
|
+
backgroundCollection: '_migronaut_background',
|
|
42
|
+
backgroundInline: false,
|
|
43
|
+
backgroundOnDrift: 'reopen',
|
|
44
|
+
backgroundDrift: 'poll',
|
|
45
|
+
backgroundShardAware: 'auto',
|
|
41
46
|
};
|
|
42
47
|
|
|
43
48
|
/** Candidate config file names, checked in priority order within the cwd */
|
|
@@ -47,6 +52,26 @@ const isNonEmptyString = (value) => typeof value === 'string' && value.length >
|
|
|
47
52
|
const isBoolean = (value) => typeof value === 'boolean';
|
|
48
53
|
const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
|
|
49
54
|
const isExtension = (value) => value === 'ts' || value === 'js';
|
|
55
|
+
const oneOf = (values) => (value) => values.includes(value);
|
|
56
|
+
const quoted = (values) => values.map((value) => `'${value}'`).join(', ');
|
|
57
|
+
|
|
58
|
+
const ON_DRIFT = ['reopen', 'report'];
|
|
59
|
+
const DRIFT_MODES = ['poll', 'stream', 'both'];
|
|
60
|
+
const SHARD_AWARE = ['auto', 'off'];
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The collections a background migration keeps its state in: the state
|
|
64
|
+
* documents themselves, their partitions and the drift watcher's resume
|
|
65
|
+
* tokens — the last two named after the first, so one setting moves all
|
|
66
|
+
* three (and a test of "must differ" covers them together).
|
|
67
|
+
*/
|
|
68
|
+
function backgroundCollectionNames(backgroundCollection) {
|
|
69
|
+
return {
|
|
70
|
+
state: backgroundCollection,
|
|
71
|
+
partitions: `${backgroundCollection}_partitions`,
|
|
72
|
+
watch: `${backgroundCollection}_watch`,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
50
75
|
|
|
51
76
|
function isStringList(value) {
|
|
52
77
|
if (!Array.isArray(value) || value.length === 0) return false;
|
|
@@ -169,6 +194,31 @@ const CONFIG_KEYS = [
|
|
|
169
194
|
message: 'must be a positive integer',
|
|
170
195
|
optional: true,
|
|
171
196
|
},
|
|
197
|
+
{
|
|
198
|
+
path: 'backgroundCollection',
|
|
199
|
+
check: isCollectionName,
|
|
200
|
+
message: "must be a valid collection name (no '$'/NUL, not system.*)",
|
|
201
|
+
optional: true,
|
|
202
|
+
},
|
|
203
|
+
{ path: 'backgroundInline', check: isBoolean, message: 'must be a boolean', optional: true },
|
|
204
|
+
{
|
|
205
|
+
path: 'backgroundOnDrift',
|
|
206
|
+
check: oneOf(ON_DRIFT),
|
|
207
|
+
message: `must be ${quoted(ON_DRIFT)}`,
|
|
208
|
+
optional: true,
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
path: 'backgroundDrift',
|
|
212
|
+
check: oneOf(DRIFT_MODES),
|
|
213
|
+
message: `must be ${quoted(DRIFT_MODES)}`,
|
|
214
|
+
optional: true,
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
path: 'backgroundShardAware',
|
|
218
|
+
check: oneOf(SHARD_AWARE),
|
|
219
|
+
message: `must be ${quoted(SHARD_AWARE)}`,
|
|
220
|
+
optional: true,
|
|
221
|
+
},
|
|
172
222
|
];
|
|
173
223
|
|
|
174
224
|
/**
|
|
@@ -220,26 +270,51 @@ function validateConfig(config, options = {}) {
|
|
|
220
270
|
// pure data, so this costs nothing. Definition *files* are loaded only when
|
|
221
271
|
// a converge runs: importing them here would make one broken file block
|
|
222
272
|
// every command, an emergency `down` included.
|
|
223
|
-
//
|
|
224
|
-
const bookkeeping =
|
|
225
|
-
for (
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
273
|
+
// Every bookkeeping collection has one job: sharing one would mix records.
|
|
274
|
+
const bookkeeping = bookkeepingCollections(config);
|
|
275
|
+
for (let position = 1; position < bookkeeping.length; position++) {
|
|
276
|
+
const [key, name, what] = bookkeeping[position];
|
|
277
|
+
if (name === undefined) continue;
|
|
278
|
+
for (let earlier = 0; earlier < position; earlier++) {
|
|
279
|
+
const [otherKey, otherName, otherWhat] = bookkeeping[earlier];
|
|
280
|
+
if (name !== otherName) continue;
|
|
281
|
+
issues.push({
|
|
282
|
+
path: key,
|
|
283
|
+
message: `${what ? `${what} (${name}) ` : ''}must differ from ${otherWhat ? `${otherKey}'s ${otherWhat}` : otherKey}`,
|
|
284
|
+
});
|
|
285
|
+
break;
|
|
230
286
|
}
|
|
231
287
|
}
|
|
232
288
|
if (Array.isArray(config.collections)) {
|
|
233
|
-
const reserved = [
|
|
234
|
-
|
|
235
|
-
config.lockCollection,
|
|
236
|
-
config.convergeLogCollection,
|
|
237
|
-
];
|
|
289
|
+
const reserved = [];
|
|
290
|
+
for (const [, name] of bookkeeping) if (name !== undefined) reserved.push(name);
|
|
238
291
|
for (const issue of collectionsIssues(config.collections, { reserved })) issues.push(issue);
|
|
239
292
|
}
|
|
240
293
|
return issues;
|
|
241
294
|
}
|
|
242
295
|
|
|
296
|
+
/**
|
|
297
|
+
* Every collection migronaut keeps records in, as `[configKey, name, what?]`
|
|
298
|
+
* — `what` names a collection derived from the key (the background
|
|
299
|
+
* partitions and watch collections).
|
|
300
|
+
*/
|
|
301
|
+
function bookkeepingCollections(config) {
|
|
302
|
+
const entries = [
|
|
303
|
+
['migrationsCollection', config.migrationsCollection],
|
|
304
|
+
['lockCollection', config.lockCollection],
|
|
305
|
+
['convergeLogCollection', config.convergeLogCollection],
|
|
306
|
+
];
|
|
307
|
+
if (typeof config.backgroundCollection === 'string') {
|
|
308
|
+
const names = backgroundCollectionNames(config.backgroundCollection);
|
|
309
|
+
entries.push(
|
|
310
|
+
['backgroundCollection', names.state],
|
|
311
|
+
['backgroundCollection', names.partitions, 'partitions collection'],
|
|
312
|
+
['backgroundCollection', names.watch, 'watch collection'],
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
return entries;
|
|
316
|
+
}
|
|
317
|
+
|
|
243
318
|
const TRUE_BOOLEAN_STRINGS = new Set(['true', '1', 'yes']);
|
|
244
319
|
const FALSE_BOOLEAN_STRINGS = new Set(['false', '0', 'no']);
|
|
245
320
|
|
|
@@ -361,6 +436,19 @@ const ENV_KEYS = [
|
|
|
361
436
|
path: 'searchIndexWaitTimeoutMs',
|
|
362
437
|
parse: parsePositiveInteger,
|
|
363
438
|
},
|
|
439
|
+
{ env: 'MIGRONAUT_BACKGROUND_COLLECTION', path: 'backgroundCollection', parse: parseString },
|
|
440
|
+
{ env: 'MIGRONAUT_BACKGROUND_INLINE', path: 'backgroundInline', parse: parseBoolean },
|
|
441
|
+
{
|
|
442
|
+
env: 'MIGRONAUT_BACKGROUND_ON_DRIFT',
|
|
443
|
+
path: 'backgroundOnDrift',
|
|
444
|
+
parse: parseEnum(ON_DRIFT),
|
|
445
|
+
},
|
|
446
|
+
{ env: 'MIGRONAUT_BACKGROUND_DRIFT', path: 'backgroundDrift', parse: parseEnum(DRIFT_MODES) },
|
|
447
|
+
{
|
|
448
|
+
env: 'MIGRONAUT_BACKGROUND_SHARD_AWARE',
|
|
449
|
+
path: 'backgroundShardAware',
|
|
450
|
+
parse: parseEnum(SHARD_AWARE),
|
|
451
|
+
},
|
|
364
452
|
];
|
|
365
453
|
|
|
366
454
|
/** Build a partial config from the MIGRONAUT_* environment variables */
|
|
@@ -599,6 +687,8 @@ module.exports = {
|
|
|
599
687
|
CONFIG_KEYS,
|
|
600
688
|
DEFAULT_CONFIG,
|
|
601
689
|
ENV_KEYS,
|
|
690
|
+
backgroundCollectionNames,
|
|
691
|
+
bookkeepingCollections,
|
|
602
692
|
// Re-exported from utils/collection-name.js, where it moved so that
|
|
603
693
|
// core/collections.js can use it without a require cycle through here.
|
|
604
694
|
isCollectionName,
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
const { deepEqual } = require('../utils/canonical.js');
|
|
2
|
-
const {
|
|
2
|
+
const { shardedVersionIndexKey, versionFloorConflict } = require('./versioning-spec.js');
|
|
3
|
+
const {
|
|
4
|
+
compareIndex,
|
|
5
|
+
normalizeDeclaredIndex,
|
|
6
|
+
normalizeLiveIndex,
|
|
7
|
+
restoreSpec,
|
|
8
|
+
sameSignature,
|
|
9
|
+
} = require('./index-spec.js');
|
|
3
10
|
const {
|
|
4
11
|
compareSearchIndex,
|
|
5
12
|
isBeingRemoved,
|
|
@@ -179,7 +186,50 @@ function joinReasons(first, second) {
|
|
|
179
186
|
return first ? `${first}; ${second}` : second;
|
|
180
187
|
}
|
|
181
188
|
|
|
182
|
-
|
|
189
|
+
/** Why the ordinary version index stays on a sharded collection whose version index replaced it */
|
|
190
|
+
const DISPLACED_REASON =
|
|
191
|
+
'replaced by the shard-key-prefixed version index — drop it once nothing hints it ' +
|
|
192
|
+
'(prune does, when the indexes are declared)';
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* The declared indexes as they apply to this live collection: on a sharded
|
|
196
|
+
* one, the version index takes the shard key between the version field and
|
|
197
|
+
* `_id` (versioning-spec.js). `displaced` names the ordinary version index it
|
|
198
|
+
* replaces, when the two differ.
|
|
199
|
+
*/
|
|
200
|
+
function indexesFor(definition, live) {
|
|
201
|
+
const indexes = definition.indexes;
|
|
202
|
+
if (indexes === undefined || !definition.versioning || !live.shardKey) return { indexes };
|
|
203
|
+
const sharded = normalizeDeclaredIndex({
|
|
204
|
+
key: shardedVersionIndexKey(definition.versioning, live.shardKey),
|
|
205
|
+
});
|
|
206
|
+
const result = [];
|
|
207
|
+
let displaced;
|
|
208
|
+
for (const index of indexes) {
|
|
209
|
+
if (index.versionIndex === true && index.name !== sharded.name) {
|
|
210
|
+
displaced = index.name;
|
|
211
|
+
result.push({ ...sharded, versionIndex: true });
|
|
212
|
+
} else {
|
|
213
|
+
result.push(index);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return { indexes: result, displaced };
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Plan the declared indexes of one collection against its live ones. With
|
|
221
|
+
* `partial` (only the version index of `versioning` is declared, not the
|
|
222
|
+
* collection's own list) the other live indexes are not managed at all: never
|
|
223
|
+
* listed, never dropped — prune does not reach them.
|
|
224
|
+
*/
|
|
225
|
+
function planIndexes(
|
|
226
|
+
declaredIndexes,
|
|
227
|
+
live,
|
|
228
|
+
{ prune: pruneOption, rebuildUnique, capabilities, partial = false, displaced },
|
|
229
|
+
row,
|
|
230
|
+
steps,
|
|
231
|
+
) {
|
|
232
|
+
const prune = pruneOption && !partial;
|
|
183
233
|
const defaultCollation = live.options?.collation;
|
|
184
234
|
const liveIndexes = [];
|
|
185
235
|
for (const raw of live.indexes) {
|
|
@@ -360,7 +410,22 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
|
|
|
360
410
|
}
|
|
361
411
|
if (group.actions.length > 0) steps.push(group);
|
|
362
412
|
|
|
413
|
+
// The ordinary version index a sharded one replaced: kept, and said so —
|
|
414
|
+
// a background migration may still be hinting it.
|
|
415
|
+
const old = displaced !== undefined ? byName.get(displaced) : undefined;
|
|
416
|
+
if (old !== undefined && !declaredNames.has(displaced) && !prune) {
|
|
417
|
+
consumed.add(displaced);
|
|
418
|
+
row({
|
|
419
|
+
target: 'index',
|
|
420
|
+
name: displaced,
|
|
421
|
+
action: 'keep',
|
|
422
|
+
reason: DISPLACED_REASON,
|
|
423
|
+
from: indexValue(old.raw),
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
|
|
363
427
|
// Last, so an index is only ever removed once everything declared exists.
|
|
428
|
+
if (partial) return;
|
|
364
429
|
for (const index of liveIndexes) {
|
|
365
430
|
if (consumed.has(index.name) || declaredNames.has(index.name)) continue;
|
|
366
431
|
if (prune && backsShardKey(index, live.shardKey)) {
|
|
@@ -554,8 +619,8 @@ function planSearchIndexes(declaredList, live, { prune, search }, row, submit, d
|
|
|
554
619
|
/**
|
|
555
620
|
* Plan one collection. `definition` is a normalized definition (see
|
|
556
621
|
* collections.js); `live` is `{ exists, type?, options?, indexes,
|
|
557
|
-
* searchIndexes? }` as converge.js reads it.
|
|
558
|
-
* `actions` are the result rows (status `'planned'`), `steps` the operations
|
|
622
|
+
* searchIndexes?, shardKey?, versionFloor? }` as converge.js reads it.
|
|
623
|
+
* Returns `{ name, actions, steps }`: `actions` are the result rows (status `'planned'`), `steps` the operations
|
|
559
624
|
* that carry them out, in execution order, each pointing at the rows it
|
|
560
625
|
* settles. `search` is `{ available, onUnavailable }` — whether the server has
|
|
561
626
|
* Atlas Search, and what a declared search index becomes when it does not.
|
|
@@ -615,7 +680,7 @@ function planCollectionSteps(
|
|
|
615
680
|
}
|
|
616
681
|
|
|
617
682
|
const desired = desiredValidator(definition);
|
|
618
|
-
const declaredIndexes = definition
|
|
683
|
+
const { indexes: declaredIndexes, displaced } = indexesFor(definition, live);
|
|
619
684
|
const declaredSearch = definition.searchIndexes;
|
|
620
685
|
const indexSteps = [];
|
|
621
686
|
const searchSubmit = [];
|
|
@@ -655,10 +720,24 @@ function planCollectionSteps(
|
|
|
655
720
|
}
|
|
656
721
|
|
|
657
722
|
if (desired !== undefined) {
|
|
658
|
-
|
|
723
|
+
// Raising versioning.min over documents still below it: refused, so the
|
|
724
|
+
// whole run stops before its first write (see versionFloorToCheck).
|
|
725
|
+
const refused = versionFloorConflict(live.versionFloor);
|
|
726
|
+
if (refused) {
|
|
727
|
+
row({ target: 'validator', name, action: 'conflict', reason: refused, to: desired });
|
|
728
|
+
} else {
|
|
729
|
+
planValidator(name, desired, liveValidator(live.options), row, steps);
|
|
730
|
+
}
|
|
659
731
|
}
|
|
660
732
|
if (declaredIndexes !== undefined) {
|
|
661
|
-
|
|
733
|
+
const partial = definition.indexesPartial === true;
|
|
734
|
+
planIndexes(
|
|
735
|
+
declaredIndexes,
|
|
736
|
+
live,
|
|
737
|
+
{ prune, rebuildUnique, capabilities, partial, displaced },
|
|
738
|
+
row,
|
|
739
|
+
indexSteps,
|
|
740
|
+
);
|
|
662
741
|
}
|
|
663
742
|
if (declaredSearch !== undefined) {
|
|
664
743
|
planSearchIndexes(declaredSearch, live, { prune, search }, row, searchSubmit, searchDrops);
|
package/src/core/converge.js
CHANGED
|
@@ -29,6 +29,8 @@ const {
|
|
|
29
29
|
warnIgnored,
|
|
30
30
|
} = require('./converge-search-run.js');
|
|
31
31
|
const { inPlaceCapabilities } = require('./index-spec.js');
|
|
32
|
+
const { belowVersionFilter } = require('../versioning/document.js');
|
|
33
|
+
const { isVersioningIndexKey, versionFloorToCheck } = require('./versioning-spec.js');
|
|
32
34
|
const {
|
|
33
35
|
INDEX_NOT_FOUND,
|
|
34
36
|
NAMESPACE_NOT_FOUND,
|
|
@@ -74,6 +76,53 @@ const HINTS = {
|
|
|
74
76
|
'first (the index was left as it was)',
|
|
75
77
|
};
|
|
76
78
|
|
|
79
|
+
/**
|
|
80
|
+
* How long the version floor probe may scan. With the version index it reads
|
|
81
|
+
* one key; without it (the index is created by the same converge that raises
|
|
82
|
+
* the floor) it may have to scan the collection — bounded, and a timeout
|
|
83
|
+
* refuses the raise rather than risk it.
|
|
84
|
+
*/
|
|
85
|
+
const FLOOR_PROBE_TIMEOUT_MS = 60_000;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Before converge raises `versioning.min`: is any document still below it?
|
|
89
|
+
* Sets `live.versionFloor` to `{ min, below }` (`below: 'unknown'` with
|
|
90
|
+
* `error` when the probe failed) for the planner to refuse the raise on. One
|
|
91
|
+
* `findOne` with only `_id` projected — nothing about the document is kept.
|
|
92
|
+
*/
|
|
93
|
+
async function probeVersionFloor(db, definition, live) {
|
|
94
|
+
const min = versionFloorToCheck(definition, live);
|
|
95
|
+
if (min === null) return;
|
|
96
|
+
const { versioning } = definition;
|
|
97
|
+
let hint;
|
|
98
|
+
for (const index of live.indexes) {
|
|
99
|
+
if (isVersioningIndexKey(index.key, versioning)) {
|
|
100
|
+
hint = index.name;
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const found = await db
|
|
106
|
+
.collection(definition.name)
|
|
107
|
+
.findOne(belowVersionFilter(versioning, min), {
|
|
108
|
+
projection: { _id: 1 },
|
|
109
|
+
maxTimeMS: FLOOR_PROBE_TIMEOUT_MS,
|
|
110
|
+
...(hint !== undefined ? { hint } : {}),
|
|
111
|
+
...READ_OPTIONS,
|
|
112
|
+
});
|
|
113
|
+
live.versionFloor = { min, below: found !== null };
|
|
114
|
+
} catch (error) {
|
|
115
|
+
live.versionFloor = { min, below: 'unknown', error: errorText(error) };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Probe every versioned collection whose floor would rise — a few at a time */
|
|
120
|
+
async function probeVersionFloors(db, definitions, live) {
|
|
121
|
+
await mapLimit([...definitions.keys()], READ_CONCURRENCY, (position) =>
|
|
122
|
+
probeVersionFloor(db, definitions[position], live[position]),
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
77
126
|
/** Shard key of each named collection, behind a mongos — when `config.collections` may be read */
|
|
78
127
|
async function readShardKeys(deps, names) {
|
|
79
128
|
const keys = new Map();
|
|
@@ -577,6 +626,7 @@ async function readAndPlan(deps, options) {
|
|
|
577
626
|
}
|
|
578
627
|
}
|
|
579
628
|
if (search.declared) await readSearch(deps, server, definitions, live, search);
|
|
629
|
+
await probeVersionFloors(db, definitions, live);
|
|
580
630
|
const plans = new Array(definitions.length);
|
|
581
631
|
const ignored = [];
|
|
582
632
|
for (const [position, definition] of definitions.entries()) {
|
|
@@ -636,6 +686,7 @@ async function readFresh(run, position, phase) {
|
|
|
636
686
|
) {
|
|
637
687
|
fresh.searchIndexes = await readSearchIndexes(deps, definition.name, phase);
|
|
638
688
|
}
|
|
689
|
+
await probeVersionFloor(deps.db, definition, fresh);
|
|
639
690
|
return fresh;
|
|
640
691
|
}
|
|
641
692
|
|
|
@@ -840,6 +891,42 @@ async function applyCollection(deps, plan, result, signal, search) {
|
|
|
840
891
|
return kept;
|
|
841
892
|
}
|
|
842
893
|
|
|
894
|
+
/**
|
|
895
|
+
* After a run raised `versioning.min`: old-shape documents written while it
|
|
896
|
+
* ran (an old pod still deploying) got past the probe. Converge cannot undo
|
|
897
|
+
* the validator, so it says so — those documents now fail validation on
|
|
898
|
+
* their next strict write, and a background migration has to pick them up.
|
|
899
|
+
*/
|
|
900
|
+
async function warnFloorBreach(run, position) {
|
|
901
|
+
const { deps, definitions, live, result } = run;
|
|
902
|
+
const floor = live[position].versionFloor;
|
|
903
|
+
if (floor?.below !== false) return;
|
|
904
|
+
const raised = result.collections[position].actions.some(
|
|
905
|
+
(action) => action.target === 'validator' && action.status === 'applied',
|
|
906
|
+
);
|
|
907
|
+
if (!raised) return;
|
|
908
|
+
const definition = definitions[position];
|
|
909
|
+
let found;
|
|
910
|
+
try {
|
|
911
|
+
found = await deps.db
|
|
912
|
+
.collection(definition.name)
|
|
913
|
+
.findOne(belowVersionFilter(definition.versioning, floor.min), {
|
|
914
|
+
projection: { _id: 1 },
|
|
915
|
+
maxTimeMS: FLOOR_PROBE_TIMEOUT_MS,
|
|
916
|
+
...READ_OPTIONS,
|
|
917
|
+
});
|
|
918
|
+
} catch {
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
if (found === null) return;
|
|
922
|
+
deps.logger.warn(
|
|
923
|
+
`⚠ ${definition.name}: documents below version ${floor.min} were written while converge ` +
|
|
924
|
+
'raised versioning.min — an old release is still writing; run the background migration ' +
|
|
925
|
+
'again once it is gone',
|
|
926
|
+
deps.fields({ collection: definition.name, min: floor.min }),
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
|
|
843
930
|
/**
|
|
844
931
|
* The verify phase — the fixed-point check: what was just applied must now
|
|
845
932
|
* compare as unchanged. Anything that does not would be "changed" again on
|
|
@@ -864,6 +951,7 @@ async function verifyFixedPoint(run, position, kept, signal) {
|
|
|
864
951
|
after = planFor(definition, await readFresh(run, position, 'apply'));
|
|
865
952
|
}
|
|
866
953
|
refreshBuilds(rows, after.actions);
|
|
954
|
+
await warnFloorBreach(run, position);
|
|
867
955
|
for (const action of after.actions) {
|
|
868
956
|
if (!CHANGE_ACTIONS.has(action.action)) continue;
|
|
869
957
|
if (action.action === 'drop' && kept.has(action.name)) continue;
|