@alexify/migronaut 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +41 -3
  3. package/bullmq.d.ts +484 -8
  4. package/index.d.ts +1264 -9
  5. package/migronaut.schema.json +93 -1
  6. package/package.json +9 -2
  7. package/src/bullmq/background-processor.js +541 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +348 -21
  11. package/src/bullmq/producer.js +185 -13
  12. package/src/bullmq/service.js +484 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/create.js +6 -0
  15. package/src/cli/exit-codes.js +6 -0
  16. package/src/cli/index.js +2 -0
  17. package/src/core/audit.js +11 -1
  18. package/src/core/background-audit.js +139 -0
  19. package/src/core/background-drift.js +126 -0
  20. package/src/core/background-dry-run.js +375 -0
  21. package/src/core/background-engine.js +849 -0
  22. package/src/core/background-kit.js +432 -0
  23. package/src/core/background-partition.js +298 -0
  24. package/src/core/background-runner.js +305 -0
  25. package/src/core/background-sandbox.js +701 -0
  26. package/src/core/background-shard.js +542 -0
  27. package/src/core/background-spec.js +597 -0
  28. package/src/core/background-store.js +951 -0
  29. package/src/core/background-throttle.js +269 -0
  30. package/src/core/background-watch-plan.js +164 -0
  31. package/src/core/background-watch-store.js +78 -0
  32. package/src/core/background-watch.js +610 -0
  33. package/src/core/background.js +1127 -0
  34. package/src/core/bson-peer.js +23 -0
  35. package/src/core/changelog.js +32 -0
  36. package/src/core/collections.js +78 -8
  37. package/src/core/config.js +102 -12
  38. package/src/core/converge-plan.js +86 -7
  39. package/src/core/converge.js +88 -0
  40. package/src/core/lock.js +48 -21
  41. package/src/core/migration-logger.js +279 -0
  42. package/src/core/migrator.js +1027 -22
  43. package/src/core/options.js +36 -0
  44. package/src/core/run-recorder.js +6 -1
  45. package/src/core/run.js +26 -12
  46. package/src/core/runner.js +34 -8
  47. package/src/core/server-info.js +9 -2
  48. package/src/core/shard-info.js +76 -0
  49. package/src/core/versioning-spec.js +181 -0
  50. package/src/errors/index.js +88 -0
  51. package/src/index.js +16 -0
  52. package/src/utils/error.js +11 -2
  53. package/src/utils/job-ref.js +44 -0
  54. package/src/utils/loader.js +77 -9
  55. package/src/utils/migration-name.js +33 -1
  56. package/src/utils/redact.js +140 -3
  57. package/src/utils/telemetry.js +110 -0
  58. package/src/utils/template.js +62 -1
  59. package/src/versioning/config.js +155 -0
  60. package/src/versioning/document.js +326 -0
  61. package/src/versioning/index.js +50 -0
  62. package/src/versioning/internal.js +279 -0
  63. package/src/versioning/mongoose.js +151 -0
  64. package/src/versioning/occ.js +318 -0
  65. package/src/versioning/registry.js +187 -0
  66. package/src/versioning/upcaster.js +213 -0
  67. package/versioning.d.ts +666 -0
  68. package/versioning.js +1 -0
@@ -0,0 +1,23 @@
1
+ /**
2
+ * The driver's BSON, from the `mongodb` peer — loaded on first use, never at
3
+ * startup, and from this one module only (pinned by a test), so nothing else
4
+ * under src/ grows its own dependency on the driver's internals.
5
+ *
6
+ * What it is for: measuring a document the way the server will (a step's
7
+ * checkpoint must stay small) and writing one as relaxed EJSON for a dry
8
+ * run's report — both things only BSON itself can do exactly.
9
+ */
10
+ let bson;
11
+
12
+ function loadBson() {
13
+ bson ??= require('mongodb').BSON;
14
+ return bson;
15
+ }
16
+
17
+ /** The size of `value` as a BSON document, in bytes */
18
+ const bsonSize = (value) => loadBson().calculateObjectSize(value);
19
+
20
+ /** `value` as relaxed EJSON — plain JSON a person can read, types tagged where JSON has none */
21
+ const toRelaxedEjson = (value) => loadBson().EJSON.serialize(value, { relaxed: true });
22
+
23
+ module.exports = { bsonSize, loadBson, toRelaxedEjson };
@@ -109,6 +109,26 @@ class Changelog {
109
109
  return failed;
110
110
  }
111
111
 
112
+ /**
113
+ * Which of `names` are applied from a history that predates migronaut — a
114
+ * `baseline` or an import from migrate-mongo: never run here, so a
115
+ * background migration among them has no state, and counts as done.
116
+ */
117
+ async getAdoptedNames(db, names) {
118
+ if (names.length === 0) return [];
119
+ const docs = await this.#coll(db)
120
+ .find({
121
+ status: 'applied',
122
+ origin: { $in: ['baseline', 'migrate-mongo'] },
123
+ name: { $in: names },
124
+ })
125
+ .project({ name: 1, _id: 0 })
126
+ .toArray();
127
+ const adopted = [];
128
+ for (const doc of docs) adopted.push(doc.name);
129
+ return adopted;
130
+ }
131
+
112
132
  /** Return a single record by migration name, or null */
113
133
  async getByName(db, name) {
114
134
  return this.#coll(db).findOne({ name });
@@ -292,6 +312,18 @@ class Changelog {
292
312
  await this.#coll(db).bulkWrite(ops, { ordered: false });
293
313
  }
294
314
 
315
+ /**
316
+ * Re-pin an applied record to the file now on disk — `background repin`,
317
+ * so the strict drift check agrees with what is actually running.
318
+ */
319
+ async setChecksum(db, name, checksum) {
320
+ const result = await this.#coll(db).updateOne(
321
+ { name, status: 'applied' },
322
+ { $set: { checksum } },
323
+ );
324
+ return result.matchedCount === 1;
325
+ }
326
+
295
327
  /**
296
328
  * Mark a migration as reverted. Sets `status='reverted'` and `revertedAt=now`.
297
329
  * Never deletes the record — preserves the full audit history.
@@ -8,6 +8,13 @@ const { errorText } = require('../utils/error.js');
8
8
  const { importUserFile, tsLoadMessageOrNull } = require('../utils/loader.js');
9
9
  const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('./index-spec.js');
10
10
  const { normalizeDeclaredSearchIndex, searchIndexIssues } = require('./search-index-spec.js');
11
+ const { resolveVersioning, versioningIssues } = require('../versioning/config.js');
12
+ const {
13
+ isVersioningIndexKey,
14
+ mergeVersioningValidator,
15
+ validatorVersioningIssues,
16
+ versioningIndex,
17
+ } = require('./versioning-spec.js');
11
18
 
12
19
  /**
13
20
  * Declared collections: validating a definition, normalizing it for the
@@ -29,6 +36,7 @@ const DEFINITION_KEYS = [
29
36
  'validationLevel',
30
37
  'validationAction',
31
38
  'prune',
39
+ 'versioning',
32
40
  ];
33
41
  const DEFINITION_KEY_SET = new Set(DEFINITION_KEYS);
34
42
  const VALIDATION_LEVELS = ['off', 'strict', 'moderate'];
@@ -108,6 +116,40 @@ function searchIndexListIssues(searchIndexes, base, issues) {
108
116
  }
109
117
  }
110
118
 
119
+ /**
120
+ * Validate the `versioning` key and how it fits the rest of the definition:
121
+ * a declared validator must leave the managed fields to it, and a declared
122
+ * index must not duplicate the version index. Returns whether the definition
123
+ * is versioned (valid or not), so the validator checks below know a
124
+ * validator is coming.
125
+ */
126
+ function versioningDefinitionIssues(definition, base, issues) {
127
+ if (definition.versioning === undefined) return false;
128
+ const path = join(base, 'versioning');
129
+ const found = versioningIssues(definition.versioning, path);
130
+ issues.push(...found);
131
+ if (found.length > 0) return true;
132
+ const versioning = resolveVersioning(definition.versioning);
133
+ if (isPlainObject(definition.validator) || definition.validator === null) {
134
+ for (const message of validatorVersioningIssues(definition.validator, versioning)) {
135
+ issues.push({ path: join(base, 'validator'), message });
136
+ }
137
+ }
138
+ if (versioning.index && Array.isArray(definition.indexes)) {
139
+ for (const [position, index] of definition.indexes.entries()) {
140
+ if (isPlainObject(index) && isVersioningIndexKey(index.key, versioning)) {
141
+ issues.push({
142
+ path: `${join(base, 'indexes')}[${position}]`,
143
+ message:
144
+ 'is the version index, which versioning declares itself — remove it, or set ' +
145
+ 'versioning.index: false',
146
+ });
147
+ }
148
+ }
149
+ }
150
+ return true;
151
+ }
152
+
111
153
  /**
112
154
  * Validate one collection definition, returning `{ path, message }` issues
113
155
  * (empty when valid). `path` prefixes every issue (`collections[2]`, or
@@ -149,17 +191,19 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
149
191
  if (
150
192
  definition.indexes === undefined &&
151
193
  definition.searchIndexes === undefined &&
152
- definition.validator === undefined
194
+ definition.validator === undefined &&
195
+ definition.versioning === undefined
153
196
  ) {
154
197
  issues.push({
155
198
  path: self,
156
- message: 'declares no indexes, searchIndexes or validator — nothing to manage',
199
+ message: 'declares no indexes, searchIndexes, validator or versioning — nothing to manage',
157
200
  });
158
201
  }
159
202
  if (definition.indexes !== undefined) indexListIssues(definition.indexes, base, issues);
160
203
  if (definition.searchIndexes !== undefined) {
161
204
  searchIndexListIssues(definition.searchIndexes, base, issues);
162
205
  }
206
+ const versioning = versioningDefinitionIssues(definition, base, issues);
163
207
 
164
208
  const { validator } = definition;
165
209
  if (validator !== undefined && validator !== null) {
@@ -170,7 +214,8 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
170
214
  if (reason) report('validator', reason);
171
215
  }
172
216
  }
173
- const hasValidator = isPlainObject(validator) && !isEmptyObject(validator);
217
+ // The versioning rules are a validator of their own.
218
+ const hasValidator = (isPlainObject(validator) && !isEmptyObject(validator)) || versioning;
174
219
  for (const [key, allowed] of [
175
220
  ['validationLevel', VALIDATION_LEVELS],
176
221
  ['validationAction', VALIDATION_ACTIONS],
@@ -222,13 +267,39 @@ function collectionsIssues(list, { reserved = [] } = {}) {
222
267
  * names, server-form keys, the spec to send), search indexes with their
223
268
  * default name and type, the validator cleaned for the wire.
224
269
  * `indexes`/`searchIndexes`/`validator` stay `undefined` when not managed.
270
+ * `versioning` is resolved and folded in: its rules merged into the
271
+ * validator, its index appended to `indexes` — with `indexesPartial` when it
272
+ * is the only index declared, so the others stay unmanaged.
225
273
  */
226
274
  function normalizeDefinition(definition, { name, source } = {}) {
227
275
  const { validator } = definition;
276
+ let wireValidator = validator === undefined || validator === null ? validator : toWire(validator);
277
+ let indexes = definition.indexes?.map((index) => normalizeDeclaredIndex(index));
278
+ let validationLevel = definition.validationLevel;
279
+ let versioning;
280
+ let indexesPartial = false;
281
+ if (definition.versioning !== undefined) {
282
+ // Folded into an ordinary validator and an ordinary index, so the planner
283
+ // needs to know nothing about versioning.
284
+ versioning = resolveVersioning(definition.versioning);
285
+ // A validator synthesized for versioning alone applies `moderate`: an
286
+ // update to a legacy document that predates the rules stays possible.
287
+ if (wireValidator === undefined || isEmptyObject(wireValidator)) validationLevel ??= 'moderate';
288
+ wireValidator = mergeVersioningValidator(wireValidator, versioning);
289
+ if (versioning.index) {
290
+ // Marked: on a sharded collection the planner swaps in the shard-key-prefixed form.
291
+ const index = { ...normalizeDeclaredIndex(versioningIndex(versioning)), versionIndex: true };
292
+ // Declaring the version index alone must not make every other index of
293
+ // the collection "undeclared" — and so a candidate for prune.
294
+ indexesPartial = indexes === undefined;
295
+ indexes = [...(indexes ?? []), index];
296
+ }
297
+ }
228
298
  return {
229
299
  name: definition.name ?? name,
230
300
  source,
231
- indexes: definition.indexes?.map((index) => normalizeDeclaredIndex(index)),
301
+ indexes,
302
+ ...(indexesPartial ? { indexesPartial } : {}),
232
303
  ...(definition.searchIndexes !== undefined
233
304
  ? {
234
305
  searchIndexes: definition.searchIndexes.map((index) =>
@@ -236,14 +307,13 @@ function normalizeDefinition(definition, { name, source } = {}) {
236
307
  ),
237
308
  }
238
309
  : {}),
239
- validator: validator === undefined || validator === null ? validator : toWire(validator),
240
- ...(definition.validationLevel !== undefined
241
- ? { validationLevel: definition.validationLevel }
242
- : {}),
310
+ validator: wireValidator,
311
+ ...(validationLevel !== undefined ? { validationLevel } : {}),
243
312
  ...(definition.validationAction !== undefined
244
313
  ? { validationAction: definition.validationAction }
245
314
  : {}),
246
315
  ...(definition.prune !== undefined ? { prune: definition.prune } : {}),
316
+ ...(versioning !== undefined ? { versioning } : {}),
247
317
  };
248
318
  }
249
319
 
@@ -38,6 +38,11 @@ const DEFAULT_CONFIG = {
38
38
  onSearchUnavailable: 'fail',
39
39
  waitForSearchIndexes: false,
40
40
  searchIndexWaitTimeoutMs: 600_000,
41
+ backgroundCollection: '_migronaut_background',
42
+ backgroundInline: false,
43
+ backgroundOnDrift: 'reopen',
44
+ backgroundDrift: 'poll',
45
+ backgroundShardAware: 'auto',
41
46
  };
42
47
 
43
48
  /** Candidate config file names, checked in priority order within the cwd */
@@ -47,6 +52,26 @@ const isNonEmptyString = (value) => typeof value === 'string' && value.length >
47
52
  const isBoolean = (value) => typeof value === 'boolean';
48
53
  const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
49
54
  const isExtension = (value) => value === 'ts' || value === 'js';
55
+ const oneOf = (values) => (value) => values.includes(value);
56
+ const quoted = (values) => values.map((value) => `'${value}'`).join(', ');
57
+
58
+ const ON_DRIFT = ['reopen', 'report'];
59
+ const DRIFT_MODES = ['poll', 'stream', 'both'];
60
+ const SHARD_AWARE = ['auto', 'off'];
61
+
62
+ /**
63
+ * The collections a background migration keeps its state in: the state
64
+ * documents themselves, their partitions and the drift watcher's resume
65
+ * tokens — the last two named after the first, so one setting moves all
66
+ * three (and a test of "must differ" covers them together).
67
+ */
68
+ function backgroundCollectionNames(backgroundCollection) {
69
+ return {
70
+ state: backgroundCollection,
71
+ partitions: `${backgroundCollection}_partitions`,
72
+ watch: `${backgroundCollection}_watch`,
73
+ };
74
+ }
50
75
 
51
76
  function isStringList(value) {
52
77
  if (!Array.isArray(value) || value.length === 0) return false;
@@ -169,6 +194,31 @@ const CONFIG_KEYS = [
169
194
  message: 'must be a positive integer',
170
195
  optional: true,
171
196
  },
197
+ {
198
+ path: 'backgroundCollection',
199
+ check: isCollectionName,
200
+ message: "must be a valid collection name (no '$'/NUL, not system.*)",
201
+ optional: true,
202
+ },
203
+ { path: 'backgroundInline', check: isBoolean, message: 'must be a boolean', optional: true },
204
+ {
205
+ path: 'backgroundOnDrift',
206
+ check: oneOf(ON_DRIFT),
207
+ message: `must be ${quoted(ON_DRIFT)}`,
208
+ optional: true,
209
+ },
210
+ {
211
+ path: 'backgroundDrift',
212
+ check: oneOf(DRIFT_MODES),
213
+ message: `must be ${quoted(DRIFT_MODES)}`,
214
+ optional: true,
215
+ },
216
+ {
217
+ path: 'backgroundShardAware',
218
+ check: oneOf(SHARD_AWARE),
219
+ message: `must be ${quoted(SHARD_AWARE)}`,
220
+ optional: true,
221
+ },
172
222
  ];
173
223
 
174
224
  /**
@@ -220,26 +270,51 @@ function validateConfig(config, options = {}) {
220
270
  // pure data, so this costs nothing. Definition *files* are loaded only when
221
271
  // a converge runs: importing them here would make one broken file block
222
272
  // every command, an emergency `down` included.
223
- // Three bookkeeping collections, three jobs: sharing one would mix records.
224
- const bookkeeping = ['migrationsCollection', 'lockCollection', 'convergeLogCollection'];
225
- for (const [position, key] of bookkeeping.entries()) {
226
- for (const other of bookkeeping.slice(0, position)) {
227
- if (config[key] !== undefined && config[key] === config[other]) {
228
- issues.push({ path: key, message: `must differ from ${other}` });
229
- }
273
+ // Every bookkeeping collection has one job: sharing one would mix records.
274
+ const bookkeeping = bookkeepingCollections(config);
275
+ for (let position = 1; position < bookkeeping.length; position++) {
276
+ const [key, name, what] = bookkeeping[position];
277
+ if (name === undefined) continue;
278
+ for (let earlier = 0; earlier < position; earlier++) {
279
+ const [otherKey, otherName, otherWhat] = bookkeeping[earlier];
280
+ if (name !== otherName) continue;
281
+ issues.push({
282
+ path: key,
283
+ message: `${what ? `${what} (${name}) ` : ''}must differ from ${otherWhat ? `${otherKey}'s ${otherWhat}` : otherKey}`,
284
+ });
285
+ break;
230
286
  }
231
287
  }
232
288
  if (Array.isArray(config.collections)) {
233
- const reserved = [
234
- config.migrationsCollection,
235
- config.lockCollection,
236
- config.convergeLogCollection,
237
- ];
289
+ const reserved = [];
290
+ for (const [, name] of bookkeeping) if (name !== undefined) reserved.push(name);
238
291
  for (const issue of collectionsIssues(config.collections, { reserved })) issues.push(issue);
239
292
  }
240
293
  return issues;
241
294
  }
242
295
 
296
+ /**
297
+ * Every collection migronaut keeps records in, as `[configKey, name, what?]`
298
+ * — `what` names a collection derived from the key (the background
299
+ * partitions and watch collections).
300
+ */
301
+ function bookkeepingCollections(config) {
302
+ const entries = [
303
+ ['migrationsCollection', config.migrationsCollection],
304
+ ['lockCollection', config.lockCollection],
305
+ ['convergeLogCollection', config.convergeLogCollection],
306
+ ];
307
+ if (typeof config.backgroundCollection === 'string') {
308
+ const names = backgroundCollectionNames(config.backgroundCollection);
309
+ entries.push(
310
+ ['backgroundCollection', names.state],
311
+ ['backgroundCollection', names.partitions, 'partitions collection'],
312
+ ['backgroundCollection', names.watch, 'watch collection'],
313
+ );
314
+ }
315
+ return entries;
316
+ }
317
+
243
318
  const TRUE_BOOLEAN_STRINGS = new Set(['true', '1', 'yes']);
244
319
  const FALSE_BOOLEAN_STRINGS = new Set(['false', '0', 'no']);
245
320
 
@@ -361,6 +436,19 @@ const ENV_KEYS = [
361
436
  path: 'searchIndexWaitTimeoutMs',
362
437
  parse: parsePositiveInteger,
363
438
  },
439
+ { env: 'MIGRONAUT_BACKGROUND_COLLECTION', path: 'backgroundCollection', parse: parseString },
440
+ { env: 'MIGRONAUT_BACKGROUND_INLINE', path: 'backgroundInline', parse: parseBoolean },
441
+ {
442
+ env: 'MIGRONAUT_BACKGROUND_ON_DRIFT',
443
+ path: 'backgroundOnDrift',
444
+ parse: parseEnum(ON_DRIFT),
445
+ },
446
+ { env: 'MIGRONAUT_BACKGROUND_DRIFT', path: 'backgroundDrift', parse: parseEnum(DRIFT_MODES) },
447
+ {
448
+ env: 'MIGRONAUT_BACKGROUND_SHARD_AWARE',
449
+ path: 'backgroundShardAware',
450
+ parse: parseEnum(SHARD_AWARE),
451
+ },
364
452
  ];
365
453
 
366
454
  /** Build a partial config from the MIGRONAUT_* environment variables */
@@ -599,6 +687,8 @@ module.exports = {
599
687
  CONFIG_KEYS,
600
688
  DEFAULT_CONFIG,
601
689
  ENV_KEYS,
690
+ backgroundCollectionNames,
691
+ bookkeepingCollections,
602
692
  // Re-exported from utils/collection-name.js, where it moved so that
603
693
  // core/collections.js can use it without a require cycle through here.
604
694
  isCollectionName,
@@ -1,5 +1,12 @@
1
1
  const { deepEqual } = require('../utils/canonical.js');
2
- const { compareIndex, normalizeLiveIndex, restoreSpec, sameSignature } = require('./index-spec.js');
2
+ const { shardedVersionIndexKey, versionFloorConflict } = require('./versioning-spec.js');
3
+ const {
4
+ compareIndex,
5
+ normalizeDeclaredIndex,
6
+ normalizeLiveIndex,
7
+ restoreSpec,
8
+ sameSignature,
9
+ } = require('./index-spec.js');
3
10
  const {
4
11
  compareSearchIndex,
5
12
  isBeingRemoved,
@@ -179,7 +186,50 @@ function joinReasons(first, second) {
179
186
  return first ? `${first}; ${second}` : second;
180
187
  }
181
188
 
182
- function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities }, row, steps) {
189
+ /** Why the ordinary version index stays on a sharded collection whose version index replaced it */
190
+ const DISPLACED_REASON =
191
+ 'replaced by the shard-key-prefixed version index — drop it once nothing hints it ' +
192
+ '(prune does, when the indexes are declared)';
193
+
194
+ /**
195
+ * The declared indexes as they apply to this live collection: on a sharded
196
+ * one, the version index takes the shard key between the version field and
197
+ * `_id` (versioning-spec.js). `displaced` names the ordinary version index it
198
+ * replaces, when the two differ.
199
+ */
200
+ function indexesFor(definition, live) {
201
+ const indexes = definition.indexes;
202
+ if (indexes === undefined || !definition.versioning || !live.shardKey) return { indexes };
203
+ const sharded = normalizeDeclaredIndex({
204
+ key: shardedVersionIndexKey(definition.versioning, live.shardKey),
205
+ });
206
+ const result = [];
207
+ let displaced;
208
+ for (const index of indexes) {
209
+ if (index.versionIndex === true && index.name !== sharded.name) {
210
+ displaced = index.name;
211
+ result.push({ ...sharded, versionIndex: true });
212
+ } else {
213
+ result.push(index);
214
+ }
215
+ }
216
+ return { indexes: result, displaced };
217
+ }
218
+
219
+ /**
220
+ * Plan the declared indexes of one collection against its live ones. With
221
+ * `partial` (only the version index of `versioning` is declared, not the
222
+ * collection's own list) the other live indexes are not managed at all: never
223
+ * listed, never dropped — prune does not reach them.
224
+ */
225
+ function planIndexes(
226
+ declaredIndexes,
227
+ live,
228
+ { prune: pruneOption, rebuildUnique, capabilities, partial = false, displaced },
229
+ row,
230
+ steps,
231
+ ) {
232
+ const prune = pruneOption && !partial;
183
233
  const defaultCollation = live.options?.collation;
184
234
  const liveIndexes = [];
185
235
  for (const raw of live.indexes) {
@@ -360,7 +410,22 @@ function planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities
360
410
  }
361
411
  if (group.actions.length > 0) steps.push(group);
362
412
 
413
+ // The ordinary version index a sharded one replaced: kept, and said so —
414
+ // a background migration may still be hinting it.
415
+ const old = displaced !== undefined ? byName.get(displaced) : undefined;
416
+ if (old !== undefined && !declaredNames.has(displaced) && !prune) {
417
+ consumed.add(displaced);
418
+ row({
419
+ target: 'index',
420
+ name: displaced,
421
+ action: 'keep',
422
+ reason: DISPLACED_REASON,
423
+ from: indexValue(old.raw),
424
+ });
425
+ }
426
+
363
427
  // Last, so an index is only ever removed once everything declared exists.
428
+ if (partial) return;
364
429
  for (const index of liveIndexes) {
365
430
  if (consumed.has(index.name) || declaredNames.has(index.name)) continue;
366
431
  if (prune && backsShardKey(index, live.shardKey)) {
@@ -554,8 +619,8 @@ function planSearchIndexes(declaredList, live, { prune, search }, row, submit, d
554
619
  /**
555
620
  * Plan one collection. `definition` is a normalized definition (see
556
621
  * collections.js); `live` is `{ exists, type?, options?, indexes,
557
- * searchIndexes? }` as converge.js reads it. Returns `{ name, actions, steps }`:
558
- * `actions` are the result rows (status `'planned'`), `steps` the operations
622
+ * searchIndexes?, shardKey?, versionFloor? }` as converge.js reads it.
623
+ * Returns `{ name, actions, steps }`: `actions` are the result rows (status `'planned'`), `steps` the operations
559
624
  * that carry them out, in execution order, each pointing at the rows it
560
625
  * settles. `search` is `{ available, onUnavailable }` — whether the server has
561
626
  * Atlas Search, and what a declared search index becomes when it does not.
@@ -615,7 +680,7 @@ function planCollectionSteps(
615
680
  }
616
681
 
617
682
  const desired = desiredValidator(definition);
618
- const declaredIndexes = definition.indexes;
683
+ const { indexes: declaredIndexes, displaced } = indexesFor(definition, live);
619
684
  const declaredSearch = definition.searchIndexes;
620
685
  const indexSteps = [];
621
686
  const searchSubmit = [];
@@ -655,10 +720,24 @@ function planCollectionSteps(
655
720
  }
656
721
 
657
722
  if (desired !== undefined) {
658
- planValidator(name, desired, liveValidator(live.options), row, steps);
723
+ // Raising versioning.min over documents still below it: refused, so the
724
+ // whole run stops before its first write (see versionFloorToCheck).
725
+ const refused = versionFloorConflict(live.versionFloor);
726
+ if (refused) {
727
+ row({ target: 'validator', name, action: 'conflict', reason: refused, to: desired });
728
+ } else {
729
+ planValidator(name, desired, liveValidator(live.options), row, steps);
730
+ }
659
731
  }
660
732
  if (declaredIndexes !== undefined) {
661
- planIndexes(declaredIndexes, live, { prune, rebuildUnique, capabilities }, row, indexSteps);
733
+ const partial = definition.indexesPartial === true;
734
+ planIndexes(
735
+ declaredIndexes,
736
+ live,
737
+ { prune, rebuildUnique, capabilities, partial, displaced },
738
+ row,
739
+ indexSteps,
740
+ );
662
741
  }
663
742
  if (declaredSearch !== undefined) {
664
743
  planSearchIndexes(declaredSearch, live, { prune, search }, row, searchSubmit, searchDrops);
@@ -29,6 +29,8 @@ const {
29
29
  warnIgnored,
30
30
  } = require('./converge-search-run.js');
31
31
  const { inPlaceCapabilities } = require('./index-spec.js');
32
+ const { belowVersionFilter } = require('../versioning/document.js');
33
+ const { isVersioningIndexKey, versionFloorToCheck } = require('./versioning-spec.js');
32
34
  const {
33
35
  INDEX_NOT_FOUND,
34
36
  NAMESPACE_NOT_FOUND,
@@ -74,6 +76,53 @@ const HINTS = {
74
76
  'first (the index was left as it was)',
75
77
  };
76
78
 
79
+ /**
80
+ * How long the version floor probe may scan. With the version index it reads
81
+ * one key; without it (the index is created by the same converge that raises
82
+ * the floor) it may have to scan the collection — bounded, and a timeout
83
+ * refuses the raise rather than risk it.
84
+ */
85
+ const FLOOR_PROBE_TIMEOUT_MS = 60_000;
86
+
87
+ /**
88
+ * Before converge raises `versioning.min`: is any document still below it?
89
+ * Sets `live.versionFloor` to `{ min, below }` (`below: 'unknown'` with
90
+ * `error` when the probe failed) for the planner to refuse the raise on. One
91
+ * `findOne` with only `_id` projected — nothing about the document is kept.
92
+ */
93
+ async function probeVersionFloor(db, definition, live) {
94
+ const min = versionFloorToCheck(definition, live);
95
+ if (min === null) return;
96
+ const { versioning } = definition;
97
+ let hint;
98
+ for (const index of live.indexes) {
99
+ if (isVersioningIndexKey(index.key, versioning)) {
100
+ hint = index.name;
101
+ break;
102
+ }
103
+ }
104
+ try {
105
+ const found = await db
106
+ .collection(definition.name)
107
+ .findOne(belowVersionFilter(versioning, min), {
108
+ projection: { _id: 1 },
109
+ maxTimeMS: FLOOR_PROBE_TIMEOUT_MS,
110
+ ...(hint !== undefined ? { hint } : {}),
111
+ ...READ_OPTIONS,
112
+ });
113
+ live.versionFloor = { min, below: found !== null };
114
+ } catch (error) {
115
+ live.versionFloor = { min, below: 'unknown', error: errorText(error) };
116
+ }
117
+ }
118
+
119
+ /** Probe every versioned collection whose floor would rise — a few at a time */
120
+ async function probeVersionFloors(db, definitions, live) {
121
+ await mapLimit([...definitions.keys()], READ_CONCURRENCY, (position) =>
122
+ probeVersionFloor(db, definitions[position], live[position]),
123
+ );
124
+ }
125
+
77
126
  /** Shard key of each named collection, behind a mongos — when `config.collections` may be read */
78
127
  async function readShardKeys(deps, names) {
79
128
  const keys = new Map();
@@ -577,6 +626,7 @@ async function readAndPlan(deps, options) {
577
626
  }
578
627
  }
579
628
  if (search.declared) await readSearch(deps, server, definitions, live, search);
629
+ await probeVersionFloors(db, definitions, live);
580
630
  const plans = new Array(definitions.length);
581
631
  const ignored = [];
582
632
  for (const [position, definition] of definitions.entries()) {
@@ -636,6 +686,7 @@ async function readFresh(run, position, phase) {
636
686
  ) {
637
687
  fresh.searchIndexes = await readSearchIndexes(deps, definition.name, phase);
638
688
  }
689
+ await probeVersionFloor(deps.db, definition, fresh);
639
690
  return fresh;
640
691
  }
641
692
 
@@ -840,6 +891,42 @@ async function applyCollection(deps, plan, result, signal, search) {
840
891
  return kept;
841
892
  }
842
893
 
894
+ /**
895
+ * After a run raised `versioning.min`: old-shape documents written while it
896
+ * ran (an old pod still deploying) got past the probe. Converge cannot undo
897
+ * the validator, so it says so — those documents now fail validation on
898
+ * their next strict write, and a background migration has to pick them up.
899
+ */
900
+ async function warnFloorBreach(run, position) {
901
+ const { deps, definitions, live, result } = run;
902
+ const floor = live[position].versionFloor;
903
+ if (floor?.below !== false) return;
904
+ const raised = result.collections[position].actions.some(
905
+ (action) => action.target === 'validator' && action.status === 'applied',
906
+ );
907
+ if (!raised) return;
908
+ const definition = definitions[position];
909
+ let found;
910
+ try {
911
+ found = await deps.db
912
+ .collection(definition.name)
913
+ .findOne(belowVersionFilter(definition.versioning, floor.min), {
914
+ projection: { _id: 1 },
915
+ maxTimeMS: FLOOR_PROBE_TIMEOUT_MS,
916
+ ...READ_OPTIONS,
917
+ });
918
+ } catch {
919
+ return;
920
+ }
921
+ if (found === null) return;
922
+ deps.logger.warn(
923
+ `⚠ ${definition.name}: documents below version ${floor.min} were written while converge ` +
924
+ 'raised versioning.min — an old release is still writing; run the background migration ' +
925
+ 'again once it is gone',
926
+ deps.fields({ collection: definition.name, min: floor.min }),
927
+ );
928
+ }
929
+
843
930
  /**
844
931
  * The verify phase — the fixed-point check: what was just applied must now
845
932
  * compare as unchanged. Anything that does not would be "changed" again on
@@ -864,6 +951,7 @@ async function verifyFixedPoint(run, position, kept, signal) {
864
951
  after = planFor(definition, await readFresh(run, position, 'apply'));
865
952
  }
866
953
  refreshBuilds(rows, after.actions);
954
+ await warnFloorBreach(run, position);
867
955
  for (const action of after.actions) {
868
956
  if (!CHANGE_ACTIONS.has(action.action)) continue;
869
957
  if (action.action === 'drop' && kept.has(action.name)) continue;