@alexify/migronaut 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +33 -2
- package/bullmq.d.ts +449 -6
- package/index.d.ts +1010 -9
- package/migronaut.schema.json +93 -1
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +128 -14
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migrator.js +904 -12
- package/src/core/options.js +16 -0
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +107 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/src/core/options.js
CHANGED
|
@@ -152,6 +152,20 @@ function assertConvergeAfterUpValid(converge, filename, to) {
|
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
/**
|
|
156
|
+
* `onBackgroundPending`: what a run does at a migration that `requires` a
|
|
157
|
+
* background migration not completed yet — `'error'` (throw
|
|
158
|
+
* BackgroundPendingError) or `'stop'` (end the run there, cleanly).
|
|
159
|
+
*/
|
|
160
|
+
function assertBackgroundPendingValid(onBackgroundPending) {
|
|
161
|
+
if (onBackgroundPending === undefined) return;
|
|
162
|
+
if (onBackgroundPending !== 'error' && onBackgroundPending !== 'stop') {
|
|
163
|
+
throw new ConfigInvalidError("onBackgroundPending must be 'error' or 'stop'", {
|
|
164
|
+
onBackgroundPending,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
155
169
|
/** `up(filename, options)` */
|
|
156
170
|
function assertUpOptions(filename, options) {
|
|
157
171
|
assertFilename(filename);
|
|
@@ -167,6 +181,7 @@ function assertUpOptions(filename, options) {
|
|
|
167
181
|
assertOrderedValid(options.ordered, filename);
|
|
168
182
|
assertConvergeAfterUpValid(options.converge, filename, options.to);
|
|
169
183
|
assertChecksumValid(options.checksum, filename);
|
|
184
|
+
assertBackgroundPendingValid(options.onBackgroundPending);
|
|
170
185
|
assertActorValid(options);
|
|
171
186
|
}
|
|
172
187
|
|
|
@@ -254,6 +269,7 @@ function assertImportOptions(options) {
|
|
|
254
269
|
}
|
|
255
270
|
|
|
256
271
|
module.exports = {
|
|
272
|
+
assertActorValid,
|
|
257
273
|
assertConvergeOptions,
|
|
258
274
|
assertDownOptions,
|
|
259
275
|
assertDryRunOptions,
|
package/src/core/run.js
CHANGED
|
@@ -41,6 +41,7 @@ const { MigratorKit, RECORD_LOCK_WAIT } = require('./migrator.js');
|
|
|
41
41
|
async function runMigrations(config = {}, options = {}) {
|
|
42
42
|
const {
|
|
43
43
|
noLock,
|
|
44
|
+
onBackgroundPending,
|
|
44
45
|
onLockHeld = 'throw',
|
|
45
46
|
// Left undefined unless given: the default then follows the holder's TTL.
|
|
46
47
|
lockWaitTimeoutMs,
|
|
@@ -71,6 +72,11 @@ async function runMigrations(config = {}, options = {}) {
|
|
|
71
72
|
kit.on('converge:end', (event) => {
|
|
72
73
|
if (event.trigger === 'up' && event.success) converge = event.result;
|
|
73
74
|
});
|
|
75
|
+
// With onBackgroundPending: 'stop', where the run stopped and what for.
|
|
76
|
+
const waiting = [];
|
|
77
|
+
kit.on('background:waiting', (event) => {
|
|
78
|
+
waiting.push({ migration: event.migration, waitsFor: event.waitsFor });
|
|
79
|
+
});
|
|
74
80
|
|
|
75
81
|
// An abort reaches the run wherever it is: the wait loop sees the signal
|
|
76
82
|
// between polls, and kit.stop() stops a run that is setting up or between
|
|
@@ -90,20 +96,28 @@ async function runMigrations(config = {}, options = {}) {
|
|
|
90
96
|
waited,
|
|
91
97
|
waitedMs,
|
|
92
98
|
attempts,
|
|
93
|
-
} = await withLockWait(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
} = await withLockWait(
|
|
100
|
+
() =>
|
|
101
|
+
kit.up(undefined, {
|
|
102
|
+
...(noLock ? { noLock: true } : {}),
|
|
103
|
+
...(onBackgroundPending !== undefined ? { onBackgroundPending } : {}),
|
|
104
|
+
}),
|
|
105
|
+
{
|
|
106
|
+
onLockHeld,
|
|
107
|
+
...(lockWaitTimeoutMs !== undefined ? { lockWaitTimeoutMs } : {}),
|
|
108
|
+
...(lockPollIntervalMs !== undefined ? { lockPollIntervalMs } : {}),
|
|
109
|
+
// Resolved AFTER connect, from the kit's own merged config: a `logger:
|
|
110
|
+
// null` in the config file must silence the wait lines too, not only
|
|
111
|
+
// the kit's own.
|
|
112
|
+
logger: kit.logger,
|
|
113
|
+
...(signal ? { signal } : {}),
|
|
114
|
+
onSettle: (wait) => kit[RECORD_LOCK_WAIT](wait),
|
|
115
|
+
},
|
|
116
|
+
);
|
|
104
117
|
return {
|
|
105
118
|
applied,
|
|
106
|
-
upToDate: applied.length === 0,
|
|
119
|
+
upToDate: applied.length === 0 && waiting.length === 0,
|
|
120
|
+
...(waiting.length > 0 ? { waiting } : {}),
|
|
107
121
|
waited,
|
|
108
122
|
waitedMs,
|
|
109
123
|
attempts,
|
package/src/core/runner.js
CHANGED
package/src/core/server-info.js
CHANGED
|
@@ -25,16 +25,23 @@ const READ_CONCURRENCY = 8;
|
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* What the server is: a mongos in front of shards (its shard keys matter to
|
|
28
|
-
* prune),
|
|
28
|
+
* prune), its topology (`replicaSet`, `sharded`, `standalone` — transactions
|
|
29
|
+
* need one of the first two), and its version (what it can change in place). Best-effort — a
|
|
29
30
|
* server that refuses to say gets the conservative answer: no in-place
|
|
30
31
|
* extras, no shard-key handling.
|
|
31
32
|
*/
|
|
32
33
|
async function readServer(db) {
|
|
33
|
-
const server = { mongos: false, version: undefined };
|
|
34
|
+
const server = { mongos: false, version: undefined, topology: undefined };
|
|
34
35
|
if (typeof db.admin !== 'function') return server;
|
|
35
36
|
try {
|
|
36
37
|
const hello = await db.admin().command({ hello: 1 });
|
|
37
38
|
server.mongos = hello?.msg === 'isdbgrid';
|
|
39
|
+
// What transactions need: a replica set member or a mongos.
|
|
40
|
+
server.topology = server.mongos
|
|
41
|
+
? 'sharded'
|
|
42
|
+
: typeof hello?.setName === 'string'
|
|
43
|
+
? 'replicaSet'
|
|
44
|
+
: 'standalone';
|
|
38
45
|
} catch {
|
|
39
46
|
// Unknown — treated as a replica set or standalone.
|
|
40
47
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
const { READ_OPTIONS } = require('./server-info.js');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What the cluster says about a collection's sharding — read from the
|
|
5
|
+
* `config` database through a mongos. Every read here needs `clusterMonitor`
|
|
6
|
+
* (or more): a user without it gets `undefined` ("unknown"), never an error,
|
|
7
|
+
* and the caller falls back to what works without it.
|
|
8
|
+
*
|
|
9
|
+
* - `readShardKey` — the shard key, or `null` for a collection that is not
|
|
10
|
+
* sharded (an 8.0 `unsplittable` one included: tracked, but on one shard
|
|
11
|
+
* under `{ _id: 1 }`, which is no key to partition or target by);
|
|
12
|
+
* - `readChunks` — the chunks of a sharded collection, in key order.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** The server's "not authorized" — the one refusal that means "unknown", not "broken" */
|
|
16
|
+
const UNAUTHORIZED = 13;
|
|
17
|
+
|
|
18
|
+
const isUnauthorized = (error) => error?.code === UNAUTHORIZED;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `{ key, uuid, timestamp, unsplittable }` of a sharded collection, `null` when
|
|
22
|
+
* it is not sharded, `undefined` when `config.collections` may not be read.
|
|
23
|
+
*/
|
|
24
|
+
async function readShardKey(client, dbName, collection) {
|
|
25
|
+
let entry;
|
|
26
|
+
try {
|
|
27
|
+
entry = await client
|
|
28
|
+
.db('config')
|
|
29
|
+
.collection('collections')
|
|
30
|
+
.findOne(
|
|
31
|
+
{ _id: `${dbName}.${collection}` },
|
|
32
|
+
{ projection: { key: 1, uuid: 1, timestamp: 1, unsplittable: 1 }, ...READ_OPTIONS },
|
|
33
|
+
);
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (isUnauthorized(error)) return undefined;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
if (entry === null || entry.unsplittable === true || entry.key === undefined) return null;
|
|
39
|
+
return {
|
|
40
|
+
key: entry.key,
|
|
41
|
+
uuid: entry.uuid,
|
|
42
|
+
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The chunks of a sharded collection (`readShardKey`'s result), in key order:
|
|
48
|
+
* `[{ min, max, shard }]` — or `undefined` when `config.chunks` may not be
|
|
49
|
+
* read. By `uuid` (5.0+), then by namespace (a cluster upgraded from 4.4 that
|
|
50
|
+
* never refreshed its chunks).
|
|
51
|
+
*/
|
|
52
|
+
async function readChunks(client, dbName, collection, sharding) {
|
|
53
|
+
const chunks = client.db('config').collection('chunks');
|
|
54
|
+
// Hashed bounds are NumberLongs: promoted to numbers, they lose precision
|
|
55
|
+
// past 2^53 (ARCHITECTURE §6.8).
|
|
56
|
+
const options = {
|
|
57
|
+
projection: { min: 1, max: 1, shard: 1 },
|
|
58
|
+
...READ_OPTIONS,
|
|
59
|
+
promoteLongs: false,
|
|
60
|
+
};
|
|
61
|
+
try {
|
|
62
|
+
let rows = await chunks.find({ uuid: sharding.uuid }, options).sort({ min: 1 }).toArray();
|
|
63
|
+
if (rows.length === 0) {
|
|
64
|
+
rows = await chunks
|
|
65
|
+
.find({ ns: `${dbName}.${collection}` }, options)
|
|
66
|
+
.sort({ min: 1 })
|
|
67
|
+
.toArray();
|
|
68
|
+
}
|
|
69
|
+
return rows;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (isUnauthorized(error)) return undefined;
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
module.exports = { isUnauthorized, readChunks, readShardKey };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
const { isPlainObject } = require('../utils/canonical.js');
|
|
2
|
+
const { versionIndexKey } = require('../versioning/document.js');
|
|
3
|
+
const { toCount } = require('../versioning/internal.js');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a collection's `versioning` block asks of the database: the validator
|
|
7
|
+
* rules for the version and revision fields, merged into whatever validator
|
|
8
|
+
* the definition declares, and the index every version-filtered scan uses.
|
|
9
|
+
* Pure — collections.js folds the result into the normalized definition, so
|
|
10
|
+
* the planner sees an ordinary validator and an ordinary index.
|
|
11
|
+
*
|
|
12
|
+
* The version rule has a `minimum` but never a `maximum`: during a rolling
|
|
13
|
+
* deploy (or after a rollback) a newer release writes a higher version than
|
|
14
|
+
* the declaration knows, and refusing that write would turn a deploy into an
|
|
15
|
+
* outage. A revision outgrows `int` after 2³¹ writes — `$inc` turns it into a
|
|
16
|
+
* `long` — so both are accepted.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** `{ required, properties }` for the managed fields, in a fixed order */
|
|
20
|
+
function versioningRules(versioning) {
|
|
21
|
+
const { field, min, revisionField } = versioning;
|
|
22
|
+
const properties = { [field]: { bsonType: 'int', minimum: min } };
|
|
23
|
+
const required = [field];
|
|
24
|
+
if (revisionField !== null) {
|
|
25
|
+
properties[revisionField] = { bsonType: ['int', 'long'], minimum: 0 };
|
|
26
|
+
required.push(revisionField);
|
|
27
|
+
}
|
|
28
|
+
// `min: 0` adapts a collection whose documents predate versioning: the
|
|
29
|
+
// fields are typed when present, but nothing requires them yet.
|
|
30
|
+
return min === 0 ? { properties } : { required, properties };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The managed field names a validator already constrains itself */
|
|
34
|
+
function managedFieldsIn(validator, versioning) {
|
|
35
|
+
const managed = [versioning.field];
|
|
36
|
+
if (versioning.revisionField !== null) managed.push(versioning.revisionField);
|
|
37
|
+
const found = new Set();
|
|
38
|
+
const schema = validator.$jsonSchema;
|
|
39
|
+
const properties = isPlainObject(schema) ? schema.properties : undefined;
|
|
40
|
+
const required = new Set(
|
|
41
|
+
isPlainObject(schema) && Array.isArray(schema.required) ? schema.required : [],
|
|
42
|
+
);
|
|
43
|
+
for (const name of managed) {
|
|
44
|
+
if (Object.hasOwn(validator, name)) found.add(name);
|
|
45
|
+
if (isPlainObject(properties) && Object.hasOwn(properties, name)) found.add(name);
|
|
46
|
+
if (required.has(name)) found.add(name);
|
|
47
|
+
}
|
|
48
|
+
return [...found];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Why a declared validator cannot carry the versioning rules — it constrains
|
|
53
|
+
* a managed field itself, or its `$jsonSchema` is not a schema object. Empty
|
|
54
|
+
* when the merge can go ahead.
|
|
55
|
+
*/
|
|
56
|
+
function validatorVersioningIssues(validator, versioning) {
|
|
57
|
+
if (validator === undefined) return [];
|
|
58
|
+
if (validator === null || !isPlainObject(validator)) {
|
|
59
|
+
return ['is null, which would remove the versioning rules — drop the validator key instead'];
|
|
60
|
+
}
|
|
61
|
+
const issues = [];
|
|
62
|
+
if (validator.$jsonSchema !== undefined && !isPlainObject(validator.$jsonSchema)) {
|
|
63
|
+
issues.push('$jsonSchema must be an object to take the versioning rules');
|
|
64
|
+
} else if (
|
|
65
|
+
isPlainObject(validator.$jsonSchema) &&
|
|
66
|
+
validator.$jsonSchema.required !== undefined &&
|
|
67
|
+
!Array.isArray(validator.$jsonSchema.required)
|
|
68
|
+
) {
|
|
69
|
+
issues.push('$jsonSchema.required must be an array to take the versioning rules');
|
|
70
|
+
}
|
|
71
|
+
for (const name of managedFieldsIn(validator, versioning)) {
|
|
72
|
+
issues.push(`constrains "${name}", which is managed by versioning — remove that rule`);
|
|
73
|
+
}
|
|
74
|
+
return issues;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The validator to declare: the versioning rules alone (no validator
|
|
79
|
+
* declared), merged into the declared `$jsonSchema` (the declared `required`
|
|
80
|
+
* and `properties` first, ours after), or — next to query operators — added
|
|
81
|
+
* as a top-level `$jsonSchema`, which the server combines with them.
|
|
82
|
+
*/
|
|
83
|
+
function mergeVersioningValidator(validator, versioning) {
|
|
84
|
+
const rules = versioningRules(versioning);
|
|
85
|
+
if (validator === undefined || Object.keys(validator).length === 0) {
|
|
86
|
+
return { $jsonSchema: rules };
|
|
87
|
+
}
|
|
88
|
+
const schema = validator.$jsonSchema;
|
|
89
|
+
if (!isPlainObject(schema)) return { ...validator, $jsonSchema: rules };
|
|
90
|
+
const merged = { ...schema };
|
|
91
|
+
if (rules.required) merged.required = [...(schema.required ?? []), ...rules.required];
|
|
92
|
+
merged.properties = { ...schema.properties, ...rules.properties };
|
|
93
|
+
return { ...validator, $jsonSchema: merged };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The declared version index — `{ [field]: 1, _id: 1 }`, named by its key */
|
|
97
|
+
const versioningIndex = (versioning) => ({ key: versionIndexKey(versioning) });
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The version index of a sharded collection: the shard key between the
|
|
101
|
+
* version field and `_id` — `{ __v: 1, region: 1, _id: 1 }` — so a batch
|
|
102
|
+
* over one chunk's range is an index range, not a filter over every old
|
|
103
|
+
* document of the shard. A hashed field stays hashed; `_id` is not repeated
|
|
104
|
+
* when the key holds it. On `{ _id: 1 }` it is the ordinary version index.
|
|
105
|
+
*/
|
|
106
|
+
function shardedVersionIndexKey(versioning, shardKey) {
|
|
107
|
+
const key = { [versioning.field]: 1 };
|
|
108
|
+
for (const [field, value] of Object.entries(shardKey)) {
|
|
109
|
+
if (field !== versioning.field) key[field] = value;
|
|
110
|
+
}
|
|
111
|
+
if (!('_id' in key)) key._id = 1;
|
|
112
|
+
return key;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Whether an index key is the version index's (same fields, same order, ascending) */
|
|
116
|
+
function isVersioningIndexKey(key, versioning) {
|
|
117
|
+
if (!isPlainObject(key)) return false;
|
|
118
|
+
const entries = Object.entries(key);
|
|
119
|
+
const expected = Object.entries(versionIndexKey(versioning));
|
|
120
|
+
if (entries.length !== expected.length) return false;
|
|
121
|
+
for (let i = 0; i < entries.length; i++) {
|
|
122
|
+
if (entries[i][0] !== expected[i][0] || Number(entries[i][1]) !== expected[i][1]) return false;
|
|
123
|
+
}
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The version floor the live validator enforces — the `minimum` of the version
|
|
129
|
+
* field's `$jsonSchema` rule — or `null` when it enforces none.
|
|
130
|
+
*/
|
|
131
|
+
function liveVersionFloor(options, versioning) {
|
|
132
|
+
const schema = options?.validator?.$jsonSchema;
|
|
133
|
+
const rule = isPlainObject(schema?.properties) ? schema.properties[versioning.field] : undefined;
|
|
134
|
+
return isPlainObject(rule) ? toCount(rule.minimum) : null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The `min` converge must check the data against before it raises the floor,
|
|
139
|
+
* or `null` when there is nothing to check: no versioning, `min: 0`, a
|
|
140
|
+
* collection that does not exist yet (no documents), or a floor already that
|
|
141
|
+
* high. Only a rising floor costs a read — the steady state costs nothing.
|
|
142
|
+
*/
|
|
143
|
+
function versionFloorToCheck(definition, live) {
|
|
144
|
+
const versioning = definition.versioning;
|
|
145
|
+
if (!versioning || versioning.min === 0 || !live.exists) return null;
|
|
146
|
+
if (live.type !== undefined && live.type !== 'collection') return null;
|
|
147
|
+
const floor = liveVersionFloor(live.options, versioning);
|
|
148
|
+
return floor !== null && floor >= versioning.min ? null : versioning.min;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Why the version floor cannot be raised — `live.versionFloor` is what
|
|
153
|
+
* converge read: `{ min, below: true | false | 'unknown', error? }` — or
|
|
154
|
+
* `undefined` when it can. The document ids are never named: they may be PII.
|
|
155
|
+
*/
|
|
156
|
+
function versionFloorConflict(floor) {
|
|
157
|
+
if (!floor || floor.below === false) return undefined;
|
|
158
|
+
if (floor.below === true) {
|
|
159
|
+
return (
|
|
160
|
+
`documents below version ${floor.min} remain — raising versioning.min would leave them ` +
|
|
161
|
+
'invalid; let the background migration that upgrades them finish (migronaut background ' +
|
|
162
|
+
'status), then converge again'
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return (
|
|
166
|
+
`could not check for documents below version ${floor.min} (${floor.error}) — converge ` +
|
|
167
|
+
'with the old min first so the version index exists, then raise it'
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
module.exports = {
|
|
172
|
+
isVersioningIndexKey,
|
|
173
|
+
shardedVersionIndexKey,
|
|
174
|
+
liveVersionFloor,
|
|
175
|
+
versionFloorConflict,
|
|
176
|
+
versionFloorToCheck,
|
|
177
|
+
mergeVersioningValidator,
|
|
178
|
+
validatorVersioningIssues,
|
|
179
|
+
versioningIndex,
|
|
180
|
+
versioningRules,
|
|
181
|
+
};
|
package/src/errors/index.js
CHANGED
|
@@ -260,6 +260,88 @@ class ConvergeFailedError extends MigronautError {
|
|
|
260
260
|
}
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Thrown by the optimistic-concurrency helpers (`@alexify/migronaut/versioning`)
|
|
265
|
+
* when a revision-guarded write matched nothing. `context.reason` says why:
|
|
266
|
+
* `'conflict'` (the document exists at another revision — `context.actual`),
|
|
267
|
+
* `'not-found'` (no document matches the filter at all) or `'unknown'` (the
|
|
268
|
+
* follow-up read was skipped or could not tell). `context.expected` is the
|
|
269
|
+
* revision the caller held. The filter is never copied in — it may carry PII.
|
|
270
|
+
*/
|
|
271
|
+
class RevisionConflictError extends MigronautError {
|
|
272
|
+
constructor(message, context, options) {
|
|
273
|
+
super('REVISION_CONFLICT', message, context, options);
|
|
274
|
+
this.name = 'RevisionConflictError';
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Thrown by an upcaster that cannot bring a document to the current shape:
|
|
280
|
+
* `context.reason` is `'newer'` (written by a newer release), `'below-min'`
|
|
281
|
+
* (older than the oldest shape still supported) or `'invalid'` (the version
|
|
282
|
+
* field is not a non-negative integer, or a step returned something that is
|
|
283
|
+
* not a document).
|
|
284
|
+
*/
|
|
285
|
+
class ShapeVersionError extends MigronautError {
|
|
286
|
+
constructor(message, context, options) {
|
|
287
|
+
super('SHAPE_VERSION_UNSUPPORTED', message, context, options);
|
|
288
|
+
this.name = 'ShapeVersionError';
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Thrown when a migration `requires` a background migration that has not
|
|
294
|
+
* completed yet — or whose collection still holds documents of the old shape.
|
|
295
|
+
* Nothing was run: `context.waitsFor` names the background migrations it
|
|
296
|
+
* waits for, with their status.
|
|
297
|
+
*/
|
|
298
|
+
class BackgroundPendingError extends MigronautError {
|
|
299
|
+
constructor(message, context, options) {
|
|
300
|
+
super('BACKGROUND_PENDING', message, context, options);
|
|
301
|
+
this.name = 'BackgroundPendingError';
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Thrown when a background migration ended `failed` — a partition used up its
|
|
307
|
+
* slice failures, the document error budget ran out, or old-shape documents
|
|
308
|
+
* kept appearing for `maxPasses` passes. `context.migration` names it and
|
|
309
|
+
* `context.lastError` says what happened last.
|
|
310
|
+
*/
|
|
311
|
+
class BackgroundFailedError extends MigronautError {
|
|
312
|
+
constructor(message, context, options) {
|
|
313
|
+
super('BACKGROUND_FAILED', message, context, options);
|
|
314
|
+
this.name = 'BackgroundFailedError';
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Thrown when a control action does not fit the background migration's state
|
|
320
|
+
* — pausing a completed one, resuming one that is not paused, retrying one
|
|
321
|
+
* that is still running. `context.status` is the state it found and
|
|
322
|
+
* `context.action` what was asked.
|
|
323
|
+
*/
|
|
324
|
+
class BackgroundConflictError extends MigronautError {
|
|
325
|
+
constructor(message, context, options) {
|
|
326
|
+
super('BACKGROUND_CONFLICT', message, context, options);
|
|
327
|
+
this.name = 'BackgroundConflictError';
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Thrown by the dry-run sandbox when a step reaches for something it cannot
|
|
333
|
+
* run inside an always-aborted transaction — DDL, an admin command, another
|
|
334
|
+
* session, `$out`/`$merge`, a migronaut-internal collection. `context.method`
|
|
335
|
+
* names the call and `context.reason` the rule it broke. A dry run reports
|
|
336
|
+
* every refusal even when the step caught the error itself.
|
|
337
|
+
*/
|
|
338
|
+
class SandboxRefusedError extends MigronautError {
|
|
339
|
+
constructor(message, context, options) {
|
|
340
|
+
super('SANDBOX_REFUSED', message, context, options);
|
|
341
|
+
this.name = 'SandboxRefusedError';
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
263
345
|
module.exports = {
|
|
264
346
|
MigronautError,
|
|
265
347
|
LockAlreadyHeldError,
|
|
@@ -286,4 +368,10 @@ module.exports = {
|
|
|
286
368
|
QueueJobInvalidError,
|
|
287
369
|
QueueJobFailedError,
|
|
288
370
|
ConvergeFailedError,
|
|
371
|
+
RevisionConflictError,
|
|
372
|
+
ShapeVersionError,
|
|
373
|
+
BackgroundPendingError,
|
|
374
|
+
BackgroundFailedError,
|
|
375
|
+
BackgroundConflictError,
|
|
376
|
+
SandboxRefusedError,
|
|
289
377
|
};
|
package/src/index.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
const { EXIT_CODES } = require('./cli/exit-codes.js');
|
|
2
|
+
const { startBackgroundRunner } = require('./core/background-runner.js');
|
|
2
3
|
const { MigratorKit } = require('./core/migrator.js');
|
|
3
4
|
const { pendingMigrations, runMigrations } = require('./core/run.js');
|
|
4
5
|
const { createLogger } = require('./utils/logger.js');
|
|
5
6
|
const {
|
|
7
|
+
BackgroundConflictError,
|
|
8
|
+
BackgroundFailedError,
|
|
9
|
+
BackgroundPendingError,
|
|
6
10
|
ChecksumMismatchError,
|
|
7
11
|
ConfigFileExistsError,
|
|
8
12
|
ConfigInvalidError,
|
|
@@ -27,7 +31,10 @@ const {
|
|
|
27
31
|
OutOfOrderMigrationError,
|
|
28
32
|
QueueJobFailedError,
|
|
29
33
|
QueueJobInvalidError,
|
|
34
|
+
RevisionConflictError,
|
|
30
35
|
RunAbortedError,
|
|
36
|
+
SandboxRefusedError,
|
|
37
|
+
ShapeVersionError,
|
|
31
38
|
} = require('./errors/index.js');
|
|
32
39
|
|
|
33
40
|
module.exports = {
|
|
@@ -46,7 +53,13 @@ module.exports = {
|
|
|
46
53
|
// The CLI's exit-code map, for wrappers that mirror its semantics
|
|
47
54
|
EXIT_CODES,
|
|
48
55
|
|
|
56
|
+
// Background migrations driven from inside the application (experimental)
|
|
57
|
+
startBackgroundRunner,
|
|
58
|
+
|
|
49
59
|
// Error classes
|
|
60
|
+
BackgroundConflictError,
|
|
61
|
+
BackgroundFailedError,
|
|
62
|
+
BackgroundPendingError,
|
|
50
63
|
ChecksumMismatchError,
|
|
51
64
|
ConfigFileExistsError,
|
|
52
65
|
ConfigInvalidError,
|
|
@@ -71,5 +84,8 @@ module.exports = {
|
|
|
71
84
|
OutOfOrderMigrationError,
|
|
72
85
|
QueueJobFailedError,
|
|
73
86
|
QueueJobInvalidError,
|
|
87
|
+
RevisionConflictError,
|
|
74
88
|
RunAbortedError,
|
|
89
|
+
SandboxRefusedError,
|
|
90
|
+
ShapeVersionError,
|
|
75
91
|
};
|
package/src/utils/error.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const { MigronautError } = require('../errors/index.js');
|
|
2
|
-
const { redactUris } = require('./redact.js');
|
|
2
|
+
const { redactOutbound, redactUris } = require('./redact.js');
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Human-readable message from any thrown value, with URI credentials masked.
|
|
@@ -25,4 +25,13 @@ function errorWithCause(error) {
|
|
|
25
25
|
return cause ? `${message} — ${cause}` : message;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
/**
|
|
29
|
+
* {@link errorText} for an error about the application's data — a background
|
|
30
|
+
* migration's document errors and failed slices, kept in its state and
|
|
31
|
+
* logged: the values a server error quotes (an E11000's duplicate key — an
|
|
32
|
+
* email, a phone number) are masked too. Migronaut never logs a document's
|
|
33
|
+
* contents; the index name still says which constraint was violated.
|
|
34
|
+
*/
|
|
35
|
+
const documentErrorText = (error) => redactOutbound(errorText(error));
|
|
36
|
+
|
|
37
|
+
module.exports = { documentErrorText, errorText, errorWithCause };
|
package/src/utils/loader.js
CHANGED
|
@@ -3,6 +3,7 @@ const path = require('node:path');
|
|
|
3
3
|
const { pathToFileURL } = require('node:url');
|
|
4
4
|
const { MigrationFileNotFoundError, MigrationInvalidExportError } = require('../errors/index.js');
|
|
5
5
|
const { errorText } = require('./error.js');
|
|
6
|
+
const { requiresIssues } = require('./migration-name.js');
|
|
6
7
|
|
|
7
8
|
/** TypeScript source extensions that require a TS-capable runtime to import */
|
|
8
9
|
const TS_EXTENSIONS = new Set(['.ts', '.mts', '.cts']);
|
|
@@ -96,16 +97,13 @@ function importUserFile(filepath, options = {}) {
|
|
|
96
97
|
}
|
|
97
98
|
|
|
98
99
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* Handles all three supported formats:
|
|
102
|
-
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
103
|
-
* - CommonJS default export (`module.exports = { up, down }`)
|
|
100
|
+
* Import a migration file: its module, resolved — the default export of a
|
|
101
|
+
* CommonJS file, the namespace of an ES module with named exports.
|
|
104
102
|
*
|
|
105
103
|
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
106
|
-
* @throws {MigrationInvalidExportError} when
|
|
104
|
+
* @throws {MigrationInvalidExportError} when TypeScript cannot be loaded
|
|
107
105
|
*/
|
|
108
|
-
async function
|
|
106
|
+
async function importMigrationModule(filepath, options = {}) {
|
|
109
107
|
try {
|
|
110
108
|
await fs.access(filepath);
|
|
111
109
|
} catch {
|
|
@@ -123,7 +121,48 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
123
121
|
throw error;
|
|
124
122
|
}
|
|
125
123
|
// `mod.default ?? mod` handles the CommonJS default-export case
|
|
126
|
-
|
|
124
|
+
return imported.default ?? imported;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The `requires` export, validated against the file's own name */
|
|
128
|
+
function readRequires(resolved, filepath) {
|
|
129
|
+
if (resolved.requires === undefined) return {};
|
|
130
|
+
const issues = requiresIssues(resolved.requires, path.basename(filepath));
|
|
131
|
+
if (issues.length > 0) {
|
|
132
|
+
throw new MigrationInvalidExportError(`Invalid ${issues[0].path}: ${issues[0].message}`, {
|
|
133
|
+
filepath,
|
|
134
|
+
issues,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return { requires: [...resolved.requires] };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* What a migration module exports, validated: a regular migration
|
|
142
|
+
* (`{ up, down, useTransaction?, timeoutMs?, description?, requires? }`) or a
|
|
143
|
+
* background one (`{ kind: 'background', background, description?,
|
|
144
|
+
* requires? }` — its spec is validated where the collection's versioning is
|
|
145
|
+
* known). A file with both `background` and `up`/`down` is refused: the
|
|
146
|
+
* expand steps belong in a migration of their own.
|
|
147
|
+
*
|
|
148
|
+
* @throws {MigrationInvalidExportError}
|
|
149
|
+
*/
|
|
150
|
+
function resolveMigrationExports(resolved, filepath) {
|
|
151
|
+
if (resolved.background !== undefined) {
|
|
152
|
+
if (resolved.up !== undefined || resolved.down !== undefined) {
|
|
153
|
+
throw new MigrationInvalidExportError(
|
|
154
|
+
'A background migration exports no up() or down() — put the expand steps in a ' +
|
|
155
|
+
'migration of their own',
|
|
156
|
+
{ filepath },
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
kind: 'background',
|
|
161
|
+
background: resolved.background,
|
|
162
|
+
...(typeof resolved.description === 'string' ? { description: resolved.description } : {}),
|
|
163
|
+
...readRequires(resolved, filepath),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
127
166
|
|
|
128
167
|
if (!isFunction(resolved.up) || !isFunction(resolved.down)) {
|
|
129
168
|
throw new MigrationInvalidExportError('Migration must export async up() and down() functions', {
|
|
@@ -142,8 +181,37 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
142
181
|
if (typeof resolved.description === 'string') {
|
|
143
182
|
migration.description = resolved.description;
|
|
144
183
|
}
|
|
184
|
+
Object.assign(migration, readRequires(resolved, filepath));
|
|
145
185
|
|
|
146
186
|
return migration;
|
|
147
187
|
}
|
|
148
188
|
|
|
149
|
-
|
|
189
|
+
/**
|
|
190
|
+
* Dynamically load a migration file and validate its exports.
|
|
191
|
+
*
|
|
192
|
+
* Handles all three supported formats:
|
|
193
|
+
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
194
|
+
* - CommonJS default export (`module.exports = { up, down }`)
|
|
195
|
+
*
|
|
196
|
+
* A background migration (`kind: 'background'`) is returned like any other:
|
|
197
|
+
* the caller tells it apart by its kind.
|
|
198
|
+
*
|
|
199
|
+
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
200
|
+
* @throws {MigrationInvalidExportError} when up/down are not both functions
|
|
201
|
+
*/
|
|
202
|
+
async function loadMigrationFile(filepath, options = {}) {
|
|
203
|
+
const migration = resolveMigrationExports(
|
|
204
|
+
await importMigrationModule(filepath, options),
|
|
205
|
+
filepath,
|
|
206
|
+
);
|
|
207
|
+
return migration;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
module.exports = {
|
|
211
|
+
importMigrationModule,
|
|
212
|
+
importUserFile,
|
|
213
|
+
loadMigrationFile,
|
|
214
|
+
resolveMigrationExports,
|
|
215
|
+
tsLoadErrorOrNull,
|
|
216
|
+
tsLoadMessageOrNull,
|
|
217
|
+
};
|