@alexify/migronaut 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/README.md +68 -10
- package/bullmq.d.ts +465 -7
- package/index.d.ts +1272 -18
- package/migronaut.schema.json +150 -2
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +153 -15
- package/src/bullmq/producer.js +202 -27
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/converge.js +38 -10
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/cli/table.js +68 -9
- package/src/core/audit.js +98 -3
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +125 -31
- package/src/core/config.js +133 -13
- package/src/core/converge-plan.js +343 -61
- package/src/core/converge-search-run.js +440 -0
- package/src/core/converge-search.js +404 -0
- package/src/core/converge.js +428 -183
- package/src/core/index-spec.js +27 -16
- package/src/core/lock.js +97 -32
- package/src/core/migrator.js +951 -26
- package/src/core/options.js +32 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/search-index-spec.js +758 -0
- package/src/core/server-info.js +70 -0
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +97 -5
- package/src/index.js +16 -0
- package/src/utils/canonical.js +34 -1
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +125 -1
- package/src/utils/template.js +69 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What converge and audit read about the server, and how: the read options
|
|
3
|
+
* forced onto every read, the pace of reading many collections, the server's
|
|
4
|
+
* version and topology, and the error codes for a namespace or an index that
|
|
5
|
+
* is not there. Mechanism only — no logger, no decisions.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Read options forced onto both reads: the primary (a secondary may not have
|
|
10
|
+
* an index build yet), and BSON values as plain JavaScript — an injected
|
|
11
|
+
* client configured with `promoteValues: false` or `useBigInt64: true` would
|
|
12
|
+
* otherwise hand back `Int32` objects or `1n`, and everything would compare as
|
|
13
|
+
* changed.
|
|
14
|
+
*/
|
|
15
|
+
const READ_OPTIONS = Object.freeze({
|
|
16
|
+
readPreference: 'primary',
|
|
17
|
+
promoteLongs: true,
|
|
18
|
+
promoteValues: true,
|
|
19
|
+
useBigInt64: false,
|
|
20
|
+
bsonRegExp: false,
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
/** listIndexes calls in flight while reading many collections — a pace, not a pool */
|
|
24
|
+
const READ_CONCURRENCY = 8;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* What the server is: a mongos in front of shards (its shard keys matter to
|
|
28
|
+
* prune), its topology (`replicaSet`, `sharded`, `standalone` — transactions
|
|
29
|
+
* need one of the first two), and its version (what it can change in place). Best-effort — a
|
|
30
|
+
* server that refuses to say gets the conservative answer: no in-place
|
|
31
|
+
* extras, no shard-key handling.
|
|
32
|
+
*/
|
|
33
|
+
async function readServer(db) {
|
|
34
|
+
const server = { mongos: false, version: undefined, topology: undefined };
|
|
35
|
+
if (typeof db.admin !== 'function') return server;
|
|
36
|
+
try {
|
|
37
|
+
const hello = await db.admin().command({ hello: 1 });
|
|
38
|
+
server.mongos = hello?.msg === 'isdbgrid';
|
|
39
|
+
// What transactions need: a replica set member or a mongos.
|
|
40
|
+
server.topology = server.mongos
|
|
41
|
+
? 'sharded'
|
|
42
|
+
: typeof hello?.setName === 'string'
|
|
43
|
+
? 'replicaSet'
|
|
44
|
+
: 'standalone';
|
|
45
|
+
} catch {
|
|
46
|
+
// Unknown — treated as a replica set or standalone.
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
const info = await db.admin().command({ buildInfo: 1 });
|
|
50
|
+
const [major, minor, patch] = Array.isArray(info?.versionArray) ? info.versionArray : [];
|
|
51
|
+
if (Number.isInteger(major) && Number.isInteger(minor)) {
|
|
52
|
+
server.version = { major, minor, ...(Number.isInteger(patch) ? { patch } : {}) };
|
|
53
|
+
}
|
|
54
|
+
} catch {
|
|
55
|
+
// Unknown version: only the always-available in-place changes.
|
|
56
|
+
}
|
|
57
|
+
return server;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const NAMESPACE_NOT_FOUND = 26;
|
|
61
|
+
|
|
62
|
+
const INDEX_NOT_FOUND = 27;
|
|
63
|
+
|
|
64
|
+
module.exports = {
|
|
65
|
+
INDEX_NOT_FOUND,
|
|
66
|
+
NAMESPACE_NOT_FOUND,
|
|
67
|
+
READ_CONCURRENCY,
|
|
68
|
+
READ_OPTIONS,
|
|
69
|
+
readServer,
|
|
70
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
const { READ_OPTIONS } = require('./server-info.js');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What the cluster says about a collection's sharding — read from the
|
|
5
|
+
* `config` database through a mongos. Every read here needs `clusterMonitor`
|
|
6
|
+
* (or more): a user without it gets `undefined` ("unknown"), never an error,
|
|
7
|
+
* and the caller falls back to what works without it.
|
|
8
|
+
*
|
|
9
|
+
* - `readShardKey` — the shard key, or `null` for a collection that is not
|
|
10
|
+
* sharded (an 8.0 `unsplittable` one included: tracked, but on one shard
|
|
11
|
+
* under `{ _id: 1 }`, which is no key to partition or target by);
|
|
12
|
+
* - `readChunks` — the chunks of a sharded collection, in key order.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** The server's "not authorized" — the one refusal that means "unknown", not "broken" */
|
|
16
|
+
const UNAUTHORIZED = 13;
|
|
17
|
+
|
|
18
|
+
const isUnauthorized = (error) => error?.code === UNAUTHORIZED;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `{ key, uuid, timestamp, unsplittable }` of a sharded collection, `null` when
|
|
22
|
+
* it is not sharded, `undefined` when `config.collections` may not be read.
|
|
23
|
+
*/
|
|
24
|
+
async function readShardKey(client, dbName, collection) {
|
|
25
|
+
let entry;
|
|
26
|
+
try {
|
|
27
|
+
entry = await client
|
|
28
|
+
.db('config')
|
|
29
|
+
.collection('collections')
|
|
30
|
+
.findOne(
|
|
31
|
+
{ _id: `${dbName}.${collection}` },
|
|
32
|
+
{ projection: { key: 1, uuid: 1, timestamp: 1, unsplittable: 1 }, ...READ_OPTIONS },
|
|
33
|
+
);
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (isUnauthorized(error)) return undefined;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
if (entry === null || entry.unsplittable === true || entry.key === undefined) return null;
|
|
39
|
+
return {
|
|
40
|
+
key: entry.key,
|
|
41
|
+
uuid: entry.uuid,
|
|
42
|
+
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The chunks of a sharded collection (`readShardKey`'s result), in key order:
|
|
48
|
+
* `[{ min, max, shard }]` — or `undefined` when `config.chunks` may not be
|
|
49
|
+
* read. By `uuid` (5.0+), then by namespace (a cluster upgraded from 4.4 that
|
|
50
|
+
* never refreshed its chunks).
|
|
51
|
+
*/
|
|
52
|
+
async function readChunks(client, dbName, collection, sharding) {
|
|
53
|
+
const chunks = client.db('config').collection('chunks');
|
|
54
|
+
// Hashed bounds are NumberLongs: promoted to numbers, they lose precision
|
|
55
|
+
// past 2^53 (ARCHITECTURE §6.8).
|
|
56
|
+
const options = {
|
|
57
|
+
projection: { min: 1, max: 1, shard: 1 },
|
|
58
|
+
...READ_OPTIONS,
|
|
59
|
+
promoteLongs: false,
|
|
60
|
+
};
|
|
61
|
+
try {
|
|
62
|
+
let rows = await chunks.find({ uuid: sharding.uuid }, options).sort({ min: 1 }).toArray();
|
|
63
|
+
if (rows.length === 0) {
|
|
64
|
+
rows = await chunks
|
|
65
|
+
.find({ ns: `${dbName}.${collection}` }, options)
|
|
66
|
+
.sort({ min: 1 })
|
|
67
|
+
.toArray();
|
|
68
|
+
}
|
|
69
|
+
return rows;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (isUnauthorized(error)) return undefined;
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
module.exports = { isUnauthorized, readChunks, readShardKey };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
const { isPlainObject } = require('../utils/canonical.js');
|
|
2
|
+
const { versionIndexKey } = require('../versioning/document.js');
|
|
3
|
+
const { toCount } = require('../versioning/internal.js');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a collection's `versioning` block asks of the database: the validator
|
|
7
|
+
* rules for the version and revision fields, merged into whatever validator
|
|
8
|
+
* the definition declares, and the index every version-filtered scan uses.
|
|
9
|
+
* Pure — collections.js folds the result into the normalized definition, so
|
|
10
|
+
* the planner sees an ordinary validator and an ordinary index.
|
|
11
|
+
*
|
|
12
|
+
* The version rule has a `minimum` but never a `maximum`: during a rolling
|
|
13
|
+
* deploy (or after a rollback) a newer release writes a higher version than
|
|
14
|
+
* the declaration knows, and refusing that write would turn a deploy into an
|
|
15
|
+
* outage. A revision outgrows `int` after 2³¹ writes — `$inc` turns it into a
|
|
16
|
+
* `long` — so both are accepted.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** `{ required, properties }` for the managed fields, in a fixed order */
|
|
20
|
+
function versioningRules(versioning) {
|
|
21
|
+
const { field, min, revisionField } = versioning;
|
|
22
|
+
const properties = { [field]: { bsonType: 'int', minimum: min } };
|
|
23
|
+
const required = [field];
|
|
24
|
+
if (revisionField !== null) {
|
|
25
|
+
properties[revisionField] = { bsonType: ['int', 'long'], minimum: 0 };
|
|
26
|
+
required.push(revisionField);
|
|
27
|
+
}
|
|
28
|
+
// `min: 0` adapts a collection whose documents predate versioning: the
|
|
29
|
+
// fields are typed when present, but nothing requires them yet.
|
|
30
|
+
return min === 0 ? { properties } : { required, properties };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The managed field names a validator already constrains itself */
|
|
34
|
+
function managedFieldsIn(validator, versioning) {
|
|
35
|
+
const managed = [versioning.field];
|
|
36
|
+
if (versioning.revisionField !== null) managed.push(versioning.revisionField);
|
|
37
|
+
const found = new Set();
|
|
38
|
+
const schema = validator.$jsonSchema;
|
|
39
|
+
const properties = isPlainObject(schema) ? schema.properties : undefined;
|
|
40
|
+
const required = new Set(
|
|
41
|
+
isPlainObject(schema) && Array.isArray(schema.required) ? schema.required : [],
|
|
42
|
+
);
|
|
43
|
+
for (const name of managed) {
|
|
44
|
+
if (Object.hasOwn(validator, name)) found.add(name);
|
|
45
|
+
if (isPlainObject(properties) && Object.hasOwn(properties, name)) found.add(name);
|
|
46
|
+
if (required.has(name)) found.add(name);
|
|
47
|
+
}
|
|
48
|
+
return [...found];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Why a declared validator cannot carry the versioning rules — it constrains
|
|
53
|
+
* a managed field itself, or its `$jsonSchema` is not a schema object. Empty
|
|
54
|
+
* when the merge can go ahead.
|
|
55
|
+
*/
|
|
56
|
+
function validatorVersioningIssues(validator, versioning) {
|
|
57
|
+
if (validator === undefined) return [];
|
|
58
|
+
if (validator === null || !isPlainObject(validator)) {
|
|
59
|
+
return ['is null, which would remove the versioning rules — drop the validator key instead'];
|
|
60
|
+
}
|
|
61
|
+
const issues = [];
|
|
62
|
+
if (validator.$jsonSchema !== undefined && !isPlainObject(validator.$jsonSchema)) {
|
|
63
|
+
issues.push('$jsonSchema must be an object to take the versioning rules');
|
|
64
|
+
} else if (
|
|
65
|
+
isPlainObject(validator.$jsonSchema) &&
|
|
66
|
+
validator.$jsonSchema.required !== undefined &&
|
|
67
|
+
!Array.isArray(validator.$jsonSchema.required)
|
|
68
|
+
) {
|
|
69
|
+
issues.push('$jsonSchema.required must be an array to take the versioning rules');
|
|
70
|
+
}
|
|
71
|
+
for (const name of managedFieldsIn(validator, versioning)) {
|
|
72
|
+
issues.push(`constrains "${name}", which is managed by versioning — remove that rule`);
|
|
73
|
+
}
|
|
74
|
+
return issues;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The validator to declare: the versioning rules alone (no validator
|
|
79
|
+
* declared), merged into the declared `$jsonSchema` (the declared `required`
|
|
80
|
+
* and `properties` first, ours after), or — next to query operators — added
|
|
81
|
+
* as a top-level `$jsonSchema`, which the server combines with them.
|
|
82
|
+
*/
|
|
83
|
+
function mergeVersioningValidator(validator, versioning) {
|
|
84
|
+
const rules = versioningRules(versioning);
|
|
85
|
+
if (validator === undefined || Object.keys(validator).length === 0) {
|
|
86
|
+
return { $jsonSchema: rules };
|
|
87
|
+
}
|
|
88
|
+
const schema = validator.$jsonSchema;
|
|
89
|
+
if (!isPlainObject(schema)) return { ...validator, $jsonSchema: rules };
|
|
90
|
+
const merged = { ...schema };
|
|
91
|
+
if (rules.required) merged.required = [...(schema.required ?? []), ...rules.required];
|
|
92
|
+
merged.properties = { ...schema.properties, ...rules.properties };
|
|
93
|
+
return { ...validator, $jsonSchema: merged };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The declared version index — `{ [field]: 1, _id: 1 }`, named by its key */
|
|
97
|
+
const versioningIndex = (versioning) => ({ key: versionIndexKey(versioning) });
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The version index of a sharded collection: the shard key between the
|
|
101
|
+
* version field and `_id` — `{ __v: 1, region: 1, _id: 1 }` — so a batch
|
|
102
|
+
* over one chunk's range is an index range, not a filter over every old
|
|
103
|
+
* document of the shard. A hashed field stays hashed; `_id` is not repeated
|
|
104
|
+
* when the key holds it. On `{ _id: 1 }` it is the ordinary version index.
|
|
105
|
+
*/
|
|
106
|
+
function shardedVersionIndexKey(versioning, shardKey) {
|
|
107
|
+
const key = { [versioning.field]: 1 };
|
|
108
|
+
for (const [field, value] of Object.entries(shardKey)) {
|
|
109
|
+
if (field !== versioning.field) key[field] = value;
|
|
110
|
+
}
|
|
111
|
+
if (!('_id' in key)) key._id = 1;
|
|
112
|
+
return key;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Whether an index key is the version index's (same fields, same order, ascending) */
|
|
116
|
+
function isVersioningIndexKey(key, versioning) {
|
|
117
|
+
if (!isPlainObject(key)) return false;
|
|
118
|
+
const entries = Object.entries(key);
|
|
119
|
+
const expected = Object.entries(versionIndexKey(versioning));
|
|
120
|
+
if (entries.length !== expected.length) return false;
|
|
121
|
+
for (let i = 0; i < entries.length; i++) {
|
|
122
|
+
if (entries[i][0] !== expected[i][0] || Number(entries[i][1]) !== expected[i][1]) return false;
|
|
123
|
+
}
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The version floor the live validator enforces — the `minimum` of the version
|
|
129
|
+
* field's `$jsonSchema` rule — or `null` when it enforces none.
|
|
130
|
+
*/
|
|
131
|
+
function liveVersionFloor(options, versioning) {
|
|
132
|
+
const schema = options?.validator?.$jsonSchema;
|
|
133
|
+
const rule = isPlainObject(schema?.properties) ? schema.properties[versioning.field] : undefined;
|
|
134
|
+
return isPlainObject(rule) ? toCount(rule.minimum) : null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The `min` converge must check the data against before it raises the floor,
|
|
139
|
+
* or `null` when there is nothing to check: no versioning, `min: 0`, a
|
|
140
|
+
* collection that does not exist yet (no documents), or a floor already that
|
|
141
|
+
* high. Only a rising floor costs a read — the steady state costs nothing.
|
|
142
|
+
*/
|
|
143
|
+
function versionFloorToCheck(definition, live) {
|
|
144
|
+
const versioning = definition.versioning;
|
|
145
|
+
if (!versioning || versioning.min === 0 || !live.exists) return null;
|
|
146
|
+
if (live.type !== undefined && live.type !== 'collection') return null;
|
|
147
|
+
const floor = liveVersionFloor(live.options, versioning);
|
|
148
|
+
return floor !== null && floor >= versioning.min ? null : versioning.min;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Why the version floor cannot be raised — `live.versionFloor` is what
|
|
153
|
+
* converge read: `{ min, below: true | false | 'unknown', error? }` — or
|
|
154
|
+
* `undefined` when it can. The document ids are never named: they may be PII.
|
|
155
|
+
*/
|
|
156
|
+
function versionFloorConflict(floor) {
|
|
157
|
+
if (!floor || floor.below === false) return undefined;
|
|
158
|
+
if (floor.below === true) {
|
|
159
|
+
return (
|
|
160
|
+
`documents below version ${floor.min} remain — raising versioning.min would leave them ` +
|
|
161
|
+
'invalid; let the background migration that upgrades them finish (migronaut background ' +
|
|
162
|
+
'status), then converge again'
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return (
|
|
166
|
+
`could not check for documents below version ${floor.min} (${floor.error}) — converge ` +
|
|
167
|
+
'with the old min first so the version index exists, then raise it'
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
module.exports = {
|
|
172
|
+
isVersioningIndexKey,
|
|
173
|
+
shardedVersionIndexKey,
|
|
174
|
+
liveVersionFloor,
|
|
175
|
+
versionFloorConflict,
|
|
176
|
+
versionFloorToCheck,
|
|
177
|
+
mergeVersioningValidator,
|
|
178
|
+
validatorVersioningIssues,
|
|
179
|
+
versioningIndex,
|
|
180
|
+
versioningRules,
|
|
181
|
+
};
|
package/src/errors/index.js
CHANGED
|
@@ -243,11 +243,15 @@ class QueueJobFailedError extends MigronautError {
|
|
|
243
243
|
|
|
244
244
|
/**
|
|
245
245
|
* Thrown by `converge` when the database cannot be brought to the declared
|
|
246
|
-
* state:
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
246
|
+
* state. `context.phase` says where it stopped: `'plan'` (a conflict refused
|
|
247
|
+
* the run before any write), `'replan'` (a collection changed while the run
|
|
248
|
+
* was under way), `'apply'` (a step failed) or `'wait'` (the search index
|
|
249
|
+
* builds did not finish — `context.reason`: `'failed'`, `'timeout'` or
|
|
250
|
+
* `'unreadable'`). A search index list that could not be read is reported in
|
|
251
|
+
* the phase that read it. `context.converge` is the converge result so far —
|
|
252
|
+
* which steps were applied, which failed, which were never reached — and
|
|
253
|
+
* `context.hint`, when present, says what usually fixes the server error
|
|
254
|
+
* behind it.
|
|
251
255
|
*/
|
|
252
256
|
class ConvergeFailedError extends MigronautError {
|
|
253
257
|
constructor(message, context, options) {
|
|
@@ -256,6 +260,88 @@ class ConvergeFailedError extends MigronautError {
|
|
|
256
260
|
}
|
|
257
261
|
}
|
|
258
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Thrown by the optimistic-concurrency helpers (`@alexify/migronaut/versioning`)
|
|
265
|
+
* when a revision-guarded write matched nothing. `context.reason` says why:
|
|
266
|
+
* `'conflict'` (the document exists at another revision — `context.actual`),
|
|
267
|
+
* `'not-found'` (no document matches the filter at all) or `'unknown'` (the
|
|
268
|
+
* follow-up read was skipped or could not tell). `context.expected` is the
|
|
269
|
+
* revision the caller held. The filter is never copied in — it may carry PII.
|
|
270
|
+
*/
|
|
271
|
+
class RevisionConflictError extends MigronautError {
|
|
272
|
+
constructor(message, context, options) {
|
|
273
|
+
super('REVISION_CONFLICT', message, context, options);
|
|
274
|
+
this.name = 'RevisionConflictError';
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Thrown by an upcaster that cannot bring a document to the current shape:
|
|
280
|
+
* `context.reason` is `'newer'` (written by a newer release), `'below-min'`
|
|
281
|
+
* (older than the oldest shape still supported) or `'invalid'` (the version
|
|
282
|
+
* field is not a non-negative integer, or a step returned something that is
|
|
283
|
+
* not a document).
|
|
284
|
+
*/
|
|
285
|
+
class ShapeVersionError extends MigronautError {
|
|
286
|
+
constructor(message, context, options) {
|
|
287
|
+
super('SHAPE_VERSION_UNSUPPORTED', message, context, options);
|
|
288
|
+
this.name = 'ShapeVersionError';
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Thrown when a migration `requires` a background migration that has not
|
|
294
|
+
* completed yet — or whose collection still holds documents of the old shape.
|
|
295
|
+
* Nothing was run: `context.waitsFor` names the background migrations it
|
|
296
|
+
* waits for, with their status.
|
|
297
|
+
*/
|
|
298
|
+
class BackgroundPendingError extends MigronautError {
|
|
299
|
+
constructor(message, context, options) {
|
|
300
|
+
super('BACKGROUND_PENDING', message, context, options);
|
|
301
|
+
this.name = 'BackgroundPendingError';
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Thrown when a background migration ended `failed` — a partition used up its
|
|
307
|
+
* slice failures, the document error budget ran out, or old-shape documents
|
|
308
|
+
* kept appearing for `maxPasses` passes. `context.migration` names it and
|
|
309
|
+
* `context.lastError` says what happened last.
|
|
310
|
+
*/
|
|
311
|
+
class BackgroundFailedError extends MigronautError {
|
|
312
|
+
constructor(message, context, options) {
|
|
313
|
+
super('BACKGROUND_FAILED', message, context, options);
|
|
314
|
+
this.name = 'BackgroundFailedError';
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Thrown when a control action does not fit the background migration's state
|
|
320
|
+
* — pausing a completed one, resuming one that is not paused, retrying one
|
|
321
|
+
* that is still running. `context.status` is the state it found and
|
|
322
|
+
* `context.action` what was asked.
|
|
323
|
+
*/
|
|
324
|
+
class BackgroundConflictError extends MigronautError {
|
|
325
|
+
constructor(message, context, options) {
|
|
326
|
+
super('BACKGROUND_CONFLICT', message, context, options);
|
|
327
|
+
this.name = 'BackgroundConflictError';
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Thrown by the dry-run sandbox when a step reaches for something it cannot
|
|
333
|
+
* run inside an always-aborted transaction — DDL, an admin command, another
|
|
334
|
+
* session, `$out`/`$merge`, a migronaut-internal collection. `context.method`
|
|
335
|
+
* names the call and `context.reason` the rule it broke. A dry run reports
|
|
336
|
+
* every refusal even when the step caught the error itself.
|
|
337
|
+
*/
|
|
338
|
+
class SandboxRefusedError extends MigronautError {
|
|
339
|
+
constructor(message, context, options) {
|
|
340
|
+
super('SANDBOX_REFUSED', message, context, options);
|
|
341
|
+
this.name = 'SandboxRefusedError';
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
259
345
|
module.exports = {
|
|
260
346
|
MigronautError,
|
|
261
347
|
LockAlreadyHeldError,
|
|
@@ -282,4 +368,10 @@ module.exports = {
|
|
|
282
368
|
QueueJobInvalidError,
|
|
283
369
|
QueueJobFailedError,
|
|
284
370
|
ConvergeFailedError,
|
|
371
|
+
RevisionConflictError,
|
|
372
|
+
ShapeVersionError,
|
|
373
|
+
BackgroundPendingError,
|
|
374
|
+
BackgroundFailedError,
|
|
375
|
+
BackgroundConflictError,
|
|
376
|
+
SandboxRefusedError,
|
|
285
377
|
};
|
package/src/index.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
const { EXIT_CODES } = require('./cli/exit-codes.js');
|
|
2
|
+
const { startBackgroundRunner } = require('./core/background-runner.js');
|
|
2
3
|
const { MigratorKit } = require('./core/migrator.js');
|
|
3
4
|
const { pendingMigrations, runMigrations } = require('./core/run.js');
|
|
4
5
|
const { createLogger } = require('./utils/logger.js');
|
|
5
6
|
const {
|
|
7
|
+
BackgroundConflictError,
|
|
8
|
+
BackgroundFailedError,
|
|
9
|
+
BackgroundPendingError,
|
|
6
10
|
ChecksumMismatchError,
|
|
7
11
|
ConfigFileExistsError,
|
|
8
12
|
ConfigInvalidError,
|
|
@@ -27,7 +31,10 @@ const {
|
|
|
27
31
|
OutOfOrderMigrationError,
|
|
28
32
|
QueueJobFailedError,
|
|
29
33
|
QueueJobInvalidError,
|
|
34
|
+
RevisionConflictError,
|
|
30
35
|
RunAbortedError,
|
|
36
|
+
SandboxRefusedError,
|
|
37
|
+
ShapeVersionError,
|
|
31
38
|
} = require('./errors/index.js');
|
|
32
39
|
|
|
33
40
|
module.exports = {
|
|
@@ -46,7 +53,13 @@ module.exports = {
|
|
|
46
53
|
// The CLI's exit-code map, for wrappers that mirror its semantics
|
|
47
54
|
EXIT_CODES,
|
|
48
55
|
|
|
56
|
+
// Background migrations driven from inside the application (experimental)
|
|
57
|
+
startBackgroundRunner,
|
|
58
|
+
|
|
49
59
|
// Error classes
|
|
60
|
+
BackgroundConflictError,
|
|
61
|
+
BackgroundFailedError,
|
|
62
|
+
BackgroundPendingError,
|
|
50
63
|
ChecksumMismatchError,
|
|
51
64
|
ConfigFileExistsError,
|
|
52
65
|
ConfigInvalidError,
|
|
@@ -71,5 +84,8 @@ module.exports = {
|
|
|
71
84
|
OutOfOrderMigrationError,
|
|
72
85
|
QueueJobFailedError,
|
|
73
86
|
QueueJobInvalidError,
|
|
87
|
+
RevisionConflictError,
|
|
74
88
|
RunAbortedError,
|
|
89
|
+
SandboxRefusedError,
|
|
90
|
+
ShapeVersionError,
|
|
75
91
|
};
|
package/src/utils/canonical.js
CHANGED
|
@@ -100,6 +100,31 @@ function regExpIssue(value, seen = new Set()) {
|
|
|
100
100
|
return null;
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Why `value` cannot be sent as declared — a function, a symbol, a cycle, or a
|
|
105
|
+
* RegExp the driver would change (see {@link regExpIssue}) — or null. Walks
|
|
106
|
+
* plain objects and arrays, the shapes a declaration is written in.
|
|
107
|
+
*/
|
|
108
|
+
function unsendable(value, seen = new Set()) {
|
|
109
|
+
if (seen.size === 0) {
|
|
110
|
+
const issue = regExpIssue(value);
|
|
111
|
+
if (issue) return issue;
|
|
112
|
+
}
|
|
113
|
+
const type = typeof value;
|
|
114
|
+
if (type === 'function') return 'must not contain functions';
|
|
115
|
+
if (type === 'symbol') return 'must not contain symbols';
|
|
116
|
+
if (value === null || type !== 'object') return null;
|
|
117
|
+
if (seen.has(value)) return 'must not contain circular references';
|
|
118
|
+
seen.add(value);
|
|
119
|
+
const items = Array.isArray(value) ? value : isPlainObject(value) ? Object.values(value) : [];
|
|
120
|
+
for (const item of items) {
|
|
121
|
+
const reason = unsendable(item, seen);
|
|
122
|
+
if (reason) return reason;
|
|
123
|
+
}
|
|
124
|
+
seen.delete(value);
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
103
128
|
/**
|
|
104
129
|
* Assign without invoking setters: a key named `__proto__` (JSON.parse makes
|
|
105
130
|
* one an own property) must stay a key, not replace the object's prototype —
|
|
@@ -176,4 +201,12 @@ function toWire(value) {
|
|
|
176
201
|
return out;
|
|
177
202
|
}
|
|
178
203
|
|
|
179
|
-
module.exports = {
|
|
204
|
+
module.exports = {
|
|
205
|
+
assign,
|
|
206
|
+
canonical,
|
|
207
|
+
deepEqual,
|
|
208
|
+
isPlainObject,
|
|
209
|
+
regExpIssue,
|
|
210
|
+
toWire,
|
|
211
|
+
unsendable,
|
|
212
|
+
};
|
package/src/utils/error.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const { MigronautError } = require('../errors/index.js');
|
|
2
|
-
const { redactUris } = require('./redact.js');
|
|
2
|
+
const { redactOutbound, redactUris } = require('./redact.js');
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Human-readable message from any thrown value, with URI credentials masked.
|
|
@@ -25,4 +25,13 @@ function errorWithCause(error) {
|
|
|
25
25
|
return cause ? `${message} — ${cause}` : message;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
/**
|
|
29
|
+
* {@link errorText} for an error about the application's data — a background
|
|
30
|
+
* migration's document errors and failed slices, kept in its state and
|
|
31
|
+
* logged: the values a server error quotes (an E11000's duplicate key — an
|
|
32
|
+
* email, a phone number) are masked too. Migronaut never logs a document's
|
|
33
|
+
* contents; the index name still says which constraint was violated.
|
|
34
|
+
*/
|
|
35
|
+
const documentErrorText = (error) => redactOutbound(errorText(error));
|
|
36
|
+
|
|
37
|
+
module.exports = { documentErrorText, errorText, errorWithCause };
|
package/src/utils/loader.js
CHANGED
|
@@ -3,6 +3,7 @@ const path = require('node:path');
|
|
|
3
3
|
const { pathToFileURL } = require('node:url');
|
|
4
4
|
const { MigrationFileNotFoundError, MigrationInvalidExportError } = require('../errors/index.js');
|
|
5
5
|
const { errorText } = require('./error.js');
|
|
6
|
+
const { requiresIssues } = require('./migration-name.js');
|
|
6
7
|
|
|
7
8
|
/** TypeScript source extensions that require a TS-capable runtime to import */
|
|
8
9
|
const TS_EXTENSIONS = new Set(['.ts', '.mts', '.cts']);
|
|
@@ -96,16 +97,13 @@ function importUserFile(filepath, options = {}) {
|
|
|
96
97
|
}
|
|
97
98
|
|
|
98
99
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* Handles all three supported formats:
|
|
102
|
-
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
103
|
-
* - CommonJS default export (`module.exports = { up, down }`)
|
|
100
|
+
* Import a migration file: its module, resolved — the default export of a
|
|
101
|
+
* CommonJS file, the namespace of an ES module with named exports.
|
|
104
102
|
*
|
|
105
103
|
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
106
|
-
* @throws {MigrationInvalidExportError} when
|
|
104
|
+
* @throws {MigrationInvalidExportError} when TypeScript cannot be loaded
|
|
107
105
|
*/
|
|
108
|
-
async function
|
|
106
|
+
async function importMigrationModule(filepath, options = {}) {
|
|
109
107
|
try {
|
|
110
108
|
await fs.access(filepath);
|
|
111
109
|
} catch {
|
|
@@ -123,7 +121,48 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
123
121
|
throw error;
|
|
124
122
|
}
|
|
125
123
|
// `mod.default ?? mod` handles the CommonJS default-export case
|
|
126
|
-
|
|
124
|
+
return imported.default ?? imported;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The `requires` export, validated against the file's own name */
|
|
128
|
+
function readRequires(resolved, filepath) {
|
|
129
|
+
if (resolved.requires === undefined) return {};
|
|
130
|
+
const issues = requiresIssues(resolved.requires, path.basename(filepath));
|
|
131
|
+
if (issues.length > 0) {
|
|
132
|
+
throw new MigrationInvalidExportError(`Invalid ${issues[0].path}: ${issues[0].message}`, {
|
|
133
|
+
filepath,
|
|
134
|
+
issues,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return { requires: [...resolved.requires] };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* What a migration module exports, validated: a regular migration
|
|
142
|
+
* (`{ up, down, useTransaction?, timeoutMs?, description?, requires? }`) or a
|
|
143
|
+
* background one (`{ kind: 'background', background, description?,
|
|
144
|
+
* requires? }` — its spec is validated where the collection's versioning is
|
|
145
|
+
* known). A file with both `background` and `up`/`down` is refused: the
|
|
146
|
+
* expand steps belong in a migration of their own.
|
|
147
|
+
*
|
|
148
|
+
* @throws {MigrationInvalidExportError}
|
|
149
|
+
*/
|
|
150
|
+
function resolveMigrationExports(resolved, filepath) {
|
|
151
|
+
if (resolved.background !== undefined) {
|
|
152
|
+
if (resolved.up !== undefined || resolved.down !== undefined) {
|
|
153
|
+
throw new MigrationInvalidExportError(
|
|
154
|
+
'A background migration exports no up() or down() — put the expand steps in a ' +
|
|
155
|
+
'migration of their own',
|
|
156
|
+
{ filepath },
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
kind: 'background',
|
|
161
|
+
background: resolved.background,
|
|
162
|
+
...(typeof resolved.description === 'string' ? { description: resolved.description } : {}),
|
|
163
|
+
...readRequires(resolved, filepath),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
127
166
|
|
|
128
167
|
if (!isFunction(resolved.up) || !isFunction(resolved.down)) {
|
|
129
168
|
throw new MigrationInvalidExportError('Migration must export async up() and down() functions', {
|
|
@@ -142,8 +181,37 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
142
181
|
if (typeof resolved.description === 'string') {
|
|
143
182
|
migration.description = resolved.description;
|
|
144
183
|
}
|
|
184
|
+
Object.assign(migration, readRequires(resolved, filepath));
|
|
145
185
|
|
|
146
186
|
return migration;
|
|
147
187
|
}
|
|
148
188
|
|
|
149
|
-
|
|
189
|
+
/**
|
|
190
|
+
* Dynamically load a migration file and validate its exports.
|
|
191
|
+
*
|
|
192
|
+
* Handles all three supported formats:
|
|
193
|
+
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
194
|
+
* - CommonJS default export (`module.exports = { up, down }`)
|
|
195
|
+
*
|
|
196
|
+
* A background migration (`kind: 'background'`) is returned like any other:
|
|
197
|
+
* the caller tells it apart by its kind.
|
|
198
|
+
*
|
|
199
|
+
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
200
|
+
* @throws {MigrationInvalidExportError} when up/down are not both functions
|
|
201
|
+
*/
|
|
202
|
+
async function loadMigrationFile(filepath, options = {}) {
|
|
203
|
+
const migration = resolveMigrationExports(
|
|
204
|
+
await importMigrationModule(filepath, options),
|
|
205
|
+
filepath,
|
|
206
|
+
);
|
|
207
|
+
return migration;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
module.exports = {
|
|
211
|
+
importMigrationModule,
|
|
212
|
+
importUserFile,
|
|
213
|
+
loadMigrationFile,
|
|
214
|
+
resolveMigrationExports,
|
|
215
|
+
tsLoadErrorOrNull,
|
|
216
|
+
tsLoadMessageOrNull,
|
|
217
|
+
};
|