@alexify/migronaut 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +41 -3
- package/bullmq.d.ts +484 -8
- package/index.d.ts +1264 -9
- package/migronaut.schema.json +93 -1
- package/package.json +9 -2
- package/src/bullmq/background-processor.js +541 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +348 -21
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +484 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +375 -0
- package/src/core/background-engine.js +849 -0
- package/src/core/background-kit.js +432 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +610 -0
- package/src/core/background.js +1127 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migration-logger.js +279 -0
- package/src/core/migrator.js +1027 -22
- package/src/core/options.js +36 -0
- package/src/core/run-recorder.js +6 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +34 -8
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/job-ref.js +44 -0
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/redact.js +140 -3
- package/src/utils/telemetry.js +110 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/src/core/options.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
const { ConfigInvalidError, MigrationInvalidNameError } = require('../errors/index.js');
|
|
2
2
|
const { actorIssue } = require('../utils/actor.js');
|
|
3
|
+
const { isPlainObject } = require('../utils/canonical.js');
|
|
3
4
|
const { isCollectionName } = require('../utils/collection-name.js');
|
|
5
|
+
const { jobRefIssue } = require('../utils/job-ref.js');
|
|
4
6
|
|
|
5
7
|
/**
|
|
6
8
|
* Validation of the options the kit's run methods take. Pure — no config, no
|
|
@@ -101,6 +103,20 @@ function assertActorValid(options) {
|
|
|
101
103
|
}
|
|
102
104
|
}
|
|
103
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Validate `job`: the queue job a run works for, `{ id, groupId? }` — bound
|
|
108
|
+
* into the run's correlation, so it must be small and exactly that shape.
|
|
109
|
+
*/
|
|
110
|
+
function assertJobValid(job, options) {
|
|
111
|
+
const issue = jobRefIssue(job, options);
|
|
112
|
+
if (issue) {
|
|
113
|
+
// For the error's context: the keys of what was given, not its values.
|
|
114
|
+
throw new ConfigInvalidError(issue, {
|
|
115
|
+
job: isPlainObject(job) ? Object.keys(job).join(', ') : typeof job,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
104
120
|
/**
|
|
105
121
|
* Validate `checksum`: the SHA-256 the caller expects the named file to have
|
|
106
122
|
* — how a queue job says which version of the file it was planned with.
|
|
@@ -152,6 +168,20 @@ function assertConvergeAfterUpValid(converge, filename, to) {
|
|
|
152
168
|
}
|
|
153
169
|
}
|
|
154
170
|
|
|
171
|
+
/**
|
|
172
|
+
* `onBackgroundPending`: what a run does at a migration that `requires` a
|
|
173
|
+
* background migration not completed yet — `'error'` (throw
|
|
174
|
+
* BackgroundPendingError) or `'stop'` (end the run there, cleanly).
|
|
175
|
+
*/
|
|
176
|
+
function assertBackgroundPendingValid(onBackgroundPending) {
|
|
177
|
+
if (onBackgroundPending === undefined) return;
|
|
178
|
+
if (onBackgroundPending !== 'error' && onBackgroundPending !== 'stop') {
|
|
179
|
+
throw new ConfigInvalidError("onBackgroundPending must be 'error' or 'stop'", {
|
|
180
|
+
onBackgroundPending,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
155
185
|
/** `up(filename, options)` */
|
|
156
186
|
function assertUpOptions(filename, options) {
|
|
157
187
|
assertFilename(filename);
|
|
@@ -167,7 +197,9 @@ function assertUpOptions(filename, options) {
|
|
|
167
197
|
assertOrderedValid(options.ordered, filename);
|
|
168
198
|
assertConvergeAfterUpValid(options.converge, filename, options.to);
|
|
169
199
|
assertChecksumValid(options.checksum, filename);
|
|
200
|
+
assertBackgroundPendingValid(options.onBackgroundPending);
|
|
170
201
|
assertActorValid(options);
|
|
202
|
+
assertJobValid(options.job);
|
|
171
203
|
}
|
|
172
204
|
|
|
173
205
|
/** `down(filename, options)` */
|
|
@@ -178,12 +210,14 @@ function assertDownOptions(filename, options) {
|
|
|
178
210
|
assertToValid(options.to, filename, options);
|
|
179
211
|
assertOrderedValid(options.ordered, filename);
|
|
180
212
|
assertActorValid(options);
|
|
213
|
+
assertJobValid(options.job);
|
|
181
214
|
}
|
|
182
215
|
|
|
183
216
|
/** `redo(filename, options)` */
|
|
184
217
|
function assertRedoOptions(filename, options) {
|
|
185
218
|
assertFilename(filename);
|
|
186
219
|
assertActorValid(options);
|
|
220
|
+
assertJobValid(options.job);
|
|
187
221
|
}
|
|
188
222
|
|
|
189
223
|
/** `dryRun(direction, filename, options)` */
|
|
@@ -254,6 +288,8 @@ function assertImportOptions(options) {
|
|
|
254
288
|
}
|
|
255
289
|
|
|
256
290
|
module.exports = {
|
|
291
|
+
assertActorValid,
|
|
292
|
+
assertJobValid,
|
|
257
293
|
assertConvergeOptions,
|
|
258
294
|
assertDownOptions,
|
|
259
295
|
assertDryRunOptions,
|
package/src/core/run-recorder.js
CHANGED
|
@@ -17,6 +17,8 @@ const { ATTRIBUTES } = require('../utils/telemetry.js');
|
|
|
17
17
|
class RunRecorder {
|
|
18
18
|
#info;
|
|
19
19
|
#runId;
|
|
20
|
+
/** The queue job the run works for — `{ jobId?, groupId? }` — for its span */
|
|
21
|
+
#job;
|
|
20
22
|
#telemetry;
|
|
21
23
|
#emit;
|
|
22
24
|
#logger;
|
|
@@ -29,9 +31,10 @@ class RunRecorder {
|
|
|
29
31
|
/** Set once the lock is held and the run span is open */
|
|
30
32
|
#span;
|
|
31
33
|
|
|
32
|
-
constructor({ info, runId, telemetry, emit, logger, fields }) {
|
|
34
|
+
constructor({ info, runId, job = {}, telemetry, emit, logger, fields }) {
|
|
33
35
|
this.#info = info;
|
|
34
36
|
this.#runId = runId;
|
|
37
|
+
this.#job = job;
|
|
35
38
|
this.#telemetry = telemetry;
|
|
36
39
|
this.#emit = emit;
|
|
37
40
|
this.#logger = logger;
|
|
@@ -69,6 +72,8 @@ class RunRecorder {
|
|
|
69
72
|
[ATTRIBUTES.RUN_ID]: this.#runId,
|
|
70
73
|
[ATTRIBUTES.RUN_COMMAND]: this.#info.command,
|
|
71
74
|
[ATTRIBUTES.RUN_DIRECTION]: this.#info.direction,
|
|
75
|
+
[ATTRIBUTES.JOB_ID]: this.#job.jobId,
|
|
76
|
+
[ATTRIBUTES.JOB_GROUP_ID]: this.#job.groupId,
|
|
72
77
|
[ATTRIBUTES.LOCK_ACQUIRE_MS]: this.#acquired?.acquireMs,
|
|
73
78
|
[ATTRIBUTES.LOCK_SKIPPED]: this.#acquired?.skipped,
|
|
74
79
|
};
|
package/src/core/run.js
CHANGED
|
@@ -41,6 +41,7 @@ const { MigratorKit, RECORD_LOCK_WAIT } = require('./migrator.js');
|
|
|
41
41
|
async function runMigrations(config = {}, options = {}) {
|
|
42
42
|
const {
|
|
43
43
|
noLock,
|
|
44
|
+
onBackgroundPending,
|
|
44
45
|
onLockHeld = 'throw',
|
|
45
46
|
// Left undefined unless given: the default then follows the holder's TTL.
|
|
46
47
|
lockWaitTimeoutMs,
|
|
@@ -71,6 +72,11 @@ async function runMigrations(config = {}, options = {}) {
|
|
|
71
72
|
kit.on('converge:end', (event) => {
|
|
72
73
|
if (event.trigger === 'up' && event.success) converge = event.result;
|
|
73
74
|
});
|
|
75
|
+
// With onBackgroundPending: 'stop', where the run stopped and what for.
|
|
76
|
+
const waiting = [];
|
|
77
|
+
kit.on('background:waiting', (event) => {
|
|
78
|
+
waiting.push({ migration: event.migration, waitsFor: event.waitsFor });
|
|
79
|
+
});
|
|
74
80
|
|
|
75
81
|
// An abort reaches the run wherever it is: the wait loop sees the signal
|
|
76
82
|
// between polls, and kit.stop() stops a run that is setting up or between
|
|
@@ -90,20 +96,28 @@ async function runMigrations(config = {}, options = {}) {
|
|
|
90
96
|
waited,
|
|
91
97
|
waitedMs,
|
|
92
98
|
attempts,
|
|
93
|
-
} = await withLockWait(
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
} = await withLockWait(
|
|
100
|
+
() =>
|
|
101
|
+
kit.up(undefined, {
|
|
102
|
+
...(noLock ? { noLock: true } : {}),
|
|
103
|
+
...(onBackgroundPending !== undefined ? { onBackgroundPending } : {}),
|
|
104
|
+
}),
|
|
105
|
+
{
|
|
106
|
+
onLockHeld,
|
|
107
|
+
...(lockWaitTimeoutMs !== undefined ? { lockWaitTimeoutMs } : {}),
|
|
108
|
+
...(lockPollIntervalMs !== undefined ? { lockPollIntervalMs } : {}),
|
|
109
|
+
// Resolved AFTER connect, from the kit's own merged config: a `logger:
|
|
110
|
+
// null` in the config file must silence the wait lines too, not only
|
|
111
|
+
// the kit's own.
|
|
112
|
+
logger: kit.logger,
|
|
113
|
+
...(signal ? { signal } : {}),
|
|
114
|
+
onSettle: (wait) => kit[RECORD_LOCK_WAIT](wait),
|
|
115
|
+
},
|
|
116
|
+
);
|
|
104
117
|
return {
|
|
105
118
|
applied,
|
|
106
|
-
upToDate: applied.length === 0,
|
|
119
|
+
upToDate: applied.length === 0 && waiting.length === 0,
|
|
120
|
+
...(waiting.length > 0 ? { waiting } : {}),
|
|
107
121
|
waited,
|
|
108
122
|
waitedMs,
|
|
109
123
|
attempts,
|
package/src/core/runner.js
CHANGED
|
@@ -84,9 +84,15 @@ async function withTimeout(promise, timeoutMs, name, direction, onTimeout) {
|
|
|
84
84
|
* body succeeded and only the (non-transactional) changelog write failed, the
|
|
85
85
|
* error carries `context.phase = 'changelog-write'` and `onError` is not fired —
|
|
86
86
|
* the migration itself did not fail.
|
|
87
|
+
*
|
|
88
|
+
* `attemptContext(attempt)` returns what the caller adds to the context of
|
|
89
|
+
* each attempt (`ctx.run`, `ctx.logger`): a retried transaction runs the body
|
|
90
|
+
* again, and that run gets a context of its own. Resolves to
|
|
91
|
+
* `{ duration, attempts }`; a failure carries `attempts` in its context.
|
|
87
92
|
*/
|
|
88
93
|
async function runMigration(params) {
|
|
89
94
|
const { name, migration, direction, context, useTransaction, hooks, onSuccess, logger } = params;
|
|
95
|
+
const { attemptContext } = params;
|
|
90
96
|
const fn = direction === 'up' ? migration.up : migration.down;
|
|
91
97
|
// A per-file `export const timeoutMs` overrides the global setting.
|
|
92
98
|
const timeoutMs = migration.timeoutMs ?? params.timeoutMs;
|
|
@@ -102,7 +108,18 @@ async function runMigration(params) {
|
|
|
102
108
|
const signal = context.signal
|
|
103
109
|
? AbortSignal.any([context.signal, timedOut.signal])
|
|
104
110
|
: timedOut.signal;
|
|
105
|
-
|
|
111
|
+
// A fresh context per attempt, not one mutated in place: a body from an
|
|
112
|
+
// earlier attempt that is still running (a timeout does not stop it) keeps
|
|
113
|
+
// the attempt it started with.
|
|
114
|
+
const contextOf = (session, attempt) => ({
|
|
115
|
+
...context,
|
|
116
|
+
signal,
|
|
117
|
+
...(session ? { session } : {}),
|
|
118
|
+
...attemptContext?.(attempt),
|
|
119
|
+
});
|
|
120
|
+
let attempts = 0;
|
|
121
|
+
// The context the body last ran with — what onError gets.
|
|
122
|
+
let runtimeContext;
|
|
106
123
|
const onTimeout = (timeoutError) => timedOut.abort(timeoutError);
|
|
107
124
|
let duration = 0;
|
|
108
125
|
// 'body' while the migration's own code runs; 'changelog' once it committed
|
|
@@ -114,25 +131,28 @@ async function runMigration(params) {
|
|
|
114
131
|
try {
|
|
115
132
|
if (useTransaction) {
|
|
116
133
|
session = context.client.startSession();
|
|
117
|
-
runtimeContext = { ...runtimeContext, session };
|
|
118
134
|
// withTransaction may run the body more than once when the driver retries
|
|
119
135
|
// a transient failure, so duration is re-measured on each attempt. The
|
|
120
136
|
// changelog write stays inside the transaction, so a failure there
|
|
121
137
|
// aborts the body's writes too — 'body' phase is accurate throughout.
|
|
122
138
|
await session.withTransaction(async () => {
|
|
123
139
|
const attemptStart = Date.now();
|
|
140
|
+
attempts += 1;
|
|
141
|
+
runtimeContext = contextOf(session, attempts);
|
|
124
142
|
await withTimeout(fn(runtimeContext), timeoutMs, name, direction, onTimeout);
|
|
125
143
|
duration = Date.now() - attemptStart;
|
|
126
144
|
await onSuccess?.(duration, session);
|
|
127
145
|
});
|
|
128
146
|
} else {
|
|
147
|
+
attempts += 1;
|
|
148
|
+
runtimeContext = contextOf(undefined, attempts);
|
|
129
149
|
await withTimeout(fn(runtimeContext), timeoutMs, name, direction, onTimeout);
|
|
130
150
|
duration = Date.now() - start;
|
|
131
151
|
phase = 'changelog';
|
|
132
152
|
await onSuccess?.(duration, undefined);
|
|
133
153
|
}
|
|
134
154
|
|
|
135
|
-
return { duration };
|
|
155
|
+
return { duration, attempts };
|
|
136
156
|
} catch (error) {
|
|
137
157
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
138
158
|
// Failures deserve timing data as much as successes — a slow-then-failing
|
|
@@ -162,15 +182,21 @@ async function runMigration(params) {
|
|
|
162
182
|
if (hooks?.onError) {
|
|
163
183
|
// A throwing onError hook must not replace the real cause.
|
|
164
184
|
try {
|
|
165
|
-
|
|
185
|
+
// A transaction that failed before its body ever ran (a session that
|
|
186
|
+
// would not start) still hands onError a whole context: a first
|
|
187
|
+
// attempt's, not counted as one.
|
|
188
|
+
await hooks.onError(name, err, runtimeContext ?? contextOf(session, 1));
|
|
166
189
|
} catch (hookError) {
|
|
167
190
|
const message = errorText(hookError);
|
|
168
191
|
logger?.warn(`⚠ onError hook failed for ${name}: ${message}`);
|
|
169
192
|
}
|
|
170
193
|
}
|
|
171
194
|
|
|
195
|
+
// How many times the body ran (the driver retries a transient
|
|
196
|
+
// transaction error by running it again) — absent if it never started.
|
|
197
|
+
const attemptsField = attempts > 0 ? { attempts } : {};
|
|
172
198
|
if (err instanceof MigrationTimeoutError) {
|
|
173
|
-
err.context = { durationMs: elapsed, ...err.context };
|
|
199
|
+
err.context = { durationMs: elapsed, ...attemptsField, ...err.context };
|
|
174
200
|
throw err;
|
|
175
201
|
}
|
|
176
202
|
// A standalone deployment refusing the transaction is a topology problem,
|
|
@@ -179,7 +205,7 @@ async function runMigration(params) {
|
|
|
179
205
|
throw new TransactionsUnsupportedError(
|
|
180
206
|
`Cannot run ${name} in a transaction — this deployment is standalone. ` +
|
|
181
207
|
'Set useTransaction: false, or run against a replica set / mongos.',
|
|
182
|
-
{ name, direction, durationMs: elapsed, cause: err.message },
|
|
208
|
+
{ name, direction, durationMs: elapsed, ...attemptsField, cause: err.message },
|
|
183
209
|
{ cause: err },
|
|
184
210
|
);
|
|
185
211
|
}
|
|
@@ -187,7 +213,7 @@ async function runMigration(params) {
|
|
|
187
213
|
`Migration ${direction} failed: ${name}`,
|
|
188
214
|
// The message is duplicated into context because that is what survives
|
|
189
215
|
// JSON serialization; `cause` keeps the real Error (and its stack).
|
|
190
|
-
{ name, direction, durationMs: elapsed, cause: err.message },
|
|
216
|
+
{ name, direction, durationMs: elapsed, ...attemptsField, cause: err.message },
|
|
191
217
|
{ cause: err },
|
|
192
218
|
);
|
|
193
219
|
} finally {
|
|
@@ -197,4 +223,4 @@ async function runMigration(params) {
|
|
|
197
223
|
}
|
|
198
224
|
}
|
|
199
225
|
|
|
200
|
-
module.exports = { runMigration };
|
|
226
|
+
module.exports = { isTransactionsUnsupported, runMigration };
|
package/src/core/server-info.js
CHANGED
|
@@ -25,16 +25,23 @@ const READ_CONCURRENCY = 8;
|
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* What the server is: a mongos in front of shards (its shard keys matter to
|
|
28
|
-
* prune),
|
|
28
|
+
* prune), its topology (`replicaSet`, `sharded`, `standalone` — transactions
|
|
29
|
+
* need one of the first two), and its version (what it can change in place). Best-effort — a
|
|
29
30
|
* server that refuses to say gets the conservative answer: no in-place
|
|
30
31
|
* extras, no shard-key handling.
|
|
31
32
|
*/
|
|
32
33
|
async function readServer(db) {
|
|
33
|
-
const server = { mongos: false, version: undefined };
|
|
34
|
+
const server = { mongos: false, version: undefined, topology: undefined };
|
|
34
35
|
if (typeof db.admin !== 'function') return server;
|
|
35
36
|
try {
|
|
36
37
|
const hello = await db.admin().command({ hello: 1 });
|
|
37
38
|
server.mongos = hello?.msg === 'isdbgrid';
|
|
39
|
+
// What transactions need: a replica set member or a mongos.
|
|
40
|
+
server.topology = server.mongos
|
|
41
|
+
? 'sharded'
|
|
42
|
+
: typeof hello?.setName === 'string'
|
|
43
|
+
? 'replicaSet'
|
|
44
|
+
: 'standalone';
|
|
38
45
|
} catch {
|
|
39
46
|
// Unknown — treated as a replica set or standalone.
|
|
40
47
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
const { READ_OPTIONS } = require('./server-info.js');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What the cluster says about a collection's sharding — read from the
|
|
5
|
+
* `config` database through a mongos. Every read here needs `clusterMonitor`
|
|
6
|
+
* (or more): a user without it gets `undefined` ("unknown"), never an error,
|
|
7
|
+
* and the caller falls back to what works without it.
|
|
8
|
+
*
|
|
9
|
+
* - `readShardKey` — the shard key, or `null` for a collection that is not
|
|
10
|
+
* sharded (an 8.0 `unsplittable` one included: tracked, but on one shard
|
|
11
|
+
* under `{ _id: 1 }`, which is no key to partition or target by);
|
|
12
|
+
* - `readChunks` — the chunks of a sharded collection, in key order.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** The server's "not authorized" — the one refusal that means "unknown", not "broken" */
|
|
16
|
+
const UNAUTHORIZED = 13;
|
|
17
|
+
|
|
18
|
+
const isUnauthorized = (error) => error?.code === UNAUTHORIZED;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `{ key, uuid, timestamp, unsplittable }` of a sharded collection, `null` when
|
|
22
|
+
* it is not sharded, `undefined` when `config.collections` may not be read.
|
|
23
|
+
*/
|
|
24
|
+
async function readShardKey(client, dbName, collection) {
|
|
25
|
+
let entry;
|
|
26
|
+
try {
|
|
27
|
+
entry = await client
|
|
28
|
+
.db('config')
|
|
29
|
+
.collection('collections')
|
|
30
|
+
.findOne(
|
|
31
|
+
{ _id: `${dbName}.${collection}` },
|
|
32
|
+
{ projection: { key: 1, uuid: 1, timestamp: 1, unsplittable: 1 }, ...READ_OPTIONS },
|
|
33
|
+
);
|
|
34
|
+
} catch (error) {
|
|
35
|
+
if (isUnauthorized(error)) return undefined;
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
if (entry === null || entry.unsplittable === true || entry.key === undefined) return null;
|
|
39
|
+
return {
|
|
40
|
+
key: entry.key,
|
|
41
|
+
uuid: entry.uuid,
|
|
42
|
+
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The chunks of a sharded collection (`readShardKey`'s result), in key order:
|
|
48
|
+
* `[{ min, max, shard }]` — or `undefined` when `config.chunks` may not be
|
|
49
|
+
* read. By `uuid` (5.0+), then by namespace (a cluster upgraded from 4.4 that
|
|
50
|
+
* never refreshed its chunks).
|
|
51
|
+
*/
|
|
52
|
+
async function readChunks(client, dbName, collection, sharding) {
|
|
53
|
+
const chunks = client.db('config').collection('chunks');
|
|
54
|
+
// Hashed bounds are NumberLongs: promoted to numbers, they lose precision
|
|
55
|
+
// past 2^53 (ARCHITECTURE §6.8).
|
|
56
|
+
const options = {
|
|
57
|
+
projection: { min: 1, max: 1, shard: 1 },
|
|
58
|
+
...READ_OPTIONS,
|
|
59
|
+
promoteLongs: false,
|
|
60
|
+
};
|
|
61
|
+
try {
|
|
62
|
+
let rows = await chunks.find({ uuid: sharding.uuid }, options).sort({ min: 1 }).toArray();
|
|
63
|
+
if (rows.length === 0) {
|
|
64
|
+
rows = await chunks
|
|
65
|
+
.find({ ns: `${dbName}.${collection}` }, options)
|
|
66
|
+
.sort({ min: 1 })
|
|
67
|
+
.toArray();
|
|
68
|
+
}
|
|
69
|
+
return rows;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (isUnauthorized(error)) return undefined;
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
module.exports = { isUnauthorized, readChunks, readShardKey };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
const { isPlainObject } = require('../utils/canonical.js');
|
|
2
|
+
const { versionIndexKey } = require('../versioning/document.js');
|
|
3
|
+
const { toCount } = require('../versioning/internal.js');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a collection's `versioning` block asks of the database: the validator
|
|
7
|
+
* rules for the version and revision fields, merged into whatever validator
|
|
8
|
+
* the definition declares, and the index every version-filtered scan uses.
|
|
9
|
+
* Pure — collections.js folds the result into the normalized definition, so
|
|
10
|
+
* the planner sees an ordinary validator and an ordinary index.
|
|
11
|
+
*
|
|
12
|
+
* The version rule has a `minimum` but never a `maximum`: during a rolling
|
|
13
|
+
* deploy (or after a rollback) a newer release writes a higher version than
|
|
14
|
+
* the declaration knows, and refusing that write would turn a deploy into an
|
|
15
|
+
* outage. A revision outgrows `int` after 2³¹ writes — `$inc` turns it into a
|
|
16
|
+
* `long` — so both are accepted.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** `{ required, properties }` for the managed fields, in a fixed order */
|
|
20
|
+
function versioningRules(versioning) {
|
|
21
|
+
const { field, min, revisionField } = versioning;
|
|
22
|
+
const properties = { [field]: { bsonType: 'int', minimum: min } };
|
|
23
|
+
const required = [field];
|
|
24
|
+
if (revisionField !== null) {
|
|
25
|
+
properties[revisionField] = { bsonType: ['int', 'long'], minimum: 0 };
|
|
26
|
+
required.push(revisionField);
|
|
27
|
+
}
|
|
28
|
+
// `min: 0` adapts a collection whose documents predate versioning: the
|
|
29
|
+
// fields are typed when present, but nothing requires them yet.
|
|
30
|
+
return min === 0 ? { properties } : { required, properties };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The managed field names a validator already constrains itself */
|
|
34
|
+
function managedFieldsIn(validator, versioning) {
|
|
35
|
+
const managed = [versioning.field];
|
|
36
|
+
if (versioning.revisionField !== null) managed.push(versioning.revisionField);
|
|
37
|
+
const found = new Set();
|
|
38
|
+
const schema = validator.$jsonSchema;
|
|
39
|
+
const properties = isPlainObject(schema) ? schema.properties : undefined;
|
|
40
|
+
const required = new Set(
|
|
41
|
+
isPlainObject(schema) && Array.isArray(schema.required) ? schema.required : [],
|
|
42
|
+
);
|
|
43
|
+
for (const name of managed) {
|
|
44
|
+
if (Object.hasOwn(validator, name)) found.add(name);
|
|
45
|
+
if (isPlainObject(properties) && Object.hasOwn(properties, name)) found.add(name);
|
|
46
|
+
if (required.has(name)) found.add(name);
|
|
47
|
+
}
|
|
48
|
+
return [...found];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Why a declared validator cannot carry the versioning rules — it constrains
|
|
53
|
+
* a managed field itself, or its `$jsonSchema` is not a schema object. Empty
|
|
54
|
+
* when the merge can go ahead.
|
|
55
|
+
*/
|
|
56
|
+
function validatorVersioningIssues(validator, versioning) {
|
|
57
|
+
if (validator === undefined) return [];
|
|
58
|
+
if (validator === null || !isPlainObject(validator)) {
|
|
59
|
+
return ['is null, which would remove the versioning rules — drop the validator key instead'];
|
|
60
|
+
}
|
|
61
|
+
const issues = [];
|
|
62
|
+
if (validator.$jsonSchema !== undefined && !isPlainObject(validator.$jsonSchema)) {
|
|
63
|
+
issues.push('$jsonSchema must be an object to take the versioning rules');
|
|
64
|
+
} else if (
|
|
65
|
+
isPlainObject(validator.$jsonSchema) &&
|
|
66
|
+
validator.$jsonSchema.required !== undefined &&
|
|
67
|
+
!Array.isArray(validator.$jsonSchema.required)
|
|
68
|
+
) {
|
|
69
|
+
issues.push('$jsonSchema.required must be an array to take the versioning rules');
|
|
70
|
+
}
|
|
71
|
+
for (const name of managedFieldsIn(validator, versioning)) {
|
|
72
|
+
issues.push(`constrains "${name}", which is managed by versioning — remove that rule`);
|
|
73
|
+
}
|
|
74
|
+
return issues;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The validator to declare: the versioning rules alone (no validator
|
|
79
|
+
* declared), merged into the declared `$jsonSchema` (the declared `required`
|
|
80
|
+
* and `properties` first, ours after), or — next to query operators — added
|
|
81
|
+
* as a top-level `$jsonSchema`, which the server combines with them.
|
|
82
|
+
*/
|
|
83
|
+
function mergeVersioningValidator(validator, versioning) {
|
|
84
|
+
const rules = versioningRules(versioning);
|
|
85
|
+
if (validator === undefined || Object.keys(validator).length === 0) {
|
|
86
|
+
return { $jsonSchema: rules };
|
|
87
|
+
}
|
|
88
|
+
const schema = validator.$jsonSchema;
|
|
89
|
+
if (!isPlainObject(schema)) return { ...validator, $jsonSchema: rules };
|
|
90
|
+
const merged = { ...schema };
|
|
91
|
+
if (rules.required) merged.required = [...(schema.required ?? []), ...rules.required];
|
|
92
|
+
merged.properties = { ...schema.properties, ...rules.properties };
|
|
93
|
+
return { ...validator, $jsonSchema: merged };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The declared version index — `{ [field]: 1, _id: 1 }`, named by its key */
|
|
97
|
+
const versioningIndex = (versioning) => ({ key: versionIndexKey(versioning) });
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The version index of a sharded collection: the shard key between the
|
|
101
|
+
* version field and `_id` — `{ __v: 1, region: 1, _id: 1 }` — so a batch
|
|
102
|
+
* over one chunk's range is an index range, not a filter over every old
|
|
103
|
+
* document of the shard. A hashed field stays hashed; `_id` is not repeated
|
|
104
|
+
* when the key holds it. On `{ _id: 1 }` it is the ordinary version index.
|
|
105
|
+
*/
|
|
106
|
+
function shardedVersionIndexKey(versioning, shardKey) {
|
|
107
|
+
const key = { [versioning.field]: 1 };
|
|
108
|
+
for (const [field, value] of Object.entries(shardKey)) {
|
|
109
|
+
if (field !== versioning.field) key[field] = value;
|
|
110
|
+
}
|
|
111
|
+
if (!('_id' in key)) key._id = 1;
|
|
112
|
+
return key;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Whether an index key is the version index's (same fields, same order, ascending) */
|
|
116
|
+
function isVersioningIndexKey(key, versioning) {
|
|
117
|
+
if (!isPlainObject(key)) return false;
|
|
118
|
+
const entries = Object.entries(key);
|
|
119
|
+
const expected = Object.entries(versionIndexKey(versioning));
|
|
120
|
+
if (entries.length !== expected.length) return false;
|
|
121
|
+
for (let i = 0; i < entries.length; i++) {
|
|
122
|
+
if (entries[i][0] !== expected[i][0] || Number(entries[i][1]) !== expected[i][1]) return false;
|
|
123
|
+
}
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The version floor the live validator enforces — the `minimum` of the version
|
|
129
|
+
* field's `$jsonSchema` rule — or `null` when it enforces none.
|
|
130
|
+
*/
|
|
131
|
+
function liveVersionFloor(options, versioning) {
|
|
132
|
+
const schema = options?.validator?.$jsonSchema;
|
|
133
|
+
const rule = isPlainObject(schema?.properties) ? schema.properties[versioning.field] : undefined;
|
|
134
|
+
return isPlainObject(rule) ? toCount(rule.minimum) : null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The `min` converge must check the data against before it raises the floor,
|
|
139
|
+
* or `null` when there is nothing to check: no versioning, `min: 0`, a
|
|
140
|
+
* collection that does not exist yet (no documents), or a floor already that
|
|
141
|
+
* high. Only a rising floor costs a read — the steady state costs nothing.
|
|
142
|
+
*/
|
|
143
|
+
function versionFloorToCheck(definition, live) {
|
|
144
|
+
const versioning = definition.versioning;
|
|
145
|
+
if (!versioning || versioning.min === 0 || !live.exists) return null;
|
|
146
|
+
if (live.type !== undefined && live.type !== 'collection') return null;
|
|
147
|
+
const floor = liveVersionFloor(live.options, versioning);
|
|
148
|
+
return floor !== null && floor >= versioning.min ? null : versioning.min;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Why the version floor cannot be raised — `live.versionFloor` is what
|
|
153
|
+
* converge read: `{ min, below: true | false | 'unknown', error? }` — or
|
|
154
|
+
* `undefined` when it can. The document ids are never named: they may be PII.
|
|
155
|
+
*/
|
|
156
|
+
function versionFloorConflict(floor) {
|
|
157
|
+
if (!floor || floor.below === false) return undefined;
|
|
158
|
+
if (floor.below === true) {
|
|
159
|
+
return (
|
|
160
|
+
`documents below version ${floor.min} remain — raising versioning.min would leave them ` +
|
|
161
|
+
'invalid; let the background migration that upgrades them finish (migronaut background ' +
|
|
162
|
+
'status), then converge again'
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return (
|
|
166
|
+
`could not check for documents below version ${floor.min} (${floor.error}) — converge ` +
|
|
167
|
+
'with the old min first so the version index exists, then raise it'
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
module.exports = {
|
|
172
|
+
isVersioningIndexKey,
|
|
173
|
+
shardedVersionIndexKey,
|
|
174
|
+
liveVersionFloor,
|
|
175
|
+
versionFloorConflict,
|
|
176
|
+
versionFloorToCheck,
|
|
177
|
+
mergeVersioningValidator,
|
|
178
|
+
validatorVersioningIssues,
|
|
179
|
+
versioningIndex,
|
|
180
|
+
versioningRules,
|
|
181
|
+
};
|
package/src/errors/index.js
CHANGED
|
@@ -260,6 +260,88 @@ class ConvergeFailedError extends MigronautError {
|
|
|
260
260
|
}
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Thrown by the optimistic-concurrency helpers (`@alexify/migronaut/versioning`)
|
|
265
|
+
* when a revision-guarded write matched nothing. `context.reason` says why:
|
|
266
|
+
* `'conflict'` (the document exists at another revision — `context.actual`),
|
|
267
|
+
* `'not-found'` (no document matches the filter at all) or `'unknown'` (the
|
|
268
|
+
* follow-up read was skipped or could not tell). `context.expected` is the
|
|
269
|
+
* revision the caller held. The filter is never copied in — it may carry PII.
|
|
270
|
+
*/
|
|
271
|
+
class RevisionConflictError extends MigronautError {
|
|
272
|
+
constructor(message, context, options) {
|
|
273
|
+
super('REVISION_CONFLICT', message, context, options);
|
|
274
|
+
this.name = 'RevisionConflictError';
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Thrown by an upcaster that cannot bring a document to the current shape:
|
|
280
|
+
* `context.reason` is `'newer'` (written by a newer release), `'below-min'`
|
|
281
|
+
* (older than the oldest shape still supported) or `'invalid'` (the version
|
|
282
|
+
* field is not a non-negative integer, or a step returned something that is
|
|
283
|
+
* not a document).
|
|
284
|
+
*/
|
|
285
|
+
class ShapeVersionError extends MigronautError {
|
|
286
|
+
constructor(message, context, options) {
|
|
287
|
+
super('SHAPE_VERSION_UNSUPPORTED', message, context, options);
|
|
288
|
+
this.name = 'ShapeVersionError';
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Thrown when a migration `requires` a background migration that has not
|
|
294
|
+
* completed yet — or whose collection still holds documents of the old shape.
|
|
295
|
+
* Nothing was run: `context.waitsFor` names the background migrations it
|
|
296
|
+
* waits for, with their status.
|
|
297
|
+
*/
|
|
298
|
+
class BackgroundPendingError extends MigronautError {
|
|
299
|
+
constructor(message, context, options) {
|
|
300
|
+
super('BACKGROUND_PENDING', message, context, options);
|
|
301
|
+
this.name = 'BackgroundPendingError';
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Thrown when a background migration ended `failed` — a partition used up its
|
|
307
|
+
* slice failures, the document error budget ran out, or old-shape documents
|
|
308
|
+
* kept appearing for `maxPasses` passes. `context.migration` names it and
|
|
309
|
+
* `context.lastError` says what happened last.
|
|
310
|
+
*/
|
|
311
|
+
class BackgroundFailedError extends MigronautError {
|
|
312
|
+
constructor(message, context, options) {
|
|
313
|
+
super('BACKGROUND_FAILED', message, context, options);
|
|
314
|
+
this.name = 'BackgroundFailedError';
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Thrown when a control action does not fit the background migration's state
|
|
320
|
+
* — pausing a completed one, resuming one that is not paused, retrying one
|
|
321
|
+
* that is still running. `context.status` is the state it found and
|
|
322
|
+
* `context.action` what was asked.
|
|
323
|
+
*/
|
|
324
|
+
class BackgroundConflictError extends MigronautError {
|
|
325
|
+
constructor(message, context, options) {
|
|
326
|
+
super('BACKGROUND_CONFLICT', message, context, options);
|
|
327
|
+
this.name = 'BackgroundConflictError';
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Thrown by the dry-run sandbox when a step reaches for something it cannot
|
|
333
|
+
* run inside an always-aborted transaction — DDL, an admin command, another
|
|
334
|
+
* session, `$out`/`$merge`, a migronaut-internal collection. `context.method`
|
|
335
|
+
* names the call and `context.reason` the rule it broke. A dry run reports
|
|
336
|
+
* every refusal even when the step caught the error itself.
|
|
337
|
+
*/
|
|
338
|
+
class SandboxRefusedError extends MigronautError {
|
|
339
|
+
constructor(message, context, options) {
|
|
340
|
+
super('SANDBOX_REFUSED', message, context, options);
|
|
341
|
+
this.name = 'SandboxRefusedError';
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
263
345
|
module.exports = {
|
|
264
346
|
MigronautError,
|
|
265
347
|
LockAlreadyHeldError,
|
|
@@ -286,4 +368,10 @@ module.exports = {
|
|
|
286
368
|
QueueJobInvalidError,
|
|
287
369
|
QueueJobFailedError,
|
|
288
370
|
ConvergeFailedError,
|
|
371
|
+
RevisionConflictError,
|
|
372
|
+
ShapeVersionError,
|
|
373
|
+
BackgroundPendingError,
|
|
374
|
+
BackgroundFailedError,
|
|
375
|
+
BackgroundConflictError,
|
|
376
|
+
SandboxRefusedError,
|
|
289
377
|
};
|