@alexify/migronaut 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +33 -2
- package/bullmq.d.ts +449 -6
- package/index.d.ts +1010 -9
- package/migronaut.schema.json +93 -1
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +128 -14
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migrator.js +904 -12
- package/src/core/options.js +16 -0
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +107 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/src/core/lock.js
CHANGED
|
@@ -8,9 +8,15 @@ const { errorText } = require('../utils/error.js');
|
|
|
8
8
|
const { randomId } = require('../utils/id.js');
|
|
9
9
|
const { safeUsername } = require('../utils/user.js');
|
|
10
10
|
|
|
11
|
-
/** Fixed `_id` of the singleton lock document */
|
|
11
|
+
/** Fixed `_id` of the singleton migration lock document */
|
|
12
12
|
const LOCK_ID = 'migronaut_lock';
|
|
13
13
|
|
|
14
|
+
/** What the migration lock is called in messages — another lock passes its own `label` */
|
|
15
|
+
const LOCK_LABEL = 'migration lock';
|
|
16
|
+
|
|
17
|
+
/** `migration lock` → `Migration lock`, for the start of a sentence */
|
|
18
|
+
const sentence = (label) => label.charAt(0).toUpperCase() + label.slice(1);
|
|
19
|
+
|
|
14
20
|
/** Returns true when an error is a MongoDB duplicate-key error (code 11000) */
|
|
15
21
|
function isDuplicateKeyError(error) {
|
|
16
22
|
return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000;
|
|
@@ -40,11 +46,19 @@ function toLockInfo(doc) {
|
|
|
40
46
|
* MongoDB-native distributed lock backed by a single document, using an atomic
|
|
41
47
|
* upsert as a test-and-set. A lock older than `ttlSeconds` is considered stale
|
|
42
48
|
* and may be reclaimed.
|
|
49
|
+
*
|
|
50
|
+
* `options.id` is the lock document's `_id` (default: the migration lock's)
|
|
51
|
+
* and `options.label` what messages call it — so other locks (a background
|
|
52
|
+
* migration's coordinator, `background:<name>`) live next to the migration
|
|
53
|
+
* lock in the same collection without ever touching it: `unlock`, the audit
|
|
54
|
+
* and `forceRelease` of the migration lock only see its own document.
|
|
43
55
|
*/
|
|
44
56
|
class MigrationLock {
|
|
45
57
|
#db;
|
|
46
58
|
#collectionName;
|
|
47
59
|
#ttlSeconds;
|
|
60
|
+
#id;
|
|
61
|
+
#label;
|
|
48
62
|
/** Token naming this instance as the current holder; set on acquire, cleared on release */
|
|
49
63
|
#owner;
|
|
50
64
|
/**
|
|
@@ -55,10 +69,22 @@ class MigrationLock {
|
|
|
55
69
|
*/
|
|
56
70
|
#nonce;
|
|
57
71
|
|
|
58
|
-
constructor(db, collectionName, ttlSeconds) {
|
|
72
|
+
constructor(db, collectionName, ttlSeconds, { id = LOCK_ID, label = LOCK_LABEL } = {}) {
|
|
59
73
|
this.#db = db;
|
|
60
74
|
this.#collectionName = collectionName;
|
|
61
75
|
this.#ttlSeconds = ttlSeconds;
|
|
76
|
+
this.#id = id;
|
|
77
|
+
this.#label = label;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The lock document's `_id` */
|
|
81
|
+
get id() {
|
|
82
|
+
return this.#id;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** What messages call this lock */
|
|
86
|
+
get label() {
|
|
87
|
+
return this.#label;
|
|
62
88
|
}
|
|
63
89
|
|
|
64
90
|
/** The token identifying this holder, or undefined when the lock is not held */
|
|
@@ -91,7 +117,7 @@ class MigrationLock {
|
|
|
91
117
|
// strings: a pipeline expression would otherwise interpret a leading `$`
|
|
92
118
|
// in a value as a field path.
|
|
93
119
|
const lockDoc = {
|
|
94
|
-
_id:
|
|
120
|
+
_id: this.#id,
|
|
95
121
|
lockedAt: '$$NOW',
|
|
96
122
|
pid: { $literal: process.pid },
|
|
97
123
|
host: { $literal: os.hostname() },
|
|
@@ -109,7 +135,7 @@ class MigrationLock {
|
|
|
109
135
|
// or the holder is stale; otherwise keep the current document untouched.
|
|
110
136
|
// The read-back below tells those outcomes apart.
|
|
111
137
|
result = await collection.updateOne(
|
|
112
|
-
{ _id:
|
|
138
|
+
{ _id: this.#id },
|
|
113
139
|
[
|
|
114
140
|
{
|
|
115
141
|
$replaceWith: {
|
|
@@ -131,8 +157,8 @@ class MigrationLock {
|
|
|
131
157
|
} catch (error) {
|
|
132
158
|
if (isDuplicateKeyError(error)) {
|
|
133
159
|
// Two processes raced the very first insert; the loser lands here.
|
|
134
|
-
const holder = await collection.findOne({ _id:
|
|
135
|
-
throw new LockAlreadyHeldError(
|
|
160
|
+
const holder = await collection.findOne({ _id: this.#id });
|
|
161
|
+
throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
|
|
136
162
|
holder: toLockInfo(holder) ?? undefined,
|
|
137
163
|
ttlMs: this.ttlMs,
|
|
138
164
|
});
|
|
@@ -155,9 +181,9 @@ class MigrationLock {
|
|
|
155
181
|
// writer's document wins; either way the loser reads a different one here
|
|
156
182
|
// and backs off instead of running concurrently. The nonce is what makes
|
|
157
183
|
// that hold when both carry the same owner token.
|
|
158
|
-
const current = await collection.findOne({ _id:
|
|
184
|
+
const current = await collection.findOne({ _id: this.#id });
|
|
159
185
|
if (!current || current.owner !== owner || current.nonce !== nonce) {
|
|
160
|
-
throw new LockAlreadyHeldError(
|
|
186
|
+
throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
|
|
161
187
|
holder: toLockInfo(current) ?? undefined,
|
|
162
188
|
ttlMs: this.ttlMs,
|
|
163
189
|
});
|
|
@@ -178,7 +204,7 @@ class MigrationLock {
|
|
|
178
204
|
}
|
|
179
205
|
const result = await this.#db
|
|
180
206
|
.collection(this.#collectionName)
|
|
181
|
-
.updateOne({ _id:
|
|
207
|
+
.updateOne({ _id: this.#id, owner: this.#owner, nonce: this.#nonce }, [
|
|
182
208
|
{ $set: { lockedAt: '$$NOW' } },
|
|
183
209
|
]);
|
|
184
210
|
return result.matchedCount === 1;
|
|
@@ -186,7 +212,7 @@ class MigrationLock {
|
|
|
186
212
|
|
|
187
213
|
/** Read the current lock document, or null when no lock is held */
|
|
188
214
|
async inspect() {
|
|
189
|
-
return this.#db.collection(this.#collectionName).findOne({ _id:
|
|
215
|
+
return this.#db.collection(this.#collectionName).findOne({ _id: this.#id });
|
|
190
216
|
}
|
|
191
217
|
|
|
192
218
|
/**
|
|
@@ -196,8 +222,8 @@ class MigrationLock {
|
|
|
196
222
|
*/
|
|
197
223
|
async forceRelease() {
|
|
198
224
|
const collection = this.#db.collection(this.#collectionName);
|
|
199
|
-
const existing = await collection.findOne({ _id:
|
|
200
|
-
await collection.deleteOne({ _id:
|
|
225
|
+
const existing = await collection.findOne({ _id: this.#id });
|
|
226
|
+
await collection.deleteOne({ _id: this.#id });
|
|
201
227
|
return existing;
|
|
202
228
|
}
|
|
203
229
|
|
|
@@ -213,14 +239,14 @@ class MigrationLock {
|
|
|
213
239
|
*/
|
|
214
240
|
async release() {
|
|
215
241
|
if (!this.#owner) return;
|
|
216
|
-
const filter = { _id:
|
|
242
|
+
const filter = { _id: this.#id, owner: this.#owner, nonce: this.#nonce };
|
|
217
243
|
try {
|
|
218
244
|
await this.#db.collection(this.#collectionName).deleteOne(filter);
|
|
219
245
|
this.#owner = undefined;
|
|
220
246
|
this.#nonce = undefined;
|
|
221
247
|
} catch (error) {
|
|
222
248
|
throw new LockReleaseFailedError(
|
|
223
|
-
|
|
249
|
+
`Failed to release ${this.#label}`,
|
|
224
250
|
{ error: errorText(error) },
|
|
225
251
|
{ cause: error },
|
|
226
252
|
);
|
|
@@ -253,6 +279,9 @@ class MigrationLock {
|
|
|
253
279
|
*/
|
|
254
280
|
async function runWithLock(lock, options, fn) {
|
|
255
281
|
const controller = new AbortController();
|
|
282
|
+
// Any object with acquire/renew/release/ttlMs is a lock here — a partition
|
|
283
|
+
// lease included; `label` names it in the messages.
|
|
284
|
+
const label = typeof lock?.label === 'string' ? lock.label : LOCK_LABEL;
|
|
256
285
|
|
|
257
286
|
if (options.noLock) {
|
|
258
287
|
options.logger.warn('⚠ Running without a lock (--no-lock) — concurrent runs are unsafe', {
|
|
@@ -278,15 +307,13 @@ async function runWithLock(lock, options, fn) {
|
|
|
278
307
|
const loseLock = (reason) => {
|
|
279
308
|
// The most alert-worthy line this module emits — structured fields so a
|
|
280
309
|
// JSON sink can trigger on it without parsing the human string.
|
|
281
|
-
options.logger.warn(`⚠ Lost the
|
|
310
|
+
options.logger.warn(`⚠ Lost the ${label} mid-run (${reason})`, {
|
|
282
311
|
event: 'lock:lost',
|
|
283
312
|
reason,
|
|
284
313
|
});
|
|
285
314
|
options.onLockLostEvent?.(reason);
|
|
286
315
|
if (abortOnLoss && !controller.signal.aborted) {
|
|
287
|
-
controller.abort(
|
|
288
|
-
new LockLostError('Lost the migration lock mid-run', { reason, aborted: true }),
|
|
289
|
-
);
|
|
316
|
+
controller.abort(new LockLostError(`Lost the ${label} mid-run`, { reason, aborted: true }));
|
|
290
317
|
}
|
|
291
318
|
};
|
|
292
319
|
|
|
@@ -381,7 +408,7 @@ async function runWithLock(lock, options, fn) {
|
|
|
381
408
|
} catch (releaseError) {
|
|
382
409
|
const message = errorText(releaseError);
|
|
383
410
|
options.logger.warn(
|
|
384
|
-
`⚠ Failed to release the
|
|
411
|
+
`⚠ Failed to release the ${label} early: ${message} — it frees itself once ` +
|
|
385
412
|
'its TTL runs out',
|
|
386
413
|
{ event: 'lock:release-failed', error: message, early: true },
|
|
387
414
|
);
|
|
@@ -424,7 +451,7 @@ async function runWithLock(lock, options, fn) {
|
|
|
424
451
|
throw releaseError;
|
|
425
452
|
}
|
|
426
453
|
const message = errorText(releaseError);
|
|
427
|
-
options.logger.warn(`⚠ Failed to release the
|
|
454
|
+
options.logger.warn(`⚠ Failed to release the ${label}: ${message}`, {
|
|
428
455
|
event: 'lock:release-failed',
|
|
429
456
|
error: message,
|
|
430
457
|
});
|
|
@@ -434,4 +461,4 @@ async function runWithLock(lock, options, fn) {
|
|
|
434
461
|
return result;
|
|
435
462
|
}
|
|
436
463
|
|
|
437
|
-
module.exports = { LOCK_ID, MigrationLock, runWithLock, toLockInfo };
|
|
464
|
+
module.exports = { LOCK_ID, LOCK_LABEL, MigrationLock, runWithLock, toLockInfo };
|