@alexify/migronaut 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CHANGELOG.md +107 -0
  2. package/README.md +33 -2
  3. package/bullmq.d.ts +449 -6
  4. package/index.d.ts +1010 -9
  5. package/migronaut.schema.json +93 -1
  6. package/package.json +8 -2
  7. package/src/bullmq/background-processor.js +469 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +128 -14
  11. package/src/bullmq/producer.js +185 -13
  12. package/src/bullmq/service.js +480 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/create.js +6 -0
  15. package/src/cli/exit-codes.js +6 -0
  16. package/src/cli/index.js +2 -0
  17. package/src/core/audit.js +11 -1
  18. package/src/core/background-audit.js +139 -0
  19. package/src/core/background-drift.js +126 -0
  20. package/src/core/background-dry-run.js +366 -0
  21. package/src/core/background-engine.js +818 -0
  22. package/src/core/background-kit.js +425 -0
  23. package/src/core/background-partition.js +298 -0
  24. package/src/core/background-runner.js +305 -0
  25. package/src/core/background-sandbox.js +701 -0
  26. package/src/core/background-shard.js +542 -0
  27. package/src/core/background-spec.js +597 -0
  28. package/src/core/background-store.js +951 -0
  29. package/src/core/background-throttle.js +269 -0
  30. package/src/core/background-watch-plan.js +164 -0
  31. package/src/core/background-watch-store.js +78 -0
  32. package/src/core/background-watch.js +605 -0
  33. package/src/core/background.js +1121 -0
  34. package/src/core/bson-peer.js +23 -0
  35. package/src/core/changelog.js +32 -0
  36. package/src/core/collections.js +78 -8
  37. package/src/core/config.js +102 -12
  38. package/src/core/converge-plan.js +86 -7
  39. package/src/core/converge.js +88 -0
  40. package/src/core/lock.js +48 -21
  41. package/src/core/migrator.js +904 -12
  42. package/src/core/options.js +16 -0
  43. package/src/core/run.js +26 -12
  44. package/src/core/runner.js +1 -1
  45. package/src/core/server-info.js +9 -2
  46. package/src/core/shard-info.js +76 -0
  47. package/src/core/versioning-spec.js +181 -0
  48. package/src/errors/index.js +88 -0
  49. package/src/index.js +16 -0
  50. package/src/utils/error.js +11 -2
  51. package/src/utils/loader.js +77 -9
  52. package/src/utils/migration-name.js +33 -1
  53. package/src/utils/telemetry.js +107 -0
  54. package/src/utils/template.js +62 -1
  55. package/src/versioning/config.js +155 -0
  56. package/src/versioning/document.js +326 -0
  57. package/src/versioning/index.js +50 -0
  58. package/src/versioning/internal.js +279 -0
  59. package/src/versioning/mongoose.js +151 -0
  60. package/src/versioning/occ.js +318 -0
  61. package/src/versioning/registry.js +187 -0
  62. package/src/versioning/upcaster.js +213 -0
  63. package/versioning.d.ts +666 -0
  64. package/versioning.js +1 -0
package/src/core/lock.js CHANGED
@@ -8,9 +8,15 @@ const { errorText } = require('../utils/error.js');
8
8
  const { randomId } = require('../utils/id.js');
9
9
  const { safeUsername } = require('../utils/user.js');
10
10
 
11
- /** Fixed `_id` of the singleton lock document */
11
+ /** Fixed `_id` of the singleton migration lock document */
12
12
  const LOCK_ID = 'migronaut_lock';
13
13
 
14
+ /** What the migration lock is called in messages — another lock passes its own `label` */
15
+ const LOCK_LABEL = 'migration lock';
16
+
17
+ /** `migration lock` → `Migration lock`, for the start of a sentence */
18
+ const sentence = (label) => label.charAt(0).toUpperCase() + label.slice(1);
19
+
14
20
  /** Returns true when an error is a MongoDB duplicate-key error (code 11000) */
15
21
  function isDuplicateKeyError(error) {
16
22
  return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000;
@@ -40,11 +46,19 @@ function toLockInfo(doc) {
40
46
  * MongoDB-native distributed lock backed by a single document, using an atomic
41
47
  * upsert as a test-and-set. A lock older than `ttlSeconds` is considered stale
42
48
  * and may be reclaimed.
49
+ *
50
+ * `options.id` is the lock document's `_id` (default: the migration lock's)
51
+ * and `options.label` what messages call it — so other locks (a background
52
+ * migration's coordinator, `background:<name>`) live next to the migration
53
+ * lock in the same collection without ever touching it: `unlock`, the audit
54
+ * and `forceRelease` of the migration lock only see its own document.
43
55
  */
44
56
  class MigrationLock {
45
57
  #db;
46
58
  #collectionName;
47
59
  #ttlSeconds;
60
+ #id;
61
+ #label;
48
62
  /** Token naming this instance as the current holder; set on acquire, cleared on release */
49
63
  #owner;
50
64
  /**
@@ -55,10 +69,22 @@ class MigrationLock {
55
69
  */
56
70
  #nonce;
57
71
 
58
- constructor(db, collectionName, ttlSeconds) {
72
+ constructor(db, collectionName, ttlSeconds, { id = LOCK_ID, label = LOCK_LABEL } = {}) {
59
73
  this.#db = db;
60
74
  this.#collectionName = collectionName;
61
75
  this.#ttlSeconds = ttlSeconds;
76
+ this.#id = id;
77
+ this.#label = label;
78
+ }
79
+
80
+ /** The lock document's `_id` */
81
+ get id() {
82
+ return this.#id;
83
+ }
84
+
85
+ /** What messages call this lock */
86
+ get label() {
87
+ return this.#label;
62
88
  }
63
89
 
64
90
  /** The token identifying this holder, or undefined when the lock is not held */
@@ -91,7 +117,7 @@ class MigrationLock {
91
117
  // strings: a pipeline expression would otherwise interpret a leading `$`
92
118
  // in a value as a field path.
93
119
  const lockDoc = {
94
- _id: LOCK_ID,
120
+ _id: this.#id,
95
121
  lockedAt: '$$NOW',
96
122
  pid: { $literal: process.pid },
97
123
  host: { $literal: os.hostname() },
@@ -109,7 +135,7 @@ class MigrationLock {
109
135
  // or the holder is stale; otherwise keep the current document untouched.
110
136
  // The read-back below tells those outcomes apart.
111
137
  result = await collection.updateOne(
112
- { _id: LOCK_ID },
138
+ { _id: this.#id },
113
139
  [
114
140
  {
115
141
  $replaceWith: {
@@ -131,8 +157,8 @@ class MigrationLock {
131
157
  } catch (error) {
132
158
  if (isDuplicateKeyError(error)) {
133
159
  // Two processes raced the very first insert; the loser lands here.
134
- const holder = await collection.findOne({ _id: LOCK_ID });
135
- throw new LockAlreadyHeldError('Migration lock is already held', {
160
+ const holder = await collection.findOne({ _id: this.#id });
161
+ throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
136
162
  holder: toLockInfo(holder) ?? undefined,
137
163
  ttlMs: this.ttlMs,
138
164
  });
@@ -155,9 +181,9 @@ class MigrationLock {
155
181
  // writer's document wins; either way the loser reads a different one here
156
182
  // and backs off instead of running concurrently. The nonce is what makes
157
183
  // that hold when both carry the same owner token.
158
- const current = await collection.findOne({ _id: LOCK_ID });
184
+ const current = await collection.findOne({ _id: this.#id });
159
185
  if (!current || current.owner !== owner || current.nonce !== nonce) {
160
- throw new LockAlreadyHeldError('Migration lock is already held', {
186
+ throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
161
187
  holder: toLockInfo(current) ?? undefined,
162
188
  ttlMs: this.ttlMs,
163
189
  });
@@ -178,7 +204,7 @@ class MigrationLock {
178
204
  }
179
205
  const result = await this.#db
180
206
  .collection(this.#collectionName)
181
- .updateOne({ _id: LOCK_ID, owner: this.#owner, nonce: this.#nonce }, [
207
+ .updateOne({ _id: this.#id, owner: this.#owner, nonce: this.#nonce }, [
182
208
  { $set: { lockedAt: '$$NOW' } },
183
209
  ]);
184
210
  return result.matchedCount === 1;
@@ -186,7 +212,7 @@ class MigrationLock {
186
212
 
187
213
  /** Read the current lock document, or null when no lock is held */
188
214
  async inspect() {
189
- return this.#db.collection(this.#collectionName).findOne({ _id: LOCK_ID });
215
+ return this.#db.collection(this.#collectionName).findOne({ _id: this.#id });
190
216
  }
191
217
 
192
218
  /**
@@ -196,8 +222,8 @@ class MigrationLock {
196
222
  */
197
223
  async forceRelease() {
198
224
  const collection = this.#db.collection(this.#collectionName);
199
- const existing = await collection.findOne({ _id: LOCK_ID });
200
- await collection.deleteOne({ _id: LOCK_ID });
225
+ const existing = await collection.findOne({ _id: this.#id });
226
+ await collection.deleteOne({ _id: this.#id });
201
227
  return existing;
202
228
  }
203
229
 
@@ -213,14 +239,14 @@ class MigrationLock {
213
239
  */
214
240
  async release() {
215
241
  if (!this.#owner) return;
216
- const filter = { _id: LOCK_ID, owner: this.#owner, nonce: this.#nonce };
242
+ const filter = { _id: this.#id, owner: this.#owner, nonce: this.#nonce };
217
243
  try {
218
244
  await this.#db.collection(this.#collectionName).deleteOne(filter);
219
245
  this.#owner = undefined;
220
246
  this.#nonce = undefined;
221
247
  } catch (error) {
222
248
  throw new LockReleaseFailedError(
223
- 'Failed to release migration lock',
249
+ `Failed to release ${this.#label}`,
224
250
  { error: errorText(error) },
225
251
  { cause: error },
226
252
  );
@@ -253,6 +279,9 @@ class MigrationLock {
253
279
  */
254
280
  async function runWithLock(lock, options, fn) {
255
281
  const controller = new AbortController();
282
+ // Any object with acquire/renew/release/ttlMs is a lock here — a partition
283
+ // lease included; `label` names it in the messages.
284
+ const label = typeof lock?.label === 'string' ? lock.label : LOCK_LABEL;
256
285
 
257
286
  if (options.noLock) {
258
287
  options.logger.warn('⚠ Running without a lock (--no-lock) — concurrent runs are unsafe', {
@@ -278,15 +307,13 @@ async function runWithLock(lock, options, fn) {
278
307
  const loseLock = (reason) => {
279
308
  // The most alert-worthy line this module emits — structured fields so a
280
309
  // JSON sink can trigger on it without parsing the human string.
281
- options.logger.warn(`⚠ Lost the migration lock mid-run (${reason})`, {
310
+ options.logger.warn(`⚠ Lost the ${label} mid-run (${reason})`, {
282
311
  event: 'lock:lost',
283
312
  reason,
284
313
  });
285
314
  options.onLockLostEvent?.(reason);
286
315
  if (abortOnLoss && !controller.signal.aborted) {
287
- controller.abort(
288
- new LockLostError('Lost the migration lock mid-run', { reason, aborted: true }),
289
- );
316
+ controller.abort(new LockLostError(`Lost the ${label} mid-run`, { reason, aborted: true }));
290
317
  }
291
318
  };
292
319
 
@@ -381,7 +408,7 @@ async function runWithLock(lock, options, fn) {
381
408
  } catch (releaseError) {
382
409
  const message = errorText(releaseError);
383
410
  options.logger.warn(
384
- `⚠ Failed to release the migration lock early: ${message} — it frees itself once ` +
411
+ `⚠ Failed to release the ${label} early: ${message} — it frees itself once ` +
385
412
  'its TTL runs out',
386
413
  { event: 'lock:release-failed', error: message, early: true },
387
414
  );
@@ -424,7 +451,7 @@ async function runWithLock(lock, options, fn) {
424
451
  throw releaseError;
425
452
  }
426
453
  const message = errorText(releaseError);
427
- options.logger.warn(`⚠ Failed to release the migration lock: ${message}`, {
454
+ options.logger.warn(`⚠ Failed to release the ${label}: ${message}`, {
428
455
  event: 'lock:release-failed',
429
456
  error: message,
430
457
  });
@@ -434,4 +461,4 @@ async function runWithLock(lock, options, fn) {
434
461
  return result;
435
462
  }
436
463
 
437
- module.exports = { LOCK_ID, MigrationLock, runWithLock, toLockInfo };
464
+ module.exports = { LOCK_ID, LOCK_LABEL, MigrationLock, runWithLock, toLockInfo };