@alexify/migronaut 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/README.md +68 -10
  3. package/bullmq.d.ts +465 -7
  4. package/index.d.ts +1272 -18
  5. package/migronaut.schema.json +150 -2
  6. package/package.json +8 -2
  7. package/src/bullmq/background-processor.js +469 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +153 -15
  11. package/src/bullmq/producer.js +202 -27
  12. package/src/bullmq/service.js +480 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/converge.js +38 -10
  15. package/src/cli/commands/create.js +6 -0
  16. package/src/cli/exit-codes.js +6 -0
  17. package/src/cli/index.js +2 -0
  18. package/src/cli/table.js +68 -9
  19. package/src/core/audit.js +98 -3
  20. package/src/core/background-audit.js +139 -0
  21. package/src/core/background-drift.js +126 -0
  22. package/src/core/background-dry-run.js +366 -0
  23. package/src/core/background-engine.js +818 -0
  24. package/src/core/background-kit.js +425 -0
  25. package/src/core/background-partition.js +298 -0
  26. package/src/core/background-runner.js +305 -0
  27. package/src/core/background-sandbox.js +701 -0
  28. package/src/core/background-shard.js +542 -0
  29. package/src/core/background-spec.js +597 -0
  30. package/src/core/background-store.js +951 -0
  31. package/src/core/background-throttle.js +269 -0
  32. package/src/core/background-watch-plan.js +164 -0
  33. package/src/core/background-watch-store.js +78 -0
  34. package/src/core/background-watch.js +605 -0
  35. package/src/core/background.js +1121 -0
  36. package/src/core/bson-peer.js +23 -0
  37. package/src/core/changelog.js +32 -0
  38. package/src/core/collections.js +125 -31
  39. package/src/core/config.js +133 -13
  40. package/src/core/converge-plan.js +343 -61
  41. package/src/core/converge-search-run.js +440 -0
  42. package/src/core/converge-search.js +404 -0
  43. package/src/core/converge.js +428 -183
  44. package/src/core/index-spec.js +27 -16
  45. package/src/core/lock.js +97 -32
  46. package/src/core/migrator.js +951 -26
  47. package/src/core/options.js +32 -1
  48. package/src/core/run.js +26 -12
  49. package/src/core/runner.js +1 -1
  50. package/src/core/search-index-spec.js +758 -0
  51. package/src/core/server-info.js +70 -0
  52. package/src/core/shard-info.js +76 -0
  53. package/src/core/versioning-spec.js +181 -0
  54. package/src/errors/index.js +97 -5
  55. package/src/index.js +16 -0
  56. package/src/utils/canonical.js +34 -1
  57. package/src/utils/error.js +11 -2
  58. package/src/utils/loader.js +77 -9
  59. package/src/utils/migration-name.js +33 -1
  60. package/src/utils/telemetry.js +125 -1
  61. package/src/utils/template.js +69 -1
  62. package/src/versioning/config.js +155 -0
  63. package/src/versioning/document.js +326 -0
  64. package/src/versioning/index.js +50 -0
  65. package/src/versioning/internal.js +279 -0
  66. package/src/versioning/mongoose.js +151 -0
  67. package/src/versioning/occ.js +318 -0
  68. package/src/versioning/registry.js +187 -0
  69. package/src/versioning/upcaster.js +213 -0
  70. package/versioning.d.ts +666 -0
  71. package/versioning.js +1 -0
package/bullmq.d.ts CHANGED
@@ -1,6 +1,12 @@
1
1
  import type {
2
2
  AuditReport,
3
+ BackgroundCounters,
4
+ BackgroundSliceResult,
5
+ BackgroundStatus,
6
+ BackgroundVerifyResult,
7
+ BackgroundWatcher,
3
8
  CollectionConvergeResult,
9
+ ConvergeSearchSummary,
4
10
  ConvergeUnstable,
5
11
  LockInfo,
6
12
  MigratorKit,
@@ -9,6 +15,7 @@ import type {
9
15
  MigronautErrorCode,
10
16
  OnLockHeld,
11
17
  StatusRow,
18
+ WatchBackgroundOptions,
12
19
  } from './index.js';
13
20
 
14
21
  // ─── Structural BullMQ surface ─────────────────────────────────────────────────
@@ -46,12 +53,24 @@ export interface BullMQJobLike<Data = any, Result = any> {
46
53
  log(row: string): Promise<number>;
47
54
  getState(): Promise<string>;
48
55
  waitUntilFinished(queueEvents: any, ttl?: number): Promise<Result>;
56
+ /** Used to put back a job a shutdown stopped before it began */
57
+ moveToWait?(token?: string): Promise<unknown>;
58
+ /** Background jobs: continue later (a lane between slices, a coordinator polling) */
59
+ moveToDelayed?(timestamp: number, token?: string): Promise<void>;
60
+ /** Background coordinators: wait for the lanes they spawned (BullMQ ≥ 5 parents) */
61
+ moveToWaitingChildren?(token: string, opts?: any): Promise<boolean>;
62
+ updateData?(data: Data): Promise<void>;
63
+ getIgnoredChildrenFailures?(): Promise<{ [jobKey: string]: string }>;
49
64
  }
50
65
 
51
66
  /** See {@link BullMQJobLike} for why this is structural */
52
67
  export interface BullMQQueueLike {
53
68
  name: string;
54
- addBulk(jobs: (MigrationJobSpec | ConvergeJobSpec)[]): Promise<BullMQJobLike[]>;
69
+ /** `prefix:name` — what a child job's `parent.queue` names. Background coordinators need it */
70
+ readonly qualifiedName?: string;
71
+ addBulk(
72
+ jobs: (MigrationJobSpec | ConvergeJobSpec | BackgroundJobSpec | BackgroundLaneJobSpec)[],
73
+ ): Promise<BullMQJobLike[]>;
55
74
  getJob(id: string): Promise<BullMQJobLike | undefined>;
56
75
  pause(): Promise<void>;
57
76
  resume(): Promise<void>;
@@ -62,6 +81,8 @@ export interface BullMQQueueLike {
62
81
  upsertJobScheduler?(id: string, repeat: any, template?: any): Promise<unknown>;
63
82
  /** BullMQ ≥ 5.16 — needed by `unschedule()` */
64
83
  removeJobScheduler?(id: string): Promise<boolean>;
84
+ /** BullMQ ≥ 5.16 — whether the drift watch's schedule exists already */
85
+ getJobScheduler?(id: string): Promise<unknown>;
65
86
  }
66
87
 
67
88
  /** See {@link BullMQJobLike} for why this is structural */
@@ -100,13 +121,25 @@ export type BullMQQueueEventsClass<E extends BullMQQueueEventsLike = BullMQQueue
100
121
  // ─── Job contract ──────────────────────────────────────────────────────────────
101
122
 
102
123
  export type MigrationJobName = 'up' | 'down' | 'sync' | 'converge';
124
+ /** The job names of a background queue */
125
+ export type BackgroundJobName = 'background' | 'background-lane' | 'background-verify';
103
126
 
104
127
  /**
105
128
  * Job names: `up`/`down` carry one migration each; `sync` plans and enqueues
106
129
  * what is pending; `converge` brings the declared collections to their
107
- * declared state.
130
+ * declared state. On the background queue: `background` (a background
131
+ * migration's coordinator), `background-lane` (one of its lanes, a child of
132
+ * the coordinator) and `background-verify` (the drift watch).
108
133
  */
109
- export const JOB_NAMES: Readonly<{ UP: 'up'; DOWN: 'down'; SYNC: 'sync'; CONVERGE: 'converge' }>;
134
+ export const JOB_NAMES: Readonly<{
135
+ UP: 'up';
136
+ DOWN: 'down';
137
+ SYNC: 'sync';
138
+ CONVERGE: 'converge';
139
+ BACKGROUND: 'background';
140
+ BACKGROUND_LANE: 'background-lane';
141
+ BACKGROUND_VERIFY: 'background-verify';
142
+ }>;
110
143
  /**
111
144
  * Version stamped on every job's data as `v`. A worker accepts every version
112
145
  * from {@link MIN_JOB_DATA_VERSION} up to its own and refuses a newer one —
@@ -119,6 +152,16 @@ export const DEFAULT_QUEUE_NAME: 'migronaut';
119
152
  export const DEFAULT_SCHEDULER_ID: 'migronaut-sync';
120
153
  /** Default id of a `schedule({ job: 'converge' })` schedule */
121
154
  export const DEFAULT_CONVERGE_SCHEDULER_ID: 'migronaut-converge';
155
+ /**
156
+ * Id of the drift watch's schedule on the background queue
157
+ * @experimental New in 2.3
158
+ */
159
+ export const DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID: 'migronaut-background-verify';
160
+ /**
161
+ * The background queue that serves a migration queue: `<queueName>-background`
162
+ * @experimental New in 2.3
163
+ */
164
+ export function backgroundQueueName(queueName: string): string;
122
165
 
123
166
  /**
124
167
  * Data of an `up` or `down` job. Stored in Redis — re-validated by the worker as untrusted input
@@ -186,6 +229,106 @@ export interface ConvergeJobData {
186
229
  reason?: string;
187
230
  }
188
231
 
232
+ /**
233
+ * Data of a `background` job — the coordinator of one background migration.
234
+ * Deduplicated on the migration: every heal collapses into the chain alive.
235
+ * @experimental New in 2.3
236
+ */
237
+ export interface BackgroundJobData {
238
+ v: 1;
239
+ kind: 'background';
240
+ migration: string;
241
+ /** The chain's round, minted on its first run — an older round bows out */
242
+ round?: number;
243
+ /** How many times this chain spawned lanes — part of their ids */
244
+ spawn?: number;
245
+ /** A takeover of a coordinator nothing has heard from for `stallMs` */
246
+ takeover?: true;
247
+ requestedBy?: string;
248
+ reason?: string;
249
+ }
250
+
251
+ /**
252
+ * Data of a `background-lane` job — one lane of a background migration
253
+ * @experimental New in 2.3
254
+ */
255
+ export interface BackgroundLaneJobData {
256
+ v: 1;
257
+ kind: 'background-lane';
258
+ migration: string;
259
+ registration: string;
260
+ generation: number;
261
+ round: number;
262
+ spawn: number;
263
+ /** 0 to 63 */
264
+ lane: number;
265
+ /** Failed slices in a row — the lane backs off, and gives up after `maxLaneRetries` */
266
+ retry?: number;
267
+ }
268
+
269
+ /**
270
+ * Data of a `background-verify` job — a drift-watch tick
271
+ * @experimental New in 2.3
272
+ */
273
+ export interface BackgroundVerifyJobData {
274
+ v: 1;
275
+ kind: 'background-verify';
276
+ }
277
+
278
+ /**
279
+ * What a completed `background` job returns: the coordinator chain ended
280
+ * @experimental New in 2.3
281
+ */
282
+ export interface BackgroundJobResult {
283
+ kind: 'background';
284
+ migration: string;
285
+ /**
286
+ * The background migration's status — or `superseded` (a newer round took
287
+ * over), `unregistered`. Run outside a Worker (no token to move the job
288
+ * with), the coordinator's next step instead: `wait`, `busy` or `process`,
289
+ * with `retryAfterMs`. Absent when a shutdown came first.
290
+ */
291
+ status?:
292
+ | BackgroundStatus['status']
293
+ | 'blocked'
294
+ | 'superseded'
295
+ | 'unregistered'
296
+ | 'wait'
297
+ | 'busy'
298
+ | 'process';
299
+ round?: number;
300
+ /** Outside a Worker: when to run the job again */
301
+ retryAfterMs?: number;
302
+ }
303
+
304
+ /**
305
+ * What a completed `background-lane` job returns: nothing left for it to claim
306
+ * (`exhausted`), the background migration stopped (`paused`, `cancelled`,
307
+ * `failed`), no plan yet (`stale`) — or `gave-up` after `maxLaneRetries`
308
+ * failed slices in a row (each counted on its partition, in MongoDB)
309
+ * @experimental New in 2.3
310
+ */
311
+ export interface BackgroundLaneJobResult {
312
+ kind: 'background-lane';
313
+ migration: string;
314
+ /** Outside a Worker also `retry` (a failed slice, to try again after `retryAfterMs`) */
315
+ outcome: BackgroundSliceResult['outcome'] | 'gave-up' | 'retry';
316
+ counters?: BackgroundCounters;
317
+ code?: MigronautErrorCode;
318
+ /** Outside a Worker (no token to move the job with): when to run it again */
319
+ retryAfterMs?: number;
320
+ }
321
+
322
+ /**
323
+ * What a completed `background-verify` job returns
324
+ * @experimental New in 2.3
325
+ */
326
+ export interface BackgroundVerifyJobResult extends BackgroundVerifyResult {
327
+ kind: 'background-verify';
328
+ /** Coordinators this tick added (or found alive) — the heal */
329
+ enqueued: number;
330
+ }
331
+
189
332
  /**
190
333
  * What a completed `up`/`down` job returns
191
334
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
@@ -202,6 +345,8 @@ export interface MigrationJobResult {
202
345
  reason?: string;
203
346
  /** Time (ms) spent waiting for the MongoDB migration lock */
204
347
  lockWaitMs: number;
348
+ /** The coordinators enqueued for what the job registered — on a queue with a background side */
349
+ background?: { migration: string; jobId: string }[];
205
350
  }
206
351
 
207
352
  /**
@@ -223,7 +368,19 @@ export interface SyncJobResult {
223
368
  * and its file has not changed since — a schedule's circuit breaker. A fix
224
369
  * (a changed file) or an explicit `enqueueUp(name)` resumes the line.
225
370
  */
226
- held?: { migration: string; reason: string; failedAt?: Date };
371
+ held?: {
372
+ migration: string;
373
+ reason: string;
374
+ failedAt?: Date;
375
+ };
376
+ /**
377
+ * Present when the next migration waits for background migrations that
378
+ * have not completed (`requires`): the tick enqueued what comes before it,
379
+ * and the line goes on once they complete. Not a failure — `held` is that.
380
+ */
381
+ waiting?: { migration: string; waitsFor: string[] };
382
+ /** The heal of the background side: coordinators added, or found alive */
383
+ background?: { enqueued: number };
227
384
  }
228
385
 
229
386
  /**
@@ -237,6 +394,13 @@ export interface ConvergeJobResult {
237
394
  inSync: boolean;
238
395
  collections: CollectionConvergeResult[];
239
396
  unstable?: ConvergeUnstable[];
397
+ /**
398
+ * Atlas Search availability and the declared search indexes still building
399
+ * — when the worker's definitions declare search indexes. Whether the job
400
+ * waits for them is the worker kit's `waitForSearchIndexes`.
401
+ * @experimental New in 2.2
402
+ */
403
+ search?: ConvergeSearchSummary;
240
404
  runId?: string;
241
405
  /** Time (ms) spent waiting for the MongoDB migration lock */
242
406
  lockWaitMs: number;
@@ -247,7 +411,8 @@ export interface ConvergeJobResult {
247
411
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
248
412
  */
249
413
  export interface MigrationJobProgress {
250
- phase: 'lock-wait' | 'running' | 'completed' | 'failed';
414
+ /** `search-wait` (New in 2.2): a converge job waiting for search index builds */
415
+ phase: 'lock-wait' | 'running' | 'search-wait' | 'completed' | 'failed';
251
416
  migration?: string;
252
417
  direction?: 'up' | 'down';
253
418
  groupId?: string;
@@ -256,7 +421,13 @@ export interface MigrationJobProgress {
256
421
  kind?: 'sync' | 'converge';
257
422
  /** `lock-wait` only */
258
423
  attempts?: number;
424
+ /** `lock-wait` and `search-wait` */
259
425
  waitedMs?: number;
426
+ /**
427
+ * `search-wait` only — how many search indexes the wait is for
428
+ * @experimental New in 2.2
429
+ */
430
+ searchIndexes?: number;
260
431
  /**
261
432
  * `failed` only — the typed error code, so nobody has to parse
262
433
  * `failedReason`; `'UNKNOWN'` for an error that is not migronaut's.
@@ -307,6 +478,39 @@ export interface ConvergeJobSpec {
307
478
  opts: { attempts: 1; deduplication: { id: string }; [option: string]: unknown };
308
479
  }
309
480
 
481
+ /**
482
+ * Per-job options passed through to every background job — retention and
483
+ * logging. Besides what a migration job refuses, the four options of a
484
+ * parent's child-failure policy are refused: the adapter owns them.
485
+ * @experimental New in 2.3
486
+ */
487
+ export interface BackgroundJobOptions extends MigrationJobOptions {
488
+ failParentOnFailure?: never;
489
+ continueParentOnFailure?: never;
490
+ ignoreDependencyOnFailure?: never;
491
+ removeDependencyOnFailure?: never;
492
+ }
493
+
494
+ /**
495
+ * A coordinator job as handed to `queue.addBulk`
496
+ * @experimental New in 2.3
497
+ */
498
+ export interface BackgroundJobSpec {
499
+ name: 'background';
500
+ data: BackgroundJobData;
501
+ opts: { attempts: number; deduplication: { id: string }; [option: string]: unknown };
502
+ }
503
+
504
+ /**
505
+ * A lane job as handed to `queue.addBulk`
506
+ * @experimental New in 2.3
507
+ */
508
+ export interface BackgroundLaneJobSpec {
509
+ name: 'background-lane';
510
+ data: BackgroundLaneJobData;
511
+ opts: { attempts: 1; [option: string]: unknown };
512
+ }
513
+
310
514
  /** A planned, not yet enqueued, group */
311
515
  export interface MigrationPlan {
312
516
  /** Id of this enqueue call, in the kit's `generateId` format (a UUID by default) */
@@ -319,6 +523,11 @@ export interface MigrationPlan {
319
523
  jobs: MigrationJobSpec[];
320
524
  /** The converge job that ends an `up` group — see `EnqueueUpOptions.converge` */
321
525
  converge?: ConvergeJobSpec;
526
+ /**
527
+ * The first pending file that requires a background migration not
528
+ * completed yet: the plan ends before it (and has no converge job)
529
+ */
530
+ waiting?: { migration: string; waitsFor: string[] };
322
531
  }
323
532
 
324
533
  /** {@link parseJobData}'s normalized result */
@@ -343,6 +552,40 @@ export type ParsedJobData =
343
552
  */
344
553
  export function parseJobData(job: { id?: string; name: string; data: unknown }): ParsedJobData;
345
554
 
555
+ /** {@link parseBackgroundJobData}'s normalized result */
556
+ export type ParsedBackgroundJobData =
557
+ | {
558
+ kind: 'background';
559
+ migration: string;
560
+ round?: number;
561
+ spawn?: number;
562
+ takeover?: true;
563
+ requestedBy?: string;
564
+ reason?: string;
565
+ }
566
+ | {
567
+ kind: 'background-lane';
568
+ migration: string;
569
+ registration: string;
570
+ generation: number;
571
+ round: number;
572
+ spawn: number;
573
+ lane: number;
574
+ retry: number;
575
+ }
576
+ | { kind: 'background-verify' };
577
+
578
+ /**
579
+ * Validate a job read back from a background queue and return a normalized
580
+ * copy. Throws `QueueJobInvalidError` for anything outside the contract.
581
+ * @experimental New in 2.3
582
+ */
583
+ export function parseBackgroundJobData(job: {
584
+ id?: string;
585
+ name: string;
586
+ data: unknown;
587
+ }): ParsedBackgroundJobData;
588
+
346
589
  /** The deduplication id a migration's job carries — never contains `:` */
347
590
  export function dedupId(direction: 'up' | 'down', migration: string): string;
348
591
 
@@ -464,6 +707,8 @@ export interface MigrationGroup {
464
707
  deduplicated: string[];
465
708
  /** The converge job ending the group, or null. `deduplicated`: a peer's identical job */
466
709
  converge: { id: string; deduplicated: boolean } | null;
710
+ /** See {@link MigrationPlan.waiting} — the group stops before this file */
711
+ waiting?: { migration: string; waitsFor: string[] };
467
712
  /**
468
713
  * Resolve when every job has finished — the converge job last; reject with
469
714
  * `QueueJobFailedError` at the first one that fails or outlives `timeoutMs`.
@@ -508,6 +753,17 @@ export type ScheduleOptions = (
508
753
  id?: string;
509
754
  to?: never;
510
755
  }
756
+ | {
757
+ /**
758
+ * Each tick runs the drift watch on the background queue (and heals
759
+ * it) — overrides the schedule `startBackgroundWorker()` registers.
760
+ * Needs the `background` option. @experimental
761
+ */
762
+ job: 'background-verify';
763
+ /** Scheduler id. Default `'migronaut-background-verify'` */
764
+ id?: string;
765
+ to?: never;
766
+ }
511
767
  );
512
768
 
513
769
  /** Options for {@link MigrationQueue.startWorker} — passed to the Worker constructor */
@@ -536,7 +792,13 @@ export interface MigrationJobView {
536
792
  state: string;
537
793
  /** As stored — see {@link MigrationJobProgress} for what the adapter writes */
538
794
  progress: MigrationJobProgress | number;
539
- returnvalue?: MigrationJobResult | SyncJobResult | ConvergeJobResult;
795
+ returnvalue?:
796
+ | MigrationJobResult
797
+ | SyncJobResult
798
+ | ConvergeJobResult
799
+ | BackgroundJobResult
800
+ | BackgroundLaneJobResult
801
+ | BackgroundVerifyJobResult;
540
802
  failedReason?: string;
541
803
  attemptsMade: number;
542
804
  timestamp?: number;
@@ -578,6 +840,16 @@ export interface CreateMigrationProcessorOptions {
578
840
  jobOptions?: MigrationJobOptions;
579
841
  /** What a job may ask for beyond the ordinary — see {@link MigrationJobPermissions} */
580
842
  allow?: MigrationJobPermissions;
843
+ /**
844
+ * The background queue: what an `up` (or `down`) job registers gets its
845
+ * coordinator there at once, and every `sync` tick heals it. @experimental
846
+ */
847
+ background?: {
848
+ queue: BullMQQueueLike;
849
+ jobOptions?: BackgroundJobOptions;
850
+ /** See {@link EnqueueBackgroundOptions.stallMs} */
851
+ stallMs?: number;
852
+ };
581
853
  }
582
854
 
583
855
  /**
@@ -617,6 +889,62 @@ export function createMigrationProcessor(
617
889
  options?: CreateMigrationProcessorOptions,
618
890
  ): MigrationProcessor;
619
891
 
892
+ /** Options for {@link createBackgroundProcessor} */
893
+ export interface CreateBackgroundProcessorOptions {
894
+ config?: Partial<MigronautConfig>;
895
+ kitOptions?: MigratorKitOptions;
896
+ kit?: MigratorKit;
897
+ /** The background queue — the coordinators add their lanes to it. Required */
898
+ queue: BullMQQueueLike;
899
+ jobOptions?: BackgroundJobOptions;
900
+ /** A lane's slice (ms). Default: each background migration's own `sliceMs` */
901
+ sliceMs?: number;
902
+ /**
903
+ * `'auto'` (default): lanes are children of their coordinator, which waits
904
+ * for them (`moveToWaitingChildren`) when the queue and its jobs support
905
+ * it. `false`: lanes on their own, and the coordinator polls MongoDB.
906
+ */
907
+ children?: 'auto' | false;
908
+ /** How often a coordinator without children looks again (ms). Default 5000 */
909
+ pollIntervalMs?: number;
910
+ /** See {@link EnqueueBackgroundOptions.stallMs} */
911
+ stallMs?: number;
912
+ /** Failed slices in a row before a lane gives up (0–100). Default 8 */
913
+ maxLaneRetries?: number;
914
+ }
915
+
916
+ /**
917
+ * The function a Worker on the background queue runs. Jobs run side by side.
918
+ * @experimental New in 2.3
919
+ */
920
+ export interface BackgroundProcessor {
921
+ (
922
+ job: BullMQJobLike<BackgroundJobData | BackgroundLaneJobData | BackgroundVerifyJobData>,
923
+ token?: string,
924
+ signal?: AbortSignal,
925
+ ): Promise<BackgroundJobResult | BackgroundLaneJobResult | BackgroundVerifyJobResult>;
926
+ readonly kit: MigratorKit;
927
+ /**
928
+ * Stop: a lane stops at its next batch, checkpoints, releases its lease and
929
+ * goes back to the queue (moved to delayed); a coordinator bows out and
930
+ * comes back. Irreversible.
931
+ */
932
+ shutdown(reason?: string): void;
933
+ /** `shutdown()`, let the jobs in flight settle, disconnect a kit the processor created */
934
+ close(): Promise<void>;
935
+ /** A coordinator for every background migration with work to do — what a worker does at start */
936
+ heal(): Promise<BackgroundEnqueueResult>;
937
+ }
938
+
939
+ /**
940
+ * Build the processor for a Worker on the background queue you construct
941
+ * yourself.
942
+ * @experimental New in 2.3
943
+ */
944
+ export function createBackgroundProcessor(
945
+ options: CreateBackgroundProcessorOptions,
946
+ ): BackgroundProcessor;
947
+
620
948
  // ─── Producer building blocks ──────────────────────────────────────────────────
621
949
 
622
950
  /** Plan an `up` group without enqueuing it */
@@ -665,6 +993,39 @@ export function enqueueConverge(
665
993
  },
666
994
  ): Promise<ConvergeHandle>;
667
995
 
996
+ /** Options for `enqueueBackground` */
997
+ export interface EnqueueBackgroundOptions {
998
+ /**
999
+ * A running background migration nothing has moved for this long (ms) — no
1000
+ * live lease, no checkpoint, no coordinator step — also gets a takeover
1001
+ * coordinator, whose newer round retires the stuck one. Default 900000
1002
+ * (15 minutes); at least 1000.
1003
+ */
1004
+ stallMs?: number;
1005
+ requestedBy?: string;
1006
+ reason?: string;
1007
+ }
1008
+
1009
+ /**
1010
+ * What `enqueueBackground` resolves with
1011
+ * @experimental New in 2.3
1012
+ */
1013
+ export interface BackgroundEnqueueResult {
1014
+ /** One per coordinator added — or absorbed by the one already alive (same id) */
1015
+ jobs: { migration: string; id: string; takeover?: true }[];
1016
+ }
1017
+
1018
+ /**
1019
+ * Enqueue the coordinator of one background migration (`migration`) — or of
1020
+ * every one with work to do — on a background queue you own. Idempotent.
1021
+ * @experimental New in 2.3
1022
+ */
1023
+ export function enqueueBackground(
1024
+ queue: BullMQQueueLike,
1025
+ kit: MigratorKit,
1026
+ options?: EnqueueBackgroundOptions & { migration?: string; jobOptions?: BackgroundJobOptions },
1027
+ ): Promise<BackgroundEnqueueResult>;
1028
+
668
1029
  /** Wait for a group's jobs — what `MigrationGroup.wait()` calls */
669
1030
  export function waitForGroup(options: {
670
1031
  queue: BullMQQueueLike;
@@ -747,6 +1108,48 @@ export interface CreateMigrationQueueOptions<
747
1108
  * refuse fails at the call. Give every producer and worker the same policy.
748
1109
  */
749
1110
  allow?: MigrationJobPermissions;
1111
+ /**
1112
+ * Background migrations on a queue of their own (`<queueName>-background`):
1113
+ * a coordinator job each, with lanes as its children. `true` takes every
1114
+ * default. @experimental New in 2.3
1115
+ */
1116
+ background?: boolean | BackgroundQueueOptions;
1117
+ }
1118
+
1119
+ /**
1120
+ * The `background` option of {@link createMigrationQueue}
1121
+ * @experimental New in 2.3
1122
+ */
1123
+ export interface BackgroundQueueOptions {
1124
+ /** Default `<queueName>-background` (or the injected queue's name) */
1125
+ queueName?: string;
1126
+ /** A background Queue instance you own — never closed by `close()` */
1127
+ queue?: BullMQQueueLike;
1128
+ jobOptions?: BackgroundJobOptions;
1129
+ /** Defaults for `startBackgroundWorker()` — concurrency 2 unless given */
1130
+ workerOptions?: { concurrency?: number; [option: string]: unknown };
1131
+ /** See {@link CreateBackgroundProcessorOptions} */
1132
+ sliceMs?: number;
1133
+ children?: 'auto' | false;
1134
+ pollIntervalMs?: number;
1135
+ /** See {@link EnqueueBackgroundOptions.stallMs} */
1136
+ stallMs?: number;
1137
+ /**
1138
+ * The drift watch's schedule, registered by `startBackgroundWorker()` (ms,
1139
+ * ≥ 1000). Default 600000 (10 minutes) — registered only when no schedule
1140
+ * exists yet, so one set with `schedule({ job: 'background-verify' })`
1141
+ * stays; given explicitly, it is re-registered at every start. `false`
1142
+ * registers none.
1143
+ */
1144
+ verifyIntervalMs?: number | false;
1145
+ /** See {@link CreateBackgroundProcessorOptions.maxLaneRetries} */
1146
+ maxLaneRetries?: number;
1147
+ /**
1148
+ * Host the live drift watcher in the background worker's process — `true`,
1149
+ * or its options. Default: when the kit's `backgroundDrift` is `'stream'`
1150
+ * or `'both'`. Closed first by `close()`.
1151
+ */
1152
+ watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
750
1153
  }
751
1154
 
752
1155
  /**
@@ -772,6 +1175,26 @@ export class MigrationQueue<
772
1175
  readonly queueName: string;
773
1176
  /** The processor, for attaching to a Worker you construct yourself */
774
1177
  readonly processor: MigrationProcessor;
1178
+ /**
1179
+ * The background queue (`background` option), if any
1180
+ * @experimental New in 2.3
1181
+ */
1182
+ readonly backgroundQueue: BullMQQueueLike | undefined;
1183
+ /**
1184
+ * The worker started by {@link startBackgroundWorker}, if any
1185
+ * @experimental New in 2.3
1186
+ */
1187
+ readonly backgroundWorker: W | undefined;
1188
+ /**
1189
+ * The background processor, for a Worker you construct yourself
1190
+ * @experimental New in 2.3
1191
+ */
1192
+ readonly backgroundProcessor: BackgroundProcessor | undefined;
1193
+ /**
1194
+ * The live drift watcher `startBackgroundWorker()` started, if any
1195
+ * @experimental New in 2.3
1196
+ */
1197
+ readonly backgroundWatcher: BackgroundWatcher | undefined;
775
1198
 
776
1199
  /**
777
1200
  * Enqueue pending migrations — all, up to `options.to`, or the one
@@ -791,6 +1214,33 @@ export class MigrationQueue<
791
1214
  */
792
1215
  enqueueConverge(options?: EnqueueConvergeOptions): Promise<ConvergeHandle>;
793
1216
 
1217
+ /**
1218
+ * Enqueue the coordinator of one background migration — or of every one
1219
+ * with work to do. Idempotent. Needs the `background` option. @experimental
1220
+ */
1221
+ enqueueBackground(
1222
+ name?: string,
1223
+ options?: EnqueueBackgroundOptions,
1224
+ ): Promise<BackgroundEnqueueResult>;
1225
+ /**
1226
+ * A background migration's status, read from MongoDB — `null` when not registered
1227
+ * @experimental New in 2.3
1228
+ */
1229
+ backgroundStatus(name: string): Promise<BackgroundStatus | null>;
1230
+ /**
1231
+ * Every background migration's status
1232
+ * @experimental New in 2.3
1233
+ */
1234
+ backgroundStatus(): Promise<BackgroundStatus[]>;
1235
+ /**
1236
+ * The drift watch, now — and, with the `background` option, a coordinator
1237
+ * for whatever it reopened. @experimental
1238
+ */
1239
+ verifyBackground(options?: {
1240
+ onDrift?: 'reopen' | 'report';
1241
+ collections?: string[];
1242
+ }): Promise<BackgroundVerifyResult>;
1243
+
794
1244
  /** Full migration status, read from MongoDB */
795
1245
  status(): Promise<StatusRow[]>;
796
1246
  /** Migrations not applied yet */
@@ -806,6 +1256,13 @@ export class MigrationQueue<
806
1256
  * that failed, tries again.
807
1257
  */
808
1258
  startWorker(options?: StartWorkerOptions): Promise<W>;
1259
+ /**
1260
+ * Start the background worker (concurrency 2 by default): coordinators and
1261
+ * lanes side by side. Registers the drift watch's schedule and heals — a
1262
+ * coordinator for every background migration with work to do. Needs the
1263
+ * `background` option. @experimental
1264
+ */
1265
+ startBackgroundWorker(options?: { concurrency?: number; [option: string]: unknown }): Promise<W>;
809
1266
  /** Stop workers from picking up new jobs; the job in flight finishes */
810
1267
  pause(): Promise<void>;
811
1268
  resume(): Promise<void>;
@@ -820,7 +1277,8 @@ export class MigrationQueue<
820
1277
  schedule(options: ScheduleOptions): Promise<void>;
821
1278
  /**
822
1279
  * Remove a schedule — the sync one by default; pass
823
- * {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another.
1280
+ * {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another. The
1281
+ * background queue's schedules (the drift watch's) are looked for too.
824
1282
  * Resolves whether one existed.
825
1283
  */
826
1284
  unschedule(id?: string): Promise<boolean>;