@alexify/migronaut 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +41 -3
- package/bullmq.d.ts +484 -8
- package/index.d.ts +1264 -9
- package/migronaut.schema.json +93 -1
- package/package.json +9 -2
- package/src/bullmq/background-processor.js +541 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +348 -21
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +484 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +375 -0
- package/src/core/background-engine.js +849 -0
- package/src/core/background-kit.js +432 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +610 -0
- package/src/core/background.js +1127 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migration-logger.js +279 -0
- package/src/core/migrator.js +1027 -22
- package/src/core/options.js +36 -0
- package/src/core/run-recorder.js +6 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +34 -8
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/job-ref.js +44 -0
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/redact.js +140 -3
- package/src/utils/telemetry.js +110 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/bullmq.d.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
AuditReport,
|
|
3
|
+
BackgroundCounters,
|
|
4
|
+
BackgroundSliceResult,
|
|
5
|
+
BackgroundStatus,
|
|
6
|
+
BackgroundVerifyResult,
|
|
7
|
+
BackgroundWatcher,
|
|
3
8
|
CollectionConvergeResult,
|
|
4
9
|
ConvergeSearchSummary,
|
|
5
10
|
ConvergeUnstable,
|
|
@@ -10,6 +15,7 @@ import type {
|
|
|
10
15
|
MigronautErrorCode,
|
|
11
16
|
OnLockHeld,
|
|
12
17
|
StatusRow,
|
|
18
|
+
WatchBackgroundOptions,
|
|
13
19
|
} from './index.js';
|
|
14
20
|
|
|
15
21
|
// ─── Structural BullMQ surface ─────────────────────────────────────────────────
|
|
@@ -47,12 +53,24 @@ export interface BullMQJobLike<Data = any, Result = any> {
|
|
|
47
53
|
log(row: string): Promise<number>;
|
|
48
54
|
getState(): Promise<string>;
|
|
49
55
|
waitUntilFinished(queueEvents: any, ttl?: number): Promise<Result>;
|
|
56
|
+
/** Used to put back a job a shutdown stopped before it began */
|
|
57
|
+
moveToWait?(token?: string): Promise<unknown>;
|
|
58
|
+
/** Background jobs: continue later (a lane between slices, a coordinator polling) */
|
|
59
|
+
moveToDelayed?(timestamp: number, token?: string): Promise<void>;
|
|
60
|
+
/** Background coordinators: wait for the lanes they spawned (BullMQ ≥ 5 parents) */
|
|
61
|
+
moveToWaitingChildren?(token: string, opts?: any): Promise<boolean>;
|
|
62
|
+
updateData?(data: Data): Promise<void>;
|
|
63
|
+
getIgnoredChildrenFailures?(): Promise<{ [jobKey: string]: string }>;
|
|
50
64
|
}
|
|
51
65
|
|
|
52
66
|
/** See {@link BullMQJobLike} for why this is structural */
|
|
53
67
|
export interface BullMQQueueLike {
|
|
54
68
|
name: string;
|
|
55
|
-
|
|
69
|
+
/** `prefix:name` — what a child job's `parent.queue` names. Background coordinators need it */
|
|
70
|
+
readonly qualifiedName?: string;
|
|
71
|
+
addBulk(
|
|
72
|
+
jobs: (MigrationJobSpec | ConvergeJobSpec | BackgroundJobSpec | BackgroundLaneJobSpec)[],
|
|
73
|
+
): Promise<BullMQJobLike[]>;
|
|
56
74
|
getJob(id: string): Promise<BullMQJobLike | undefined>;
|
|
57
75
|
pause(): Promise<void>;
|
|
58
76
|
resume(): Promise<void>;
|
|
@@ -63,6 +81,8 @@ export interface BullMQQueueLike {
|
|
|
63
81
|
upsertJobScheduler?(id: string, repeat: any, template?: any): Promise<unknown>;
|
|
64
82
|
/** BullMQ ≥ 5.16 — needed by `unschedule()` */
|
|
65
83
|
removeJobScheduler?(id: string): Promise<boolean>;
|
|
84
|
+
/** BullMQ ≥ 5.16 — whether the drift watch's schedule exists already */
|
|
85
|
+
getJobScheduler?(id: string): Promise<unknown>;
|
|
66
86
|
}
|
|
67
87
|
|
|
68
88
|
/** See {@link BullMQJobLike} for why this is structural */
|
|
@@ -101,13 +121,25 @@ export type BullMQQueueEventsClass<E extends BullMQQueueEventsLike = BullMQQueue
|
|
|
101
121
|
// ─── Job contract ──────────────────────────────────────────────────────────────
|
|
102
122
|
|
|
103
123
|
export type MigrationJobName = 'up' | 'down' | 'sync' | 'converge';
|
|
124
|
+
/** The job names of a background queue */
|
|
125
|
+
export type BackgroundJobName = 'background' | 'background-lane' | 'background-verify';
|
|
104
126
|
|
|
105
127
|
/**
|
|
106
128
|
* Job names: `up`/`down` carry one migration each; `sync` plans and enqueues
|
|
107
129
|
* what is pending; `converge` brings the declared collections to their
|
|
108
|
-
* declared state.
|
|
130
|
+
* declared state. On the background queue: `background` (a background
|
|
131
|
+
* migration's coordinator), `background-lane` (one of its lanes, a child of
|
|
132
|
+
* the coordinator) and `background-verify` (the drift watch).
|
|
109
133
|
*/
|
|
110
|
-
export const JOB_NAMES: Readonly<{
|
|
134
|
+
export const JOB_NAMES: Readonly<{
|
|
135
|
+
UP: 'up';
|
|
136
|
+
DOWN: 'down';
|
|
137
|
+
SYNC: 'sync';
|
|
138
|
+
CONVERGE: 'converge';
|
|
139
|
+
BACKGROUND: 'background';
|
|
140
|
+
BACKGROUND_LANE: 'background-lane';
|
|
141
|
+
BACKGROUND_VERIFY: 'background-verify';
|
|
142
|
+
}>;
|
|
111
143
|
/**
|
|
112
144
|
* Version stamped on every job's data as `v`. A worker accepts every version
|
|
113
145
|
* from {@link MIN_JOB_DATA_VERSION} up to its own and refuses a newer one —
|
|
@@ -120,6 +152,16 @@ export const DEFAULT_QUEUE_NAME: 'migronaut';
|
|
|
120
152
|
export const DEFAULT_SCHEDULER_ID: 'migronaut-sync';
|
|
121
153
|
/** Default id of a `schedule({ job: 'converge' })` schedule */
|
|
122
154
|
export const DEFAULT_CONVERGE_SCHEDULER_ID: 'migronaut-converge';
|
|
155
|
+
/**
|
|
156
|
+
* Id of the drift watch's schedule on the background queue
|
|
157
|
+
* @experimental New in 2.3
|
|
158
|
+
*/
|
|
159
|
+
export const DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID: 'migronaut-background-verify';
|
|
160
|
+
/**
|
|
161
|
+
* The background queue that serves a migration queue: `<queueName>-background`
|
|
162
|
+
* @experimental New in 2.3
|
|
163
|
+
*/
|
|
164
|
+
export function backgroundQueueName(queueName: string): string;
|
|
123
165
|
|
|
124
166
|
/**
|
|
125
167
|
* Data of an `up` or `down` job. Stored in Redis — re-validated by the worker as untrusted input
|
|
@@ -187,6 +229,106 @@ export interface ConvergeJobData {
|
|
|
187
229
|
reason?: string;
|
|
188
230
|
}
|
|
189
231
|
|
|
232
|
+
/**
|
|
233
|
+
* Data of a `background` job — the coordinator of one background migration.
|
|
234
|
+
* Deduplicated on the migration: every heal collapses into the chain alive.
|
|
235
|
+
* @experimental New in 2.3
|
|
236
|
+
*/
|
|
237
|
+
export interface BackgroundJobData {
|
|
238
|
+
v: 1;
|
|
239
|
+
kind: 'background';
|
|
240
|
+
migration: string;
|
|
241
|
+
/** The chain's round, minted on its first run — an older round bows out */
|
|
242
|
+
round?: number;
|
|
243
|
+
/** How many times this chain spawned lanes — part of their ids */
|
|
244
|
+
spawn?: number;
|
|
245
|
+
/** A takeover of a coordinator nothing has heard from for `stallMs` */
|
|
246
|
+
takeover?: true;
|
|
247
|
+
requestedBy?: string;
|
|
248
|
+
reason?: string;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Data of a `background-lane` job — one lane of a background migration
|
|
253
|
+
* @experimental New in 2.3
|
|
254
|
+
*/
|
|
255
|
+
export interface BackgroundLaneJobData {
|
|
256
|
+
v: 1;
|
|
257
|
+
kind: 'background-lane';
|
|
258
|
+
migration: string;
|
|
259
|
+
registration: string;
|
|
260
|
+
generation: number;
|
|
261
|
+
round: number;
|
|
262
|
+
spawn: number;
|
|
263
|
+
/** 0 to 63 */
|
|
264
|
+
lane: number;
|
|
265
|
+
/** Failed slices in a row — the lane backs off, and gives up after `maxLaneRetries` */
|
|
266
|
+
retry?: number;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Data of a `background-verify` job — a drift-watch tick
|
|
271
|
+
* @experimental New in 2.3
|
|
272
|
+
*/
|
|
273
|
+
export interface BackgroundVerifyJobData {
|
|
274
|
+
v: 1;
|
|
275
|
+
kind: 'background-verify';
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* What a completed `background` job returns: the coordinator chain ended
|
|
280
|
+
* @experimental New in 2.3
|
|
281
|
+
*/
|
|
282
|
+
export interface BackgroundJobResult {
|
|
283
|
+
kind: 'background';
|
|
284
|
+
migration: string;
|
|
285
|
+
/**
|
|
286
|
+
* The background migration's status — or `superseded` (a newer round took
|
|
287
|
+
* over), `unregistered`. Run outside a Worker (no token to move the job
|
|
288
|
+
* with), the coordinator's next step instead: `wait`, `busy` or `process`,
|
|
289
|
+
* with `retryAfterMs`. Absent when a shutdown came first.
|
|
290
|
+
*/
|
|
291
|
+
status?:
|
|
292
|
+
| BackgroundStatus['status']
|
|
293
|
+
| 'blocked'
|
|
294
|
+
| 'superseded'
|
|
295
|
+
| 'unregistered'
|
|
296
|
+
| 'wait'
|
|
297
|
+
| 'busy'
|
|
298
|
+
| 'process';
|
|
299
|
+
round?: number;
|
|
300
|
+
/** Outside a Worker: when to run the job again */
|
|
301
|
+
retryAfterMs?: number;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* What a completed `background-lane` job returns: nothing left for it to claim
|
|
306
|
+
* (`exhausted`), the background migration stopped (`paused`, `cancelled`,
|
|
307
|
+
* `failed`), no plan yet (`stale`) — or `gave-up` after `maxLaneRetries`
|
|
308
|
+
* failed slices in a row (each counted on its partition, in MongoDB)
|
|
309
|
+
* @experimental New in 2.3
|
|
310
|
+
*/
|
|
311
|
+
export interface BackgroundLaneJobResult {
|
|
312
|
+
kind: 'background-lane';
|
|
313
|
+
migration: string;
|
|
314
|
+
/** Outside a Worker also `retry` (a failed slice, to try again after `retryAfterMs`) */
|
|
315
|
+
outcome: BackgroundSliceResult['outcome'] | 'gave-up' | 'retry';
|
|
316
|
+
counters?: BackgroundCounters;
|
|
317
|
+
code?: MigronautErrorCode;
|
|
318
|
+
/** Outside a Worker (no token to move the job with): when to run it again */
|
|
319
|
+
retryAfterMs?: number;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* What a completed `background-verify` job returns
|
|
324
|
+
* @experimental New in 2.3
|
|
325
|
+
*/
|
|
326
|
+
export interface BackgroundVerifyJobResult extends BackgroundVerifyResult {
|
|
327
|
+
kind: 'background-verify';
|
|
328
|
+
/** Coordinators this tick added (or found alive) — the heal */
|
|
329
|
+
enqueued: number;
|
|
330
|
+
}
|
|
331
|
+
|
|
190
332
|
/**
|
|
191
333
|
* What a completed `up`/`down` job returns
|
|
192
334
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
@@ -203,6 +345,8 @@ export interface MigrationJobResult {
|
|
|
203
345
|
reason?: string;
|
|
204
346
|
/** Time (ms) spent waiting for the MongoDB migration lock */
|
|
205
347
|
lockWaitMs: number;
|
|
348
|
+
/** The coordinators enqueued for what the job registered — on a queue with a background side */
|
|
349
|
+
background?: { migration: string; jobId: string }[];
|
|
206
350
|
}
|
|
207
351
|
|
|
208
352
|
/**
|
|
@@ -224,7 +368,19 @@ export interface SyncJobResult {
|
|
|
224
368
|
* and its file has not changed since — a schedule's circuit breaker. A fix
|
|
225
369
|
* (a changed file) or an explicit `enqueueUp(name)` resumes the line.
|
|
226
370
|
*/
|
|
227
|
-
held?: {
|
|
371
|
+
held?: {
|
|
372
|
+
migration: string;
|
|
373
|
+
reason: string;
|
|
374
|
+
failedAt?: Date;
|
|
375
|
+
};
|
|
376
|
+
/**
|
|
377
|
+
* Present when the next migration waits for background migrations that
|
|
378
|
+
* have not completed (`requires`): the tick enqueued what comes before it,
|
|
379
|
+
* and the line goes on once they complete. Not a failure — `held` is that.
|
|
380
|
+
*/
|
|
381
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
382
|
+
/** The heal of the background side: coordinators added, or found alive */
|
|
383
|
+
background?: { enqueued: number };
|
|
228
384
|
}
|
|
229
385
|
|
|
230
386
|
/**
|
|
@@ -322,6 +478,39 @@ export interface ConvergeJobSpec {
|
|
|
322
478
|
opts: { attempts: 1; deduplication: { id: string }; [option: string]: unknown };
|
|
323
479
|
}
|
|
324
480
|
|
|
481
|
+
/**
|
|
482
|
+
* Per-job options passed through to every background job — retention and
|
|
483
|
+
* logging. Besides what a migration job refuses, the four options of a
|
|
484
|
+
* parent's child-failure policy are refused: the adapter owns them.
|
|
485
|
+
* @experimental New in 2.3
|
|
486
|
+
*/
|
|
487
|
+
export interface BackgroundJobOptions extends MigrationJobOptions {
|
|
488
|
+
failParentOnFailure?: never;
|
|
489
|
+
continueParentOnFailure?: never;
|
|
490
|
+
ignoreDependencyOnFailure?: never;
|
|
491
|
+
removeDependencyOnFailure?: never;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* A coordinator job as handed to `queue.addBulk`
|
|
496
|
+
* @experimental New in 2.3
|
|
497
|
+
*/
|
|
498
|
+
export interface BackgroundJobSpec {
|
|
499
|
+
name: 'background';
|
|
500
|
+
data: BackgroundJobData;
|
|
501
|
+
opts: { attempts: number; deduplication: { id: string }; [option: string]: unknown };
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* A lane job as handed to `queue.addBulk`
|
|
506
|
+
* @experimental New in 2.3
|
|
507
|
+
*/
|
|
508
|
+
export interface BackgroundLaneJobSpec {
|
|
509
|
+
name: 'background-lane';
|
|
510
|
+
data: BackgroundLaneJobData;
|
|
511
|
+
opts: { attempts: 1; [option: string]: unknown };
|
|
512
|
+
}
|
|
513
|
+
|
|
325
514
|
/** A planned, not yet enqueued, group */
|
|
326
515
|
export interface MigrationPlan {
|
|
327
516
|
/** Id of this enqueue call, in the kit's `generateId` format (a UUID by default) */
|
|
@@ -334,6 +523,11 @@ export interface MigrationPlan {
|
|
|
334
523
|
jobs: MigrationJobSpec[];
|
|
335
524
|
/** The converge job that ends an `up` group — see `EnqueueUpOptions.converge` */
|
|
336
525
|
converge?: ConvergeJobSpec;
|
|
526
|
+
/**
|
|
527
|
+
* The first pending file that requires a background migration not
|
|
528
|
+
* completed yet: the plan ends before it (and has no converge job)
|
|
529
|
+
*/
|
|
530
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
337
531
|
}
|
|
338
532
|
|
|
339
533
|
/** {@link parseJobData}'s normalized result */
|
|
@@ -358,6 +552,40 @@ export type ParsedJobData =
|
|
|
358
552
|
*/
|
|
359
553
|
export function parseJobData(job: { id?: string; name: string; data: unknown }): ParsedJobData;
|
|
360
554
|
|
|
555
|
+
/** {@link parseBackgroundJobData}'s normalized result */
|
|
556
|
+
export type ParsedBackgroundJobData =
|
|
557
|
+
| {
|
|
558
|
+
kind: 'background';
|
|
559
|
+
migration: string;
|
|
560
|
+
round?: number;
|
|
561
|
+
spawn?: number;
|
|
562
|
+
takeover?: true;
|
|
563
|
+
requestedBy?: string;
|
|
564
|
+
reason?: string;
|
|
565
|
+
}
|
|
566
|
+
| {
|
|
567
|
+
kind: 'background-lane';
|
|
568
|
+
migration: string;
|
|
569
|
+
registration: string;
|
|
570
|
+
generation: number;
|
|
571
|
+
round: number;
|
|
572
|
+
spawn: number;
|
|
573
|
+
lane: number;
|
|
574
|
+
retry: number;
|
|
575
|
+
}
|
|
576
|
+
| { kind: 'background-verify' };
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Validate a job read back from a background queue and return a normalized
|
|
580
|
+
* copy. Throws `QueueJobInvalidError` for anything outside the contract.
|
|
581
|
+
* @experimental New in 2.3
|
|
582
|
+
*/
|
|
583
|
+
export function parseBackgroundJobData(job: {
|
|
584
|
+
id?: string;
|
|
585
|
+
name: string;
|
|
586
|
+
data: unknown;
|
|
587
|
+
}): ParsedBackgroundJobData;
|
|
588
|
+
|
|
361
589
|
/** The deduplication id a migration's job carries — never contains `:` */
|
|
362
590
|
export function dedupId(direction: 'up' | 'down', migration: string): string;
|
|
363
591
|
|
|
@@ -479,6 +707,8 @@ export interface MigrationGroup {
|
|
|
479
707
|
deduplicated: string[];
|
|
480
708
|
/** The converge job ending the group, or null. `deduplicated`: a peer's identical job */
|
|
481
709
|
converge: { id: string; deduplicated: boolean } | null;
|
|
710
|
+
/** See {@link MigrationPlan.waiting} — the group stops before this file */
|
|
711
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
482
712
|
/**
|
|
483
713
|
* Resolve when every job has finished — the converge job last; reject with
|
|
484
714
|
* `QueueJobFailedError` at the first one that fails or outlives `timeoutMs`.
|
|
@@ -523,6 +753,17 @@ export type ScheduleOptions = (
|
|
|
523
753
|
id?: string;
|
|
524
754
|
to?: never;
|
|
525
755
|
}
|
|
756
|
+
| {
|
|
757
|
+
/**
|
|
758
|
+
* Each tick runs the drift watch on the background queue (and heals
|
|
759
|
+
* it) — overrides the schedule `startBackgroundWorker()` registers.
|
|
760
|
+
* Needs the `background` option. @experimental
|
|
761
|
+
*/
|
|
762
|
+
job: 'background-verify';
|
|
763
|
+
/** Scheduler id. Default `'migronaut-background-verify'` */
|
|
764
|
+
id?: string;
|
|
765
|
+
to?: never;
|
|
766
|
+
}
|
|
526
767
|
);
|
|
527
768
|
|
|
528
769
|
/** Options for {@link MigrationQueue.startWorker} — passed to the Worker constructor */
|
|
@@ -551,7 +792,13 @@ export interface MigrationJobView {
|
|
|
551
792
|
state: string;
|
|
552
793
|
/** As stored — see {@link MigrationJobProgress} for what the adapter writes */
|
|
553
794
|
progress: MigrationJobProgress | number;
|
|
554
|
-
returnvalue?:
|
|
795
|
+
returnvalue?:
|
|
796
|
+
| MigrationJobResult
|
|
797
|
+
| SyncJobResult
|
|
798
|
+
| ConvergeJobResult
|
|
799
|
+
| BackgroundJobResult
|
|
800
|
+
| BackgroundLaneJobResult
|
|
801
|
+
| BackgroundVerifyJobResult;
|
|
555
802
|
failedReason?: string;
|
|
556
803
|
attemptsMade: number;
|
|
557
804
|
timestamp?: number;
|
|
@@ -593,6 +840,23 @@ export interface CreateMigrationProcessorOptions {
|
|
|
593
840
|
jobOptions?: MigrationJobOptions;
|
|
594
841
|
/** What a job may ask for beyond the ordinary — see {@link MigrationJobPermissions} */
|
|
595
842
|
allow?: MigrationJobPermissions;
|
|
843
|
+
/**
|
|
844
|
+
* How many of a migration's `userland: true` lines one job's log takes (a
|
|
845
|
+
* lane: one slice) — the rest are counted in one closing row. `0` mirrors
|
|
846
|
+
* none; `migration:log` still carries every line. Default 1000
|
|
847
|
+
* @experimental New in 2.4
|
|
848
|
+
*/
|
|
849
|
+
userlandLogRows?: number;
|
|
850
|
+
/**
|
|
851
|
+
* The background queue: what an `up` (or `down`) job registers gets its
|
|
852
|
+
* coordinator there at once, and every `sync` tick heals it. @experimental
|
|
853
|
+
*/
|
|
854
|
+
background?: {
|
|
855
|
+
queue: BullMQQueueLike;
|
|
856
|
+
jobOptions?: BackgroundJobOptions;
|
|
857
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
858
|
+
stallMs?: number;
|
|
859
|
+
};
|
|
596
860
|
}
|
|
597
861
|
|
|
598
862
|
/**
|
|
@@ -600,6 +864,11 @@ export interface CreateMigrationProcessorOptions {
|
|
|
600
864
|
* declares exactly three parameters, which is what makes BullMQ hand it the
|
|
601
865
|
* cancellation signal. Jobs are processed one at a time even when the Worker
|
|
602
866
|
* is configured for more.
|
|
867
|
+
*
|
|
868
|
+
* Each run is told which job it works for (the kit's `job` option), so a
|
|
869
|
+
* migration's `ctx.run` and every `migration:log` event carry `jobId` and
|
|
870
|
+
* `groupId`, and the migration's `userland: true` lines are written into the
|
|
871
|
+
* job's log next to the processor's own rows (`✎ …`).
|
|
603
872
|
*/
|
|
604
873
|
export interface MigrationProcessor {
|
|
605
874
|
(
|
|
@@ -607,7 +876,10 @@ export interface MigrationProcessor {
|
|
|
607
876
|
token?: string,
|
|
608
877
|
signal?: AbortSignal,
|
|
609
878
|
): Promise<MigrationJobResult | SyncJobResult | ConvergeJobResult>;
|
|
610
|
-
/**
|
|
879
|
+
/**
|
|
880
|
+
* The kit running the jobs — subscribe to its events for metrics, and to
|
|
881
|
+
* `migration:log` for what the migrations log for your users
|
|
882
|
+
*/
|
|
611
883
|
readonly kit: MigratorKit;
|
|
612
884
|
/**
|
|
613
885
|
* Stop taking the lock. Irreversible. A job that has not started its
|
|
@@ -632,6 +904,70 @@ export function createMigrationProcessor(
|
|
|
632
904
|
options?: CreateMigrationProcessorOptions,
|
|
633
905
|
): MigrationProcessor;
|
|
634
906
|
|
|
907
|
+
/** Options for {@link createBackgroundProcessor} */
|
|
908
|
+
export interface CreateBackgroundProcessorOptions {
|
|
909
|
+
config?: Partial<MigronautConfig>;
|
|
910
|
+
kitOptions?: MigratorKitOptions;
|
|
911
|
+
kit?: MigratorKit;
|
|
912
|
+
/** The background queue — the coordinators add their lanes to it. Required */
|
|
913
|
+
queue: BullMQQueueLike;
|
|
914
|
+
jobOptions?: BackgroundJobOptions;
|
|
915
|
+
/** A lane's slice (ms). Default: each background migration's own `sliceMs` */
|
|
916
|
+
sliceMs?: number;
|
|
917
|
+
/**
|
|
918
|
+
* `'auto'` (default): lanes are children of their coordinator, which waits
|
|
919
|
+
* for them (`moveToWaitingChildren`) when the queue and its jobs support
|
|
920
|
+
* it. `false`: lanes on their own, and the coordinator polls MongoDB.
|
|
921
|
+
*/
|
|
922
|
+
children?: 'auto' | false;
|
|
923
|
+
/** How often a coordinator without children looks again (ms). Default 5000 */
|
|
924
|
+
pollIntervalMs?: number;
|
|
925
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
926
|
+
stallMs?: number;
|
|
927
|
+
/** Failed slices in a row before a lane gives up (0–100). Default 8 */
|
|
928
|
+
maxLaneRetries?: number;
|
|
929
|
+
/**
|
|
930
|
+
* See {@link CreateMigrationProcessorOptions.userlandLogRows} — counted per slice
|
|
931
|
+
* @experimental New in 2.4
|
|
932
|
+
*/
|
|
933
|
+
userlandLogRows?: number;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/**
|
|
937
|
+
* The function a Worker on the background queue runs. Jobs run side by side.
|
|
938
|
+
* A lane's slice is told its job (`runBackgroundSlice`'s `job`), so its
|
|
939
|
+
* `migration:log` events carry `jobId` and its `userland: true` lines are
|
|
940
|
+
* written into the lane job's log (`✎ …`).
|
|
941
|
+
* @experimental New in 2.3
|
|
942
|
+
*/
|
|
943
|
+
export interface BackgroundProcessor {
|
|
944
|
+
(
|
|
945
|
+
job: BullMQJobLike<BackgroundJobData | BackgroundLaneJobData | BackgroundVerifyJobData>,
|
|
946
|
+
token?: string,
|
|
947
|
+
signal?: AbortSignal,
|
|
948
|
+
): Promise<BackgroundJobResult | BackgroundLaneJobResult | BackgroundVerifyJobResult>;
|
|
949
|
+
readonly kit: MigratorKit;
|
|
950
|
+
/**
|
|
951
|
+
* Stop: a lane stops at its next batch, checkpoints, releases its lease and
|
|
952
|
+
* goes back to the queue (moved to delayed); a coordinator bows out and
|
|
953
|
+
* comes back. Irreversible.
|
|
954
|
+
*/
|
|
955
|
+
shutdown(reason?: string): void;
|
|
956
|
+
/** `shutdown()`, let the jobs in flight settle, disconnect a kit the processor created */
|
|
957
|
+
close(): Promise<void>;
|
|
958
|
+
/** A coordinator for every background migration with work to do — what a worker does at start */
|
|
959
|
+
heal(): Promise<BackgroundEnqueueResult>;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* Build the processor for a Worker on the background queue you construct
|
|
964
|
+
* yourself.
|
|
965
|
+
* @experimental New in 2.3
|
|
966
|
+
*/
|
|
967
|
+
export function createBackgroundProcessor(
|
|
968
|
+
options: CreateBackgroundProcessorOptions,
|
|
969
|
+
): BackgroundProcessor;
|
|
970
|
+
|
|
635
971
|
// ─── Producer building blocks ──────────────────────────────────────────────────
|
|
636
972
|
|
|
637
973
|
/** Plan an `up` group without enqueuing it */
|
|
@@ -680,6 +1016,39 @@ export function enqueueConverge(
|
|
|
680
1016
|
},
|
|
681
1017
|
): Promise<ConvergeHandle>;
|
|
682
1018
|
|
|
1019
|
+
/** Options for `enqueueBackground` */
|
|
1020
|
+
export interface EnqueueBackgroundOptions {
|
|
1021
|
+
/**
|
|
1022
|
+
* A running background migration nothing has moved for this long (ms) — no
|
|
1023
|
+
* live lease, no checkpoint, no coordinator step — also gets a takeover
|
|
1024
|
+
* coordinator, whose newer round retires the stuck one. Default 900000
|
|
1025
|
+
* (15 minutes); at least 1000.
|
|
1026
|
+
*/
|
|
1027
|
+
stallMs?: number;
|
|
1028
|
+
requestedBy?: string;
|
|
1029
|
+
reason?: string;
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* What `enqueueBackground` resolves with
|
|
1034
|
+
* @experimental New in 2.3
|
|
1035
|
+
*/
|
|
1036
|
+
export interface BackgroundEnqueueResult {
|
|
1037
|
+
/** One per coordinator added — or absorbed by the one already alive (same id) */
|
|
1038
|
+
jobs: { migration: string; id: string; takeover?: true }[];
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* Enqueue the coordinator of one background migration (`migration`) — or of
|
|
1043
|
+
* every one with work to do — on a background queue you own. Idempotent.
|
|
1044
|
+
* @experimental New in 2.3
|
|
1045
|
+
*/
|
|
1046
|
+
export function enqueueBackground(
|
|
1047
|
+
queue: BullMQQueueLike,
|
|
1048
|
+
kit: MigratorKit,
|
|
1049
|
+
options?: EnqueueBackgroundOptions & { migration?: string; jobOptions?: BackgroundJobOptions },
|
|
1050
|
+
): Promise<BackgroundEnqueueResult>;
|
|
1051
|
+
|
|
683
1052
|
/** Wait for a group's jobs — what `MigrationGroup.wait()` calls */
|
|
684
1053
|
export function waitForGroup(options: {
|
|
685
1054
|
queue: BullMQQueueLike;
|
|
@@ -762,6 +1131,54 @@ export interface CreateMigrationQueueOptions<
|
|
|
762
1131
|
* refuse fails at the call. Give every producer and worker the same policy.
|
|
763
1132
|
*/
|
|
764
1133
|
allow?: MigrationJobPermissions;
|
|
1134
|
+
/**
|
|
1135
|
+
* See {@link CreateMigrationProcessorOptions.userlandLogRows} — for the
|
|
1136
|
+
* migration jobs and, with `background`, the lanes
|
|
1137
|
+
* @experimental New in 2.4
|
|
1138
|
+
*/
|
|
1139
|
+
userlandLogRows?: number;
|
|
1140
|
+
/**
|
|
1141
|
+
* Background migrations on a queue of their own (`<queueName>-background`):
|
|
1142
|
+
* a coordinator job each, with lanes as its children. `true` takes every
|
|
1143
|
+
* default. @experimental New in 2.3
|
|
1144
|
+
*/
|
|
1145
|
+
background?: boolean | BackgroundQueueOptions;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
/**
|
|
1149
|
+
* The `background` option of {@link createMigrationQueue}
|
|
1150
|
+
* @experimental New in 2.3
|
|
1151
|
+
*/
|
|
1152
|
+
export interface BackgroundQueueOptions {
|
|
1153
|
+
/** Default `<queueName>-background` (or the injected queue's name) */
|
|
1154
|
+
queueName?: string;
|
|
1155
|
+
/** A background Queue instance you own — never closed by `close()` */
|
|
1156
|
+
queue?: BullMQQueueLike;
|
|
1157
|
+
jobOptions?: BackgroundJobOptions;
|
|
1158
|
+
/** Defaults for `startBackgroundWorker()` — concurrency 2 unless given */
|
|
1159
|
+
workerOptions?: { concurrency?: number; [option: string]: unknown };
|
|
1160
|
+
/** See {@link CreateBackgroundProcessorOptions} */
|
|
1161
|
+
sliceMs?: number;
|
|
1162
|
+
children?: 'auto' | false;
|
|
1163
|
+
pollIntervalMs?: number;
|
|
1164
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
1165
|
+
stallMs?: number;
|
|
1166
|
+
/**
|
|
1167
|
+
* The drift watch's schedule, registered by `startBackgroundWorker()` (ms,
|
|
1168
|
+
* ≥ 1000). Default 600000 (10 minutes) — registered only when no schedule
|
|
1169
|
+
* exists yet, so one set with `schedule({ job: 'background-verify' })`
|
|
1170
|
+
* stays; given explicitly, it is re-registered at every start. `false`
|
|
1171
|
+
* registers none.
|
|
1172
|
+
*/
|
|
1173
|
+
verifyIntervalMs?: number | false;
|
|
1174
|
+
/** See {@link CreateBackgroundProcessorOptions.maxLaneRetries} */
|
|
1175
|
+
maxLaneRetries?: number;
|
|
1176
|
+
/**
|
|
1177
|
+
* Host the live drift watcher in the background worker's process — `true`,
|
|
1178
|
+
* or its options. Default: when the kit's `backgroundDrift` is `'stream'`
|
|
1179
|
+
* or `'both'`. Closed first by `close()`.
|
|
1180
|
+
*/
|
|
1181
|
+
watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
|
|
765
1182
|
}
|
|
766
1183
|
|
|
767
1184
|
/**
|
|
@@ -776,7 +1193,11 @@ export class MigrationQueue<
|
|
|
776
1193
|
> {
|
|
777
1194
|
constructor(options: CreateMigrationQueueOptions<Q, W, E>);
|
|
778
1195
|
|
|
779
|
-
/**
|
|
1196
|
+
/**
|
|
1197
|
+
* The kit behind the queue — `kit.on('migration:success', …)` for metrics,
|
|
1198
|
+
* `kit.on('migration:log', …)` (in the worker's process) to keep what the
|
|
1199
|
+
* migrations log for your users
|
|
1200
|
+
*/
|
|
780
1201
|
readonly kit: MigratorKit;
|
|
781
1202
|
/** Your Queue, with its own type */
|
|
782
1203
|
readonly queue: Q;
|
|
@@ -787,6 +1208,26 @@ export class MigrationQueue<
|
|
|
787
1208
|
readonly queueName: string;
|
|
788
1209
|
/** The processor, for attaching to a Worker you construct yourself */
|
|
789
1210
|
readonly processor: MigrationProcessor;
|
|
1211
|
+
/**
|
|
1212
|
+
* The background queue (`background` option), if any
|
|
1213
|
+
* @experimental New in 2.3
|
|
1214
|
+
*/
|
|
1215
|
+
readonly backgroundQueue: BullMQQueueLike | undefined;
|
|
1216
|
+
/**
|
|
1217
|
+
* The worker started by {@link startBackgroundWorker}, if any
|
|
1218
|
+
* @experimental New in 2.3
|
|
1219
|
+
*/
|
|
1220
|
+
readonly backgroundWorker: W | undefined;
|
|
1221
|
+
/**
|
|
1222
|
+
* The background processor, for a Worker you construct yourself
|
|
1223
|
+
* @experimental New in 2.3
|
|
1224
|
+
*/
|
|
1225
|
+
readonly backgroundProcessor: BackgroundProcessor | undefined;
|
|
1226
|
+
/**
|
|
1227
|
+
* The live drift watcher `startBackgroundWorker()` started, if any
|
|
1228
|
+
* @experimental New in 2.3
|
|
1229
|
+
*/
|
|
1230
|
+
readonly backgroundWatcher: BackgroundWatcher | undefined;
|
|
790
1231
|
|
|
791
1232
|
/**
|
|
792
1233
|
* Enqueue pending migrations — all, up to `options.to`, or the one
|
|
@@ -806,6 +1247,33 @@ export class MigrationQueue<
|
|
|
806
1247
|
*/
|
|
807
1248
|
enqueueConverge(options?: EnqueueConvergeOptions): Promise<ConvergeHandle>;
|
|
808
1249
|
|
|
1250
|
+
/**
|
|
1251
|
+
* Enqueue the coordinator of one background migration — or of every one
|
|
1252
|
+
* with work to do. Idempotent. Needs the `background` option. @experimental
|
|
1253
|
+
*/
|
|
1254
|
+
enqueueBackground(
|
|
1255
|
+
name?: string,
|
|
1256
|
+
options?: EnqueueBackgroundOptions,
|
|
1257
|
+
): Promise<BackgroundEnqueueResult>;
|
|
1258
|
+
/**
|
|
1259
|
+
* A background migration's status, read from MongoDB — `null` when not registered
|
|
1260
|
+
* @experimental New in 2.3
|
|
1261
|
+
*/
|
|
1262
|
+
backgroundStatus(name: string): Promise<BackgroundStatus | null>;
|
|
1263
|
+
/**
|
|
1264
|
+
* Every background migration's status
|
|
1265
|
+
* @experimental New in 2.3
|
|
1266
|
+
*/
|
|
1267
|
+
backgroundStatus(): Promise<BackgroundStatus[]>;
|
|
1268
|
+
/**
|
|
1269
|
+
* The drift watch, now — and, with the `background` option, a coordinator
|
|
1270
|
+
* for whatever it reopened. @experimental
|
|
1271
|
+
*/
|
|
1272
|
+
verifyBackground(options?: {
|
|
1273
|
+
onDrift?: 'reopen' | 'report';
|
|
1274
|
+
collections?: string[];
|
|
1275
|
+
}): Promise<BackgroundVerifyResult>;
|
|
1276
|
+
|
|
809
1277
|
/** Full migration status, read from MongoDB */
|
|
810
1278
|
status(): Promise<StatusRow[]>;
|
|
811
1279
|
/** Migrations not applied yet */
|
|
@@ -821,6 +1289,13 @@ export class MigrationQueue<
|
|
|
821
1289
|
* that failed, tries again.
|
|
822
1290
|
*/
|
|
823
1291
|
startWorker(options?: StartWorkerOptions): Promise<W>;
|
|
1292
|
+
/**
|
|
1293
|
+
* Start the background worker (concurrency 2 by default): coordinators and
|
|
1294
|
+
* lanes side by side. Registers the drift watch's schedule and heals — a
|
|
1295
|
+
* coordinator for every background migration with work to do. Needs the
|
|
1296
|
+
* `background` option. @experimental
|
|
1297
|
+
*/
|
|
1298
|
+
startBackgroundWorker(options?: { concurrency?: number; [option: string]: unknown }): Promise<W>;
|
|
824
1299
|
/** Stop workers from picking up new jobs; the job in flight finishes */
|
|
825
1300
|
pause(): Promise<void>;
|
|
826
1301
|
resume(): Promise<void>;
|
|
@@ -835,7 +1310,8 @@ export class MigrationQueue<
|
|
|
835
1310
|
schedule(options: ScheduleOptions): Promise<void>;
|
|
836
1311
|
/**
|
|
837
1312
|
* Remove a schedule — the sync one by default; pass
|
|
838
|
-
* {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another.
|
|
1313
|
+
* {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another. The
|
|
1314
|
+
* background queue's schedules (the drift watch's) are looked for too.
|
|
839
1315
|
* Resolves whether one existed.
|
|
840
1316
|
*/
|
|
841
1317
|
unschedule(id?: string): Promise<boolean>;
|