@alexify/migronaut 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/README.md +68 -10
- package/bullmq.d.ts +465 -7
- package/index.d.ts +1272 -18
- package/migronaut.schema.json +150 -2
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +153 -15
- package/src/bullmq/producer.js +202 -27
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/converge.js +38 -10
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/cli/table.js +68 -9
- package/src/core/audit.js +98 -3
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +125 -31
- package/src/core/config.js +133 -13
- package/src/core/converge-plan.js +343 -61
- package/src/core/converge-search-run.js +440 -0
- package/src/core/converge-search.js +404 -0
- package/src/core/converge.js +428 -183
- package/src/core/index-spec.js +27 -16
- package/src/core/lock.js +97 -32
- package/src/core/migrator.js +951 -26
- package/src/core/options.js +32 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/search-index-spec.js +758 -0
- package/src/core/server-info.js +70 -0
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +97 -5
- package/src/index.js +16 -0
- package/src/utils/canonical.js +34 -1
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +125 -1
- package/src/utils/template.js +69 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/bullmq.d.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
AuditReport,
|
|
3
|
+
BackgroundCounters,
|
|
4
|
+
BackgroundSliceResult,
|
|
5
|
+
BackgroundStatus,
|
|
6
|
+
BackgroundVerifyResult,
|
|
7
|
+
BackgroundWatcher,
|
|
3
8
|
CollectionConvergeResult,
|
|
9
|
+
ConvergeSearchSummary,
|
|
4
10
|
ConvergeUnstable,
|
|
5
11
|
LockInfo,
|
|
6
12
|
MigratorKit,
|
|
@@ -9,6 +15,7 @@ import type {
|
|
|
9
15
|
MigronautErrorCode,
|
|
10
16
|
OnLockHeld,
|
|
11
17
|
StatusRow,
|
|
18
|
+
WatchBackgroundOptions,
|
|
12
19
|
} from './index.js';
|
|
13
20
|
|
|
14
21
|
// ─── Structural BullMQ surface ─────────────────────────────────────────────────
|
|
@@ -46,12 +53,24 @@ export interface BullMQJobLike<Data = any, Result = any> {
|
|
|
46
53
|
log(row: string): Promise<number>;
|
|
47
54
|
getState(): Promise<string>;
|
|
48
55
|
waitUntilFinished(queueEvents: any, ttl?: number): Promise<Result>;
|
|
56
|
+
/** Used to put back a job a shutdown stopped before it began */
|
|
57
|
+
moveToWait?(token?: string): Promise<unknown>;
|
|
58
|
+
/** Background jobs: continue later (a lane between slices, a coordinator polling) */
|
|
59
|
+
moveToDelayed?(timestamp: number, token?: string): Promise<void>;
|
|
60
|
+
/** Background coordinators: wait for the lanes they spawned (BullMQ ≥ 5 parents) */
|
|
61
|
+
moveToWaitingChildren?(token: string, opts?: any): Promise<boolean>;
|
|
62
|
+
updateData?(data: Data): Promise<void>;
|
|
63
|
+
getIgnoredChildrenFailures?(): Promise<{ [jobKey: string]: string }>;
|
|
49
64
|
}
|
|
50
65
|
|
|
51
66
|
/** See {@link BullMQJobLike} for why this is structural */
|
|
52
67
|
export interface BullMQQueueLike {
|
|
53
68
|
name: string;
|
|
54
|
-
|
|
69
|
+
/** `prefix:name` — what a child job's `parent.queue` names. Background coordinators need it */
|
|
70
|
+
readonly qualifiedName?: string;
|
|
71
|
+
addBulk(
|
|
72
|
+
jobs: (MigrationJobSpec | ConvergeJobSpec | BackgroundJobSpec | BackgroundLaneJobSpec)[],
|
|
73
|
+
): Promise<BullMQJobLike[]>;
|
|
55
74
|
getJob(id: string): Promise<BullMQJobLike | undefined>;
|
|
56
75
|
pause(): Promise<void>;
|
|
57
76
|
resume(): Promise<void>;
|
|
@@ -62,6 +81,8 @@ export interface BullMQQueueLike {
|
|
|
62
81
|
upsertJobScheduler?(id: string, repeat: any, template?: any): Promise<unknown>;
|
|
63
82
|
/** BullMQ ≥ 5.16 — needed by `unschedule()` */
|
|
64
83
|
removeJobScheduler?(id: string): Promise<boolean>;
|
|
84
|
+
/** BullMQ ≥ 5.16 — whether the drift watch's schedule exists already */
|
|
85
|
+
getJobScheduler?(id: string): Promise<unknown>;
|
|
65
86
|
}
|
|
66
87
|
|
|
67
88
|
/** See {@link BullMQJobLike} for why this is structural */
|
|
@@ -100,13 +121,25 @@ export type BullMQQueueEventsClass<E extends BullMQQueueEventsLike = BullMQQueue
|
|
|
100
121
|
// ─── Job contract ──────────────────────────────────────────────────────────────
|
|
101
122
|
|
|
102
123
|
export type MigrationJobName = 'up' | 'down' | 'sync' | 'converge';
|
|
124
|
+
/** The job names of a background queue */
|
|
125
|
+
export type BackgroundJobName = 'background' | 'background-lane' | 'background-verify';
|
|
103
126
|
|
|
104
127
|
/**
|
|
105
128
|
* Job names: `up`/`down` carry one migration each; `sync` plans and enqueues
|
|
106
129
|
* what is pending; `converge` brings the declared collections to their
|
|
107
|
-
* declared state.
|
|
130
|
+
* declared state. On the background queue: `background` (a background
|
|
131
|
+
* migration's coordinator), `background-lane` (one of its lanes, a child of
|
|
132
|
+
* the coordinator) and `background-verify` (the drift watch).
|
|
108
133
|
*/
|
|
109
|
-
export const JOB_NAMES: Readonly<{
|
|
134
|
+
export const JOB_NAMES: Readonly<{
|
|
135
|
+
UP: 'up';
|
|
136
|
+
DOWN: 'down';
|
|
137
|
+
SYNC: 'sync';
|
|
138
|
+
CONVERGE: 'converge';
|
|
139
|
+
BACKGROUND: 'background';
|
|
140
|
+
BACKGROUND_LANE: 'background-lane';
|
|
141
|
+
BACKGROUND_VERIFY: 'background-verify';
|
|
142
|
+
}>;
|
|
110
143
|
/**
|
|
111
144
|
* Version stamped on every job's data as `v`. A worker accepts every version
|
|
112
145
|
* from {@link MIN_JOB_DATA_VERSION} up to its own and refuses a newer one —
|
|
@@ -119,6 +152,16 @@ export const DEFAULT_QUEUE_NAME: 'migronaut';
|
|
|
119
152
|
export const DEFAULT_SCHEDULER_ID: 'migronaut-sync';
|
|
120
153
|
/** Default id of a `schedule({ job: 'converge' })` schedule */
|
|
121
154
|
export const DEFAULT_CONVERGE_SCHEDULER_ID: 'migronaut-converge';
|
|
155
|
+
/**
|
|
156
|
+
* Id of the drift watch's schedule on the background queue
|
|
157
|
+
* @experimental New in 2.3
|
|
158
|
+
*/
|
|
159
|
+
export const DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID: 'migronaut-background-verify';
|
|
160
|
+
/**
|
|
161
|
+
* The background queue that serves a migration queue: `<queueName>-background`
|
|
162
|
+
* @experimental New in 2.3
|
|
163
|
+
*/
|
|
164
|
+
export function backgroundQueueName(queueName: string): string;
|
|
122
165
|
|
|
123
166
|
/**
|
|
124
167
|
* Data of an `up` or `down` job. Stored in Redis — re-validated by the worker as untrusted input
|
|
@@ -186,6 +229,106 @@ export interface ConvergeJobData {
|
|
|
186
229
|
reason?: string;
|
|
187
230
|
}
|
|
188
231
|
|
|
232
|
+
/**
|
|
233
|
+
* Data of a `background` job — the coordinator of one background migration.
|
|
234
|
+
* Deduplicated on the migration: every heal collapses into the chain alive.
|
|
235
|
+
* @experimental New in 2.3
|
|
236
|
+
*/
|
|
237
|
+
export interface BackgroundJobData {
|
|
238
|
+
v: 1;
|
|
239
|
+
kind: 'background';
|
|
240
|
+
migration: string;
|
|
241
|
+
/** The chain's round, minted on its first run — an older round bows out */
|
|
242
|
+
round?: number;
|
|
243
|
+
/** How many times this chain spawned lanes — part of their ids */
|
|
244
|
+
spawn?: number;
|
|
245
|
+
/** A takeover of a coordinator nothing has heard from for `stallMs` */
|
|
246
|
+
takeover?: true;
|
|
247
|
+
requestedBy?: string;
|
|
248
|
+
reason?: string;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Data of a `background-lane` job — one lane of a background migration
|
|
253
|
+
* @experimental New in 2.3
|
|
254
|
+
*/
|
|
255
|
+
export interface BackgroundLaneJobData {
|
|
256
|
+
v: 1;
|
|
257
|
+
kind: 'background-lane';
|
|
258
|
+
migration: string;
|
|
259
|
+
registration: string;
|
|
260
|
+
generation: number;
|
|
261
|
+
round: number;
|
|
262
|
+
spawn: number;
|
|
263
|
+
/** 0 to 63 */
|
|
264
|
+
lane: number;
|
|
265
|
+
/** Failed slices in a row — the lane backs off, and gives up after `maxLaneRetries` */
|
|
266
|
+
retry?: number;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Data of a `background-verify` job — a drift-watch tick
|
|
271
|
+
* @experimental New in 2.3
|
|
272
|
+
*/
|
|
273
|
+
export interface BackgroundVerifyJobData {
|
|
274
|
+
v: 1;
|
|
275
|
+
kind: 'background-verify';
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* What a completed `background` job returns: the coordinator chain ended
|
|
280
|
+
* @experimental New in 2.3
|
|
281
|
+
*/
|
|
282
|
+
export interface BackgroundJobResult {
|
|
283
|
+
kind: 'background';
|
|
284
|
+
migration: string;
|
|
285
|
+
/**
|
|
286
|
+
* The background migration's status — or `superseded` (a newer round took
|
|
287
|
+
* over), `unregistered`. Run outside a Worker (no token to move the job
|
|
288
|
+
* with), the coordinator's next step instead: `wait`, `busy` or `process`,
|
|
289
|
+
* with `retryAfterMs`. Absent when a shutdown came first.
|
|
290
|
+
*/
|
|
291
|
+
status?:
|
|
292
|
+
| BackgroundStatus['status']
|
|
293
|
+
| 'blocked'
|
|
294
|
+
| 'superseded'
|
|
295
|
+
| 'unregistered'
|
|
296
|
+
| 'wait'
|
|
297
|
+
| 'busy'
|
|
298
|
+
| 'process';
|
|
299
|
+
round?: number;
|
|
300
|
+
/** Outside a Worker: when to run the job again */
|
|
301
|
+
retryAfterMs?: number;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* What a completed `background-lane` job returns: nothing left for it to claim
|
|
306
|
+
* (`exhausted`), the background migration stopped (`paused`, `cancelled`,
|
|
307
|
+
* `failed`), no plan yet (`stale`) — or `gave-up` after `maxLaneRetries`
|
|
308
|
+
* failed slices in a row (each counted on its partition, in MongoDB)
|
|
309
|
+
* @experimental New in 2.3
|
|
310
|
+
*/
|
|
311
|
+
export interface BackgroundLaneJobResult {
|
|
312
|
+
kind: 'background-lane';
|
|
313
|
+
migration: string;
|
|
314
|
+
/** Outside a Worker also `retry` (a failed slice, to try again after `retryAfterMs`) */
|
|
315
|
+
outcome: BackgroundSliceResult['outcome'] | 'gave-up' | 'retry';
|
|
316
|
+
counters?: BackgroundCounters;
|
|
317
|
+
code?: MigronautErrorCode;
|
|
318
|
+
/** Outside a Worker (no token to move the job with): when to run it again */
|
|
319
|
+
retryAfterMs?: number;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* What a completed `background-verify` job returns
|
|
324
|
+
* @experimental New in 2.3
|
|
325
|
+
*/
|
|
326
|
+
export interface BackgroundVerifyJobResult extends BackgroundVerifyResult {
|
|
327
|
+
kind: 'background-verify';
|
|
328
|
+
/** Coordinators this tick added (or found alive) — the heal */
|
|
329
|
+
enqueued: number;
|
|
330
|
+
}
|
|
331
|
+
|
|
189
332
|
/**
|
|
190
333
|
* What a completed `up`/`down` job returns
|
|
191
334
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
@@ -202,6 +345,8 @@ export interface MigrationJobResult {
|
|
|
202
345
|
reason?: string;
|
|
203
346
|
/** Time (ms) spent waiting for the MongoDB migration lock */
|
|
204
347
|
lockWaitMs: number;
|
|
348
|
+
/** The coordinators enqueued for what the job registered — on a queue with a background side */
|
|
349
|
+
background?: { migration: string; jobId: string }[];
|
|
205
350
|
}
|
|
206
351
|
|
|
207
352
|
/**
|
|
@@ -223,7 +368,19 @@ export interface SyncJobResult {
|
|
|
223
368
|
* and its file has not changed since — a schedule's circuit breaker. A fix
|
|
224
369
|
* (a changed file) or an explicit `enqueueUp(name)` resumes the line.
|
|
225
370
|
*/
|
|
226
|
-
held?: {
|
|
371
|
+
held?: {
|
|
372
|
+
migration: string;
|
|
373
|
+
reason: string;
|
|
374
|
+
failedAt?: Date;
|
|
375
|
+
};
|
|
376
|
+
/**
|
|
377
|
+
* Present when the next migration waits for background migrations that
|
|
378
|
+
* have not completed (`requires`): the tick enqueued what comes before it,
|
|
379
|
+
* and the line goes on once they complete. Not a failure — `held` is that.
|
|
380
|
+
*/
|
|
381
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
382
|
+
/** The heal of the background side: coordinators added, or found alive */
|
|
383
|
+
background?: { enqueued: number };
|
|
227
384
|
}
|
|
228
385
|
|
|
229
386
|
/**
|
|
@@ -237,6 +394,13 @@ export interface ConvergeJobResult {
|
|
|
237
394
|
inSync: boolean;
|
|
238
395
|
collections: CollectionConvergeResult[];
|
|
239
396
|
unstable?: ConvergeUnstable[];
|
|
397
|
+
/**
|
|
398
|
+
* Atlas Search availability and the declared search indexes still building
|
|
399
|
+
* — when the worker's definitions declare search indexes. Whether the job
|
|
400
|
+
* waits for them is the worker kit's `waitForSearchIndexes`.
|
|
401
|
+
* @experimental New in 2.2
|
|
402
|
+
*/
|
|
403
|
+
search?: ConvergeSearchSummary;
|
|
240
404
|
runId?: string;
|
|
241
405
|
/** Time (ms) spent waiting for the MongoDB migration lock */
|
|
242
406
|
lockWaitMs: number;
|
|
@@ -247,7 +411,8 @@ export interface ConvergeJobResult {
|
|
|
247
411
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
248
412
|
*/
|
|
249
413
|
export interface MigrationJobProgress {
|
|
250
|
-
|
|
414
|
+
/** `search-wait` (New in 2.2): a converge job waiting for search index builds */
|
|
415
|
+
phase: 'lock-wait' | 'running' | 'search-wait' | 'completed' | 'failed';
|
|
251
416
|
migration?: string;
|
|
252
417
|
direction?: 'up' | 'down';
|
|
253
418
|
groupId?: string;
|
|
@@ -256,7 +421,13 @@ export interface MigrationJobProgress {
|
|
|
256
421
|
kind?: 'sync' | 'converge';
|
|
257
422
|
/** `lock-wait` only */
|
|
258
423
|
attempts?: number;
|
|
424
|
+
/** `lock-wait` and `search-wait` */
|
|
259
425
|
waitedMs?: number;
|
|
426
|
+
/**
|
|
427
|
+
* `search-wait` only — how many search indexes the wait is for
|
|
428
|
+
* @experimental New in 2.2
|
|
429
|
+
*/
|
|
430
|
+
searchIndexes?: number;
|
|
260
431
|
/**
|
|
261
432
|
* `failed` only — the typed error code, so nobody has to parse
|
|
262
433
|
* `failedReason`; `'UNKNOWN'` for an error that is not migronaut's.
|
|
@@ -307,6 +478,39 @@ export interface ConvergeJobSpec {
|
|
|
307
478
|
opts: { attempts: 1; deduplication: { id: string }; [option: string]: unknown };
|
|
308
479
|
}
|
|
309
480
|
|
|
481
|
+
/**
|
|
482
|
+
* Per-job options passed through to every background job — retention and
|
|
483
|
+
* logging. Besides what a migration job refuses, the four options of a
|
|
484
|
+
* parent's child-failure policy are refused: the adapter owns them.
|
|
485
|
+
* @experimental New in 2.3
|
|
486
|
+
*/
|
|
487
|
+
export interface BackgroundJobOptions extends MigrationJobOptions {
|
|
488
|
+
failParentOnFailure?: never;
|
|
489
|
+
continueParentOnFailure?: never;
|
|
490
|
+
ignoreDependencyOnFailure?: never;
|
|
491
|
+
removeDependencyOnFailure?: never;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* A coordinator job as handed to `queue.addBulk`
|
|
496
|
+
* @experimental New in 2.3
|
|
497
|
+
*/
|
|
498
|
+
export interface BackgroundJobSpec {
|
|
499
|
+
name: 'background';
|
|
500
|
+
data: BackgroundJobData;
|
|
501
|
+
opts: { attempts: number; deduplication: { id: string }; [option: string]: unknown };
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* A lane job as handed to `queue.addBulk`
|
|
506
|
+
* @experimental New in 2.3
|
|
507
|
+
*/
|
|
508
|
+
export interface BackgroundLaneJobSpec {
|
|
509
|
+
name: 'background-lane';
|
|
510
|
+
data: BackgroundLaneJobData;
|
|
511
|
+
opts: { attempts: 1; [option: string]: unknown };
|
|
512
|
+
}
|
|
513
|
+
|
|
310
514
|
/** A planned, not yet enqueued, group */
|
|
311
515
|
export interface MigrationPlan {
|
|
312
516
|
/** Id of this enqueue call, in the kit's `generateId` format (a UUID by default) */
|
|
@@ -319,6 +523,11 @@ export interface MigrationPlan {
|
|
|
319
523
|
jobs: MigrationJobSpec[];
|
|
320
524
|
/** The converge job that ends an `up` group — see `EnqueueUpOptions.converge` */
|
|
321
525
|
converge?: ConvergeJobSpec;
|
|
526
|
+
/**
|
|
527
|
+
* The first pending file that requires a background migration not
|
|
528
|
+
* completed yet: the plan ends before it (and has no converge job)
|
|
529
|
+
*/
|
|
530
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
322
531
|
}
|
|
323
532
|
|
|
324
533
|
/** {@link parseJobData}'s normalized result */
|
|
@@ -343,6 +552,40 @@ export type ParsedJobData =
|
|
|
343
552
|
*/
|
|
344
553
|
export function parseJobData(job: { id?: string; name: string; data: unknown }): ParsedJobData;
|
|
345
554
|
|
|
555
|
+
/** {@link parseBackgroundJobData}'s normalized result */
|
|
556
|
+
export type ParsedBackgroundJobData =
|
|
557
|
+
| {
|
|
558
|
+
kind: 'background';
|
|
559
|
+
migration: string;
|
|
560
|
+
round?: number;
|
|
561
|
+
spawn?: number;
|
|
562
|
+
takeover?: true;
|
|
563
|
+
requestedBy?: string;
|
|
564
|
+
reason?: string;
|
|
565
|
+
}
|
|
566
|
+
| {
|
|
567
|
+
kind: 'background-lane';
|
|
568
|
+
migration: string;
|
|
569
|
+
registration: string;
|
|
570
|
+
generation: number;
|
|
571
|
+
round: number;
|
|
572
|
+
spawn: number;
|
|
573
|
+
lane: number;
|
|
574
|
+
retry: number;
|
|
575
|
+
}
|
|
576
|
+
| { kind: 'background-verify' };
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Validate a job read back from a background queue and return a normalized
|
|
580
|
+
* copy. Throws `QueueJobInvalidError` for anything outside the contract.
|
|
581
|
+
* @experimental New in 2.3
|
|
582
|
+
*/
|
|
583
|
+
export function parseBackgroundJobData(job: {
|
|
584
|
+
id?: string;
|
|
585
|
+
name: string;
|
|
586
|
+
data: unknown;
|
|
587
|
+
}): ParsedBackgroundJobData;
|
|
588
|
+
|
|
346
589
|
/** The deduplication id a migration's job carries — never contains `:` */
|
|
347
590
|
export function dedupId(direction: 'up' | 'down', migration: string): string;
|
|
348
591
|
|
|
@@ -464,6 +707,8 @@ export interface MigrationGroup {
|
|
|
464
707
|
deduplicated: string[];
|
|
465
708
|
/** The converge job ending the group, or null. `deduplicated`: a peer's identical job */
|
|
466
709
|
converge: { id: string; deduplicated: boolean } | null;
|
|
710
|
+
/** See {@link MigrationPlan.waiting} — the group stops before this file */
|
|
711
|
+
waiting?: { migration: string; waitsFor: string[] };
|
|
467
712
|
/**
|
|
468
713
|
* Resolve when every job has finished — the converge job last; reject with
|
|
469
714
|
* `QueueJobFailedError` at the first one that fails or outlives `timeoutMs`.
|
|
@@ -508,6 +753,17 @@ export type ScheduleOptions = (
|
|
|
508
753
|
id?: string;
|
|
509
754
|
to?: never;
|
|
510
755
|
}
|
|
756
|
+
| {
|
|
757
|
+
/**
|
|
758
|
+
* Each tick runs the drift watch on the background queue (and heals
|
|
759
|
+
* it) — overrides the schedule `startBackgroundWorker()` registers.
|
|
760
|
+
* Needs the `background` option. @experimental
|
|
761
|
+
*/
|
|
762
|
+
job: 'background-verify';
|
|
763
|
+
/** Scheduler id. Default `'migronaut-background-verify'` */
|
|
764
|
+
id?: string;
|
|
765
|
+
to?: never;
|
|
766
|
+
}
|
|
511
767
|
);
|
|
512
768
|
|
|
513
769
|
/** Options for {@link MigrationQueue.startWorker} — passed to the Worker constructor */
|
|
@@ -536,7 +792,13 @@ export interface MigrationJobView {
|
|
|
536
792
|
state: string;
|
|
537
793
|
/** As stored — see {@link MigrationJobProgress} for what the adapter writes */
|
|
538
794
|
progress: MigrationJobProgress | number;
|
|
539
|
-
returnvalue?:
|
|
795
|
+
returnvalue?:
|
|
796
|
+
| MigrationJobResult
|
|
797
|
+
| SyncJobResult
|
|
798
|
+
| ConvergeJobResult
|
|
799
|
+
| BackgroundJobResult
|
|
800
|
+
| BackgroundLaneJobResult
|
|
801
|
+
| BackgroundVerifyJobResult;
|
|
540
802
|
failedReason?: string;
|
|
541
803
|
attemptsMade: number;
|
|
542
804
|
timestamp?: number;
|
|
@@ -578,6 +840,16 @@ export interface CreateMigrationProcessorOptions {
|
|
|
578
840
|
jobOptions?: MigrationJobOptions;
|
|
579
841
|
/** What a job may ask for beyond the ordinary — see {@link MigrationJobPermissions} */
|
|
580
842
|
allow?: MigrationJobPermissions;
|
|
843
|
+
/**
|
|
844
|
+
* The background queue: what an `up` (or `down`) job registers gets its
|
|
845
|
+
* coordinator there at once, and every `sync` tick heals it. @experimental
|
|
846
|
+
*/
|
|
847
|
+
background?: {
|
|
848
|
+
queue: BullMQQueueLike;
|
|
849
|
+
jobOptions?: BackgroundJobOptions;
|
|
850
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
851
|
+
stallMs?: number;
|
|
852
|
+
};
|
|
581
853
|
}
|
|
582
854
|
|
|
583
855
|
/**
|
|
@@ -617,6 +889,62 @@ export function createMigrationProcessor(
|
|
|
617
889
|
options?: CreateMigrationProcessorOptions,
|
|
618
890
|
): MigrationProcessor;
|
|
619
891
|
|
|
892
|
+
/** Options for {@link createBackgroundProcessor} */
|
|
893
|
+
export interface CreateBackgroundProcessorOptions {
|
|
894
|
+
config?: Partial<MigronautConfig>;
|
|
895
|
+
kitOptions?: MigratorKitOptions;
|
|
896
|
+
kit?: MigratorKit;
|
|
897
|
+
/** The background queue — the coordinators add their lanes to it. Required */
|
|
898
|
+
queue: BullMQQueueLike;
|
|
899
|
+
jobOptions?: BackgroundJobOptions;
|
|
900
|
+
/** A lane's slice (ms). Default: each background migration's own `sliceMs` */
|
|
901
|
+
sliceMs?: number;
|
|
902
|
+
/**
|
|
903
|
+
* `'auto'` (default): lanes are children of their coordinator, which waits
|
|
904
|
+
* for them (`moveToWaitingChildren`) when the queue and its jobs support
|
|
905
|
+
* it. `false`: lanes on their own, and the coordinator polls MongoDB.
|
|
906
|
+
*/
|
|
907
|
+
children?: 'auto' | false;
|
|
908
|
+
/** How often a coordinator without children looks again (ms). Default 5000 */
|
|
909
|
+
pollIntervalMs?: number;
|
|
910
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
911
|
+
stallMs?: number;
|
|
912
|
+
/** Failed slices in a row before a lane gives up (0–100). Default 8 */
|
|
913
|
+
maxLaneRetries?: number;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/**
|
|
917
|
+
* The function a Worker on the background queue runs. Jobs run side by side.
|
|
918
|
+
* @experimental New in 2.3
|
|
919
|
+
*/
|
|
920
|
+
export interface BackgroundProcessor {
|
|
921
|
+
(
|
|
922
|
+
job: BullMQJobLike<BackgroundJobData | BackgroundLaneJobData | BackgroundVerifyJobData>,
|
|
923
|
+
token?: string,
|
|
924
|
+
signal?: AbortSignal,
|
|
925
|
+
): Promise<BackgroundJobResult | BackgroundLaneJobResult | BackgroundVerifyJobResult>;
|
|
926
|
+
readonly kit: MigratorKit;
|
|
927
|
+
/**
|
|
928
|
+
* Stop: a lane stops at its next batch, checkpoints, releases its lease and
|
|
929
|
+
* goes back to the queue (moved to delayed); a coordinator bows out and
|
|
930
|
+
* comes back. Irreversible.
|
|
931
|
+
*/
|
|
932
|
+
shutdown(reason?: string): void;
|
|
933
|
+
/** `shutdown()`, let the jobs in flight settle, disconnect a kit the processor created */
|
|
934
|
+
close(): Promise<void>;
|
|
935
|
+
/** A coordinator for every background migration with work to do — what a worker does at start */
|
|
936
|
+
heal(): Promise<BackgroundEnqueueResult>;
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
/**
|
|
940
|
+
* Build the processor for a Worker on the background queue you construct
|
|
941
|
+
* yourself.
|
|
942
|
+
* @experimental New in 2.3
|
|
943
|
+
*/
|
|
944
|
+
export function createBackgroundProcessor(
|
|
945
|
+
options: CreateBackgroundProcessorOptions,
|
|
946
|
+
): BackgroundProcessor;
|
|
947
|
+
|
|
620
948
|
// ─── Producer building blocks ──────────────────────────────────────────────────
|
|
621
949
|
|
|
622
950
|
/** Plan an `up` group without enqueuing it */
|
|
@@ -665,6 +993,39 @@ export function enqueueConverge(
|
|
|
665
993
|
},
|
|
666
994
|
): Promise<ConvergeHandle>;
|
|
667
995
|
|
|
996
|
+
/** Options for `enqueueBackground` */
|
|
997
|
+
export interface EnqueueBackgroundOptions {
|
|
998
|
+
/**
|
|
999
|
+
* A running background migration nothing has moved for this long (ms) — no
|
|
1000
|
+
* live lease, no checkpoint, no coordinator step — also gets a takeover
|
|
1001
|
+
* coordinator, whose newer round retires the stuck one. Default 900000
|
|
1002
|
+
* (15 minutes); at least 1000.
|
|
1003
|
+
*/
|
|
1004
|
+
stallMs?: number;
|
|
1005
|
+
requestedBy?: string;
|
|
1006
|
+
reason?: string;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
/**
|
|
1010
|
+
* What `enqueueBackground` resolves with
|
|
1011
|
+
* @experimental New in 2.3
|
|
1012
|
+
*/
|
|
1013
|
+
export interface BackgroundEnqueueResult {
|
|
1014
|
+
/** One per coordinator added — or absorbed by the one already alive (same id) */
|
|
1015
|
+
jobs: { migration: string; id: string; takeover?: true }[];
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
/**
|
|
1019
|
+
* Enqueue the coordinator of one background migration (`migration`) — or of
|
|
1020
|
+
* every one with work to do — on a background queue you own. Idempotent.
|
|
1021
|
+
* @experimental New in 2.3
|
|
1022
|
+
*/
|
|
1023
|
+
export function enqueueBackground(
|
|
1024
|
+
queue: BullMQQueueLike,
|
|
1025
|
+
kit: MigratorKit,
|
|
1026
|
+
options?: EnqueueBackgroundOptions & { migration?: string; jobOptions?: BackgroundJobOptions },
|
|
1027
|
+
): Promise<BackgroundEnqueueResult>;
|
|
1028
|
+
|
|
668
1029
|
/** Wait for a group's jobs — what `MigrationGroup.wait()` calls */
|
|
669
1030
|
export function waitForGroup(options: {
|
|
670
1031
|
queue: BullMQQueueLike;
|
|
@@ -747,6 +1108,48 @@ export interface CreateMigrationQueueOptions<
|
|
|
747
1108
|
* refuse fails at the call. Give every producer and worker the same policy.
|
|
748
1109
|
*/
|
|
749
1110
|
allow?: MigrationJobPermissions;
|
|
1111
|
+
/**
|
|
1112
|
+
* Background migrations on a queue of their own (`<queueName>-background`):
|
|
1113
|
+
* a coordinator job each, with lanes as its children. `true` takes every
|
|
1114
|
+
* default. @experimental New in 2.3
|
|
1115
|
+
*/
|
|
1116
|
+
background?: boolean | BackgroundQueueOptions;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/**
|
|
1120
|
+
* The `background` option of {@link createMigrationQueue}
|
|
1121
|
+
* @experimental New in 2.3
|
|
1122
|
+
*/
|
|
1123
|
+
export interface BackgroundQueueOptions {
|
|
1124
|
+
/** Default `<queueName>-background` (or the injected queue's name) */
|
|
1125
|
+
queueName?: string;
|
|
1126
|
+
/** A background Queue instance you own — never closed by `close()` */
|
|
1127
|
+
queue?: BullMQQueueLike;
|
|
1128
|
+
jobOptions?: BackgroundJobOptions;
|
|
1129
|
+
/** Defaults for `startBackgroundWorker()` — concurrency 2 unless given */
|
|
1130
|
+
workerOptions?: { concurrency?: number; [option: string]: unknown };
|
|
1131
|
+
/** See {@link CreateBackgroundProcessorOptions} */
|
|
1132
|
+
sliceMs?: number;
|
|
1133
|
+
children?: 'auto' | false;
|
|
1134
|
+
pollIntervalMs?: number;
|
|
1135
|
+
/** See {@link EnqueueBackgroundOptions.stallMs} */
|
|
1136
|
+
stallMs?: number;
|
|
1137
|
+
/**
|
|
1138
|
+
* The drift watch's schedule, registered by `startBackgroundWorker()` (ms,
|
|
1139
|
+
* ≥ 1000). Default 600000 (10 minutes) — registered only when no schedule
|
|
1140
|
+
* exists yet, so one set with `schedule({ job: 'background-verify' })`
|
|
1141
|
+
* stays; given explicitly, it is re-registered at every start. `false`
|
|
1142
|
+
* registers none.
|
|
1143
|
+
*/
|
|
1144
|
+
verifyIntervalMs?: number | false;
|
|
1145
|
+
/** See {@link CreateBackgroundProcessorOptions.maxLaneRetries} */
|
|
1146
|
+
maxLaneRetries?: number;
|
|
1147
|
+
/**
|
|
1148
|
+
* Host the live drift watcher in the background worker's process — `true`,
|
|
1149
|
+
* or its options. Default: when the kit's `backgroundDrift` is `'stream'`
|
|
1150
|
+
* or `'both'`. Closed first by `close()`.
|
|
1151
|
+
*/
|
|
1152
|
+
watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
|
|
750
1153
|
}
|
|
751
1154
|
|
|
752
1155
|
/**
|
|
@@ -772,6 +1175,26 @@ export class MigrationQueue<
|
|
|
772
1175
|
readonly queueName: string;
|
|
773
1176
|
/** The processor, for attaching to a Worker you construct yourself */
|
|
774
1177
|
readonly processor: MigrationProcessor;
|
|
1178
|
+
/**
|
|
1179
|
+
* The background queue (`background` option), if any
|
|
1180
|
+
* @experimental New in 2.3
|
|
1181
|
+
*/
|
|
1182
|
+
readonly backgroundQueue: BullMQQueueLike | undefined;
|
|
1183
|
+
/**
|
|
1184
|
+
* The worker started by {@link startBackgroundWorker}, if any
|
|
1185
|
+
* @experimental New in 2.3
|
|
1186
|
+
*/
|
|
1187
|
+
readonly backgroundWorker: W | undefined;
|
|
1188
|
+
/**
|
|
1189
|
+
* The background processor, for a Worker you construct yourself
|
|
1190
|
+
* @experimental New in 2.3
|
|
1191
|
+
*/
|
|
1192
|
+
readonly backgroundProcessor: BackgroundProcessor | undefined;
|
|
1193
|
+
/**
|
|
1194
|
+
* The live drift watcher `startBackgroundWorker()` started, if any
|
|
1195
|
+
* @experimental New in 2.3
|
|
1196
|
+
*/
|
|
1197
|
+
readonly backgroundWatcher: BackgroundWatcher | undefined;
|
|
775
1198
|
|
|
776
1199
|
/**
|
|
777
1200
|
* Enqueue pending migrations — all, up to `options.to`, or the one
|
|
@@ -791,6 +1214,33 @@ export class MigrationQueue<
|
|
|
791
1214
|
*/
|
|
792
1215
|
enqueueConverge(options?: EnqueueConvergeOptions): Promise<ConvergeHandle>;
|
|
793
1216
|
|
|
1217
|
+
/**
|
|
1218
|
+
* Enqueue the coordinator of one background migration — or of every one
|
|
1219
|
+
* with work to do. Idempotent. Needs the `background` option. @experimental
|
|
1220
|
+
*/
|
|
1221
|
+
enqueueBackground(
|
|
1222
|
+
name?: string,
|
|
1223
|
+
options?: EnqueueBackgroundOptions,
|
|
1224
|
+
): Promise<BackgroundEnqueueResult>;
|
|
1225
|
+
/**
|
|
1226
|
+
* A background migration's status, read from MongoDB — `null` when not registered
|
|
1227
|
+
* @experimental New in 2.3
|
|
1228
|
+
*/
|
|
1229
|
+
backgroundStatus(name: string): Promise<BackgroundStatus | null>;
|
|
1230
|
+
/**
|
|
1231
|
+
* Every background migration's status
|
|
1232
|
+
* @experimental New in 2.3
|
|
1233
|
+
*/
|
|
1234
|
+
backgroundStatus(): Promise<BackgroundStatus[]>;
|
|
1235
|
+
/**
|
|
1236
|
+
* The drift watch, now — and, with the `background` option, a coordinator
|
|
1237
|
+
* for whatever it reopened. @experimental
|
|
1238
|
+
*/
|
|
1239
|
+
verifyBackground(options?: {
|
|
1240
|
+
onDrift?: 'reopen' | 'report';
|
|
1241
|
+
collections?: string[];
|
|
1242
|
+
}): Promise<BackgroundVerifyResult>;
|
|
1243
|
+
|
|
794
1244
|
/** Full migration status, read from MongoDB */
|
|
795
1245
|
status(): Promise<StatusRow[]>;
|
|
796
1246
|
/** Migrations not applied yet */
|
|
@@ -806,6 +1256,13 @@ export class MigrationQueue<
|
|
|
806
1256
|
* that failed, tries again.
|
|
807
1257
|
*/
|
|
808
1258
|
startWorker(options?: StartWorkerOptions): Promise<W>;
|
|
1259
|
+
/**
|
|
1260
|
+
* Start the background worker (concurrency 2 by default): coordinators and
|
|
1261
|
+
* lanes side by side. Registers the drift watch's schedule and heals — a
|
|
1262
|
+
* coordinator for every background migration with work to do. Needs the
|
|
1263
|
+
* `background` option. @experimental
|
|
1264
|
+
*/
|
|
1265
|
+
startBackgroundWorker(options?: { concurrency?: number; [option: string]: unknown }): Promise<W>;
|
|
809
1266
|
/** Stop workers from picking up new jobs; the job in flight finishes */
|
|
810
1267
|
pause(): Promise<void>;
|
|
811
1268
|
resume(): Promise<void>;
|
|
@@ -820,7 +1277,8 @@ export class MigrationQueue<
|
|
|
820
1277
|
schedule(options: ScheduleOptions): Promise<void>;
|
|
821
1278
|
/**
|
|
822
1279
|
* Remove a schedule — the sync one by default; pass
|
|
823
|
-
* {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another.
|
|
1280
|
+
* {@link DEFAULT_CONVERGE_SCHEDULER_ID} (or your own id) for another. The
|
|
1281
|
+
* background queue's schedules (the drift watch's) are looked for too.
|
|
824
1282
|
* Resolves whether one existed.
|
|
825
1283
|
*/
|
|
826
1284
|
unschedule(id?: string): Promise<boolean>;
|