@alexify/migronaut 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +41 -3
- package/bullmq.d.ts +484 -8
- package/index.d.ts +1264 -9
- package/migronaut.schema.json +93 -1
- package/package.json +9 -2
- package/src/bullmq/background-processor.js +541 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +348 -21
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +484 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +375 -0
- package/src/core/background-engine.js +849 -0
- package/src/core/background-kit.js +432 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +610 -0
- package/src/core/background.js +1127 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migration-logger.js +279 -0
- package/src/core/migrator.js +1027 -22
- package/src/core/options.js +36 -0
- package/src/core/run-recorder.js +6 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +34 -8
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/job-ref.js +44 -0
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/redact.js +140 -3
- package/src/utils/telemetry.js +110 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/index.d.ts
CHANGED
|
@@ -35,12 +35,69 @@ export interface MigrationContext {
|
|
|
35
35
|
* migronaut cannot interrupt a running function by itself.
|
|
36
36
|
*/
|
|
37
37
|
signal?: AbortSignal;
|
|
38
|
+
/**
|
|
39
|
+
* The kit's logger, with this run's correlation ({@link run}) bound into the
|
|
40
|
+
* fields of every line. A call whose fields hold `userland: true` is also
|
|
41
|
+
* emitted as the `migration:log` event, for the application to store and show
|
|
42
|
+
* its users — migronaut stores none of it:
|
|
43
|
+
*
|
|
44
|
+
* ```js
|
|
45
|
+
* logger.info('batch done', { userland: true, processed: 1000 });
|
|
46
|
+
* ```
|
|
47
|
+
*
|
|
48
|
+
* Always present when migronaut runs the migration; optional so a context
|
|
49
|
+
* built by hand (in a test) still type-checks.
|
|
50
|
+
* @experimental New in 2.4
|
|
51
|
+
*/
|
|
52
|
+
logger?: MigrationLogger;
|
|
53
|
+
/**
|
|
54
|
+
* Who this is: the run id, the migration, the direction, the transaction
|
|
55
|
+
* attempt and, when the caller named them, the queue job and the actor.
|
|
56
|
+
* Frozen; the same values `logger` binds and `migration:log` carries.
|
|
57
|
+
* Always present when migronaut runs the migration.
|
|
58
|
+
* @experimental New in 2.4
|
|
59
|
+
*/
|
|
60
|
+
run?: MigrationRunInfo;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The correlation of a run, as `ctx.run`. `migration`, `batch` and `attempt`
|
|
65
|
+
* describe one migration and are absent in `beforeAll`/`afterAll`.
|
|
66
|
+
* @experimental New in 2.4
|
|
67
|
+
*/
|
|
68
|
+
export interface MigrationRunInfo {
|
|
69
|
+
/** The run id — the same on the lock, the changelog record and every event of the run */
|
|
70
|
+
readonly id: string;
|
|
71
|
+
readonly direction: 'up' | 'down';
|
|
72
|
+
/** The migration file */
|
|
73
|
+
readonly migration?: string;
|
|
74
|
+
/** The changelog batch (`up` only) */
|
|
75
|
+
readonly batch?: number;
|
|
76
|
+
/**
|
|
77
|
+
* 1 — or more when a transaction was retried and the body runs again. Not a
|
|
78
|
+
* queue retry: a migration job is never retried.
|
|
79
|
+
*/
|
|
80
|
+
readonly attempt?: number;
|
|
81
|
+
/** The queue job that runs this migration (set by the BullMQ adapter, or the `job` option) */
|
|
82
|
+
readonly jobId?: string;
|
|
83
|
+
/** The queue group the job belongs to */
|
|
84
|
+
readonly groupId?: string;
|
|
85
|
+
/** Who asked for the run (`requestedBy` option) */
|
|
86
|
+
readonly requestedBy?: string;
|
|
87
|
+
/** Why (`reason` option) */
|
|
88
|
+
readonly reason?: string;
|
|
38
89
|
}
|
|
39
90
|
|
|
40
91
|
/** Shape of an imported migration file module */
|
|
41
92
|
export interface MigrationModule {
|
|
42
93
|
up: (ctx: MigrationContext) => Promise<void>;
|
|
43
94
|
down: (ctx: MigrationContext) => Promise<void>;
|
|
95
|
+
/**
|
|
96
|
+
* Background migrations (file names, each sorting before this file) that
|
|
97
|
+
* must have completed before this migration runs.
|
|
98
|
+
* @experimental New in 2.3
|
|
99
|
+
*/
|
|
100
|
+
requires?: readonly string[];
|
|
44
101
|
/** If true, wraps this migration in a MongoDB session + transaction */
|
|
45
102
|
useTransaction?: boolean;
|
|
46
103
|
/** Overrides `MigronautConfig.timeoutMs` for this migration only */
|
|
@@ -49,6 +106,261 @@ export interface MigrationModule {
|
|
|
49
106
|
description?: string;
|
|
50
107
|
}
|
|
51
108
|
|
|
109
|
+
// ─── Document shapes and background migrations ───────────────────────────────
|
|
110
|
+
|
|
111
|
+
/** The system field names of a versioned collection — `revisionField` is `null` without revisions */
|
|
112
|
+
export interface ShapeFieldNames {
|
|
113
|
+
field: string;
|
|
114
|
+
revisionField: string | null;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The default system field names: `__v` and `__rev` */
|
|
118
|
+
export interface DefaultShapeFieldNames {
|
|
119
|
+
field: '__v';
|
|
120
|
+
revisionField: '__rev';
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* A document type without its system fields (the version and the revision),
|
|
125
|
+
* distributed over a union. What a shape body is declared as, and what a
|
|
126
|
+
* background transformation returns: migronaut writes the system fields.
|
|
127
|
+
* @experimental New in 2.3
|
|
128
|
+
*/
|
|
129
|
+
export type Body<T, N extends ShapeFieldNames = DefaultShapeFieldNames> = T extends unknown
|
|
130
|
+
? Omit<T, N['field'] | Extract<N['revisionField'], string>>
|
|
131
|
+
: never;
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* What a background migration's transformation gets besides the document.
|
|
135
|
+
* `session`, `db` and `client` are there only in a `transaction` background
|
|
136
|
+
* migration — writes to other collections must pass `session` to commit
|
|
137
|
+
* with the batch.
|
|
138
|
+
* @experimental New in 2.3
|
|
139
|
+
*/
|
|
140
|
+
export interface BackgroundMigrationContext {
|
|
141
|
+
/** Aborted when the slice is stopping (lease lost, pause, shutdown) */
|
|
142
|
+
signal: AbortSignal;
|
|
143
|
+
/**
|
|
144
|
+
* The kit's logger with {@link background} bound into every line. Fields
|
|
145
|
+
* with `userland: true` also emit `migration:log` (`kind: 'background'`) —
|
|
146
|
+
* but `migrate` runs once per document, and again for a document a
|
|
147
|
+
* concurrent write moved: log from `migrateBatch` or a `step` rather than
|
|
148
|
+
* per document. In a dry run the lines say `dryRun: true` and nothing is
|
|
149
|
+
* emitted.
|
|
150
|
+
*/
|
|
151
|
+
logger: MigrationLogger;
|
|
152
|
+
direction: 'forward' | 'revert';
|
|
153
|
+
/** Where this runs — frozen */
|
|
154
|
+
background: BackgroundRunInfo;
|
|
155
|
+
session?: ClientSession;
|
|
156
|
+
db?: Db;
|
|
157
|
+
client?: MongoClient;
|
|
158
|
+
/** True in a dry run — the writes are rolled back */
|
|
159
|
+
dryRun?: boolean;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* `ctx.background`: which background migration, generation and partition —
|
|
164
|
+
* and the lane, its queue job and the transaction attempt, as its log lines
|
|
165
|
+
* and `migration:log` events carry them.
|
|
166
|
+
*/
|
|
167
|
+
export interface BackgroundRunInfo {
|
|
168
|
+
readonly name: string;
|
|
169
|
+
/** The plan generation — absent when the live drift watcher runs the transformation */
|
|
170
|
+
readonly generation?: number;
|
|
171
|
+
/** The partition (`''` for the drift watcher, `'dry-run'` in a dry run) */
|
|
172
|
+
readonly partition: string;
|
|
173
|
+
/**
|
|
174
|
+
* The lane's run id — its lease's owner, the runId of its `background:*`
|
|
175
|
+
* events. Absent in a dry run.
|
|
176
|
+
* @experimental New in 2.4
|
|
177
|
+
*/
|
|
178
|
+
readonly runId?: string;
|
|
179
|
+
/**
|
|
180
|
+
* The queue job working the lane — a background queue's lane job, or the
|
|
181
|
+
* migration job whose run drives it inline (`backgroundInline`)
|
|
182
|
+
* @experimental New in 2.4
|
|
183
|
+
*/
|
|
184
|
+
readonly jobId?: string;
|
|
185
|
+
/**
|
|
186
|
+
* The group of that migration job — only when a run drives it inline; a
|
|
187
|
+
* background queue's lanes have none
|
|
188
|
+
* @experimental New in 2.4
|
|
189
|
+
*/
|
|
190
|
+
readonly groupId?: string;
|
|
191
|
+
/**
|
|
192
|
+
* 1 — or more when a transactional batch or step runs again in a new
|
|
193
|
+
* transaction (a transient error, a conflict, a smaller batch)
|
|
194
|
+
* @experimental New in 2.4
|
|
195
|
+
*/
|
|
196
|
+
readonly attempt: number;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** How a background migration splits its collection into partitions */
|
|
200
|
+
export interface BackgroundPartitionSettings {
|
|
201
|
+
/** Partitions per lane (default 4), so a slow partition does not hold the pass */
|
|
202
|
+
overPartition?: number;
|
|
203
|
+
/** Default 256 */
|
|
204
|
+
maxPartitions?: number;
|
|
205
|
+
/** No partition is planned smaller than this (default 4 × batchSize) */
|
|
206
|
+
minPartitionDocs?: number;
|
|
207
|
+
/** Ids sampled to place the boundaries (default min(10 000, 100 × partitions)) */
|
|
208
|
+
sampleSize?: number;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** A transactional background migration's budget */
|
|
212
|
+
export interface BackgroundTransactionSettings {
|
|
213
|
+
/** Per batch, ≤ 50 000 (default 10 000) */
|
|
214
|
+
timeoutMs?: number;
|
|
215
|
+
/** Retries of a batch on a transient transaction error (default 5) */
|
|
216
|
+
maxRetries?: number;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** The latency-driven throttle (AIMD) */
|
|
220
|
+
export interface BackgroundAdaptiveSettings {
|
|
221
|
+
/** A batch write slower than this halves the batch (default 500) */
|
|
222
|
+
targetLatencyMs?: number;
|
|
223
|
+
/** Default 10 */
|
|
224
|
+
minBatchSize?: number;
|
|
225
|
+
/** Never above `batchSize` (the default) */
|
|
226
|
+
maxBatchSize?: number;
|
|
227
|
+
/** Default 30 000 */
|
|
228
|
+
maxPauseMs?: number;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** What a `throttle` hook is told before every batch */
|
|
232
|
+
export interface BackgroundThrottleContext {
|
|
233
|
+
name: string;
|
|
234
|
+
collection?: string;
|
|
235
|
+
generation: number;
|
|
236
|
+
partition: string;
|
|
237
|
+
batchSize: number;
|
|
238
|
+
signal: AbortSignal;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Settings every background migration may carry, with their defaults.
|
|
243
|
+
* @experimental New in 2.3
|
|
244
|
+
*/
|
|
245
|
+
export interface BackgroundMigrationSettings {
|
|
246
|
+
description?: string;
|
|
247
|
+
/** Documents per batch: 500 (100 with `transaction`) */
|
|
248
|
+
batchSize?: number;
|
|
249
|
+
/** Pause between batches: 100 ms */
|
|
250
|
+
pauseMs?: number;
|
|
251
|
+
/** How long a lane holds a partition before it yields: 30 000 ms */
|
|
252
|
+
sliceMs?: number;
|
|
253
|
+
/** Every batch write's, transactions included — default `{ w: 'majority' }` */
|
|
254
|
+
writeConcern?: { w?: number | 'majority'; j?: boolean; wtimeoutMS?: number };
|
|
255
|
+
/** Documents that may fail before the background migration does: 0 (at most 1000) */
|
|
256
|
+
maxDocumentErrors?: number;
|
|
257
|
+
/** Passes over the remaining old-shape documents before giving up: 10 */
|
|
258
|
+
maxPasses?: number;
|
|
259
|
+
/** Re-read rounds for documents a concurrent write changed under a batch: 3 */
|
|
260
|
+
maxConflictRetries?: number;
|
|
261
|
+
/** Failed slices of one partition in a row (no checkpoint between) before it fails: 3 */
|
|
262
|
+
maxSliceFailures?: number;
|
|
263
|
+
/** Wait while a secondary lags more than this: 10 000 ms (`false`: never) */
|
|
264
|
+
maxReplicationLagMs?: number | false;
|
|
265
|
+
/** Called before every batch; a number it returns is an extra pause (ms) */
|
|
266
|
+
throttle?(ctx: BackgroundThrottleContext): number | void | Promise<number | void>;
|
|
267
|
+
/** Partitions processed at once, across every process: 1 (at most 64) */
|
|
268
|
+
maxParallel?: number;
|
|
269
|
+
partitions?: BackgroundPartitionSettings;
|
|
270
|
+
/** Batch and checkpoint in one transaction (needs a replica set or mongos): false */
|
|
271
|
+
transaction?: boolean | BackgroundTransactionSettings;
|
|
272
|
+
/** The latency-driven throttle: true */
|
|
273
|
+
adaptive?: boolean | BackgroundAdaptiveSettings;
|
|
274
|
+
/** Lanes per shard on a sharded collection: 1 */
|
|
275
|
+
shardConcurrency?: number;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* A declarative background migration: every document of `collection` at
|
|
280
|
+
* version `from` (and matching `filter`) rewritten to version `to` by
|
|
281
|
+
* `migrate` (or `migrateBatch`), in partitions, behind an optimistic guard.
|
|
282
|
+
* `From` and `To` type the documents — see `BackgroundMigrationFor` in
|
|
283
|
+
* `@alexify/migronaut/versioning` for the shape-map form.
|
|
284
|
+
*
|
|
285
|
+
* The callbacks are declared as methods, so a transformation typed for the
|
|
286
|
+
* stored document (with its version) or for its body fits either way.
|
|
287
|
+
* @experimental New in 2.3
|
|
288
|
+
*/
|
|
289
|
+
export interface DeclarativeBackgroundMigration<
|
|
290
|
+
From extends object = Record<string, any>,
|
|
291
|
+
To extends object = Record<string, any>,
|
|
292
|
+
> extends BackgroundMigrationSettings {
|
|
293
|
+
collection: string;
|
|
294
|
+
/** The version rewritten — 0 for documents without a version field */
|
|
295
|
+
from: number;
|
|
296
|
+
to: number;
|
|
297
|
+
filter?: Record<string, unknown>;
|
|
298
|
+
/** The new document for one old one; the engine sets the version and bumps the revision */
|
|
299
|
+
migrate?(doc: From, ctx: BackgroundMigrationContext): To | Promise<To>;
|
|
300
|
+
/** The new documents for a batch, aligned — an `Error` fails just that document */
|
|
301
|
+
migrateBatch?(docs: From[], ctx: BackgroundMigrationContext): (To | Error)[] | Promise<(To | Error)[]>;
|
|
302
|
+
/** The way back, for `down` */
|
|
303
|
+
revert?(doc: To, ctx: BackgroundMigrationContext): From | Promise<From>;
|
|
304
|
+
revertBatch?(docs: To[], ctx: BackgroundMigrationContext): (From | Error)[] | Promise<(From | Error)[]>;
|
|
305
|
+
/** Default: the collection's `versioning.field`, else `'__v'` */
|
|
306
|
+
versionField?: string;
|
|
307
|
+
/** Default: the collection's `versioning.revisionField`, else `'__rev'` */
|
|
308
|
+
revisionField?: string;
|
|
309
|
+
/**
|
|
310
|
+
* `'revision'` (default) guards each write with the revision; a collection
|
|
311
|
+
* without revisions must say `'version-only'` — a concurrent write that
|
|
312
|
+
* leaves the version alone is then invisible to it.
|
|
313
|
+
*/
|
|
314
|
+
occ?: 'revision' | 'version-only';
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** What a `step` background migration gets */
|
|
318
|
+
export interface BackgroundStepContext extends BackgroundMigrationContext {
|
|
319
|
+
db: Db;
|
|
320
|
+
client: MongoClient;
|
|
321
|
+
/** What the previous step returned (`null` at the start) */
|
|
322
|
+
checkpoint: unknown;
|
|
323
|
+
/** Epoch ms the step should return by — the slice ends then */
|
|
324
|
+
deadline: number;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** What a `step` returns */
|
|
328
|
+
export interface BackgroundStepResult {
|
|
329
|
+
/** Saved (≤ 64 KiB of BSON) and handed to the next step */
|
|
330
|
+
checkpoint: unknown;
|
|
331
|
+
/** True once there is nothing left */
|
|
332
|
+
done: boolean;
|
|
333
|
+
processed?: number;
|
|
334
|
+
migrated?: number;
|
|
335
|
+
/** For progress, when known */
|
|
336
|
+
total?: number;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* A free-form background migration — the escape hatch: migronaut runs `step`
|
|
341
|
+
* again and again with its last checkpoint until it says `done`, owning the
|
|
342
|
+
* lease, the slices, the throttle and the controls. One partition only; the
|
|
343
|
+
* writes must be idempotent.
|
|
344
|
+
* @experimental New in 2.3
|
|
345
|
+
*/
|
|
346
|
+
export interface StepBackgroundMigration extends BackgroundMigrationSettings {
|
|
347
|
+
/** Shown in status — the collection it works on, if one */
|
|
348
|
+
collection?: string;
|
|
349
|
+
step(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
|
|
350
|
+
revertStep?(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/** A background migration, as a migration file exports it: `export const background = {…}` */
|
|
354
|
+
export type BackgroundMigration = DeclarativeBackgroundMigration | StepBackgroundMigration;
|
|
355
|
+
|
|
356
|
+
/** Shape of a background migration file module — no `up`/`down` */
|
|
357
|
+
export interface BackgroundMigrationModule {
|
|
358
|
+
background: BackgroundMigration;
|
|
359
|
+
/** Background migrations that must complete first (each sorting before this file) */
|
|
360
|
+
requires?: readonly string[];
|
|
361
|
+
description?: string;
|
|
362
|
+
}
|
|
363
|
+
|
|
52
364
|
// ─── Changelog ────────────────────────────────────────────────────────────────
|
|
53
365
|
|
|
54
366
|
export type MigrationStatus = 'applied' | 'reverted' | 'failed';
|
|
@@ -93,6 +405,12 @@ export interface MigrationRecord {
|
|
|
93
405
|
* `migronaut import` — these are not reversible by migronaut. Absent for native records.
|
|
94
406
|
*/
|
|
95
407
|
origin?: MigrationOrigin;
|
|
408
|
+
/**
|
|
409
|
+
* `'background'` for a background migration file — applying it registered
|
|
410
|
+
* the background migration; its documents are rewritten later.
|
|
411
|
+
* @experimental New in 2.3
|
|
412
|
+
*/
|
|
413
|
+
kind?: 'background';
|
|
96
414
|
}
|
|
97
415
|
|
|
98
416
|
// ─── Config ───────────────────────────────────────────────────────────────────
|
|
@@ -316,6 +634,37 @@ export interface MigronautConfig {
|
|
|
316
634
|
* @experimental New in 2.2
|
|
317
635
|
*/
|
|
318
636
|
searchIndexWaitTimeoutMs?: number;
|
|
637
|
+
/**
|
|
638
|
+
* Where background migrations keep their state — and, named after it,
|
|
639
|
+
* their partitions (`<name>_partitions`) and the drift watcher's resume
|
|
640
|
+
* tokens (`<name>_watch`). Default `'_migronaut_background'`.
|
|
641
|
+
* @experimental New in 2.3
|
|
642
|
+
*/
|
|
643
|
+
backgroundCollection?: string;
|
|
644
|
+
/**
|
|
645
|
+
* Run a background migration to the end inside the `up` that registers it,
|
|
646
|
+
* under the migration lock — for small collections and tests. Default false.
|
|
647
|
+
* @experimental New in 2.3
|
|
648
|
+
*/
|
|
649
|
+
backgroundInline?: boolean;
|
|
650
|
+
/**
|
|
651
|
+
* What the drift watch does with old-shape documents that appear after a
|
|
652
|
+
* background migration completed: `'reopen'` it (default) or only `'report'`.
|
|
653
|
+
* @experimental New in 2.3
|
|
654
|
+
*/
|
|
655
|
+
backgroundOnDrift?: 'reopen' | 'report';
|
|
656
|
+
/**
|
|
657
|
+
* How drift is watched: `'poll'` (default — a check every 10 minutes),
|
|
658
|
+
* `'stream'` (change streams, the check as a backstop) or `'both'`.
|
|
659
|
+
* @experimental New in 2.3
|
|
660
|
+
*/
|
|
661
|
+
backgroundDrift?: 'poll' | 'stream' | 'both';
|
|
662
|
+
/**
|
|
663
|
+
* Partition a sharded collection by its shard key and target each write at
|
|
664
|
+
* one shard (`'auto'`, default), or treat it like any other (`'off'`).
|
|
665
|
+
* @experimental New in 2.3
|
|
666
|
+
*/
|
|
667
|
+
backgroundShardAware?: 'auto' | 'off';
|
|
319
668
|
/** Mongoose instance — required only if your migrations use Mongoose models */
|
|
320
669
|
mongoose?: MongooseLike;
|
|
321
670
|
hooks?: MigrationHooks;
|
|
@@ -496,25 +845,65 @@ export interface CollectionDefinition {
|
|
|
496
845
|
* Every index besides `_id`. Undeclared live indexes are kept (and reported)
|
|
497
846
|
* unless `prune` is on.
|
|
498
847
|
*/
|
|
499
|
-
indexes?: IndexDefinition[];
|
|
848
|
+
indexes?: readonly IndexDefinition[];
|
|
500
849
|
/**
|
|
501
850
|
* Atlas Search and Vector Search indexes (Atlas, an Atlas CLI local
|
|
502
851
|
* deployment, or MongoDB 8.3+ with mongot). Undeclared live ones are kept
|
|
503
852
|
* unless `prune` is on; leave the key out and they are not managed at all.
|
|
504
853
|
* @experimental New in 2.2
|
|
505
854
|
*/
|
|
506
|
-
searchIndexes?: SearchIndexDefinition[];
|
|
507
|
-
/**
|
|
855
|
+
searchIndexes?: readonly SearchIndexDefinition[];
|
|
856
|
+
/**
|
|
857
|
+
* A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator.
|
|
858
|
+
* With `versioning`, its rules are merged in — and `null` is refused.
|
|
859
|
+
*/
|
|
508
860
|
validator?: Record<string, unknown> | null;
|
|
509
|
-
/**
|
|
861
|
+
/**
|
|
862
|
+
* Default: 'strict' — 'moderate' when the only rules are the ones
|
|
863
|
+
* `versioning` adds. Only with a validator (or `versioning`)
|
|
864
|
+
*/
|
|
510
865
|
validationLevel?: ValidationLevel;
|
|
511
|
-
/** Default: 'error'. Only with a validator */
|
|
866
|
+
/** Default: 'error'. Only with a validator (or `versioning`) */
|
|
512
867
|
validationAction?: ValidationAction;
|
|
513
868
|
/**
|
|
514
869
|
* Drop live indexes (and search indexes, when `searchIndexes` is declared)
|
|
515
|
-
* this definition does not declare. Default: the call's `prune`, else false
|
|
870
|
+
* this definition does not declare. Default: the call's `prune`, else false.
|
|
871
|
+
* With `versioning` and no `indexes`, only the version index is managed —
|
|
872
|
+
* prune leaves the others alone.
|
|
516
873
|
*/
|
|
517
874
|
prune?: boolean;
|
|
875
|
+
/**
|
|
876
|
+
* Document shape versioning: the version (and revision) field typed and
|
|
877
|
+
* required by the validator, and the version index background migrations
|
|
878
|
+
* scan. The source of truth `defineShapes` reads too.
|
|
879
|
+
* @experimental New in 2.3
|
|
880
|
+
*/
|
|
881
|
+
versioning?: CollectionVersioning;
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* The `versioning` block of a collection definition.
|
|
886
|
+
* @experimental New in 2.3
|
|
887
|
+
*/
|
|
888
|
+
export interface CollectionVersioning {
|
|
889
|
+
/** The shape version new documents are written at (≥ 1) */
|
|
890
|
+
current: number;
|
|
891
|
+
/**
|
|
892
|
+
* The oldest shape still allowed (default 1, ≤ `current`). `0` types the
|
|
893
|
+
* fields without requiring them — for a collection that predates
|
|
894
|
+
* versioning. Converge refuses to raise it while documents below it remain.
|
|
895
|
+
* There is deliberately no maximum: a newer release may write ahead of the
|
|
896
|
+
* declaration during a rolling deploy.
|
|
897
|
+
*/
|
|
898
|
+
min?: number;
|
|
899
|
+
/** The version field. Default `'__v'` */
|
|
900
|
+
field?: string;
|
|
901
|
+
/** Also manage a revision field for optimistic concurrency. Default `true` */
|
|
902
|
+
revision?: boolean;
|
|
903
|
+
/** The revision field. Default `'__rev'` */
|
|
904
|
+
revisionField?: string;
|
|
905
|
+
/** Declare the `{ <field>: 1, _id: 1 }` index. Default `true` */
|
|
906
|
+
index?: boolean;
|
|
518
907
|
}
|
|
519
908
|
|
|
520
909
|
/** What a `collectionsDir` file exports: a definition whose name defaults to the file name */
|
|
@@ -794,9 +1183,41 @@ export interface MigronautLogger {
|
|
|
794
1183
|
* `{ runId, migration, direction, batch, durationMs }` — so a machine-readable
|
|
795
1184
|
* logger does not have to parse the human string. A plain `(msg) => …` logger
|
|
796
1185
|
* remains valid: the extra argument is simply ignored.
|
|
1186
|
+
*
|
|
1187
|
+
* On a migration's `ctx.logger`, fields with `userland: true` also emit the
|
|
1188
|
+
* `migration:log` event (see {@link MigrationLogEvent}).
|
|
797
1189
|
*/
|
|
798
1190
|
export type LogMethod = (msg: string, fields?: Record<string, unknown>) => void;
|
|
799
1191
|
|
|
1192
|
+
/**
|
|
1193
|
+
* `ctx.logger`: the kit's logger with the run's correlation bound into every
|
|
1194
|
+
* line. Each method takes `(msg, fields?)` — or pino's own `(fields, msg?)` —
|
|
1195
|
+
* and an `Error` as the message (its message, credentials masked). Fields with
|
|
1196
|
+
* `userland: true` also emit the `migration:log` event.
|
|
1197
|
+
*
|
|
1198
|
+
* Any {@link MigronautLogger} — or a pino instance — is one, so a context built
|
|
1199
|
+
* by hand in a test can pass the logger it has.
|
|
1200
|
+
* @experimental New in 2.4
|
|
1201
|
+
*/
|
|
1202
|
+
export interface MigrationLogger {
|
|
1203
|
+
debug(
|
|
1204
|
+
msgOrFields: string | Error | Record<string, unknown>,
|
|
1205
|
+
fieldsOrMsg?: Record<string, unknown> | string,
|
|
1206
|
+
): void;
|
|
1207
|
+
info(
|
|
1208
|
+
msgOrFields: string | Error | Record<string, unknown>,
|
|
1209
|
+
fieldsOrMsg?: Record<string, unknown> | string,
|
|
1210
|
+
): void;
|
|
1211
|
+
warn(
|
|
1212
|
+
msgOrFields: string | Error | Record<string, unknown>,
|
|
1213
|
+
fieldsOrMsg?: Record<string, unknown> | string,
|
|
1214
|
+
): void;
|
|
1215
|
+
error(
|
|
1216
|
+
msgOrFields: string | Error | Record<string, unknown>,
|
|
1217
|
+
fieldsOrMsg?: Record<string, unknown> | string,
|
|
1218
|
+
): void;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
800
1221
|
// ─── Telemetry ────────────────────────────────────────────────────────────────
|
|
801
1222
|
|
|
802
1223
|
/** A span or metric attribute value — the scalar subset migronaut sets */
|
|
@@ -941,6 +1362,12 @@ export interface StatusRow {
|
|
|
941
1362
|
origin?: MigrationOrigin;
|
|
942
1363
|
/** Redacted message of the last failed attempt (status `'failed'` only) */
|
|
943
1364
|
error?: string;
|
|
1365
|
+
/**
|
|
1366
|
+
* `'background'` for a background migration file (applied = registered) —
|
|
1367
|
+
* in `status()` and `dryRun('up')` rows alike
|
|
1368
|
+
* @experimental New in 2.3
|
|
1369
|
+
*/
|
|
1370
|
+
kind?: 'background';
|
|
944
1371
|
/** When the last failed attempt was recorded (status `'failed'` only) */
|
|
945
1372
|
failedAt?: Date;
|
|
946
1373
|
/**
|
|
@@ -961,6 +1388,16 @@ export interface StatusRow {
|
|
|
961
1388
|
* (`up(file, { checksum })`).
|
|
962
1389
|
*/
|
|
963
1390
|
checksum?: string;
|
|
1391
|
+
/**
|
|
1392
|
+
* `dryRun('up')` rows: the background migrations the file requires
|
|
1393
|
+
* @experimental New in 2.3
|
|
1394
|
+
*/
|
|
1395
|
+
requires?: string[];
|
|
1396
|
+
/**
|
|
1397
|
+
* `dryRun('up')` rows: those of `requires` not completed yet
|
|
1398
|
+
* @experimental New in 2.3
|
|
1399
|
+
*/
|
|
1400
|
+
waitsFor?: string[];
|
|
964
1401
|
/** Who asked for the apply, and why — when the run said (`requestedBy` / `reason` options) */
|
|
965
1402
|
requestedBy?: string;
|
|
966
1403
|
reason?: string;
|
|
@@ -1061,7 +1498,13 @@ export type MigronautErrorCode =
|
|
|
1061
1498
|
| 'MIGRATION_BLOCKED'
|
|
1062
1499
|
| 'QUEUE_JOB_INVALID'
|
|
1063
1500
|
| 'QUEUE_JOB_FAILED'
|
|
1064
|
-
| 'CONVERGE_FAILED'
|
|
1501
|
+
| 'CONVERGE_FAILED'
|
|
1502
|
+
| 'REVISION_CONFLICT'
|
|
1503
|
+
| 'SHAPE_VERSION_UNSUPPORTED'
|
|
1504
|
+
| 'BACKGROUND_PENDING'
|
|
1505
|
+
| 'BACKGROUND_FAILED'
|
|
1506
|
+
| 'BACKGROUND_CONFLICT'
|
|
1507
|
+
| 'SANDBOX_REFUSED';
|
|
1065
1508
|
|
|
1066
1509
|
// ─── Config file format ─────────────────────────────────────────────────────────
|
|
1067
1510
|
|
|
@@ -1123,6 +1566,14 @@ export interface UpOptions {
|
|
|
1123
1566
|
* with a filename or `to`.
|
|
1124
1567
|
*/
|
|
1125
1568
|
converge?: boolean;
|
|
1569
|
+
/**
|
|
1570
|
+
* What the run does at a migration that `requires` a background migration
|
|
1571
|
+
* not completed yet: `'error'` (default) throws {@link BackgroundPendingError};
|
|
1572
|
+
* `'stop'` ends the run there, cleanly (`background:waiting`). Either way
|
|
1573
|
+
* that migration fires no hook and leaves no failed trace.
|
|
1574
|
+
* @experimental New in 2.3
|
|
1575
|
+
*/
|
|
1576
|
+
onBackgroundPending?: 'error' | 'stop';
|
|
1126
1577
|
/**
|
|
1127
1578
|
* Who asked for this run (≤ 128 characters) — stamped on the changelog
|
|
1128
1579
|
* records it writes. `executedBy` is the OS user that ran it; on a queue
|
|
@@ -1131,6 +1582,25 @@ export interface UpOptions {
|
|
|
1131
1582
|
requestedBy?: string;
|
|
1132
1583
|
/** Why (≤ 512 characters) — a ticket, a sentence; stamped like `requestedBy` */
|
|
1133
1584
|
reason?: string;
|
|
1585
|
+
/**
|
|
1586
|
+
* The queue job this run works for — bound into `ctx.run`, the run's log
|
|
1587
|
+
* lines and `migration:log`. The BullMQ adapter sets it; set it yourself
|
|
1588
|
+
* when you drive the kit from a queue of your own.
|
|
1589
|
+
* @experimental New in 2.4
|
|
1590
|
+
*/
|
|
1591
|
+
job?: JobRef;
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
/**
|
|
1595
|
+
* The queue job a run works for. Nothing is stored: the ids only correlate
|
|
1596
|
+
* what the run logs with the job a dashboard shows.
|
|
1597
|
+
* @experimental New in 2.4
|
|
1598
|
+
*/
|
|
1599
|
+
export interface JobRef {
|
|
1600
|
+
/** The job's id (≤ 1024 characters) */
|
|
1601
|
+
id: string;
|
|
1602
|
+
/** The group of jobs it was enqueued with (≤ 128 characters) */
|
|
1603
|
+
groupId?: string;
|
|
1134
1604
|
}
|
|
1135
1605
|
|
|
1136
1606
|
/** Options for {@link MigratorKit.down} */
|
|
@@ -1165,6 +1635,11 @@ export interface DownOptions {
|
|
|
1165
1635
|
requestedBy?: string;
|
|
1166
1636
|
/** Why (≤ 512 characters) — a ticket, a sentence; stamped as `revertReason` */
|
|
1167
1637
|
reason?: string;
|
|
1638
|
+
/**
|
|
1639
|
+
* The queue job this run works for — see {@link UpOptions.job}
|
|
1640
|
+
* @experimental New in 2.4
|
|
1641
|
+
*/
|
|
1642
|
+
job?: JobRef;
|
|
1168
1643
|
}
|
|
1169
1644
|
|
|
1170
1645
|
/** Payload common to every lifecycle event */
|
|
@@ -1178,6 +1653,12 @@ export interface MigrationEvent extends MigronautEventBase {
|
|
|
1178
1653
|
direction: 'up' | 'down';
|
|
1179
1654
|
batch?: number;
|
|
1180
1655
|
durationMs?: number;
|
|
1656
|
+
/**
|
|
1657
|
+
* How many times the body ran, when the driver retried its transaction (on
|
|
1658
|
+
* `migration:success` and `migration:error`; absent when it ran once)
|
|
1659
|
+
* @experimental New in 2.4
|
|
1660
|
+
*/
|
|
1661
|
+
attempts?: number;
|
|
1181
1662
|
/**
|
|
1182
1663
|
* Human-readable failure message (on `migration:error` only), with URI
|
|
1183
1664
|
* credentials already redacted — safe to ship to metrics/alerting as-is.
|
|
@@ -1299,6 +1780,82 @@ export interface ConvergeEndEvent extends MigronautEventBase {
|
|
|
1299
1780
|
error?: string;
|
|
1300
1781
|
}
|
|
1301
1782
|
|
|
1783
|
+
/** The level of a `ctx.logger` call */
|
|
1784
|
+
export type MigrationLogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
1785
|
+
|
|
1786
|
+
/**
|
|
1787
|
+
* What every `migration:log` event carries.
|
|
1788
|
+
* @experimental New in 2.4
|
|
1789
|
+
*/
|
|
1790
|
+
export interface MigrationLogEventBase {
|
|
1791
|
+
level: MigrationLogLevel;
|
|
1792
|
+
/**
|
|
1793
|
+
* The message — URI credentials and the values a server error quotes (an
|
|
1794
|
+
* E11000's duplicate key) masked — at most 2048 characters
|
|
1795
|
+
*/
|
|
1796
|
+
msg: string;
|
|
1797
|
+
/**
|
|
1798
|
+
* The call's fields without the `userland` marker, as a document a driver can
|
|
1799
|
+
* store: a copy, its strings redacted, at most 8 levels and 1000 entries
|
|
1800
|
+
* deep, strings at most 4096 characters. Dates, regular expressions and BSON
|
|
1801
|
+
* values are kept as they are, binary data up to 4096 bytes too. An `Error`
|
|
1802
|
+
* becomes `{ name, message, code?, codeName? }`; a `Map` an object, a `Set`
|
|
1803
|
+
* an array; any other instance what `JSON.stringify` would see.
|
|
1804
|
+
*/
|
|
1805
|
+
data: Record<string, unknown>;
|
|
1806
|
+
/** When the call was made, by this process's clock — a TTL index can expire on it */
|
|
1807
|
+
at: Date;
|
|
1808
|
+
/** Increasing within one `runId`: orders the events of one millisecond */
|
|
1809
|
+
seq: number;
|
|
1810
|
+
/** Present when `msg` or `data` was cut to those bounds */
|
|
1811
|
+
truncated?: true;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* A `ctx.logger` call with `userland: true` in an ordinary migration or its
|
|
1816
|
+
* hooks — `migration`, `batch` and `attempt` are absent in `beforeAll`/`afterAll`.
|
|
1817
|
+
* @experimental New in 2.4
|
|
1818
|
+
*/
|
|
1819
|
+
export interface OrdinaryMigrationLogEvent extends MigrationLogEventBase {
|
|
1820
|
+
kind: 'migration';
|
|
1821
|
+
runId: string;
|
|
1822
|
+
direction: 'up' | 'down';
|
|
1823
|
+
migration?: string;
|
|
1824
|
+
batch?: number;
|
|
1825
|
+
attempt?: number;
|
|
1826
|
+
jobId?: string;
|
|
1827
|
+
groupId?: string;
|
|
1828
|
+
requestedBy?: string;
|
|
1829
|
+
reason?: string;
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
/**
|
|
1833
|
+
* A `ctx.logger` call with `userland: true` in a background migration's
|
|
1834
|
+
* `migrate`, `migrateBatch`, `step` (or their way back). `runId` is the lane's,
|
|
1835
|
+
* the one its `background:*` events carry.
|
|
1836
|
+
* @experimental New in 2.4
|
|
1837
|
+
*/
|
|
1838
|
+
export interface BackgroundMigrationLogEvent extends MigrationLogEventBase {
|
|
1839
|
+
kind: 'background';
|
|
1840
|
+
/** The lane's run id (a dry run, which has none, emits nothing) */
|
|
1841
|
+
runId: string;
|
|
1842
|
+
migration: string;
|
|
1843
|
+
direction: 'forward' | 'revert';
|
|
1844
|
+
generation?: number;
|
|
1845
|
+
partition: string;
|
|
1846
|
+
attempt: number;
|
|
1847
|
+
jobId?: string;
|
|
1848
|
+
/** Only when a run drives it inline — see {@link BackgroundRunInfo.groupId} */
|
|
1849
|
+
groupId?: string;
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
/**
|
|
1853
|
+
* The `migration:log` event — `kind` tells an ordinary migration's from a
|
|
1854
|
+
* background migration's.
|
|
1855
|
+
* @experimental New in 2.4
|
|
1856
|
+
*/
|
|
1857
|
+
export type MigrationLogEvent = OrdinaryMigrationLogEvent | BackgroundMigrationLogEvent;
|
|
1858
|
+
|
|
1302
1859
|
/**
|
|
1303
1860
|
* Lifecycle events emitted by {@link MigratorKit}. Subscribe to feed metrics or
|
|
1304
1861
|
* alerting without parsing log lines; a listener that throws is contained and
|
|
@@ -1312,6 +1869,12 @@ export interface MigronautEvents {
|
|
|
1312
1869
|
'migration:success': (event: MigrationEvent) => void;
|
|
1313
1870
|
'migration:skipped': (event: MigrationEvent) => void;
|
|
1314
1871
|
'migration:error': (event: MigrationEvent) => void;
|
|
1872
|
+
/**
|
|
1873
|
+
* A `ctx.logger` call marked `userland: true` — for the application to keep.
|
|
1874
|
+
* Logged lines without the marker emit nothing.
|
|
1875
|
+
* @experimental New in 2.4
|
|
1876
|
+
*/
|
|
1877
|
+
'migration:log': (event: MigrationLogEvent) => void;
|
|
1315
1878
|
'lock:acquired': (event: LockEvent) => void;
|
|
1316
1879
|
'lock:released': (event: LockEvent) => void;
|
|
1317
1880
|
'lock:lost': (event: LockEvent) => void;
|
|
@@ -1319,14 +1882,52 @@ export interface MigronautEvents {
|
|
|
1319
1882
|
'converge:action': (event: ConvergeActionEvent) => void;
|
|
1320
1883
|
'converge:wait': (event: ConvergeWaitEvent) => void;
|
|
1321
1884
|
'converge:end': (event: ConvergeEndEvent) => void;
|
|
1885
|
+
/** @experimental New in 2.3 */
|
|
1886
|
+
'background:registered': (event: BackgroundRegisteredEvent) => void;
|
|
1887
|
+
/** @experimental New in 2.3 */
|
|
1888
|
+
'background:waiting': (event: BackgroundEvent) => void;
|
|
1889
|
+
/** @experimental New in 2.3 */
|
|
1890
|
+
'background:drift': (event: BackgroundEvent) => void;
|
|
1891
|
+
/**
|
|
1892
|
+
* A collection's live drift watcher changed state
|
|
1893
|
+
* @experimental New in 2.3
|
|
1894
|
+
*/
|
|
1895
|
+
'background:watch': (event: {
|
|
1896
|
+
runId?: string;
|
|
1897
|
+
collection: string;
|
|
1898
|
+
state: BackgroundWatchState;
|
|
1899
|
+
}) => void;
|
|
1900
|
+
/** @experimental New in 2.3 */
|
|
1901
|
+
'background:unblocked': (event: BackgroundEvent) => void;
|
|
1902
|
+
/** @experimental New in 2.3 */
|
|
1903
|
+
'background:partitioned': (event: BackgroundEvent) => void;
|
|
1904
|
+
/** @experimental New in 2.3 */
|
|
1905
|
+
'background:pass': (event: BackgroundEvent) => void;
|
|
1906
|
+
/** @experimental New in 2.3 */
|
|
1907
|
+
'background:slice:start': (event: BackgroundEvent) => void;
|
|
1908
|
+
/** @experimental New in 2.3 */
|
|
1909
|
+
'background:batch': (event: BackgroundEvent) => void;
|
|
1910
|
+
/** @experimental New in 2.3 */
|
|
1911
|
+
'background:slice:end': (event: BackgroundEvent) => void;
|
|
1912
|
+
/** @experimental New in 2.3 */
|
|
1913
|
+
'background:lease:lost': (event: BackgroundEvent) => void;
|
|
1914
|
+
/** @experimental New in 2.3 */
|
|
1915
|
+
'background:throttle': (event: BackgroundEvent) => void;
|
|
1916
|
+
/** @experimental New in 2.3 */
|
|
1917
|
+
'background:control': (event: BackgroundEvent) => void;
|
|
1918
|
+
/** @experimental New in 2.3 */
|
|
1919
|
+
'background:completed': (event: BackgroundEvent) => void;
|
|
1920
|
+
/** @experimental New in 2.3 */
|
|
1921
|
+
'background:failed': (event: BackgroundEvent) => void;
|
|
1322
1922
|
}
|
|
1323
1923
|
|
|
1324
1924
|
/** One check performed by {@link MigratorKit.audit} */
|
|
1325
1925
|
export interface AuditCheck {
|
|
1326
1926
|
/**
|
|
1327
1927
|
* e.g. 'config', 'connection', 'transactions', 'indexes', 'lock', 'checksums',
|
|
1328
|
-
* 'pending', 'ordering', 'runtime' —
|
|
1329
|
-
* hold search indexes
|
|
1928
|
+
* 'pending', 'ordering', 'runtime' — 'search' when declared collections
|
|
1929
|
+
* hold search indexes, and 'background' when background migrations are
|
|
1930
|
+
* registered
|
|
1330
1931
|
*/
|
|
1331
1932
|
name: string;
|
|
1332
1933
|
status: 'pass' | 'warn' | 'fail';
|
|
@@ -1356,6 +1957,11 @@ export interface RedoOptions {
|
|
|
1356
1957
|
requestedBy?: string;
|
|
1357
1958
|
/** Why — stamped like `requestedBy` */
|
|
1358
1959
|
reason?: string;
|
|
1960
|
+
/**
|
|
1961
|
+
* The queue job this run works for — both halves carry it; see {@link UpOptions.job}
|
|
1962
|
+
* @experimental New in 2.4
|
|
1963
|
+
*/
|
|
1964
|
+
job?: JobRef;
|
|
1359
1965
|
}
|
|
1360
1966
|
|
|
1361
1967
|
/** Options for {@link MigratorKit.create} */
|
|
@@ -1367,6 +1973,12 @@ export interface CreateOptions {
|
|
|
1367
1973
|
* `createExtension`. Leave unset to let the config decide (default: `'js'`).
|
|
1368
1974
|
*/
|
|
1369
1975
|
js?: boolean;
|
|
1976
|
+
/**
|
|
1977
|
+
* Generate a background migration (`export const background`) instead of
|
|
1978
|
+
* `up`/`down`. Not combined with `template`.
|
|
1979
|
+
* @experimental New in 2.3
|
|
1980
|
+
*/
|
|
1981
|
+
background?: boolean;
|
|
1370
1982
|
}
|
|
1371
1983
|
|
|
1372
1984
|
/** Options for {@link MigratorKit.init} */
|
|
@@ -1562,11 +2174,522 @@ export class MigratorKit extends EventEmitter {
|
|
|
1562
2174
|
* on and something is declared. Resolves the config; does not connect.
|
|
1563
2175
|
*/
|
|
1564
2176
|
convergesAfterUp(): Promise<boolean>;
|
|
2177
|
+
/**
|
|
2178
|
+
* How drift is watched — the `backgroundDrift` setting — which a runner or
|
|
2179
|
+
* a queue worker hosting this kit follows. Resolves the config; does not
|
|
2180
|
+
* connect. @experimental
|
|
2181
|
+
*/
|
|
2182
|
+
driftMode(): Promise<'poll' | 'stream' | 'both'>;
|
|
1565
2183
|
/**
|
|
1566
2184
|
* The converge history, newest first (`limit` 1–1000, default 20): one entry
|
|
1567
2185
|
* per converge that changed something or failed. Read-only.
|
|
1568
2186
|
*/
|
|
1569
2187
|
convergeHistory(options?: { limit?: number }): Promise<ConvergeHistoryEntry[]>;
|
|
2188
|
+
|
|
2189
|
+
// ─── Background migrations (experimental, new in 2.3) ─────────────────────
|
|
2190
|
+
// Reentrant: none of these is a run — no migration lock, no run id; one kit
|
|
2191
|
+
// may drive many at once.
|
|
2192
|
+
|
|
2193
|
+
/** One coordinator step — see {@link BackgroundCoordinatorAnswer}. @experimental */
|
|
2194
|
+
coordinateBackground(
|
|
2195
|
+
name: string,
|
|
2196
|
+
options?: { signal?: AbortSignal; driver?: BackgroundDriver },
|
|
2197
|
+
): Promise<BackgroundCoordinatorAnswer>;
|
|
2198
|
+
/** One slice of one lane: claim a partition and a slot, work it, release. @experimental */
|
|
2199
|
+
runBackgroundSlice(
|
|
2200
|
+
name: string,
|
|
2201
|
+
options?: {
|
|
2202
|
+
signal?: AbortSignal;
|
|
2203
|
+
sliceMs?: number;
|
|
2204
|
+
/**
|
|
2205
|
+
* The queue job working the lane — on its log lines and `migration:log` events
|
|
2206
|
+
* @experimental New in 2.4
|
|
2207
|
+
*/
|
|
2208
|
+
job?: Pick<JobRef, 'id'>;
|
|
2209
|
+
},
|
|
2210
|
+
): Promise<BackgroundSliceResult>;
|
|
2211
|
+
/**
|
|
2212
|
+
* Drive a background migration from this process until it is done (or one
|
|
2213
|
+
* round, `untilDone: false`) with up to `concurrency` lanes (≤ its
|
|
2214
|
+
* `maxParallel`). A failed one throws {@link BackgroundFailedError}; a stop
|
|
2215
|
+
* {@link RunAbortedError} — it goes on from there next time. @experimental
|
|
2216
|
+
*/
|
|
2217
|
+
runBackground(
|
|
2218
|
+
name: string,
|
|
2219
|
+
options?: {
|
|
2220
|
+
signal?: AbortSignal;
|
|
2221
|
+
sliceMs?: number;
|
|
2222
|
+
untilDone?: boolean;
|
|
2223
|
+
concurrency?: number;
|
|
2224
|
+
},
|
|
2225
|
+
): Promise<BackgroundStatus>;
|
|
2226
|
+
/** One background migration's status, or `null` when it is not registered. @experimental */
|
|
2227
|
+
backgroundStatus(name: string): Promise<BackgroundStatus | null>;
|
|
2228
|
+
/** Every background migration's status, oldest registration first. @experimental */
|
|
2229
|
+
backgroundStatus(): Promise<BackgroundStatus[]>;
|
|
2230
|
+
/** The partitions of a background migration's latest generation. @experimental */
|
|
2231
|
+
backgroundPartitions(name: string): Promise<BackgroundPartitionInfo[]>;
|
|
2232
|
+
/** The background migrations with work to do (blocked ones unblocked on the way). @experimental */
|
|
2233
|
+
runnableBackground(): Promise<RunnableBackground[]>;
|
|
2234
|
+
/** Pause; its lanes stop at the next batch (`wait` until they have). @experimental */
|
|
2235
|
+
pauseBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
2236
|
+
/** Resume a paused one. @experimental */
|
|
2237
|
+
resumeBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
2238
|
+
/** Cancel (`wait` until its lanes have stopped). @experimental */
|
|
2239
|
+
cancelBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
2240
|
+
/**
|
|
2241
|
+
* Retry a failed or cancelled one — the same generation, or `fromStart`;
|
|
2242
|
+
* `repin` pins the file on disk first. A completed one is reopened. @experimental
|
|
2243
|
+
*/
|
|
2244
|
+
retryBackground(
|
|
2245
|
+
name: string,
|
|
2246
|
+
options?: BackgroundControlOptions & { fromStart?: boolean; repin?: boolean },
|
|
2247
|
+
): Promise<BackgroundControlResult>;
|
|
2248
|
+
/** Pin the file on disk (checksum, spec — and the changelog's checksum). @experimental */
|
|
2249
|
+
repinBackground(
|
|
2250
|
+
name: string,
|
|
2251
|
+
options?: BackgroundControlOptions,
|
|
2252
|
+
): Promise<BackgroundControlResult & { replan: boolean; checksum: string }>;
|
|
2253
|
+
/** Clear the coordinator lock and every lease of a stuck one. @experimental */
|
|
2254
|
+
unlockBackground(name: string): Promise<{ lock: boolean; leases: number }>;
|
|
2255
|
+
/**
|
|
2256
|
+
* Dry-run a background migration, registered or not, with nothing written:
|
|
2257
|
+
* on a sample, its transformation alone — or with `validate`, the real write
|
|
2258
|
+
* path in a transaction that is always aborted. @experimental
|
|
2259
|
+
*/
|
|
2260
|
+
dryRunBackground(name: string, options?: BackgroundDryRunOptions): Promise<BackgroundDryRun>;
|
|
2261
|
+
/**
|
|
2262
|
+
* The drift watch, once: one indexed probe per completed background
|
|
2263
|
+
* migration for documents of its old shape that appeared since; a finding
|
|
2264
|
+
* reopens it (`onDrift: 'reopen'`, the `backgroundOnDrift` default) or is
|
|
2265
|
+
* only reported. No document id is returned. @experimental
|
|
2266
|
+
*/
|
|
2267
|
+
verifyBackground(options?: {
|
|
2268
|
+
onDrift?: 'reopen' | 'report';
|
|
2269
|
+
collections?: string[];
|
|
2270
|
+
}): Promise<BackgroundVerifyResult>;
|
|
2271
|
+
/**
|
|
2272
|
+
* The live drift watcher: a change stream per collection with a completed
|
|
2273
|
+
* background migration — one leader per collection across every process —
|
|
2274
|
+
* that upgrades each old-shape write moments after it lands, through the
|
|
2275
|
+
* lanes' own write path. Resolves once started; rejects with
|
|
2276
|
+
* ConfigInvalidError on a standalone server (no change streams).
|
|
2277
|
+
* @experimental
|
|
2278
|
+
*/
|
|
2279
|
+
watchBackground(options?: WatchBackgroundOptions): Promise<BackgroundWatcher>;
|
|
2280
|
+
/** What the live drift watchers recorded for a collection — `null` when it has none @experimental */
|
|
2281
|
+
backgroundWatchStatus(collection: string): Promise<BackgroundWatchStatus | null>;
|
|
2282
|
+
/** What the live drift watchers recorded, one row per watched collection @experimental */
|
|
2283
|
+
backgroundWatchStatus(): Promise<BackgroundWatchStatus[]>;
|
|
2284
|
+
}
|
|
2285
|
+
|
|
2286
|
+
/** What a collection's live drift watcher is doing */
|
|
2287
|
+
export type BackgroundWatchState =
|
|
2288
|
+
| 'following'
|
|
2289
|
+
| 'catching-up'
|
|
2290
|
+
| 'streaming'
|
|
2291
|
+
| 'history-lost'
|
|
2292
|
+
| 'overloaded'
|
|
2293
|
+
| 'restarting'
|
|
2294
|
+
| 'suspended'
|
|
2295
|
+
| 'fallback'
|
|
2296
|
+
| 'stopped';
|
|
2297
|
+
|
|
2298
|
+
/** Options of {@link MigratorKit.watchBackground} */
|
|
2299
|
+
export interface WatchBackgroundOptions {
|
|
2300
|
+
/** Only these collections. Default: every one with a completed background migration */
|
|
2301
|
+
collections?: string[];
|
|
2302
|
+
/** Stops the watcher when aborted */
|
|
2303
|
+
signal?: AbortSignal;
|
|
2304
|
+
/** `false`: only report old-shape writes, never upgrade them. Default `true` */
|
|
2305
|
+
upgrade?: boolean;
|
|
2306
|
+
/** How often the edges and the collections are read again (ms). Default 30000 */
|
|
2307
|
+
refreshMs?: number;
|
|
2308
|
+
/** The most often the resume token is saved (ms). Default 5000 */
|
|
2309
|
+
checkpointMs?: number;
|
|
2310
|
+
/** How often a follower tries to become the leader (ms, jittered). Default 10000 */
|
|
2311
|
+
leaderRetryMs?: number;
|
|
2312
|
+
/** Collections watched by this process at most; the rest stay with the poll. Default 16 */
|
|
2313
|
+
maxCollections?: number;
|
|
2314
|
+
/**
|
|
2315
|
+
* A stream this far behind (ms) gives up on its backlog: the background
|
|
2316
|
+
* migrations it serves are reopened, and it starts again from now. Default 60000
|
|
2317
|
+
*/
|
|
2318
|
+
maxLagMs?: number;
|
|
2319
|
+
/** Hears every failure (the watcher itself never throws) */
|
|
2320
|
+
onError?(error: unknown, collection?: string): void;
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
/** A running live drift watcher */
|
|
2324
|
+
export interface BackgroundWatcher {
|
|
2325
|
+
readonly running: boolean;
|
|
2326
|
+
/** What each followed collection's watcher is doing in this process */
|
|
2327
|
+
status(): {
|
|
2328
|
+
collection: string;
|
|
2329
|
+
state: BackgroundWatchState | 'starting';
|
|
2330
|
+
/** Whether this process leads the collection */
|
|
2331
|
+
leading: boolean;
|
|
2332
|
+
counters: { events: number; upgraded: number; failed: number; skipped: number };
|
|
2333
|
+
lastEventAt?: Date;
|
|
2334
|
+
}[];
|
|
2335
|
+
/** Close every stream, save its position, release its lock */
|
|
2336
|
+
stop(): Promise<void>;
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
/** A collection's live drift watcher, as stored — never its resume token */
|
|
2340
|
+
export interface BackgroundWatchStatus {
|
|
2341
|
+
collection: string;
|
|
2342
|
+
state: BackgroundWatchState | 'starting';
|
|
2343
|
+
/** The version a document should have at least */
|
|
2344
|
+
target?: number;
|
|
2345
|
+
/** The background migrations it upgrades with */
|
|
2346
|
+
edges: string[];
|
|
2347
|
+
leader?: { host: string; pid: number; at: Date };
|
|
2348
|
+
counters: { events: number; upgraded: number; failed: number; skipped: number };
|
|
2349
|
+
lastEventAt?: Date;
|
|
2350
|
+
updatedAt: Date;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
// ─── Background migration results ─────────────────────────────────────────────
|
|
2354
|
+
|
|
2355
|
+
/** Where a background migration stands */
|
|
2356
|
+
export type BackgroundState =
|
|
2357
|
+
| 'blocked'
|
|
2358
|
+
| 'pending'
|
|
2359
|
+
| 'running'
|
|
2360
|
+
| 'paused'
|
|
2361
|
+
| 'completed'
|
|
2362
|
+
| 'failed'
|
|
2363
|
+
| 'cancelled';
|
|
2364
|
+
|
|
2365
|
+
/**
|
|
2366
|
+
* Who runs a coordinator step: `{ kind, ref?, round? }` — a BullMQ round lets the newest win
|
|
2367
|
+
* @experimental New in 2.3
|
|
2368
|
+
*/
|
|
2369
|
+
export interface BackgroundDriver {
|
|
2370
|
+
kind: 'bullmq' | 'runner' | 'cli' | 'inline' | 'local';
|
|
2371
|
+
ref?: string;
|
|
2372
|
+
round?: number;
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2375
|
+
/**
|
|
2376
|
+
* One of {@link MigratorKit.runnableBackground}: what a driver needs to pick it up, or to tell it stalled
|
|
2377
|
+
* @experimental New in 2.3
|
|
2378
|
+
*/
|
|
2379
|
+
export interface RunnableBackground {
|
|
2380
|
+
migration: string;
|
|
2381
|
+
status: BackgroundState;
|
|
2382
|
+
maxParallel: number;
|
|
2383
|
+
/** Leases renewed within their TTL — lanes working right now */
|
|
2384
|
+
liveLeases: number;
|
|
2385
|
+
registeredAt: Date;
|
|
2386
|
+
startedAt?: Date;
|
|
2387
|
+
lastProgressAt?: Date;
|
|
2388
|
+
coordinator?: { kind: string; round?: number; at: Date };
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
/**
|
|
2392
|
+
* What a coordinator step says to do next
|
|
2393
|
+
* @experimental New in 2.3
|
|
2394
|
+
*/
|
|
2395
|
+
export interface BackgroundCoordinatorAnswer {
|
|
2396
|
+
next: 'process' | 'wait' | 'done' | 'busy' | 'superseded';
|
|
2397
|
+
/** `process`: lanes that could start now */
|
|
2398
|
+
lanes?: number;
|
|
2399
|
+
generation?: number;
|
|
2400
|
+
/** `process`: the registration the lanes work for (it names their jobs) */
|
|
2401
|
+
registration?: string;
|
|
2402
|
+
/** `process`: the current plan's partitions by status */
|
|
2403
|
+
counts?: BackgroundStatus['partitions'];
|
|
2404
|
+
/**
|
|
2405
|
+
* A `bullmq` driver's round — handed out by this step to a chain that
|
|
2406
|
+
* asked without one; a chain whose round is not the latest is `superseded`
|
|
2407
|
+
*/
|
|
2408
|
+
round?: number;
|
|
2409
|
+
/** `done`: where it stands */
|
|
2410
|
+
status?: BackgroundState | 'unregistered';
|
|
2411
|
+
/** `wait`: why — `checksum`, `replan-draining`, `plan-race`, … */
|
|
2412
|
+
reason?: string;
|
|
2413
|
+
/** `done` + `failed`: why */
|
|
2414
|
+
error?: string;
|
|
2415
|
+
waitsFor?: string[];
|
|
2416
|
+
retryAfterMs?: number;
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
/** Counters of a slice, a partition or a whole background migration */
|
|
2420
|
+
export interface BackgroundCounters {
|
|
2421
|
+
scanned?: number;
|
|
2422
|
+
migrated?: number;
|
|
2423
|
+
skipped?: number;
|
|
2424
|
+
conflicts?: number;
|
|
2425
|
+
failed?: number;
|
|
2426
|
+
retried?: number;
|
|
2427
|
+
batches?: number;
|
|
2428
|
+
processed?: number;
|
|
2429
|
+
txnRetries?: number;
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
/** How a lane's slice ended */
|
|
2433
|
+
export interface BackgroundSliceResult {
|
|
2434
|
+
outcome:
|
|
2435
|
+
| 'yielded'
|
|
2436
|
+
| 'exhausted'
|
|
2437
|
+
| 'busy'
|
|
2438
|
+
| 'stale'
|
|
2439
|
+
| 'paused'
|
|
2440
|
+
| 'cancelled'
|
|
2441
|
+
| 'failed'
|
|
2442
|
+
| 'stopped'
|
|
2443
|
+
| 'lost';
|
|
2444
|
+
counters: BackgroundCounters;
|
|
2445
|
+
retryAfterMs?: number;
|
|
2446
|
+
error?: BackgroundFailedError;
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
/** A background migration, as {@link MigratorKit.backgroundStatus} reports it */
|
|
2450
|
+
export interface BackgroundStatus {
|
|
2451
|
+
migration: string;
|
|
2452
|
+
status: BackgroundState;
|
|
2453
|
+
phase: 'partition' | 'process' | 'replan';
|
|
2454
|
+
direction: 'forward' | 'revert';
|
|
2455
|
+
/** Minted at every (re-)registration — `up --force`, `redo` and `down` get a new one */
|
|
2456
|
+
registration: string;
|
|
2457
|
+
mode: 'declarative' | 'step';
|
|
2458
|
+
collection?: string;
|
|
2459
|
+
from?: number;
|
|
2460
|
+
to?: number;
|
|
2461
|
+
generation: number;
|
|
2462
|
+
pass: number;
|
|
2463
|
+
maxParallel: number;
|
|
2464
|
+
transaction: boolean;
|
|
2465
|
+
totals: BackgroundCounters & { slices?: number; reclaims?: number };
|
|
2466
|
+
/** Distinct documents that failed (within `maxDocumentErrors`) */
|
|
2467
|
+
failedDocuments: number;
|
|
2468
|
+
requires: string[];
|
|
2469
|
+
waitsFor: string[];
|
|
2470
|
+
/** The current plan's partitions by status */
|
|
2471
|
+
partitions?: {
|
|
2472
|
+
total: number;
|
|
2473
|
+
pending: number;
|
|
2474
|
+
running: number;
|
|
2475
|
+
done: number;
|
|
2476
|
+
failed: number;
|
|
2477
|
+
cancelled: number;
|
|
2478
|
+
superseded: number;
|
|
2479
|
+
leased: number;
|
|
2480
|
+
};
|
|
2481
|
+
/** Leases renewed within their TTL — lanes working right now */
|
|
2482
|
+
liveLeases: number;
|
|
2483
|
+
/**
|
|
2484
|
+
* The driver of the latest coordinator step that said who it was — a queue's
|
|
2485
|
+
* coordinator chain carries its `round`, and an older round bows out
|
|
2486
|
+
*/
|
|
2487
|
+
coordinator?: { kind: string; round?: number; at: Date };
|
|
2488
|
+
/**
|
|
2489
|
+
* The current plan. `estimate` is the documents it expects to rewrite —
|
|
2490
|
+
* with `atLeast`, a count that stopped at its limit (there are more)
|
|
2491
|
+
*/
|
|
2492
|
+
plan?: {
|
|
2493
|
+
method: string;
|
|
2494
|
+
estimate: number;
|
|
2495
|
+
atLeast?: boolean;
|
|
2496
|
+
partitions: number;
|
|
2497
|
+
degraded?: string;
|
|
2498
|
+
};
|
|
2499
|
+
/**
|
|
2500
|
+
* On a sharded collection, how the plan used the shard key: `chunks` (a
|
|
2501
|
+
* partition per run of chunks on one shard), `sampled` (the key space
|
|
2502
|
+
* sampled — the chunks could not be read), or `untargeted` (partitions by
|
|
2503
|
+
* `_id`: the key could not be read, or the version index does not carry it)
|
|
2504
|
+
*/
|
|
2505
|
+
sharding?: {
|
|
2506
|
+
mode: 'chunks' | 'sampled' | 'empty' | 'untargeted';
|
|
2507
|
+
shardKey?: Record<string, 1 | 'hashed'>;
|
|
2508
|
+
hashed?: boolean;
|
|
2509
|
+
/** Shards the partitions are grouped by */
|
|
2510
|
+
groups?: number;
|
|
2511
|
+
};
|
|
2512
|
+
registeredAt: Date;
|
|
2513
|
+
startedAt?: Date;
|
|
2514
|
+
completedAt?: Date;
|
|
2515
|
+
lastProgressAt?: Date;
|
|
2516
|
+
lastError?: string;
|
|
2517
|
+
description?: string;
|
|
2518
|
+
/** The registration this one replaced (`up --force`, `redo`, `down`), as it stood then */
|
|
2519
|
+
previous?: {
|
|
2520
|
+
registration: string;
|
|
2521
|
+
status: BackgroundState;
|
|
2522
|
+
direction: 'forward' | 'revert';
|
|
2523
|
+
pass: number;
|
|
2524
|
+
totals: BackgroundCounters & { slices?: number; reclaims?: number };
|
|
2525
|
+
registeredAt: Date;
|
|
2526
|
+
completedAt?: Date;
|
|
2527
|
+
};
|
|
2528
|
+
}
|
|
2529
|
+
|
|
2530
|
+
/** One partition of a background migration */
|
|
2531
|
+
export interface BackgroundPartitionInfo {
|
|
2532
|
+
id: string;
|
|
2533
|
+
generation: number;
|
|
2534
|
+
seq: number;
|
|
2535
|
+
status: 'pending' | 'running' | 'done' | 'failed' | 'cancelled' | 'superseded';
|
|
2536
|
+
scope: Record<string, unknown>;
|
|
2537
|
+
estimate: number;
|
|
2538
|
+
counters: BackgroundCounters;
|
|
2539
|
+
group?: string;
|
|
2540
|
+
lease?: { slot: number; owner: string; host: string; pid: number; renewedAt: Date };
|
|
2541
|
+
throttle?: { batchSize: number; pauseMs: number };
|
|
2542
|
+
claims: number;
|
|
2543
|
+
reclaims: number;
|
|
2544
|
+
failures: number;
|
|
2545
|
+
lastError?: string;
|
|
2546
|
+
}
|
|
2547
|
+
|
|
2548
|
+
/** Options every background control action takes */
|
|
2549
|
+
export interface BackgroundControlOptions {
|
|
2550
|
+
/** Who asked — recorded in its history */
|
|
2551
|
+
requestedBy?: string;
|
|
2552
|
+
/** Why — recorded in its history */
|
|
2553
|
+
reason?: string;
|
|
2554
|
+
/** pause / cancel: resolve once no lane holds a lease any more */
|
|
2555
|
+
wait?: boolean;
|
|
2556
|
+
signal?: AbortSignal;
|
|
2557
|
+
}
|
|
2558
|
+
|
|
2559
|
+
/** What a control action did */
|
|
2560
|
+
export interface BackgroundControlResult {
|
|
2561
|
+
applied: 'changed' | 'unchanged';
|
|
2562
|
+
status: BackgroundState;
|
|
2563
|
+
/** `wait`: whether every lane stopped in time */
|
|
2564
|
+
stopped?: boolean;
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
/** What {@link MigratorKit.verifyBackground} found */
|
|
2568
|
+
export interface BackgroundVerifyResult {
|
|
2569
|
+
/** Completed background migrations probed */
|
|
2570
|
+
checked: number;
|
|
2571
|
+
/** Skipped: another one at work on the collection, the validator guards it, no version index */
|
|
2572
|
+
skipped: number;
|
|
2573
|
+
drift: { migration: string; collection: string; action: 'reopened' | 'reported' }[];
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
/** Options of {@link MigratorKit.dryRunBackground} */
|
|
2577
|
+
export interface BackgroundDryRunOptions {
|
|
2578
|
+
/** A random sample of this many matching documents (1–1000, default 5) */
|
|
2579
|
+
sample?: number;
|
|
2580
|
+
/** The first n matching documents by `_id`, instead of a sample */
|
|
2581
|
+
first?: number;
|
|
2582
|
+
/** Through the real write path, in the always-aborted sandbox */
|
|
2583
|
+
validate?: boolean;
|
|
2584
|
+
/** Dry-run the way back */
|
|
2585
|
+
direction?: 'forward' | 'revert';
|
|
2586
|
+
/** Step migrations: how many steps (1–50, default 1) */
|
|
2587
|
+
steps?: number;
|
|
2588
|
+
/** Step migrations: document images kept (default 20, at most 1000) */
|
|
2589
|
+
maxDocuments?: number;
|
|
2590
|
+
/** Step migrations: from no checkpoint, not the pinned one */
|
|
2591
|
+
fromStart?: boolean;
|
|
2592
|
+
/** Stop the sandbox after this long (default 50 000 ms) — steps, or a `validate` sample */
|
|
2593
|
+
deadlineMs?: number;
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2596
|
+
/** One document of a dry run, as relaxed EJSON */
|
|
2597
|
+
export interface BackgroundDryRunDocument {
|
|
2598
|
+
_id: unknown;
|
|
2599
|
+
before: Record<string, unknown>;
|
|
2600
|
+
after?: Record<string, unknown>;
|
|
2601
|
+
/** The operator update it would be written with (without `validate`) */
|
|
2602
|
+
change?: Record<string, unknown>;
|
|
2603
|
+
error?: string;
|
|
2604
|
+
/** With `validate`: what the server made of it */
|
|
2605
|
+
validation?: 'ok' | 'failed' | 'skipped';
|
|
2606
|
+
}
|
|
2607
|
+
|
|
2608
|
+
/** One operation the sandbox ran — the filter as relaxed EJSON, at most 2 KiB */
|
|
2609
|
+
export interface BackgroundSandboxOperation {
|
|
2610
|
+
seq: number;
|
|
2611
|
+
step: number;
|
|
2612
|
+
collection?: string;
|
|
2613
|
+
method: string;
|
|
2614
|
+
filter?: unknown;
|
|
2615
|
+
result?: unknown;
|
|
2616
|
+
durationMs?: number;
|
|
2617
|
+
error?: string;
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
/** A document the sandbox saw change */
|
|
2621
|
+
export interface BackgroundSandboxDocument {
|
|
2622
|
+
collection: string;
|
|
2623
|
+
_id: unknown;
|
|
2624
|
+
op: 'insert' | 'update' | 'delete' | 'unknown';
|
|
2625
|
+
before?: Record<string, unknown>;
|
|
2626
|
+
after?: Record<string, unknown>;
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
/** What {@link MigratorKit.dryRunBackground} found */
|
|
2630
|
+
export type BackgroundDryRun =
|
|
2631
|
+
| {
|
|
2632
|
+
mode: 'declarative';
|
|
2633
|
+
migration: string;
|
|
2634
|
+
direction: 'forward' | 'revert';
|
|
2635
|
+
method: 'sample' | 'first';
|
|
2636
|
+
requested: number;
|
|
2637
|
+
found: number;
|
|
2638
|
+
migrated: number;
|
|
2639
|
+
failed: number;
|
|
2640
|
+
documents: BackgroundDryRunDocument[];
|
|
2641
|
+
/** With `validate` */
|
|
2642
|
+
validated?: true;
|
|
2643
|
+
aborted?: true;
|
|
2644
|
+
ops?: BackgroundSandboxOperation[];
|
|
2645
|
+
refusals?: { method: string; reason: string; collection?: string }[];
|
|
2646
|
+
/** Documents of other collections the side writes touched */
|
|
2647
|
+
sideEffects?: BackgroundSandboxDocument[];
|
|
2648
|
+
attempts?: number;
|
|
2649
|
+
}
|
|
2650
|
+
| BackgroundStepDryRun;
|
|
2651
|
+
|
|
2652
|
+
/** A step migration's dry run: up to `steps` steps in one always-aborted transaction */
|
|
2653
|
+
export interface BackgroundStepDryRun {
|
|
2654
|
+
mode: 'step';
|
|
2655
|
+
migration: string;
|
|
2656
|
+
direction: 'forward' | 'revert';
|
|
2657
|
+
aborted: true;
|
|
2658
|
+
ok: boolean;
|
|
2659
|
+
attempts: number;
|
|
2660
|
+
stoppedBy?: 'deadline' | 'done' | 'steps';
|
|
2661
|
+
steps: {
|
|
2662
|
+
step: number;
|
|
2663
|
+
checkpointIn: unknown;
|
|
2664
|
+
checkpointOut?: unknown;
|
|
2665
|
+
done?: boolean;
|
|
2666
|
+
processed?: number;
|
|
2667
|
+
migrated?: number;
|
|
2668
|
+
error?: string;
|
|
2669
|
+
}[];
|
|
2670
|
+
ops: BackgroundSandboxOperation[];
|
|
2671
|
+
documents: BackgroundSandboxDocument[];
|
|
2672
|
+
refusals: { method: string; reason: string; collection?: string }[];
|
|
2673
|
+
leakedCursors: number;
|
|
2674
|
+
truncated: boolean;
|
|
2675
|
+
abortedBy?: string;
|
|
2676
|
+
error?: string;
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
/** `background:registered` */
|
|
2680
|
+
export interface BackgroundRegisteredEvent {
|
|
2681
|
+
runId?: string;
|
|
2682
|
+
migration: string;
|
|
2683
|
+
status: BackgroundState | 'withdrawn';
|
|
2684
|
+
direction: 'forward' | 'revert';
|
|
2685
|
+
waitsFor?: string[];
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
/** Any other `background:*` event: the migration it is about, and what happened */
|
|
2689
|
+
export interface BackgroundEvent {
|
|
2690
|
+
runId?: string;
|
|
2691
|
+
migration: string;
|
|
2692
|
+
[field: string]: unknown;
|
|
1570
2693
|
}
|
|
1571
2694
|
|
|
1572
2695
|
// ─── Programmatic entry points ─────────────────────────────────────────────────
|
|
@@ -1578,6 +2701,13 @@ export type OnLockHeld = 'throw' | 'wait';
|
|
|
1578
2701
|
export interface RunMigrationsOptions extends MigratorKitOptions {
|
|
1579
2702
|
/** Skip lock acquisition (dev only — never in production) */
|
|
1580
2703
|
noLock?: boolean;
|
|
2704
|
+
/**
|
|
2705
|
+
* At a migration that requires an unfinished background migration: throw
|
|
2706
|
+
* (`'error'`, default) or stop the run there (`'stop'`, listed in
|
|
2707
|
+
* `waiting`) — `'stop'` lets an app boot while a background migration runs.
|
|
2708
|
+
* @experimental New in 2.3
|
|
2709
|
+
*/
|
|
2710
|
+
onBackgroundPending?: 'error' | 'stop';
|
|
1581
2711
|
/**
|
|
1582
2712
|
* How to react when another process already holds the migration lock — the
|
|
1583
2713
|
* typical case when several app instances boot at once.
|
|
@@ -1632,6 +2762,12 @@ export interface MigrationSummary {
|
|
|
1632
2762
|
attempts: number;
|
|
1633
2763
|
/** The converge that ended the run — present only when `convergeAfterUp` converged */
|
|
1634
2764
|
converge?: ConvergeResult;
|
|
2765
|
+
/**
|
|
2766
|
+
* With `onBackgroundPending: 'stop'`: the migration the run stopped at and
|
|
2767
|
+
* the background migrations it waits for.
|
|
2768
|
+
* @experimental New in 2.3
|
|
2769
|
+
*/
|
|
2770
|
+
waiting?: { migration: string; waitsFor: { migration: string; status: string }[] }[];
|
|
1635
2771
|
}
|
|
1636
2772
|
|
|
1637
2773
|
/**
|
|
@@ -1668,6 +2804,56 @@ export const EXIT_CODES: Readonly<
|
|
|
1668
2804
|
>
|
|
1669
2805
|
>;
|
|
1670
2806
|
|
|
2807
|
+
// ─── Background runner ────────────────────────────────────────────────────────
|
|
2808
|
+
|
|
2809
|
+
/** Options of {@link startBackgroundRunner} */
|
|
2810
|
+
export interface BackgroundRunnerOptions {
|
|
2811
|
+
/** The kit to drive — or `config` (and `kitOptions`) for one the runner makes and closes */
|
|
2812
|
+
kit?: MigratorKit;
|
|
2813
|
+
config?: Partial<MigronautConfig>;
|
|
2814
|
+
kitOptions?: MigratorKitOptions;
|
|
2815
|
+
/** Lane loops in this process, shared by every background migration (default 1, ≤ 64) */
|
|
2816
|
+
concurrency?: number;
|
|
2817
|
+
/** How often the runnable list is read again (default 5000 ms) */
|
|
2818
|
+
pollIntervalMs?: number;
|
|
2819
|
+
/** A slice's length (default: each background migration's `sliceMs`) */
|
|
2820
|
+
sliceMs?: number;
|
|
2821
|
+
/** The drift watch's period (default 600 000 ms — 10 minutes); `false`: off */
|
|
2822
|
+
verifyIntervalMs?: number | false;
|
|
2823
|
+
/**
|
|
2824
|
+
* Host the live drift watcher in this process — `true`, or its options.
|
|
2825
|
+
* Default: when `backgroundDrift` is `'stream'` or `'both'`
|
|
2826
|
+
*/
|
|
2827
|
+
watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
|
|
2828
|
+
/** Stops the runner, as `stop()` does */
|
|
2829
|
+
signal?: AbortSignal;
|
|
2830
|
+
/** Hears every failed slice (the runner itself never throws) */
|
|
2831
|
+
onError?: (error: unknown, migration?: string) => void;
|
|
2832
|
+
}
|
|
2833
|
+
|
|
2834
|
+
/** A running {@link startBackgroundRunner} */
|
|
2835
|
+
export interface BackgroundRunner {
|
|
2836
|
+
readonly kit: MigratorKit;
|
|
2837
|
+
readonly running: boolean;
|
|
2838
|
+
/** The live drift watcher this runner hosts, once started — or undefined */
|
|
2839
|
+
readonly watcher: BackgroundWatcher | undefined;
|
|
2840
|
+
/**
|
|
2841
|
+
* Stop at the next batch, release every lease, and close the kit the runner
|
|
2842
|
+
* made. `timeoutMs`: stop waiting for a lane stuck in its transformation
|
|
2843
|
+
* (its lease expires; the work resumes from the last checkpoint)
|
|
2844
|
+
*/
|
|
2845
|
+
stop(options?: { timeoutMs?: number }): Promise<void>;
|
|
2846
|
+
}
|
|
2847
|
+
|
|
2848
|
+
/**
|
|
2849
|
+
* Drive background migrations from inside the application — no queue:
|
|
2850
|
+
* `concurrency` lane loops shared by every runnable background migration,
|
|
2851
|
+
* round-robin, plus the drift watch every `verifyIntervalMs`. Several
|
|
2852
|
+
* application instances share the work through the leases.
|
|
2853
|
+
* @experimental New in 2.3
|
|
2854
|
+
*/
|
|
2855
|
+
export function startBackgroundRunner(options?: BackgroundRunnerOptions): BackgroundRunner;
|
|
2856
|
+
|
|
1671
2857
|
// ─── Logger factory ───────────────────────────────────────────────────────────
|
|
1672
2858
|
|
|
1673
2859
|
/** Threshold accepted by {@link createLogger} — drops anything less severe */
|
|
@@ -1882,3 +3068,72 @@ export class QueueJobFailedError extends MigronautError {
|
|
|
1882
3068
|
export class ConvergeFailedError extends MigronautError {
|
|
1883
3069
|
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
1884
3070
|
}
|
|
3071
|
+
|
|
3072
|
+
/**
|
|
3073
|
+
* Thrown by the optimistic-concurrency helpers of `@alexify/migronaut/versioning`
|
|
3074
|
+
* when a revision-guarded write matched nothing. `context.reason` is
|
|
3075
|
+
* `'conflict'` (the document is at another revision — `context.actual`),
|
|
3076
|
+
* `'not-found'` (nothing matches the filter) or `'unknown'` (the follow-up read
|
|
3077
|
+
* was skipped or could not tell); `context.expected` is the revision the
|
|
3078
|
+
* caller held. The filter is never copied into the error. Experimental.
|
|
3079
|
+
*/
|
|
3080
|
+
export class RevisionConflictError extends MigronautError {
|
|
3081
|
+
readonly context?: RevisionConflictContext;
|
|
3082
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
/** {@link RevisionConflictError}'s `context` — what a caller decides on */
|
|
3086
|
+
export interface RevisionConflictContext {
|
|
3087
|
+
reason: 'conflict' | 'not-found' | 'unknown';
|
|
3088
|
+
/** The revision the caller held */
|
|
3089
|
+
expected: number;
|
|
3090
|
+
/** `conflict`: the revision the document is at */
|
|
3091
|
+
actual?: number;
|
|
3092
|
+
/** The collection's name, when the collection object has one */
|
|
3093
|
+
collection?: string;
|
|
3094
|
+
[key: string]: unknown;
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
/**
|
|
3098
|
+
* Thrown by an upcaster that cannot bring a document to the current shape:
|
|
3099
|
+
* `context.reason` is `'newer'`, `'below-min'` or `'invalid'`, with
|
|
3100
|
+
* `context.version` and `context.current`. Experimental.
|
|
3101
|
+
*/
|
|
3102
|
+
export class ShapeVersionError extends MigronautError {
|
|
3103
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3104
|
+
}
|
|
3105
|
+
|
|
3106
|
+
/**
|
|
3107
|
+
* Thrown when a migration `requires` a background migration that has not
|
|
3108
|
+
* completed — or whose collection still holds old-shape documents. Nothing was
|
|
3109
|
+
* run; `context.waitsFor` lists what it waits for. Experimental.
|
|
3110
|
+
*/
|
|
3111
|
+
export class BackgroundPendingError extends MigronautError {
|
|
3112
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
/**
|
|
3116
|
+
* Thrown when a background migration ended `failed`; `context.migration` names
|
|
3117
|
+
* it and `context.lastError` says what happened last. Experimental.
|
|
3118
|
+
*/
|
|
3119
|
+
export class BackgroundFailedError extends MigronautError {
|
|
3120
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
/**
|
|
3124
|
+
* Thrown when a control action does not fit the background migration's state;
|
|
3125
|
+
* `context.status` is the state found and `context.action` what was asked.
|
|
3126
|
+
* Experimental.
|
|
3127
|
+
*/
|
|
3128
|
+
export class BackgroundConflictError extends MigronautError {
|
|
3129
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3130
|
+
}
|
|
3131
|
+
|
|
3132
|
+
/**
|
|
3133
|
+
* Thrown by the dry-run sandbox when a step reaches for something it cannot
|
|
3134
|
+
* run inside an always-aborted transaction; `context.method` names the call
|
|
3135
|
+
* and `context.reason` the rule. Experimental.
|
|
3136
|
+
*/
|
|
3137
|
+
export class SandboxRefusedError extends MigronautError {
|
|
3138
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
3139
|
+
}
|