@alexify/migronaut 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +41 -3
  3. package/bullmq.d.ts +484 -8
  4. package/index.d.ts +1264 -9
  5. package/migronaut.schema.json +93 -1
  6. package/package.json +9 -2
  7. package/src/bullmq/background-processor.js +541 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +348 -21
  11. package/src/bullmq/producer.js +185 -13
  12. package/src/bullmq/service.js +484 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/create.js +6 -0
  15. package/src/cli/exit-codes.js +6 -0
  16. package/src/cli/index.js +2 -0
  17. package/src/core/audit.js +11 -1
  18. package/src/core/background-audit.js +139 -0
  19. package/src/core/background-drift.js +126 -0
  20. package/src/core/background-dry-run.js +375 -0
  21. package/src/core/background-engine.js +849 -0
  22. package/src/core/background-kit.js +432 -0
  23. package/src/core/background-partition.js +298 -0
  24. package/src/core/background-runner.js +305 -0
  25. package/src/core/background-sandbox.js +701 -0
  26. package/src/core/background-shard.js +542 -0
  27. package/src/core/background-spec.js +597 -0
  28. package/src/core/background-store.js +951 -0
  29. package/src/core/background-throttle.js +269 -0
  30. package/src/core/background-watch-plan.js +164 -0
  31. package/src/core/background-watch-store.js +78 -0
  32. package/src/core/background-watch.js +610 -0
  33. package/src/core/background.js +1127 -0
  34. package/src/core/bson-peer.js +23 -0
  35. package/src/core/changelog.js +32 -0
  36. package/src/core/collections.js +78 -8
  37. package/src/core/config.js +102 -12
  38. package/src/core/converge-plan.js +86 -7
  39. package/src/core/converge.js +88 -0
  40. package/src/core/lock.js +48 -21
  41. package/src/core/migration-logger.js +279 -0
  42. package/src/core/migrator.js +1027 -22
  43. package/src/core/options.js +36 -0
  44. package/src/core/run-recorder.js +6 -1
  45. package/src/core/run.js +26 -12
  46. package/src/core/runner.js +34 -8
  47. package/src/core/server-info.js +9 -2
  48. package/src/core/shard-info.js +76 -0
  49. package/src/core/versioning-spec.js +181 -0
  50. package/src/errors/index.js +88 -0
  51. package/src/index.js +16 -0
  52. package/src/utils/error.js +11 -2
  53. package/src/utils/job-ref.js +44 -0
  54. package/src/utils/loader.js +77 -9
  55. package/src/utils/migration-name.js +33 -1
  56. package/src/utils/redact.js +140 -3
  57. package/src/utils/telemetry.js +110 -0
  58. package/src/utils/template.js +62 -1
  59. package/src/versioning/config.js +155 -0
  60. package/src/versioning/document.js +326 -0
  61. package/src/versioning/index.js +50 -0
  62. package/src/versioning/internal.js +279 -0
  63. package/src/versioning/mongoose.js +151 -0
  64. package/src/versioning/occ.js +318 -0
  65. package/src/versioning/registry.js +187 -0
  66. package/src/versioning/upcaster.js +213 -0
  67. package/versioning.d.ts +666 -0
  68. package/versioning.js +1 -0
package/index.d.ts CHANGED
@@ -35,12 +35,69 @@ export interface MigrationContext {
35
35
  * migronaut cannot interrupt a running function by itself.
36
36
  */
37
37
  signal?: AbortSignal;
38
+ /**
39
+ * The kit's logger, with this run's correlation ({@link run}) bound into the
40
+ * fields of every line. A call whose fields hold `userland: true` is also
41
+ * emitted as the `migration:log` event, for the application to store and show
42
+ * its users — migronaut stores none of it:
43
+ *
44
+ * ```js
45
+ * logger.info('batch done', { userland: true, processed: 1000 });
46
+ * ```
47
+ *
48
+ * Always present when migronaut runs the migration; optional so a context
49
+ * built by hand (in a test) still type-checks.
50
+ * @experimental New in 2.4
51
+ */
52
+ logger?: MigrationLogger;
53
+ /**
54
+ * Who this is: the run id, the migration, the direction, the transaction
55
+ * attempt and, when the caller named them, the queue job and the actor.
56
+ * Frozen; the same values `logger` binds and `migration:log` carries.
57
+ * Always present when migronaut runs the migration.
58
+ * @experimental New in 2.4
59
+ */
60
+ run?: MigrationRunInfo;
61
+ }
62
+
63
+ /**
64
+ * The correlation of a run, as `ctx.run`. `migration`, `batch` and `attempt`
65
+ * describe one migration and are absent in `beforeAll`/`afterAll`.
66
+ * @experimental New in 2.4
67
+ */
68
+ export interface MigrationRunInfo {
69
+ /** The run id — the same on the lock, the changelog record and every event of the run */
70
+ readonly id: string;
71
+ readonly direction: 'up' | 'down';
72
+ /** The migration file */
73
+ readonly migration?: string;
74
+ /** The changelog batch (`up` only) */
75
+ readonly batch?: number;
76
+ /**
77
+ * 1 — or more when a transaction was retried and the body runs again. Not a
78
+ * queue retry: a migration job is never retried.
79
+ */
80
+ readonly attempt?: number;
81
+ /** The queue job that runs this migration (set by the BullMQ adapter, or the `job` option) */
82
+ readonly jobId?: string;
83
+ /** The queue group the job belongs to */
84
+ readonly groupId?: string;
85
+ /** Who asked for the run (`requestedBy` option) */
86
+ readonly requestedBy?: string;
87
+ /** Why (`reason` option) */
88
+ readonly reason?: string;
38
89
  }
39
90
 
40
91
  /** Shape of an imported migration file module */
41
92
  export interface MigrationModule {
42
93
  up: (ctx: MigrationContext) => Promise<void>;
43
94
  down: (ctx: MigrationContext) => Promise<void>;
95
+ /**
96
+ * Background migrations (file names, each sorting before this file) that
97
+ * must have completed before this migration runs.
98
+ * @experimental New in 2.3
99
+ */
100
+ requires?: readonly string[];
44
101
  /** If true, wraps this migration in a MongoDB session + transaction */
45
102
  useTransaction?: boolean;
46
103
  /** Overrides `MigronautConfig.timeoutMs` for this migration only */
@@ -49,6 +106,261 @@ export interface MigrationModule {
49
106
  description?: string;
50
107
  }
51
108
 
109
+ // ─── Document shapes and background migrations ───────────────────────────────
110
+
111
+ /** The system field names of a versioned collection — `revisionField` is `null` without revisions */
112
+ export interface ShapeFieldNames {
113
+ field: string;
114
+ revisionField: string | null;
115
+ }
116
+
117
+ /** The default system field names: `__v` and `__rev` */
118
+ export interface DefaultShapeFieldNames {
119
+ field: '__v';
120
+ revisionField: '__rev';
121
+ }
122
+
123
+ /**
124
+ * A document type without its system fields (the version and the revision),
125
+ * distributed over a union. What a shape body is declared as, and what a
126
+ * background transformation returns: migronaut writes the system fields.
127
+ * @experimental New in 2.3
128
+ */
129
+ export type Body<T, N extends ShapeFieldNames = DefaultShapeFieldNames> = T extends unknown
130
+ ? Omit<T, N['field'] | Extract<N['revisionField'], string>>
131
+ : never;
132
+
133
+ /**
134
+ * What a background migration's transformation gets besides the document.
135
+ * `session`, `db` and `client` are there only in a `transaction` background
136
+ * migration — writes to other collections must pass `session` to commit
137
+ * with the batch.
138
+ * @experimental New in 2.3
139
+ */
140
+ export interface BackgroundMigrationContext {
141
+ /** Aborted when the slice is stopping (lease lost, pause, shutdown) */
142
+ signal: AbortSignal;
143
+ /**
144
+ * The kit's logger with {@link background} bound into every line. Fields
145
+ * with `userland: true` also emit `migration:log` (`kind: 'background'`) —
146
+ * but `migrate` runs once per document, and again for a document a
147
+ * concurrent write moved: log from `migrateBatch` or a `step` rather than
148
+ * per document. In a dry run the lines say `dryRun: true` and nothing is
149
+ * emitted.
150
+ */
151
+ logger: MigrationLogger;
152
+ direction: 'forward' | 'revert';
153
+ /** Where this runs — frozen */
154
+ background: BackgroundRunInfo;
155
+ session?: ClientSession;
156
+ db?: Db;
157
+ client?: MongoClient;
158
+ /** True in a dry run — the writes are rolled back */
159
+ dryRun?: boolean;
160
+ }
161
+
162
+ /**
163
+ * `ctx.background`: which background migration, generation and partition —
164
+ * and the lane, its queue job and the transaction attempt, as its log lines
165
+ * and `migration:log` events carry them.
166
+ */
167
+ export interface BackgroundRunInfo {
168
+ readonly name: string;
169
+ /** The plan generation — absent when the live drift watcher runs the transformation */
170
+ readonly generation?: number;
171
+ /** The partition (`''` for the drift watcher, `'dry-run'` in a dry run) */
172
+ readonly partition: string;
173
+ /**
174
+ * The lane's run id — its lease's owner, the runId of its `background:*`
175
+ * events. Absent in a dry run.
176
+ * @experimental New in 2.4
177
+ */
178
+ readonly runId?: string;
179
+ /**
180
+ * The queue job working the lane — a background queue's lane job, or the
181
+ * migration job whose run drives it inline (`backgroundInline`)
182
+ * @experimental New in 2.4
183
+ */
184
+ readonly jobId?: string;
185
+ /**
186
+ * The group of that migration job — only when a run drives it inline; a
187
+ * background queue's lanes have none
188
+ * @experimental New in 2.4
189
+ */
190
+ readonly groupId?: string;
191
+ /**
192
+ * 1 — or more when a transactional batch or step runs again in a new
193
+ * transaction (a transient error, a conflict, a smaller batch)
194
+ * @experimental New in 2.4
195
+ */
196
+ readonly attempt: number;
197
+ }
198
+
199
+ /** How a background migration splits its collection into partitions */
200
+ export interface BackgroundPartitionSettings {
201
+ /** Partitions per lane (default 4), so a slow partition does not hold the pass */
202
+ overPartition?: number;
203
+ /** Default 256 */
204
+ maxPartitions?: number;
205
+ /** No partition is planned smaller than this (default 4 × batchSize) */
206
+ minPartitionDocs?: number;
207
+ /** Ids sampled to place the boundaries (default min(10 000, 100 × partitions)) */
208
+ sampleSize?: number;
209
+ }
210
+
211
+ /** A transactional background migration's budget */
212
+ export interface BackgroundTransactionSettings {
213
+ /** Per batch, ≤ 50 000 (default 10 000) */
214
+ timeoutMs?: number;
215
+ /** Retries of a batch on a transient transaction error (default 5) */
216
+ maxRetries?: number;
217
+ }
218
+
219
+ /** The latency-driven throttle (AIMD) */
220
+ export interface BackgroundAdaptiveSettings {
221
+ /** A batch write slower than this halves the batch (default 500) */
222
+ targetLatencyMs?: number;
223
+ /** Default 10 */
224
+ minBatchSize?: number;
225
+ /** Never above `batchSize` (the default) */
226
+ maxBatchSize?: number;
227
+ /** Default 30 000 */
228
+ maxPauseMs?: number;
229
+ }
230
+
231
+ /** What a `throttle` hook is told before every batch */
232
+ export interface BackgroundThrottleContext {
233
+ name: string;
234
+ collection?: string;
235
+ generation: number;
236
+ partition: string;
237
+ batchSize: number;
238
+ signal: AbortSignal;
239
+ }
240
+
241
+ /**
242
+ * Settings every background migration may carry, with their defaults.
243
+ * @experimental New in 2.3
244
+ */
245
+ export interface BackgroundMigrationSettings {
246
+ description?: string;
247
+ /** Documents per batch: 500 (100 with `transaction`) */
248
+ batchSize?: number;
249
+ /** Pause between batches: 100 ms */
250
+ pauseMs?: number;
251
+ /** How long a lane holds a partition before it yields: 30 000 ms */
252
+ sliceMs?: number;
253
+ /** Every batch write's, transactions included — default `{ w: 'majority' }` */
254
+ writeConcern?: { w?: number | 'majority'; j?: boolean; wtimeoutMS?: number };
255
+ /** Documents that may fail before the background migration does: 0 (at most 1000) */
256
+ maxDocumentErrors?: number;
257
+ /** Passes over the remaining old-shape documents before giving up: 10 */
258
+ maxPasses?: number;
259
+ /** Re-read rounds for documents a concurrent write changed under a batch: 3 */
260
+ maxConflictRetries?: number;
261
+ /** Failed slices of one partition in a row (no checkpoint between) before it fails: 3 */
262
+ maxSliceFailures?: number;
263
+ /** Wait while a secondary lags more than this: 10 000 ms (`false`: never) */
264
+ maxReplicationLagMs?: number | false;
265
+ /** Called before every batch; a number it returns is an extra pause (ms) */
266
+ throttle?(ctx: BackgroundThrottleContext): number | void | Promise<number | void>;
267
+ /** Partitions processed at once, across every process: 1 (at most 64) */
268
+ maxParallel?: number;
269
+ partitions?: BackgroundPartitionSettings;
270
+ /** Batch and checkpoint in one transaction (needs a replica set or mongos): false */
271
+ transaction?: boolean | BackgroundTransactionSettings;
272
+ /** The latency-driven throttle: true */
273
+ adaptive?: boolean | BackgroundAdaptiveSettings;
274
+ /** Lanes per shard on a sharded collection: 1 */
275
+ shardConcurrency?: number;
276
+ }
277
+
278
+ /**
279
+ * A declarative background migration: every document of `collection` at
280
+ * version `from` (and matching `filter`) rewritten to version `to` by
281
+ * `migrate` (or `migrateBatch`), in partitions, behind an optimistic guard.
282
+ * `From` and `To` type the documents — see `BackgroundMigrationFor` in
283
+ * `@alexify/migronaut/versioning` for the shape-map form.
284
+ *
285
+ * The callbacks are declared as methods, so a transformation typed for the
286
+ * stored document (with its version) or for its body fits either way.
287
+ * @experimental New in 2.3
288
+ */
289
+ export interface DeclarativeBackgroundMigration<
290
+ From extends object = Record<string, any>,
291
+ To extends object = Record<string, any>,
292
+ > extends BackgroundMigrationSettings {
293
+ collection: string;
294
+ /** The version rewritten — 0 for documents without a version field */
295
+ from: number;
296
+ to: number;
297
+ filter?: Record<string, unknown>;
298
+ /** The new document for one old one; the engine sets the version and bumps the revision */
299
+ migrate?(doc: From, ctx: BackgroundMigrationContext): To | Promise<To>;
300
+ /** The new documents for a batch, aligned — an `Error` fails just that document */
301
+ migrateBatch?(docs: From[], ctx: BackgroundMigrationContext): (To | Error)[] | Promise<(To | Error)[]>;
302
+ /** The way back, for `down` */
303
+ revert?(doc: To, ctx: BackgroundMigrationContext): From | Promise<From>;
304
+ revertBatch?(docs: To[], ctx: BackgroundMigrationContext): (From | Error)[] | Promise<(From | Error)[]>;
305
+ /** Default: the collection's `versioning.field`, else `'__v'` */
306
+ versionField?: string;
307
+ /** Default: the collection's `versioning.revisionField`, else `'__rev'` */
308
+ revisionField?: string;
309
+ /**
310
+ * `'revision'` (default) guards each write with the revision; a collection
311
+ * without revisions must say `'version-only'` — a concurrent write that
312
+ * leaves the version alone is then invisible to it.
313
+ */
314
+ occ?: 'revision' | 'version-only';
315
+ }
316
+
317
+ /** What a `step` background migration gets */
318
+ export interface BackgroundStepContext extends BackgroundMigrationContext {
319
+ db: Db;
320
+ client: MongoClient;
321
+ /** What the previous step returned (`null` at the start) */
322
+ checkpoint: unknown;
323
+ /** Epoch ms the step should return by — the slice ends then */
324
+ deadline: number;
325
+ }
326
+
327
+ /** What a `step` returns */
328
+ export interface BackgroundStepResult {
329
+ /** Saved (≤ 64 KiB of BSON) and handed to the next step */
330
+ checkpoint: unknown;
331
+ /** True once there is nothing left */
332
+ done: boolean;
333
+ processed?: number;
334
+ migrated?: number;
335
+ /** For progress, when known */
336
+ total?: number;
337
+ }
338
+
339
+ /**
340
+ * A free-form background migration — the escape hatch: migronaut runs `step`
341
+ * again and again with its last checkpoint until it says `done`, owning the
342
+ * lease, the slices, the throttle and the controls. One partition only; the
343
+ * writes must be idempotent.
344
+ * @experimental New in 2.3
345
+ */
346
+ export interface StepBackgroundMigration extends BackgroundMigrationSettings {
347
+ /** Shown in status — the collection it works on, if one */
348
+ collection?: string;
349
+ step(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
350
+ revertStep?(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
351
+ }
352
+
353
+ /** A background migration, as a migration file exports it: `export const background = {…}` */
354
+ export type BackgroundMigration = DeclarativeBackgroundMigration | StepBackgroundMigration;
355
+
356
+ /** Shape of a background migration file module — no `up`/`down` */
357
+ export interface BackgroundMigrationModule {
358
+ background: BackgroundMigration;
359
+ /** Background migrations that must complete first (each sorting before this file) */
360
+ requires?: readonly string[];
361
+ description?: string;
362
+ }
363
+
52
364
  // ─── Changelog ────────────────────────────────────────────────────────────────
53
365
 
54
366
  export type MigrationStatus = 'applied' | 'reverted' | 'failed';
@@ -93,6 +405,12 @@ export interface MigrationRecord {
93
405
  * `migronaut import` — these are not reversible by migronaut. Absent for native records.
94
406
  */
95
407
  origin?: MigrationOrigin;
408
+ /**
409
+ * `'background'` for a background migration file — applying it registered
410
+ * the background migration; its documents are rewritten later.
411
+ * @experimental New in 2.3
412
+ */
413
+ kind?: 'background';
96
414
  }
97
415
 
98
416
  // ─── Config ───────────────────────────────────────────────────────────────────
@@ -316,6 +634,37 @@ export interface MigronautConfig {
316
634
  * @experimental New in 2.2
317
635
  */
318
636
  searchIndexWaitTimeoutMs?: number;
637
+ /**
638
+ * Where background migrations keep their state — and, named after it,
639
+ * their partitions (`<name>_partitions`) and the drift watcher's resume
640
+ * tokens (`<name>_watch`). Default `'_migronaut_background'`.
641
+ * @experimental New in 2.3
642
+ */
643
+ backgroundCollection?: string;
644
+ /**
645
+ * Run a background migration to the end inside the `up` that registers it,
646
+ * under the migration lock — for small collections and tests. Default false.
647
+ * @experimental New in 2.3
648
+ */
649
+ backgroundInline?: boolean;
650
+ /**
651
+ * What the drift watch does with old-shape documents that appear after a
652
+ * background migration completed: `'reopen'` it (default) or only `'report'`.
653
+ * @experimental New in 2.3
654
+ */
655
+ backgroundOnDrift?: 'reopen' | 'report';
656
+ /**
657
+ * How drift is watched: `'poll'` (default — a check every 10 minutes),
658
+ * `'stream'` (change streams, the check as a backstop) or `'both'`.
659
+ * @experimental New in 2.3
660
+ */
661
+ backgroundDrift?: 'poll' | 'stream' | 'both';
662
+ /**
663
+ * Partition a sharded collection by its shard key and target each write at
664
+ * one shard (`'auto'`, default), or treat it like any other (`'off'`).
665
+ * @experimental New in 2.3
666
+ */
667
+ backgroundShardAware?: 'auto' | 'off';
319
668
  /** Mongoose instance — required only if your migrations use Mongoose models */
320
669
  mongoose?: MongooseLike;
321
670
  hooks?: MigrationHooks;
@@ -496,25 +845,65 @@ export interface CollectionDefinition {
496
845
  * Every index besides `_id`. Undeclared live indexes are kept (and reported)
497
846
  * unless `prune` is on.
498
847
  */
499
- indexes?: IndexDefinition[];
848
+ indexes?: readonly IndexDefinition[];
500
849
  /**
501
850
  * Atlas Search and Vector Search indexes (Atlas, an Atlas CLI local
502
851
  * deployment, or MongoDB 8.3+ with mongot). Undeclared live ones are kept
503
852
  * unless `prune` is on; leave the key out and they are not managed at all.
504
853
  * @experimental New in 2.2
505
854
  */
506
- searchIndexes?: SearchIndexDefinition[];
507
- /** A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator */
855
+ searchIndexes?: readonly SearchIndexDefinition[];
856
+ /**
857
+ * A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator.
858
+ * With `versioning`, its rules are merged in — and `null` is refused.
859
+ */
508
860
  validator?: Record<string, unknown> | null;
509
- /** Default: 'strict'. Only with a validator */
861
+ /**
862
+ * Default: 'strict' — 'moderate' when the only rules are the ones
863
+ * `versioning` adds. Only with a validator (or `versioning`)
864
+ */
510
865
  validationLevel?: ValidationLevel;
511
- /** Default: 'error'. Only with a validator */
866
+ /** Default: 'error'. Only with a validator (or `versioning`) */
512
867
  validationAction?: ValidationAction;
513
868
  /**
514
869
  * Drop live indexes (and search indexes, when `searchIndexes` is declared)
515
- * this definition does not declare. Default: the call's `prune`, else false
870
+ * this definition does not declare. Default: the call's `prune`, else false.
871
+ * With `versioning` and no `indexes`, only the version index is managed —
872
+ * prune leaves the others alone.
516
873
  */
517
874
  prune?: boolean;
875
+ /**
876
+ * Document shape versioning: the version (and revision) field typed and
877
+ * required by the validator, and the version index background migrations
878
+ * scan. The source of truth `defineShapes` reads too.
879
+ * @experimental New in 2.3
880
+ */
881
+ versioning?: CollectionVersioning;
882
+ }
883
+
884
+ /**
885
+ * The `versioning` block of a collection definition.
886
+ * @experimental New in 2.3
887
+ */
888
+ export interface CollectionVersioning {
889
+ /** The shape version new documents are written at (≥ 1) */
890
+ current: number;
891
+ /**
892
+ * The oldest shape still allowed (default 1, ≤ `current`). `0` types the
893
+ * fields without requiring them — for a collection that predates
894
+ * versioning. Converge refuses to raise it while documents below it remain.
895
+ * There is deliberately no maximum: a newer release may write ahead of the
896
+ * declaration during a rolling deploy.
897
+ */
898
+ min?: number;
899
+ /** The version field. Default `'__v'` */
900
+ field?: string;
901
+ /** Also manage a revision field for optimistic concurrency. Default `true` */
902
+ revision?: boolean;
903
+ /** The revision field. Default `'__rev'` */
904
+ revisionField?: string;
905
+ /** Declare the `{ <field>: 1, _id: 1 }` index. Default `true` */
906
+ index?: boolean;
518
907
  }
519
908
 
520
909
  /** What a `collectionsDir` file exports: a definition whose name defaults to the file name */
@@ -794,9 +1183,41 @@ export interface MigronautLogger {
794
1183
  * `{ runId, migration, direction, batch, durationMs }` — so a machine-readable
795
1184
  * logger does not have to parse the human string. A plain `(msg) => …` logger
796
1185
  * remains valid: the extra argument is simply ignored.
1186
+ *
1187
+ * On a migration's `ctx.logger`, fields with `userland: true` also emit the
1188
+ * `migration:log` event (see {@link MigrationLogEvent}).
797
1189
  */
798
1190
  export type LogMethod = (msg: string, fields?: Record<string, unknown>) => void;
799
1191
 
1192
+ /**
1193
+ * `ctx.logger`: the kit's logger with the run's correlation bound into every
1194
+ * line. Each method takes `(msg, fields?)` — or pino's own `(fields, msg?)` —
1195
+ * and an `Error` as the message (its message, credentials masked). Fields with
1196
+ * `userland: true` also emit the `migration:log` event.
1197
+ *
1198
+ * Any {@link MigronautLogger} — or a pino instance — is one, so a context built
1199
+ * by hand in a test can pass the logger it has.
1200
+ * @experimental New in 2.4
1201
+ */
1202
+ export interface MigrationLogger {
1203
+ debug(
1204
+ msgOrFields: string | Error | Record<string, unknown>,
1205
+ fieldsOrMsg?: Record<string, unknown> | string,
1206
+ ): void;
1207
+ info(
1208
+ msgOrFields: string | Error | Record<string, unknown>,
1209
+ fieldsOrMsg?: Record<string, unknown> | string,
1210
+ ): void;
1211
+ warn(
1212
+ msgOrFields: string | Error | Record<string, unknown>,
1213
+ fieldsOrMsg?: Record<string, unknown> | string,
1214
+ ): void;
1215
+ error(
1216
+ msgOrFields: string | Error | Record<string, unknown>,
1217
+ fieldsOrMsg?: Record<string, unknown> | string,
1218
+ ): void;
1219
+ }
1220
+
800
1221
  // ─── Telemetry ────────────────────────────────────────────────────────────────
801
1222
 
802
1223
  /** A span or metric attribute value — the scalar subset migronaut sets */
@@ -941,6 +1362,12 @@ export interface StatusRow {
941
1362
  origin?: MigrationOrigin;
942
1363
  /** Redacted message of the last failed attempt (status `'failed'` only) */
943
1364
  error?: string;
1365
+ /**
1366
+ * `'background'` for a background migration file (applied = registered) —
1367
+ * in `status()` and `dryRun('up')` rows alike
1368
+ * @experimental New in 2.3
1369
+ */
1370
+ kind?: 'background';
944
1371
  /** When the last failed attempt was recorded (status `'failed'` only) */
945
1372
  failedAt?: Date;
946
1373
  /**
@@ -961,6 +1388,16 @@ export interface StatusRow {
961
1388
  * (`up(file, { checksum })`).
962
1389
  */
963
1390
  checksum?: string;
1391
+ /**
1392
+ * `dryRun('up')` rows: the background migrations the file requires
1393
+ * @experimental New in 2.3
1394
+ */
1395
+ requires?: string[];
1396
+ /**
1397
+ * `dryRun('up')` rows: those of `requires` not completed yet
1398
+ * @experimental New in 2.3
1399
+ */
1400
+ waitsFor?: string[];
964
1401
  /** Who asked for the apply, and why — when the run said (`requestedBy` / `reason` options) */
965
1402
  requestedBy?: string;
966
1403
  reason?: string;
@@ -1061,7 +1498,13 @@ export type MigronautErrorCode =
1061
1498
  | 'MIGRATION_BLOCKED'
1062
1499
  | 'QUEUE_JOB_INVALID'
1063
1500
  | 'QUEUE_JOB_FAILED'
1064
- | 'CONVERGE_FAILED';
1501
+ | 'CONVERGE_FAILED'
1502
+ | 'REVISION_CONFLICT'
1503
+ | 'SHAPE_VERSION_UNSUPPORTED'
1504
+ | 'BACKGROUND_PENDING'
1505
+ | 'BACKGROUND_FAILED'
1506
+ | 'BACKGROUND_CONFLICT'
1507
+ | 'SANDBOX_REFUSED';
1065
1508
 
1066
1509
  // ─── Config file format ─────────────────────────────────────────────────────────
1067
1510
 
@@ -1123,6 +1566,14 @@ export interface UpOptions {
1123
1566
  * with a filename or `to`.
1124
1567
  */
1125
1568
  converge?: boolean;
1569
+ /**
1570
+ * What the run does at a migration that `requires` a background migration
1571
+ * not completed yet: `'error'` (default) throws {@link BackgroundPendingError};
1572
+ * `'stop'` ends the run there, cleanly (`background:waiting`). Either way
1573
+ * that migration fires no hook and leaves no failed trace.
1574
+ * @experimental New in 2.3
1575
+ */
1576
+ onBackgroundPending?: 'error' | 'stop';
1126
1577
  /**
1127
1578
  * Who asked for this run (≤ 128 characters) — stamped on the changelog
1128
1579
  * records it writes. `executedBy` is the OS user that ran it; on a queue
@@ -1131,6 +1582,25 @@ export interface UpOptions {
1131
1582
  requestedBy?: string;
1132
1583
  /** Why (≤ 512 characters) — a ticket, a sentence; stamped like `requestedBy` */
1133
1584
  reason?: string;
1585
+ /**
1586
+ * The queue job this run works for — bound into `ctx.run`, the run's log
1587
+ * lines and `migration:log`. The BullMQ adapter sets it; set it yourself
1588
+ * when you drive the kit from a queue of your own.
1589
+ * @experimental New in 2.4
1590
+ */
1591
+ job?: JobRef;
1592
+ }
1593
+
1594
+ /**
1595
+ * The queue job a run works for. Nothing is stored: the ids only correlate
1596
+ * what the run logs with the job a dashboard shows.
1597
+ * @experimental New in 2.4
1598
+ */
1599
+ export interface JobRef {
1600
+ /** The job's id (≤ 1024 characters) */
1601
+ id: string;
1602
+ /** The group of jobs it was enqueued with (≤ 128 characters) */
1603
+ groupId?: string;
1134
1604
  }
1135
1605
 
1136
1606
  /** Options for {@link MigratorKit.down} */
@@ -1165,6 +1635,11 @@ export interface DownOptions {
1165
1635
  requestedBy?: string;
1166
1636
  /** Why (≤ 512 characters) — a ticket, a sentence; stamped as `revertReason` */
1167
1637
  reason?: string;
1638
+ /**
1639
+ * The queue job this run works for — see {@link UpOptions.job}
1640
+ * @experimental New in 2.4
1641
+ */
1642
+ job?: JobRef;
1168
1643
  }
1169
1644
 
1170
1645
  /** Payload common to every lifecycle event */
@@ -1178,6 +1653,12 @@ export interface MigrationEvent extends MigronautEventBase {
1178
1653
  direction: 'up' | 'down';
1179
1654
  batch?: number;
1180
1655
  durationMs?: number;
1656
+ /**
1657
+ * How many times the body ran, when the driver retried its transaction (on
1658
+ * `migration:success` and `migration:error`; absent when it ran once)
1659
+ * @experimental New in 2.4
1660
+ */
1661
+ attempts?: number;
1181
1662
  /**
1182
1663
  * Human-readable failure message (on `migration:error` only), with URI
1183
1664
  * credentials already redacted — safe to ship to metrics/alerting as-is.
@@ -1299,6 +1780,82 @@ export interface ConvergeEndEvent extends MigronautEventBase {
1299
1780
  error?: string;
1300
1781
  }
1301
1782
 
1783
+ /** The level of a `ctx.logger` call */
1784
+ export type MigrationLogLevel = 'debug' | 'info' | 'warn' | 'error';
1785
+
1786
+ /**
1787
+ * What every `migration:log` event carries.
1788
+ * @experimental New in 2.4
1789
+ */
1790
+ export interface MigrationLogEventBase {
1791
+ level: MigrationLogLevel;
1792
+ /**
1793
+ * The message — URI credentials and the values a server error quotes (an
1794
+ * E11000's duplicate key) masked — at most 2048 characters
1795
+ */
1796
+ msg: string;
1797
+ /**
1798
+ * The call's fields without the `userland` marker, as a document a driver can
1799
+ * store: a copy, its strings redacted, at most 8 levels and 1000 entries
1800
+ * deep, strings at most 4096 characters. Dates, regular expressions and BSON
1801
+ * values are kept as they are, binary data up to 4096 bytes too. An `Error`
1802
+ * becomes `{ name, message, code?, codeName? }`; a `Map` an object, a `Set`
1803
+ * an array; any other instance what `JSON.stringify` would see.
1804
+ */
1805
+ data: Record<string, unknown>;
1806
+ /** When the call was made, by this process's clock — a TTL index can expire on it */
1807
+ at: Date;
1808
+ /** Increasing within one `runId`: orders the events of one millisecond */
1809
+ seq: number;
1810
+ /** Present when `msg` or `data` was cut to those bounds */
1811
+ truncated?: true;
1812
+ }
1813
+
1814
+ /**
1815
+ * A `ctx.logger` call with `userland: true` in an ordinary migration or its
1816
+ * hooks — `migration`, `batch` and `attempt` are absent in `beforeAll`/`afterAll`.
1817
+ * @experimental New in 2.4
1818
+ */
1819
+ export interface OrdinaryMigrationLogEvent extends MigrationLogEventBase {
1820
+ kind: 'migration';
1821
+ runId: string;
1822
+ direction: 'up' | 'down';
1823
+ migration?: string;
1824
+ batch?: number;
1825
+ attempt?: number;
1826
+ jobId?: string;
1827
+ groupId?: string;
1828
+ requestedBy?: string;
1829
+ reason?: string;
1830
+ }
1831
+
1832
+ /**
1833
+ * A `ctx.logger` call with `userland: true` in a background migration's
1834
+ * `migrate`, `migrateBatch`, `step` (or their way back). `runId` is the lane's,
1835
+ * the one its `background:*` events carry.
1836
+ * @experimental New in 2.4
1837
+ */
1838
+ export interface BackgroundMigrationLogEvent extends MigrationLogEventBase {
1839
+ kind: 'background';
1840
+ /** The lane's run id (a dry run, which has none, emits nothing) */
1841
+ runId: string;
1842
+ migration: string;
1843
+ direction: 'forward' | 'revert';
1844
+ generation?: number;
1845
+ partition: string;
1846
+ attempt: number;
1847
+ jobId?: string;
1848
+ /** Only when a run drives it inline — see {@link BackgroundRunInfo.groupId} */
1849
+ groupId?: string;
1850
+ }
1851
+
1852
+ /**
1853
+ * The `migration:log` event — `kind` tells an ordinary migration's from a
1854
+ * background migration's.
1855
+ * @experimental New in 2.4
1856
+ */
1857
+ export type MigrationLogEvent = OrdinaryMigrationLogEvent | BackgroundMigrationLogEvent;
1858
+
1302
1859
  /**
1303
1860
  * Lifecycle events emitted by {@link MigratorKit}. Subscribe to feed metrics or
1304
1861
  * alerting without parsing log lines; a listener that throws is contained and
@@ -1312,6 +1869,12 @@ export interface MigronautEvents {
1312
1869
  'migration:success': (event: MigrationEvent) => void;
1313
1870
  'migration:skipped': (event: MigrationEvent) => void;
1314
1871
  'migration:error': (event: MigrationEvent) => void;
1872
+ /**
1873
+ * A `ctx.logger` call marked `userland: true` — for the application to keep.
1874
+ * Logged lines without the marker emit nothing.
1875
+ * @experimental New in 2.4
1876
+ */
1877
+ 'migration:log': (event: MigrationLogEvent) => void;
1315
1878
  'lock:acquired': (event: LockEvent) => void;
1316
1879
  'lock:released': (event: LockEvent) => void;
1317
1880
  'lock:lost': (event: LockEvent) => void;
@@ -1319,14 +1882,52 @@ export interface MigronautEvents {
1319
1882
  'converge:action': (event: ConvergeActionEvent) => void;
1320
1883
  'converge:wait': (event: ConvergeWaitEvent) => void;
1321
1884
  'converge:end': (event: ConvergeEndEvent) => void;
1885
+ /** @experimental New in 2.3 */
1886
+ 'background:registered': (event: BackgroundRegisteredEvent) => void;
1887
+ /** @experimental New in 2.3 */
1888
+ 'background:waiting': (event: BackgroundEvent) => void;
1889
+ /** @experimental New in 2.3 */
1890
+ 'background:drift': (event: BackgroundEvent) => void;
1891
+ /**
1892
+ * A collection's live drift watcher changed state
1893
+ * @experimental New in 2.3
1894
+ */
1895
+ 'background:watch': (event: {
1896
+ runId?: string;
1897
+ collection: string;
1898
+ state: BackgroundWatchState;
1899
+ }) => void;
1900
+ /** @experimental New in 2.3 */
1901
+ 'background:unblocked': (event: BackgroundEvent) => void;
1902
+ /** @experimental New in 2.3 */
1903
+ 'background:partitioned': (event: BackgroundEvent) => void;
1904
+ /** @experimental New in 2.3 */
1905
+ 'background:pass': (event: BackgroundEvent) => void;
1906
+ /** @experimental New in 2.3 */
1907
+ 'background:slice:start': (event: BackgroundEvent) => void;
1908
+ /** @experimental New in 2.3 */
1909
+ 'background:batch': (event: BackgroundEvent) => void;
1910
+ /** @experimental New in 2.3 */
1911
+ 'background:slice:end': (event: BackgroundEvent) => void;
1912
+ /** @experimental New in 2.3 */
1913
+ 'background:lease:lost': (event: BackgroundEvent) => void;
1914
+ /** @experimental New in 2.3 */
1915
+ 'background:throttle': (event: BackgroundEvent) => void;
1916
+ /** @experimental New in 2.3 */
1917
+ 'background:control': (event: BackgroundEvent) => void;
1918
+ /** @experimental New in 2.3 */
1919
+ 'background:completed': (event: BackgroundEvent) => void;
1920
+ /** @experimental New in 2.3 */
1921
+ 'background:failed': (event: BackgroundEvent) => void;
1322
1922
  }
1323
1923
 
1324
1924
  /** One check performed by {@link MigratorKit.audit} */
1325
1925
  export interface AuditCheck {
1326
1926
  /**
1327
1927
  * e.g. 'config', 'connection', 'transactions', 'indexes', 'lock', 'checksums',
1328
- * 'pending', 'ordering', 'runtime' — and 'search' when declared collections
1329
- * hold search indexes
1928
+ * 'pending', 'ordering', 'runtime' — 'search' when declared collections
1929
+ * hold search indexes, and 'background' when background migrations are
1930
+ * registered
1330
1931
  */
1331
1932
  name: string;
1332
1933
  status: 'pass' | 'warn' | 'fail';
@@ -1356,6 +1957,11 @@ export interface RedoOptions {
1356
1957
  requestedBy?: string;
1357
1958
  /** Why — stamped like `requestedBy` */
1358
1959
  reason?: string;
1960
+ /**
1961
+ * The queue job this run works for — both halves carry it; see {@link UpOptions.job}
1962
+ * @experimental New in 2.4
1963
+ */
1964
+ job?: JobRef;
1359
1965
  }
1360
1966
 
1361
1967
  /** Options for {@link MigratorKit.create} */
@@ -1367,6 +1973,12 @@ export interface CreateOptions {
1367
1973
  * `createExtension`. Leave unset to let the config decide (default: `'js'`).
1368
1974
  */
1369
1975
  js?: boolean;
1976
+ /**
1977
+ * Generate a background migration (`export const background`) instead of
1978
+ * `up`/`down`. Not combined with `template`.
1979
+ * @experimental New in 2.3
1980
+ */
1981
+ background?: boolean;
1370
1982
  }
1371
1983
 
1372
1984
  /** Options for {@link MigratorKit.init} */
@@ -1562,11 +2174,522 @@ export class MigratorKit extends EventEmitter {
1562
2174
  * on and something is declared. Resolves the config; does not connect.
1563
2175
  */
1564
2176
  convergesAfterUp(): Promise<boolean>;
2177
+ /**
2178
+ * How drift is watched — the `backgroundDrift` setting — which a runner or
2179
+ * a queue worker hosting this kit follows. Resolves the config; does not
2180
+ * connect. @experimental
2181
+ */
2182
+ driftMode(): Promise<'poll' | 'stream' | 'both'>;
1565
2183
  /**
1566
2184
  * The converge history, newest first (`limit` 1–1000, default 20): one entry
1567
2185
  * per converge that changed something or failed. Read-only.
1568
2186
  */
1569
2187
  convergeHistory(options?: { limit?: number }): Promise<ConvergeHistoryEntry[]>;
2188
+
2189
+ // ─── Background migrations (experimental, new in 2.3) ─────────────────────
2190
+ // Reentrant: none of these is a run — no migration lock, no run id; one kit
2191
+ // may drive many at once.
2192
+
2193
+ /** One coordinator step — see {@link BackgroundCoordinatorAnswer}. @experimental */
2194
+ coordinateBackground(
2195
+ name: string,
2196
+ options?: { signal?: AbortSignal; driver?: BackgroundDriver },
2197
+ ): Promise<BackgroundCoordinatorAnswer>;
2198
+ /** One slice of one lane: claim a partition and a slot, work it, release. @experimental */
2199
+ runBackgroundSlice(
2200
+ name: string,
2201
+ options?: {
2202
+ signal?: AbortSignal;
2203
+ sliceMs?: number;
2204
+ /**
2205
+ * The queue job working the lane — on its log lines and `migration:log` events
2206
+ * @experimental New in 2.4
2207
+ */
2208
+ job?: Pick<JobRef, 'id'>;
2209
+ },
2210
+ ): Promise<BackgroundSliceResult>;
2211
+ /**
2212
+ * Drive a background migration from this process until it is done (or one
2213
+ * round, `untilDone: false`) with up to `concurrency` lanes (≤ its
2214
+ * `maxParallel`). A failed one throws {@link BackgroundFailedError}; a stop
2215
+ * {@link RunAbortedError} — it goes on from there next time. @experimental
2216
+ */
2217
+ runBackground(
2218
+ name: string,
2219
+ options?: {
2220
+ signal?: AbortSignal;
2221
+ sliceMs?: number;
2222
+ untilDone?: boolean;
2223
+ concurrency?: number;
2224
+ },
2225
+ ): Promise<BackgroundStatus>;
2226
+ /** One background migration's status, or `null` when it is not registered. @experimental */
2227
+ backgroundStatus(name: string): Promise<BackgroundStatus | null>;
2228
+ /** Every background migration's status, oldest registration first. @experimental */
2229
+ backgroundStatus(): Promise<BackgroundStatus[]>;
2230
+ /** The partitions of a background migration's latest generation. @experimental */
2231
+ backgroundPartitions(name: string): Promise<BackgroundPartitionInfo[]>;
2232
+ /** The background migrations with work to do (blocked ones unblocked on the way). @experimental */
2233
+ runnableBackground(): Promise<RunnableBackground[]>;
2234
+ /** Pause; its lanes stop at the next batch (`wait` until they have). @experimental */
2235
+ pauseBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
2236
+ /** Resume a paused one. @experimental */
2237
+ resumeBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
2238
+ /** Cancel (`wait` until its lanes have stopped). @experimental */
2239
+ cancelBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
2240
+ /**
2241
+ * Retry a failed or cancelled one — the same generation, or `fromStart`;
2242
+ * `repin` pins the file on disk first. A completed one is reopened. @experimental
2243
+ */
2244
+ retryBackground(
2245
+ name: string,
2246
+ options?: BackgroundControlOptions & { fromStart?: boolean; repin?: boolean },
2247
+ ): Promise<BackgroundControlResult>;
2248
+ /** Pin the file on disk (checksum, spec — and the changelog's checksum). @experimental */
2249
+ repinBackground(
2250
+ name: string,
2251
+ options?: BackgroundControlOptions,
2252
+ ): Promise<BackgroundControlResult & { replan: boolean; checksum: string }>;
2253
+ /** Clear the coordinator lock and every lease of a stuck one. @experimental */
2254
+ unlockBackground(name: string): Promise<{ lock: boolean; leases: number }>;
2255
+ /**
2256
+ * Dry-run a background migration, registered or not, with nothing written:
2257
+ * on a sample, its transformation alone — or with `validate`, the real write
2258
+ * path in a transaction that is always aborted. @experimental
2259
+ */
2260
+ dryRunBackground(name: string, options?: BackgroundDryRunOptions): Promise<BackgroundDryRun>;
2261
+ /**
2262
+ * The drift watch, once: one indexed probe per completed background
2263
+ * migration for documents of its old shape that appeared since; a finding
2264
+ * reopens it (`onDrift: 'reopen'`, the `backgroundOnDrift` default) or is
2265
+ * only reported. No document id is returned. @experimental
2266
+ */
2267
+ verifyBackground(options?: {
2268
+ onDrift?: 'reopen' | 'report';
2269
+ collections?: string[];
2270
+ }): Promise<BackgroundVerifyResult>;
2271
+ /**
2272
+ * The live drift watcher: a change stream per collection with a completed
2273
+ * background migration — one leader per collection across every process —
2274
+ * that upgrades each old-shape write moments after it lands, through the
2275
+ * lanes' own write path. Resolves once started; rejects with
2276
+ * ConfigInvalidError on a standalone server (no change streams).
2277
+ * @experimental
2278
+ */
2279
+ watchBackground(options?: WatchBackgroundOptions): Promise<BackgroundWatcher>;
2280
+ /** What the live drift watchers recorded for a collection — `null` when it has none @experimental */
2281
+ backgroundWatchStatus(collection: string): Promise<BackgroundWatchStatus | null>;
2282
+ /** What the live drift watchers recorded, one row per watched collection @experimental */
2283
+ backgroundWatchStatus(): Promise<BackgroundWatchStatus[]>;
2284
+ }
2285
+
2286
+ /** What a collection's live drift watcher is doing */
2287
+ export type BackgroundWatchState =
2288
+ | 'following'
2289
+ | 'catching-up'
2290
+ | 'streaming'
2291
+ | 'history-lost'
2292
+ | 'overloaded'
2293
+ | 'restarting'
2294
+ | 'suspended'
2295
+ | 'fallback'
2296
+ | 'stopped';
2297
+
2298
+ /** Options of {@link MigratorKit.watchBackground} */
2299
+ export interface WatchBackgroundOptions {
2300
+ /** Only these collections. Default: every one with a completed background migration */
2301
+ collections?: string[];
2302
+ /** Stops the watcher when aborted */
2303
+ signal?: AbortSignal;
2304
+ /** `false`: only report old-shape writes, never upgrade them. Default `true` */
2305
+ upgrade?: boolean;
2306
+ /** How often the edges and the collections are read again (ms). Default 30000 */
2307
+ refreshMs?: number;
2308
+ /** The most often the resume token is saved (ms). Default 5000 */
2309
+ checkpointMs?: number;
2310
+ /** How often a follower tries to become the leader (ms, jittered). Default 10000 */
2311
+ leaderRetryMs?: number;
2312
+ /** Collections watched by this process at most; the rest stay with the poll. Default 16 */
2313
+ maxCollections?: number;
2314
+ /**
2315
+ * A stream this far behind (ms) gives up on its backlog: the background
2316
+ * migrations it serves are reopened, and it starts again from now. Default 60000
2317
+ */
2318
+ maxLagMs?: number;
2319
+ /** Hears every failure (the watcher itself never throws) */
2320
+ onError?(error: unknown, collection?: string): void;
2321
+ }
2322
+
2323
+ /** A running live drift watcher */
2324
+ export interface BackgroundWatcher {
2325
+ readonly running: boolean;
2326
+ /** What each followed collection's watcher is doing in this process */
2327
+ status(): {
2328
+ collection: string;
2329
+ state: BackgroundWatchState | 'starting';
2330
+ /** Whether this process leads the collection */
2331
+ leading: boolean;
2332
+ counters: { events: number; upgraded: number; failed: number; skipped: number };
2333
+ lastEventAt?: Date;
2334
+ }[];
2335
+ /** Close every stream, save its position, release its lock */
2336
+ stop(): Promise<void>;
2337
+ }
2338
+
2339
+ /** A collection's live drift watcher, as stored — never its resume token */
2340
+ export interface BackgroundWatchStatus {
2341
+ collection: string;
2342
+ state: BackgroundWatchState | 'starting';
2343
+ /** The version a document should have at least */
2344
+ target?: number;
2345
+ /** The background migrations it upgrades with */
2346
+ edges: string[];
2347
+ leader?: { host: string; pid: number; at: Date };
2348
+ counters: { events: number; upgraded: number; failed: number; skipped: number };
2349
+ lastEventAt?: Date;
2350
+ updatedAt: Date;
2351
+ }
2352
+
2353
+ // ─── Background migration results ─────────────────────────────────────────────
2354
+
2355
+ /** Where a background migration stands */
2356
+ export type BackgroundState =
2357
+ | 'blocked'
2358
+ | 'pending'
2359
+ | 'running'
2360
+ | 'paused'
2361
+ | 'completed'
2362
+ | 'failed'
2363
+ | 'cancelled';
2364
+
2365
+ /**
2366
+ * Who runs a coordinator step: `{ kind, ref?, round? }` — a BullMQ round lets the newest win
2367
+ * @experimental New in 2.3
2368
+ */
2369
+ export interface BackgroundDriver {
2370
+ kind: 'bullmq' | 'runner' | 'cli' | 'inline' | 'local';
2371
+ ref?: string;
2372
+ round?: number;
2373
+ }
2374
+
2375
+ /**
2376
+ * One of {@link MigratorKit.runnableBackground}: what a driver needs to pick it up, or to tell it stalled
2377
+ * @experimental New in 2.3
2378
+ */
2379
+ export interface RunnableBackground {
2380
+ migration: string;
2381
+ status: BackgroundState;
2382
+ maxParallel: number;
2383
+ /** Leases renewed within their TTL — lanes working right now */
2384
+ liveLeases: number;
2385
+ registeredAt: Date;
2386
+ startedAt?: Date;
2387
+ lastProgressAt?: Date;
2388
+ coordinator?: { kind: string; round?: number; at: Date };
2389
+ }
2390
+
2391
+ /**
2392
+ * What a coordinator step says to do next
2393
+ * @experimental New in 2.3
2394
+ */
2395
+ export interface BackgroundCoordinatorAnswer {
2396
+ next: 'process' | 'wait' | 'done' | 'busy' | 'superseded';
2397
+ /** `process`: lanes that could start now */
2398
+ lanes?: number;
2399
+ generation?: number;
2400
+ /** `process`: the registration the lanes work for (it names their jobs) */
2401
+ registration?: string;
2402
+ /** `process`: the current plan's partitions by status */
2403
+ counts?: BackgroundStatus['partitions'];
2404
+ /**
2405
+ * A `bullmq` driver's round — handed out by this step to a chain that
2406
+ * asked without one; a chain whose round is not the latest is `superseded`
2407
+ */
2408
+ round?: number;
2409
+ /** `done`: where it stands */
2410
+ status?: BackgroundState | 'unregistered';
2411
+ /** `wait`: why — `checksum`, `replan-draining`, `plan-race`, … */
2412
+ reason?: string;
2413
+ /** `done` + `failed`: why */
2414
+ error?: string;
2415
+ waitsFor?: string[];
2416
+ retryAfterMs?: number;
2417
+ }
2418
+
2419
+ /** Counters of a slice, a partition or a whole background migration */
2420
+ export interface BackgroundCounters {
2421
+ scanned?: number;
2422
+ migrated?: number;
2423
+ skipped?: number;
2424
+ conflicts?: number;
2425
+ failed?: number;
2426
+ retried?: number;
2427
+ batches?: number;
2428
+ processed?: number;
2429
+ txnRetries?: number;
2430
+ }
2431
+
2432
+ /** How a lane's slice ended */
2433
+ export interface BackgroundSliceResult {
2434
+ outcome:
2435
+ | 'yielded'
2436
+ | 'exhausted'
2437
+ | 'busy'
2438
+ | 'stale'
2439
+ | 'paused'
2440
+ | 'cancelled'
2441
+ | 'failed'
2442
+ | 'stopped'
2443
+ | 'lost';
2444
+ counters: BackgroundCounters;
2445
+ retryAfterMs?: number;
2446
+ error?: BackgroundFailedError;
2447
+ }
2448
+
2449
+ /** A background migration, as {@link MigratorKit.backgroundStatus} reports it */
2450
+ export interface BackgroundStatus {
2451
+ migration: string;
2452
+ status: BackgroundState;
2453
+ phase: 'partition' | 'process' | 'replan';
2454
+ direction: 'forward' | 'revert';
2455
+ /** Minted at every (re-)registration — `up --force`, `redo` and `down` get a new one */
2456
+ registration: string;
2457
+ mode: 'declarative' | 'step';
2458
+ collection?: string;
2459
+ from?: number;
2460
+ to?: number;
2461
+ generation: number;
2462
+ pass: number;
2463
+ maxParallel: number;
2464
+ transaction: boolean;
2465
+ totals: BackgroundCounters & { slices?: number; reclaims?: number };
2466
+ /** Distinct documents that failed (within `maxDocumentErrors`) */
2467
+ failedDocuments: number;
2468
+ requires: string[];
2469
+ waitsFor: string[];
2470
+ /** The current plan's partitions by status */
2471
+ partitions?: {
2472
+ total: number;
2473
+ pending: number;
2474
+ running: number;
2475
+ done: number;
2476
+ failed: number;
2477
+ cancelled: number;
2478
+ superseded: number;
2479
+ leased: number;
2480
+ };
2481
+ /** Leases renewed within their TTL — lanes working right now */
2482
+ liveLeases: number;
2483
+ /**
2484
+ * The driver of the latest coordinator step that said who it was — a queue's
2485
+ * coordinator chain carries its `round`, and an older round bows out
2486
+ */
2487
+ coordinator?: { kind: string; round?: number; at: Date };
2488
+ /**
2489
+ * The current plan. `estimate` is the documents it expects to rewrite —
2490
+ * with `atLeast`, a count that stopped at its limit (there are more)
2491
+ */
2492
+ plan?: {
2493
+ method: string;
2494
+ estimate: number;
2495
+ atLeast?: boolean;
2496
+ partitions: number;
2497
+ degraded?: string;
2498
+ };
2499
+ /**
2500
+ * On a sharded collection, how the plan used the shard key: `chunks` (a
2501
+ * partition per run of chunks on one shard), `sampled` (the key space
2502
+ * sampled — the chunks could not be read), or `untargeted` (partitions by
2503
+ * `_id`: the key could not be read, or the version index does not carry it)
2504
+ */
2505
+ sharding?: {
2506
+ mode: 'chunks' | 'sampled' | 'empty' | 'untargeted';
2507
+ shardKey?: Record<string, 1 | 'hashed'>;
2508
+ hashed?: boolean;
2509
+ /** Shards the partitions are grouped by */
2510
+ groups?: number;
2511
+ };
2512
+ registeredAt: Date;
2513
+ startedAt?: Date;
2514
+ completedAt?: Date;
2515
+ lastProgressAt?: Date;
2516
+ lastError?: string;
2517
+ description?: string;
2518
+ /** The registration this one replaced (`up --force`, `redo`, `down`), as it stood then */
2519
+ previous?: {
2520
+ registration: string;
2521
+ status: BackgroundState;
2522
+ direction: 'forward' | 'revert';
2523
+ pass: number;
2524
+ totals: BackgroundCounters & { slices?: number; reclaims?: number };
2525
+ registeredAt: Date;
2526
+ completedAt?: Date;
2527
+ };
2528
+ }
2529
+
2530
+ /** One partition of a background migration */
2531
+ export interface BackgroundPartitionInfo {
2532
+ id: string;
2533
+ generation: number;
2534
+ seq: number;
2535
+ status: 'pending' | 'running' | 'done' | 'failed' | 'cancelled' | 'superseded';
2536
+ scope: Record<string, unknown>;
2537
+ estimate: number;
2538
+ counters: BackgroundCounters;
2539
+ group?: string;
2540
+ lease?: { slot: number; owner: string; host: string; pid: number; renewedAt: Date };
2541
+ throttle?: { batchSize: number; pauseMs: number };
2542
+ claims: number;
2543
+ reclaims: number;
2544
+ failures: number;
2545
+ lastError?: string;
2546
+ }
2547
+
2548
+ /** Options every background control action takes */
2549
+ export interface BackgroundControlOptions {
2550
+ /** Who asked — recorded in its history */
2551
+ requestedBy?: string;
2552
+ /** Why — recorded in its history */
2553
+ reason?: string;
2554
+ /** pause / cancel: resolve once no lane holds a lease any more */
2555
+ wait?: boolean;
2556
+ signal?: AbortSignal;
2557
+ }
2558
+
2559
+ /** What a control action did */
2560
+ export interface BackgroundControlResult {
2561
+ applied: 'changed' | 'unchanged';
2562
+ status: BackgroundState;
2563
+ /** `wait`: whether every lane stopped in time */
2564
+ stopped?: boolean;
2565
+ }
2566
+
2567
+ /** What {@link MigratorKit.verifyBackground} found */
2568
+ export interface BackgroundVerifyResult {
2569
+ /** Completed background migrations probed */
2570
+ checked: number;
2571
+ /** Skipped: another one at work on the collection, the validator guards it, no version index */
2572
+ skipped: number;
2573
+ drift: { migration: string; collection: string; action: 'reopened' | 'reported' }[];
2574
+ }
2575
+
2576
+ /** Options of {@link MigratorKit.dryRunBackground} */
2577
+ export interface BackgroundDryRunOptions {
2578
+ /** A random sample of this many matching documents (1–1000, default 5) */
2579
+ sample?: number;
2580
+ /** The first n matching documents by `_id`, instead of a sample */
2581
+ first?: number;
2582
+ /** Through the real write path, in the always-aborted sandbox */
2583
+ validate?: boolean;
2584
+ /** Dry-run the way back */
2585
+ direction?: 'forward' | 'revert';
2586
+ /** Step migrations: how many steps (1–50, default 1) */
2587
+ steps?: number;
2588
+ /** Step migrations: document images kept (default 20, at most 1000) */
2589
+ maxDocuments?: number;
2590
+ /** Step migrations: from no checkpoint, not the pinned one */
2591
+ fromStart?: boolean;
2592
+ /** Stop the sandbox after this long (default 50 000 ms) — steps, or a `validate` sample */
2593
+ deadlineMs?: number;
2594
+ }
2595
+
2596
+ /** One document of a dry run, as relaxed EJSON */
2597
+ export interface BackgroundDryRunDocument {
2598
+ _id: unknown;
2599
+ before: Record<string, unknown>;
2600
+ after?: Record<string, unknown>;
2601
+ /** The operator update it would be written with (without `validate`) */
2602
+ change?: Record<string, unknown>;
2603
+ error?: string;
2604
+ /** With `validate`: what the server made of it */
2605
+ validation?: 'ok' | 'failed' | 'skipped';
2606
+ }
2607
+
2608
+ /** One operation the sandbox ran — the filter as relaxed EJSON, at most 2 KiB */
2609
+ export interface BackgroundSandboxOperation {
2610
+ seq: number;
2611
+ step: number;
2612
+ collection?: string;
2613
+ method: string;
2614
+ filter?: unknown;
2615
+ result?: unknown;
2616
+ durationMs?: number;
2617
+ error?: string;
2618
+ }
2619
+
2620
+ /** A document the sandbox saw change */
2621
+ export interface BackgroundSandboxDocument {
2622
+ collection: string;
2623
+ _id: unknown;
2624
+ op: 'insert' | 'update' | 'delete' | 'unknown';
2625
+ before?: Record<string, unknown>;
2626
+ after?: Record<string, unknown>;
2627
+ }
2628
+
2629
+ /** What {@link MigratorKit.dryRunBackground} found */
2630
+ export type BackgroundDryRun =
2631
+ | {
2632
+ mode: 'declarative';
2633
+ migration: string;
2634
+ direction: 'forward' | 'revert';
2635
+ method: 'sample' | 'first';
2636
+ requested: number;
2637
+ found: number;
2638
+ migrated: number;
2639
+ failed: number;
2640
+ documents: BackgroundDryRunDocument[];
2641
+ /** With `validate` */
2642
+ validated?: true;
2643
+ aborted?: true;
2644
+ ops?: BackgroundSandboxOperation[];
2645
+ refusals?: { method: string; reason: string; collection?: string }[];
2646
+ /** Documents of other collections the side writes touched */
2647
+ sideEffects?: BackgroundSandboxDocument[];
2648
+ attempts?: number;
2649
+ }
2650
+ | BackgroundStepDryRun;
2651
+
2652
+ /** A step migration's dry run: up to `steps` steps in one always-aborted transaction */
2653
+ export interface BackgroundStepDryRun {
2654
+ mode: 'step';
2655
+ migration: string;
2656
+ direction: 'forward' | 'revert';
2657
+ aborted: true;
2658
+ ok: boolean;
2659
+ attempts: number;
2660
+ stoppedBy?: 'deadline' | 'done' | 'steps';
2661
+ steps: {
2662
+ step: number;
2663
+ checkpointIn: unknown;
2664
+ checkpointOut?: unknown;
2665
+ done?: boolean;
2666
+ processed?: number;
2667
+ migrated?: number;
2668
+ error?: string;
2669
+ }[];
2670
+ ops: BackgroundSandboxOperation[];
2671
+ documents: BackgroundSandboxDocument[];
2672
+ refusals: { method: string; reason: string; collection?: string }[];
2673
+ leakedCursors: number;
2674
+ truncated: boolean;
2675
+ abortedBy?: string;
2676
+ error?: string;
2677
+ }
2678
+
2679
+ /** `background:registered` */
2680
+ export interface BackgroundRegisteredEvent {
2681
+ runId?: string;
2682
+ migration: string;
2683
+ status: BackgroundState | 'withdrawn';
2684
+ direction: 'forward' | 'revert';
2685
+ waitsFor?: string[];
2686
+ }
2687
+
2688
+ /** Any other `background:*` event: the migration it is about, and what happened */
2689
+ export interface BackgroundEvent {
2690
+ runId?: string;
2691
+ migration: string;
2692
+ [field: string]: unknown;
1570
2693
  }
1571
2694
 
1572
2695
  // ─── Programmatic entry points ─────────────────────────────────────────────────
@@ -1578,6 +2701,13 @@ export type OnLockHeld = 'throw' | 'wait';
1578
2701
  export interface RunMigrationsOptions extends MigratorKitOptions {
1579
2702
  /** Skip lock acquisition (dev only — never in production) */
1580
2703
  noLock?: boolean;
2704
+ /**
2705
+ * At a migration that requires an unfinished background migration: throw
2706
+ * (`'error'`, default) or stop the run there (`'stop'`, listed in
2707
+ * `waiting`) — `'stop'` lets an app boot while a background migration runs.
2708
+ * @experimental New in 2.3
2709
+ */
2710
+ onBackgroundPending?: 'error' | 'stop';
1581
2711
  /**
1582
2712
  * How to react when another process already holds the migration lock — the
1583
2713
  * typical case when several app instances boot at once.
@@ -1632,6 +2762,12 @@ export interface MigrationSummary {
1632
2762
  attempts: number;
1633
2763
  /** The converge that ended the run — present only when `convergeAfterUp` converged */
1634
2764
  converge?: ConvergeResult;
2765
+ /**
2766
+ * With `onBackgroundPending: 'stop'`: the migration the run stopped at and
2767
+ * the background migrations it waits for.
2768
+ * @experimental New in 2.3
2769
+ */
2770
+ waiting?: { migration: string; waitsFor: { migration: string; status: string }[] }[];
1635
2771
  }
1636
2772
 
1637
2773
  /**
@@ -1668,6 +2804,56 @@ export const EXIT_CODES: Readonly<
1668
2804
  >
1669
2805
  >;
1670
2806
 
2807
+ // ─── Background runner ────────────────────────────────────────────────────────
2808
+
2809
+ /** Options of {@link startBackgroundRunner} */
2810
+ export interface BackgroundRunnerOptions {
2811
+ /** The kit to drive — or `config` (and `kitOptions`) for one the runner makes and closes */
2812
+ kit?: MigratorKit;
2813
+ config?: Partial<MigronautConfig>;
2814
+ kitOptions?: MigratorKitOptions;
2815
+ /** Lane loops in this process, shared by every background migration (default 1, ≤ 64) */
2816
+ concurrency?: number;
2817
+ /** How often the runnable list is read again (default 5000 ms) */
2818
+ pollIntervalMs?: number;
2819
+ /** A slice's length (default: each background migration's `sliceMs`) */
2820
+ sliceMs?: number;
2821
+ /** The drift watch's period (default 600 000 ms — 10 minutes); `false`: off */
2822
+ verifyIntervalMs?: number | false;
2823
+ /**
2824
+ * Host the live drift watcher in this process — `true`, or its options.
2825
+ * Default: when `backgroundDrift` is `'stream'` or `'both'`
2826
+ */
2827
+ watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
2828
+ /** Stops the runner, as `stop()` does */
2829
+ signal?: AbortSignal;
2830
+ /** Hears every failed slice (the runner itself never throws) */
2831
+ onError?: (error: unknown, migration?: string) => void;
2832
+ }
2833
+
2834
+ /** A running {@link startBackgroundRunner} */
2835
+ export interface BackgroundRunner {
2836
+ readonly kit: MigratorKit;
2837
+ readonly running: boolean;
2838
+ /** The live drift watcher this runner hosts, once started — or undefined */
2839
+ readonly watcher: BackgroundWatcher | undefined;
2840
+ /**
2841
+ * Stop at the next batch, release every lease, and close the kit the runner
2842
+ * made. `timeoutMs`: stop waiting for a lane stuck in its transformation
2843
+ * (its lease expires; the work resumes from the last checkpoint)
2844
+ */
2845
+ stop(options?: { timeoutMs?: number }): Promise<void>;
2846
+ }
2847
+
2848
+ /**
2849
+ * Drive background migrations from inside the application — no queue:
2850
+ * `concurrency` lane loops shared by every runnable background migration,
2851
+ * round-robin, plus the drift watch every `verifyIntervalMs`. Several
2852
+ * application instances share the work through the leases.
2853
+ * @experimental New in 2.3
2854
+ */
2855
+ export function startBackgroundRunner(options?: BackgroundRunnerOptions): BackgroundRunner;
2856
+
1671
2857
  // ─── Logger factory ───────────────────────────────────────────────────────────
1672
2858
 
1673
2859
  /** Threshold accepted by {@link createLogger} — drops anything less severe */
@@ -1882,3 +3068,72 @@ export class QueueJobFailedError extends MigronautError {
1882
3068
  export class ConvergeFailedError extends MigronautError {
1883
3069
  constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
1884
3070
  }
3071
+
3072
+ /**
3073
+ * Thrown by the optimistic-concurrency helpers of `@alexify/migronaut/versioning`
3074
+ * when a revision-guarded write matched nothing. `context.reason` is
3075
+ * `'conflict'` (the document is at another revision — `context.actual`),
3076
+ * `'not-found'` (nothing matches the filter) or `'unknown'` (the follow-up read
3077
+ * was skipped or could not tell); `context.expected` is the revision the
3078
+ * caller held. The filter is never copied into the error. Experimental.
3079
+ */
3080
+ export class RevisionConflictError extends MigronautError {
3081
+ readonly context?: RevisionConflictContext;
3082
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3083
+ }
3084
+
3085
+ /** {@link RevisionConflictError}'s `context` — what a caller decides on */
3086
+ export interface RevisionConflictContext {
3087
+ reason: 'conflict' | 'not-found' | 'unknown';
3088
+ /** The revision the caller held */
3089
+ expected: number;
3090
+ /** `conflict`: the revision the document is at */
3091
+ actual?: number;
3092
+ /** The collection's name, when the collection object has one */
3093
+ collection?: string;
3094
+ [key: string]: unknown;
3095
+ }
3096
+
3097
+ /**
3098
+ * Thrown by an upcaster that cannot bring a document to the current shape:
3099
+ * `context.reason` is `'newer'`, `'below-min'` or `'invalid'`, with
3100
+ * `context.version` and `context.current`. Experimental.
3101
+ */
3102
+ export class ShapeVersionError extends MigronautError {
3103
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3104
+ }
3105
+
3106
+ /**
3107
+ * Thrown when a migration `requires` a background migration that has not
3108
+ * completed — or whose collection still holds old-shape documents. Nothing was
3109
+ * run; `context.waitsFor` lists what it waits for. Experimental.
3110
+ */
3111
+ export class BackgroundPendingError extends MigronautError {
3112
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3113
+ }
3114
+
3115
+ /**
3116
+ * Thrown when a background migration ended `failed`; `context.migration` names
3117
+ * it and `context.lastError` says what happened last. Experimental.
3118
+ */
3119
+ export class BackgroundFailedError extends MigronautError {
3120
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3121
+ }
3122
+
3123
+ /**
3124
+ * Thrown when a control action does not fit the background migration's state;
3125
+ * `context.status` is the state found and `context.action` what was asked.
3126
+ * Experimental.
3127
+ */
3128
+ export class BackgroundConflictError extends MigronautError {
3129
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3130
+ }
3131
+
3132
+ /**
3133
+ * Thrown by the dry-run sandbox when a step reaches for something it cannot
3134
+ * run inside an always-aborted transaction; `context.method` names the call
3135
+ * and `context.reason` the rule. Experimental.
3136
+ */
3137
+ export class SandboxRefusedError extends MigronautError {
3138
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
3139
+ }