@alexify/migronaut 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/README.md +68 -10
  3. package/bullmq.d.ts +465 -7
  4. package/index.d.ts +1272 -18
  5. package/migronaut.schema.json +150 -2
  6. package/package.json +8 -2
  7. package/src/bullmq/background-processor.js +469 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +153 -15
  11. package/src/bullmq/producer.js +202 -27
  12. package/src/bullmq/service.js +480 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/converge.js +38 -10
  15. package/src/cli/commands/create.js +6 -0
  16. package/src/cli/exit-codes.js +6 -0
  17. package/src/cli/index.js +2 -0
  18. package/src/cli/table.js +68 -9
  19. package/src/core/audit.js +98 -3
  20. package/src/core/background-audit.js +139 -0
  21. package/src/core/background-drift.js +126 -0
  22. package/src/core/background-dry-run.js +366 -0
  23. package/src/core/background-engine.js +818 -0
  24. package/src/core/background-kit.js +425 -0
  25. package/src/core/background-partition.js +298 -0
  26. package/src/core/background-runner.js +305 -0
  27. package/src/core/background-sandbox.js +701 -0
  28. package/src/core/background-shard.js +542 -0
  29. package/src/core/background-spec.js +597 -0
  30. package/src/core/background-store.js +951 -0
  31. package/src/core/background-throttle.js +269 -0
  32. package/src/core/background-watch-plan.js +164 -0
  33. package/src/core/background-watch-store.js +78 -0
  34. package/src/core/background-watch.js +605 -0
  35. package/src/core/background.js +1121 -0
  36. package/src/core/bson-peer.js +23 -0
  37. package/src/core/changelog.js +32 -0
  38. package/src/core/collections.js +125 -31
  39. package/src/core/config.js +133 -13
  40. package/src/core/converge-plan.js +343 -61
  41. package/src/core/converge-search-run.js +440 -0
  42. package/src/core/converge-search.js +404 -0
  43. package/src/core/converge.js +428 -183
  44. package/src/core/index-spec.js +27 -16
  45. package/src/core/lock.js +97 -32
  46. package/src/core/migrator.js +951 -26
  47. package/src/core/options.js +32 -1
  48. package/src/core/run.js +26 -12
  49. package/src/core/runner.js +1 -1
  50. package/src/core/search-index-spec.js +758 -0
  51. package/src/core/server-info.js +70 -0
  52. package/src/core/shard-info.js +76 -0
  53. package/src/core/versioning-spec.js +181 -0
  54. package/src/errors/index.js +97 -5
  55. package/src/index.js +16 -0
  56. package/src/utils/canonical.js +34 -1
  57. package/src/utils/error.js +11 -2
  58. package/src/utils/loader.js +77 -9
  59. package/src/utils/migration-name.js +33 -1
  60. package/src/utils/telemetry.js +125 -1
  61. package/src/utils/template.js +69 -1
  62. package/src/versioning/config.js +155 -0
  63. package/src/versioning/document.js +326 -0
  64. package/src/versioning/index.js +50 -0
  65. package/src/versioning/internal.js +279 -0
  66. package/src/versioning/mongoose.js +151 -0
  67. package/src/versioning/occ.js +318 -0
  68. package/src/versioning/registry.js +187 -0
  69. package/src/versioning/upcaster.js +213 -0
  70. package/versioning.d.ts +666 -0
  71. package/versioning.js +1 -0
package/index.d.ts CHANGED
@@ -41,6 +41,12 @@ export interface MigrationContext {
41
41
  export interface MigrationModule {
42
42
  up: (ctx: MigrationContext) => Promise<void>;
43
43
  down: (ctx: MigrationContext) => Promise<void>;
44
+ /**
45
+ * Background migrations (file names, each sorting before this file) that
46
+ * must have completed before this migration runs.
47
+ * @experimental New in 2.3
48
+ */
49
+ requires?: readonly string[];
44
50
  /** If true, wraps this migration in a MongoDB session + transaction */
45
51
  useTransaction?: boolean;
46
52
  /** Overrides `MigronautConfig.timeoutMs` for this migration only */
@@ -49,6 +55,215 @@ export interface MigrationModule {
49
55
  description?: string;
50
56
  }
51
57
 
58
+ // ─── Document shapes and background migrations ───────────────────────────────
59
+
60
+ /** The system field names of a versioned collection — `revisionField` is `null` without revisions */
61
+ export interface ShapeFieldNames {
62
+ field: string;
63
+ revisionField: string | null;
64
+ }
65
+
66
+ /** The default system field names: `__v` and `__rev` */
67
+ export interface DefaultShapeFieldNames {
68
+ field: '__v';
69
+ revisionField: '__rev';
70
+ }
71
+
72
+ /**
73
+ * A document type without its system fields (the version and the revision),
74
+ * distributed over a union. What a shape body is declared as, and what a
75
+ * background transformation returns: migronaut writes the system fields.
76
+ * @experimental New in 2.3
77
+ */
78
+ export type Body<T, N extends ShapeFieldNames = DefaultShapeFieldNames> = T extends unknown
79
+ ? Omit<T, N['field'] | Extract<N['revisionField'], string>>
80
+ : never;
81
+
82
+ /**
83
+ * What a background migration's transformation gets besides the document.
84
+ * `session`, `db` and `client` are there only in a `transaction` background
85
+ * migration — writes to other collections must pass `session` to commit
86
+ * with the batch.
87
+ * @experimental New in 2.3
88
+ */
89
+ export interface BackgroundMigrationContext {
90
+ /** Aborted when the slice is stopping (lease lost, pause, shutdown) */
91
+ signal: AbortSignal;
92
+ logger: MigronautLogger;
93
+ direction: 'forward' | 'revert';
94
+ background: { name: string; generation: number; partition: string };
95
+ session?: ClientSession;
96
+ db?: Db;
97
+ client?: MongoClient;
98
+ /** True in a dry run — the writes are rolled back */
99
+ dryRun?: boolean;
100
+ }
101
+
102
+ /** How a background migration splits its collection into partitions */
103
+ export interface BackgroundPartitionSettings {
104
+ /** Partitions per lane (default 4), so a slow partition does not hold the pass */
105
+ overPartition?: number;
106
+ /** Default 256 */
107
+ maxPartitions?: number;
108
+ /** No partition is planned smaller than this (default 4 × batchSize) */
109
+ minPartitionDocs?: number;
110
+ /** Ids sampled to place the boundaries (default min(10 000, 100 × partitions)) */
111
+ sampleSize?: number;
112
+ }
113
+
114
+ /** A transactional background migration's budget */
115
+ export interface BackgroundTransactionSettings {
116
+ /** Per batch, ≤ 50 000 (default 10 000) */
117
+ timeoutMs?: number;
118
+ /** Retries of a batch on a transient transaction error (default 5) */
119
+ maxRetries?: number;
120
+ }
121
+
122
+ /** The latency-driven throttle (AIMD) */
123
+ export interface BackgroundAdaptiveSettings {
124
+ /** A batch write slower than this halves the batch (default 500) */
125
+ targetLatencyMs?: number;
126
+ /** Default 10 */
127
+ minBatchSize?: number;
128
+ /** Never above `batchSize` (the default) */
129
+ maxBatchSize?: number;
130
+ /** Default 30 000 */
131
+ maxPauseMs?: number;
132
+ }
133
+
134
+ /** What a `throttle` hook is told before every batch */
135
+ export interface BackgroundThrottleContext {
136
+ name: string;
137
+ collection?: string;
138
+ generation: number;
139
+ partition: string;
140
+ batchSize: number;
141
+ signal: AbortSignal;
142
+ }
143
+
144
+ /**
145
+ * Settings every background migration may carry, with their defaults.
146
+ * @experimental New in 2.3
147
+ */
148
+ export interface BackgroundMigrationSettings {
149
+ description?: string;
150
+ /** Documents per batch: 500 (100 with `transaction`) */
151
+ batchSize?: number;
152
+ /** Pause between batches: 100 ms */
153
+ pauseMs?: number;
154
+ /** How long a lane holds a partition before it yields: 30 000 ms */
155
+ sliceMs?: number;
156
+ /** Every batch write's, transactions included — default `{ w: 'majority' }` */
157
+ writeConcern?: { w?: number | 'majority'; j?: boolean; wtimeoutMS?: number };
158
+ /** Documents that may fail before the background migration does: 0 (at most 1000) */
159
+ maxDocumentErrors?: number;
160
+ /** Passes over the remaining old-shape documents before giving up: 10 */
161
+ maxPasses?: number;
162
+ /** Re-read rounds for documents a concurrent write changed under a batch: 3 */
163
+ maxConflictRetries?: number;
164
+ /** Failed slices of one partition in a row (no checkpoint between) before it fails: 3 */
165
+ maxSliceFailures?: number;
166
+ /** Wait while a secondary lags more than this: 10 000 ms (`false`: never) */
167
+ maxReplicationLagMs?: number | false;
168
+ /** Called before every batch; a number it returns is an extra pause (ms) */
169
+ throttle?(ctx: BackgroundThrottleContext): number | void | Promise<number | void>;
170
+ /** Partitions processed at once, across every process: 1 (at most 64) */
171
+ maxParallel?: number;
172
+ partitions?: BackgroundPartitionSettings;
173
+ /** Batch and checkpoint in one transaction (needs a replica set or mongos): false */
174
+ transaction?: boolean | BackgroundTransactionSettings;
175
+ /** The latency-driven throttle: true */
176
+ adaptive?: boolean | BackgroundAdaptiveSettings;
177
+ /** Lanes per shard on a sharded collection: 1 */
178
+ shardConcurrency?: number;
179
+ }
180
+
181
+ /**
182
+ * A declarative background migration: every document of `collection` at
183
+ * version `from` (and matching `filter`) rewritten to version `to` by
184
+ * `migrate` (or `migrateBatch`), in partitions, behind an optimistic guard.
185
+ * `From` and `To` type the documents — see `BackgroundMigrationFor` in
186
+ * `@alexify/migronaut/versioning` for the shape-map form.
187
+ *
188
+ * The callbacks are declared as methods, so a transformation typed for the
189
+ * stored document (with its version) or for its body fits either way.
190
+ * @experimental New in 2.3
191
+ */
192
+ export interface DeclarativeBackgroundMigration<
193
+ From extends object = Record<string, any>,
194
+ To extends object = Record<string, any>,
195
+ > extends BackgroundMigrationSettings {
196
+ collection: string;
197
+ /** The version rewritten — 0 for documents without a version field */
198
+ from: number;
199
+ to: number;
200
+ filter?: Record<string, unknown>;
201
+ /** The new document for one old one; the engine sets the version and bumps the revision */
202
+ migrate?(doc: From, ctx: BackgroundMigrationContext): To | Promise<To>;
203
+ /** The new documents for a batch, aligned — an `Error` fails just that document */
204
+ migrateBatch?(docs: From[], ctx: BackgroundMigrationContext): (To | Error)[] | Promise<(To | Error)[]>;
205
+ /** The way back, for `down` */
206
+ revert?(doc: To, ctx: BackgroundMigrationContext): From | Promise<From>;
207
+ revertBatch?(docs: To[], ctx: BackgroundMigrationContext): (From | Error)[] | Promise<(From | Error)[]>;
208
+ /** Default: the collection's `versioning.field`, else `'__v'` */
209
+ versionField?: string;
210
+ /** Default: the collection's `versioning.revisionField`, else `'__rev'` */
211
+ revisionField?: string;
212
+ /**
213
+ * `'revision'` (default) guards each write with the revision; a collection
214
+ * without revisions must say `'version-only'` — a concurrent write that
215
+ * leaves the version alone is then invisible to it.
216
+ */
217
+ occ?: 'revision' | 'version-only';
218
+ }
219
+
220
+ /** What a `step` background migration gets */
221
+ export interface BackgroundStepContext extends BackgroundMigrationContext {
222
+ db: Db;
223
+ client: MongoClient;
224
+ /** What the previous step returned (`null` at the start) */
225
+ checkpoint: unknown;
226
+ /** Epoch ms the step should return by — the slice ends then */
227
+ deadline: number;
228
+ }
229
+
230
+ /** What a `step` returns */
231
+ export interface BackgroundStepResult {
232
+ /** Saved (≤ 64 KiB of BSON) and handed to the next step */
233
+ checkpoint: unknown;
234
+ /** True once there is nothing left */
235
+ done: boolean;
236
+ processed?: number;
237
+ migrated?: number;
238
+ /** For progress, when known */
239
+ total?: number;
240
+ }
241
+
242
+ /**
243
+ * A free-form background migration — the escape hatch: migronaut runs `step`
244
+ * again and again with its last checkpoint until it says `done`, owning the
245
+ * lease, the slices, the throttle and the controls. One partition only; the
246
+ * writes must be idempotent.
247
+ * @experimental New in 2.3
248
+ */
249
+ export interface StepBackgroundMigration extends BackgroundMigrationSettings {
250
+ /** Shown in status — the collection it works on, if one */
251
+ collection?: string;
252
+ step(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
253
+ revertStep?(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
254
+ }
255
+
256
+ /** A background migration, as a migration file exports it: `export const background = {…}` */
257
+ export type BackgroundMigration = DeclarativeBackgroundMigration | StepBackgroundMigration;
258
+
259
+ /** Shape of a background migration file module — no `up`/`down` */
260
+ export interface BackgroundMigrationModule {
261
+ background: BackgroundMigration;
262
+ /** Background migrations that must complete first (each sorting before this file) */
263
+ requires?: readonly string[];
264
+ description?: string;
265
+ }
266
+
52
267
  // ─── Changelog ────────────────────────────────────────────────────────────────
53
268
 
54
269
  export type MigrationStatus = 'applied' | 'reverted' | 'failed';
@@ -93,6 +308,12 @@ export interface MigrationRecord {
93
308
  * `migronaut import` — these are not reversible by migronaut. Absent for native records.
94
309
  */
95
310
  origin?: MigrationOrigin;
311
+ /**
312
+ * `'background'` for a background migration file — applying it registered
313
+ * the background migration; its documents are rewritten later.
314
+ * @experimental New in 2.3
315
+ */
316
+ kind?: 'background';
96
317
  }
97
318
 
98
319
  // ─── Config ───────────────────────────────────────────────────────────────────
@@ -292,6 +513,61 @@ export interface MigronautConfig {
292
513
  * Default: false
293
514
  */
294
515
  convergeAfterUp?: boolean;
516
+ /**
517
+ * What converge does with declared search indexes on a server without
518
+ * Atlas Search: `'fail'` refuses the run before anything is written,
519
+ * `'skip'` converges everything else and reports them as `skip` rows.
520
+ * Default: 'fail'
521
+ * @experimental New in 2.2
522
+ */
523
+ onSearchUnavailable?: 'fail' | 'skip';
524
+ /**
525
+ * Hold every converge — the after-up one included — until each declared
526
+ * search index is queryable with its declared definition. Search indexes
527
+ * build in the background, so without it a new one is not queryable yet when
528
+ * converge returns. An index that FAILED or went STALE before the run, its
529
+ * definition unchanged, does not hold it — it is warned about instead. The
530
+ * migration lock is released while it waits. Default: false
531
+ * @experimental New in 2.2
532
+ */
533
+ waitForSearchIndexes?: boolean;
534
+ /**
535
+ * How long `waitForSearchIndexes` waits before the converge fails with
536
+ * `phase: 'wait'` (the server goes on building). Default: 600000 (10 minutes)
537
+ * @experimental New in 2.2
538
+ */
539
+ searchIndexWaitTimeoutMs?: number;
540
+ /**
541
+ * Where background migrations keep their state — and, named after it,
542
+ * their partitions (`<name>_partitions`) and the drift watcher's resume
543
+ * tokens (`<name>_watch`). Default `'_migronaut_background'`.
544
+ * @experimental New in 2.3
545
+ */
546
+ backgroundCollection?: string;
547
+ /**
548
+ * Run a background migration to the end inside the `up` that registers it,
549
+ * under the migration lock — for small collections and tests. Default false.
550
+ * @experimental New in 2.3
551
+ */
552
+ backgroundInline?: boolean;
553
+ /**
554
+ * What the drift watch does with old-shape documents that appear after a
555
+ * background migration completed: `'reopen'` it (default) or only `'report'`.
556
+ * @experimental New in 2.3
557
+ */
558
+ backgroundOnDrift?: 'reopen' | 'report';
559
+ /**
560
+ * How drift is watched: `'poll'` (default — a check every 10 minutes),
561
+ * `'stream'` (change streams, the check as a backstop) or `'both'`.
562
+ * @experimental New in 2.3
563
+ */
564
+ backgroundDrift?: 'poll' | 'stream' | 'both';
565
+ /**
566
+ * Partition a sharded collection by its shard key and target each write at
567
+ * one shard (`'auto'`, default), or treat it like any other (`'off'`).
568
+ * @experimental New in 2.3
569
+ */
570
+ backgroundShardAware?: 'auto' | 'off';
295
571
  /** Mongoose instance — required only if your migrations use Mongoose models */
296
572
  mongoose?: MongooseLike;
297
573
  hooks?: MigrationHooks;
@@ -393,9 +669,77 @@ export interface IndexDefinition {
393
669
  export type ValidationLevel = 'off' | 'strict' | 'moderate';
394
670
  export type ValidationAction = 'error' | 'warn' | 'errorAndLog';
395
671
 
672
+ /** The two kinds of Atlas search index */
673
+ export type SearchIndexType = 'search' | 'vectorSearch';
674
+
675
+ /** `mappings` of an Atlas Search definition */
676
+ export interface SearchIndexMappings {
677
+ /** Default: false */
678
+ dynamic?: boolean | { typeSet: string };
679
+ fields?: Record<string, unknown>;
680
+ }
681
+
682
+ /**
683
+ * An Atlas Search index definition, as Atlas defines it. Compared whole, with
684
+ * the documented defaults filled in; anything Atlas adds can be declared too.
685
+ */
686
+ export interface SearchDefinition {
687
+ mappings: SearchIndexMappings;
688
+ /** Default: 'lucene.standard' */
689
+ analyzer?: string;
690
+ /** Default: the analyzer */
691
+ searchAnalyzer?: string;
692
+ analyzers?: Array<Record<string, unknown>>;
693
+ synonyms?: Array<Record<string, unknown>>;
694
+ /** Default: false */
695
+ storedSource?: boolean | { include?: string[]; exclude?: string[] };
696
+ /** Default: 1 */
697
+ numPartitions?: number;
698
+ [option: string]: unknown;
699
+ }
700
+
701
+ /** One field of a Vector Search definition */
702
+ export interface VectorSearchField {
703
+ /** `'autoEmbed'` is Atlas's automated embedding (in preview); one index holds vector or autoEmbed fields, not both */
704
+ type: 'vector' | 'filter' | 'autoEmbed' | (string & {});
705
+ path: string;
706
+ numDimensions?: number;
707
+ similarity?: 'euclidean' | 'cosine' | 'dotProduct';
708
+ /** Default: 'none' ('scalar' for autoEmbed) */
709
+ quantization?: string;
710
+ /** Default: 'hnsw' */
711
+ indexingMethod?: 'hnsw' | 'flat';
712
+ /** Default: { maxEdges: 16, numEdgeCandidates: 100 } */
713
+ hnswOptions?: { maxEdges?: number; numEdgeCandidates?: number };
714
+ /** autoEmbed: the embedding model */
715
+ model?: string;
716
+ /** autoEmbed: 'text' */
717
+ modality?: string;
718
+ [option: string]: unknown;
719
+ }
720
+
721
+ /** A Vector Search index definition */
722
+ export interface VectorSearchDefinition {
723
+ fields: VectorSearchField[];
724
+ [option: string]: unknown;
725
+ }
726
+
727
+ /**
728
+ * One declared Atlas Search or Vector Search index. The name defaults to
729
+ * `'default'` and the type to `'search'`, as on the server. A change of type
730
+ * — or of an autoEmbed field's path, model, size, quantization or modality —
731
+ * cannot be made in place: converge refuses it, and the way is a new index
732
+ * under a new name (converge, then remove the old declaration and converge
733
+ * with prune).
734
+ * @experimental New in 2.2 — the shape may still change in a minor release (named in the CHANGELOG).
735
+ */
736
+ export type SearchIndexDefinition =
737
+ | { name?: string; type?: 'search'; definition: SearchDefinition }
738
+ | { name?: string; type: 'vectorSearch'; definition: VectorSearchDefinition };
739
+
396
740
  /**
397
741
  * A declared collection: the end state `converge()` keeps it in. Leave
398
- * `indexes` or `validator` out to leave that part unmanaged.
742
+ * `indexes`, `searchIndexes` or `validator` out to leave that part unmanaged.
399
743
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
400
744
  */
401
745
  export interface CollectionDefinition {
@@ -404,15 +748,65 @@ export interface CollectionDefinition {
404
748
  * Every index besides `_id`. Undeclared live indexes are kept (and reported)
405
749
  * unless `prune` is on.
406
750
  */
407
- indexes?: IndexDefinition[];
408
- /** A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator */
751
+ indexes?: readonly IndexDefinition[];
752
+ /**
753
+ * Atlas Search and Vector Search indexes (Atlas, an Atlas CLI local
754
+ * deployment, or MongoDB 8.3+ with mongot). Undeclared live ones are kept
755
+ * unless `prune` is on; leave the key out and they are not managed at all.
756
+ * @experimental New in 2.2
757
+ */
758
+ searchIndexes?: readonly SearchIndexDefinition[];
759
+ /**
760
+ * A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator.
761
+ * With `versioning`, its rules are merged in — and `null` is refused.
762
+ */
409
763
  validator?: Record<string, unknown> | null;
410
- /** Default: 'strict'. Only with a validator */
764
+ /**
765
+ * Default: 'strict' — 'moderate' when the only rules are the ones
766
+ * `versioning` adds. Only with a validator (or `versioning`)
767
+ */
411
768
  validationLevel?: ValidationLevel;
412
- /** Default: 'error'. Only with a validator */
769
+ /** Default: 'error'. Only with a validator (or `versioning`) */
413
770
  validationAction?: ValidationAction;
414
- /** Drop live indexes this definition does not declare. Default: the call's `prune`, else false */
771
+ /**
772
+ * Drop live indexes (and search indexes, when `searchIndexes` is declared)
773
+ * this definition does not declare. Default: the call's `prune`, else false.
774
+ * With `versioning` and no `indexes`, only the version index is managed —
775
+ * prune leaves the others alone.
776
+ */
415
777
  prune?: boolean;
778
+ /**
779
+ * Document shape versioning: the version (and revision) field typed and
780
+ * required by the validator, and the version index background migrations
781
+ * scan. The source of truth `defineShapes` reads too.
782
+ * @experimental New in 2.3
783
+ */
784
+ versioning?: CollectionVersioning;
785
+ }
786
+
787
+ /**
788
+ * The `versioning` block of a collection definition.
789
+ * @experimental New in 2.3
790
+ */
791
+ export interface CollectionVersioning {
792
+ /** The shape version new documents are written at (≥ 1) */
793
+ current: number;
794
+ /**
795
+ * The oldest shape still allowed (default 1, ≤ `current`). `0` types the
796
+ * fields without requiring them — for a collection that predates
797
+ * versioning. Converge refuses to raise it while documents below it remain.
798
+ * There is deliberately no maximum: a newer release may write ahead of the
799
+ * declaration during a rolling deploy.
800
+ */
801
+ min?: number;
802
+ /** The version field. Default `'__v'` */
803
+ field?: string;
804
+ /** Also manage a revision field for optimistic concurrency. Default `true` */
805
+ revision?: boolean;
806
+ /** The revision field. Default `'__rev'` */
807
+ revisionField?: string;
808
+ /** Declare the `{ <field>: 1, _id: 1 }` index. Default `true` */
809
+ index?: boolean;
416
810
  }
417
811
 
418
812
  /** What a `collectionsDir` file exports: a definition whose name defaults to the file name */
@@ -443,20 +837,33 @@ export interface ConvergeOptions {
443
837
  * CLI: `--rebuild-unique`.
444
838
  */
445
839
  rebuildUnique?: boolean;
840
+ /**
841
+ * Hold the run until every declared search index serves its
842
+ * declaration — failing on a FAILED build of an index this run created or
843
+ * changed, or after `searchIndexWaitTimeoutMs`. The migration lock is released
844
+ * when the wait starts — it only reads. Overrides the config's `waitForSearchIndexes`;
845
+ * not with `dryRun`. CLI: `--wait-search` / `--no-wait-search`.
846
+ * @experimental New in 2.2
847
+ */
848
+ waitForSearchIndexes?: boolean;
446
849
  /** Who asked for this converge — recorded in the converge history */
447
850
  requestedBy?: string;
448
851
  /** Why — recorded in the converge history */
449
852
  reason?: string;
450
853
  }
451
854
 
452
- export type ConvergeTarget = 'collection' | 'validator' | 'index';
855
+ export type ConvergeTarget = 'collection' | 'validator' | 'index' | 'searchIndex';
453
856
 
454
857
  /**
455
858
  * What converge does to one target. `keep` is an undeclared index left alone
456
859
  * (prune off); `conflict` refuses the run — an undeclared index covers the
457
860
  * declared one's key under another name, a unique index would be rebuilt
458
- * without {@link ConvergeOptions.rebuildUnique}, or the collection is a view
459
- * or a time-series collection.
861
+ * without {@link ConvergeOptions.rebuildUnique}, the collection is a view or a
862
+ * time-series collection, a search index would need a change no update can
863
+ * make (its type, an autoEmbed field's model or size), or the server has no
864
+ * Atlas Search; `skip` is a declared search index left alone on a server
865
+ * without Search (`onSearchUnavailable: 'skip'`). A search index is never
866
+ * `recreate`d: `modify` updates it in place.
460
867
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
461
868
  */
462
869
  export type ConvergeActionKind =
@@ -466,7 +873,37 @@ export type ConvergeActionKind =
466
873
  | 'drop'
467
874
  | 'keep'
468
875
  | 'unchanged'
469
- | 'conflict';
876
+ | 'conflict'
877
+ | 'skip';
878
+
879
+ /**
880
+ * The status `$listSearchIndexes` reports for a search index — `'UNKNOWN'`
881
+ * when the server reports none
882
+ */
883
+ export type SearchIndexStatus =
884
+ | 'PENDING'
885
+ | 'BUILDING'
886
+ | 'READY'
887
+ | 'FAILED'
888
+ | 'STALE'
889
+ | 'DELETING'
890
+ | 'DOES_NOT_EXIST'
891
+ | 'UNKNOWN'
892
+ | (string & {});
893
+
894
+ /**
895
+ * Where the server is with a search index: it builds in the background, so a
896
+ * created or updated one is not queryable (with its new definition) at once
897
+ * @experimental New in 2.2
898
+ */
899
+ export interface SearchIndexBuild {
900
+ status: SearchIndexStatus;
901
+ queryable: boolean;
902
+ /** The server's message — why a build FAILED, typically */
903
+ message?: string;
904
+ /** A newer definition is being built next to the one served */
905
+ updating?: true;
906
+ }
470
907
 
471
908
  /**
472
909
  * `planned` in a dry run; otherwise `applied`, `failed`, or `skipped` — no
@@ -497,6 +934,21 @@ export interface ConvergeAction {
497
934
  from?: Record<string, unknown>;
498
935
  /** What the row puts there — the declared index or validator — on rows that create or change it */
499
936
  to?: Record<string, unknown>;
937
+ /**
938
+ * A search index row's build state on the server — as read before the run,
939
+ * and after it for a row the run applied
940
+ * @experimental New in 2.2
941
+ */
942
+ build?: SearchIndexBuild;
943
+ /**
944
+ * On a search index row: the options the server reports that the
945
+ * declaration does not set and migronaut knows no default for
946
+ * (`mappings.fields.title.similarity`) — left out of the comparison, so a
947
+ * new server default does not make every converge update the index.
948
+ * Declare one to manage it.
949
+ * @experimental New in 2.2
950
+ */
951
+ ignored?: string[];
500
952
  }
501
953
 
502
954
  /**
@@ -532,6 +984,12 @@ export interface ConvergeHistoryEntry {
532
984
  /** The rows that changed, failed or refused the run — each with its collection and `from` / `to` */
533
985
  actions: Array<ConvergeAction & { collection: string }>;
534
986
  unstable?: ConvergeUnstable[];
987
+ /**
988
+ * What the run saw of Atlas Search, and how a wait for its builds ended —
989
+ * when a definition declares `searchIndexes`
990
+ * @experimental New in 2.2
991
+ */
992
+ search?: ConvergeSearchSummary;
535
993
  }
536
994
 
537
995
  /**
@@ -546,6 +1004,42 @@ export interface ConvergeUnstable {
546
1004
  reason?: string;
547
1005
  }
548
1006
 
1007
+ /**
1008
+ * A declared search index that exists but does not serve its declaration yet
1009
+ * @experimental New in 2.2
1010
+ */
1011
+ export interface SearchIndexNotReady extends SearchIndexBuild {
1012
+ collection: string;
1013
+ name: string;
1014
+ }
1015
+
1016
+ /**
1017
+ * What a converge saw of Atlas Search — present when a definition declares
1018
+ * `searchIndexes`
1019
+ * @experimental New in 2.2
1020
+ */
1021
+ export interface ConvergeSearchSummary {
1022
+ /** Whether the server has Atlas Search */
1023
+ available: boolean;
1024
+ /**
1025
+ * How that was told: `'listed'` (the server listed search indexes),
1026
+ * `'parameter'` (its search index manager setting), `'error'` (it refused
1027
+ * a search command), `'version'` (older than 6.0, not asked) or
1028
+ * `'assumed'` (it would not say — a refusal at apply time reports it)
1029
+ */
1030
+ evidence?: 'listed' | 'parameter' | 'error' | 'version' | 'assumed';
1031
+ /**
1032
+ * Declared search indexes still building, updating, stale or failed. Does
1033
+ * not count against `inSync`: a build is the server's work, not a difference.
1034
+ */
1035
+ notReady: SearchIndexNotReady[];
1036
+ /** How a wait for the builds (`waitForSearchIndexes`) ended, when there was one */
1037
+ wait?: { outcome: ConvergeWaitOutcome; waitedMs: number };
1038
+ }
1039
+
1040
+ /** How a wait for search index builds ended */
1041
+ export type ConvergeWaitOutcome = 'ready' | 'failed' | 'timeout' | 'unreadable' | 'aborted';
1042
+
549
1043
  /**
550
1044
  * Outcome of {@link MigratorKit.converge}
551
1045
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
@@ -556,11 +1050,15 @@ export interface ConvergeResult {
556
1050
  changed: number;
557
1051
  /**
558
1052
  * True when the database matches the declarations: nothing left to do and
559
- * no conflict. Undeclared indexes kept with prune off do not count against it.
1053
+ * no conflict. Undeclared indexes kept with prune off, search indexes
1054
+ * skipped on a server without Search, and search index builds still under
1055
+ * way do not count against it.
560
1056
  */
561
1057
  inSync: boolean;
562
1058
  collections: CollectionConvergeResult[];
563
1059
  unstable?: ConvergeUnstable[];
1060
+ /** @experimental New in 2.2 */
1061
+ search?: ConvergeSearchSummary;
564
1062
  }
565
1063
 
566
1064
  // ─── Logger ───────────────────────────────────────────────────────────────────
@@ -735,6 +1233,12 @@ export interface StatusRow {
735
1233
  origin?: MigrationOrigin;
736
1234
  /** Redacted message of the last failed attempt (status `'failed'` only) */
737
1235
  error?: string;
1236
+ /**
1237
+ * `'background'` for a background migration file (applied = registered) —
1238
+ * in `status()` and `dryRun('up')` rows alike
1239
+ * @experimental New in 2.3
1240
+ */
1241
+ kind?: 'background';
738
1242
  /** When the last failed attempt was recorded (status `'failed'` only) */
739
1243
  failedAt?: Date;
740
1244
  /**
@@ -755,6 +1259,16 @@ export interface StatusRow {
755
1259
  * (`up(file, { checksum })`).
756
1260
  */
757
1261
  checksum?: string;
1262
+ /**
1263
+ * `dryRun('up')` rows: the background migrations the file requires
1264
+ * @experimental New in 2.3
1265
+ */
1266
+ requires?: string[];
1267
+ /**
1268
+ * `dryRun('up')` rows: those of `requires` not completed yet
1269
+ * @experimental New in 2.3
1270
+ */
1271
+ waitsFor?: string[];
758
1272
  /** Who asked for the apply, and why — when the run said (`requestedBy` / `reason` options) */
759
1273
  requestedBy?: string;
760
1274
  reason?: string;
@@ -855,7 +1369,13 @@ export type MigronautErrorCode =
855
1369
  | 'MIGRATION_BLOCKED'
856
1370
  | 'QUEUE_JOB_INVALID'
857
1371
  | 'QUEUE_JOB_FAILED'
858
- | 'CONVERGE_FAILED';
1372
+ | 'CONVERGE_FAILED'
1373
+ | 'REVISION_CONFLICT'
1374
+ | 'SHAPE_VERSION_UNSUPPORTED'
1375
+ | 'BACKGROUND_PENDING'
1376
+ | 'BACKGROUND_FAILED'
1377
+ | 'BACKGROUND_CONFLICT'
1378
+ | 'SANDBOX_REFUSED';
859
1379
 
860
1380
  // ─── Config file format ─────────────────────────────────────────────────────────
861
1381
 
@@ -917,6 +1437,14 @@ export interface UpOptions {
917
1437
  * with a filename or `to`.
918
1438
  */
919
1439
  converge?: boolean;
1440
+ /**
1441
+ * What the run does at a migration that `requires` a background migration
1442
+ * not completed yet: `'error'` (default) throws {@link BackgroundPendingError};
1443
+ * `'stop'` ends the run there, cleanly (`background:waiting`). Either way
1444
+ * that migration fires no hook and leaves no failed trace.
1445
+ * @experimental New in 2.3
1446
+ */
1447
+ onBackgroundPending?: 'error' | 'stop';
920
1448
  /**
921
1449
  * Who asked for this run (≤ 128 characters) — stamped on the changelog
922
1450
  * records it writes. `executedBy` is the OS user that ran it; on a queue
@@ -1014,6 +1542,12 @@ export interface LockEvent extends MigronautEventBase {
1014
1542
  ttlMs?: number;
1015
1543
  /** How long acquisition took in ms (on `lock:acquired`) */
1016
1544
  acquireMs?: number;
1545
+ /**
1546
+ * True on a `lock:released` that came before the run ended: a converge gave
1547
+ * the lock up to wait for search index builds, which only reads.
1548
+ * @experimental New in 2.2
1549
+ */
1550
+ early?: true;
1017
1551
  }
1018
1552
 
1019
1553
  /** Who started a converge: the `converge` call itself, or a bulk `up` (`convergeAfterUp`) */
@@ -1053,6 +1587,26 @@ export interface ConvergeActionEvent extends MigronautEventBase {
1053
1587
  /**
1054
1588
  * @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
1055
1589
  */
1590
+ /**
1591
+ * A converge's wait for search index builds (`waitForSearchIndexes`):
1592
+ * `started` once, `progress` every 30 seconds, then how it ended — one of
1593
+ * {@link ConvergeWaitOutcome}.
1594
+ * @experimental New in 2.2
1595
+ */
1596
+ export interface ConvergeWaitEvent extends MigronautEventBase {
1597
+ status: 'started' | 'progress' | ConvergeWaitOutcome;
1598
+ /** How many search indexes the wait is for */
1599
+ searchIndexes: number;
1600
+ /** `started` only — whether the migration lock was released for the wait */
1601
+ lockReleased?: boolean;
1602
+ /** `started` only — the budget (`searchIndexWaitTimeoutMs`) */
1603
+ timeoutMs?: number;
1604
+ /** Every status but `started` */
1605
+ waitedMs?: number;
1606
+ /** `failed` and `timeout` — the indexes that did not get there */
1607
+ notReady?: SearchIndexNotReady[];
1608
+ }
1609
+
1056
1610
  export interface ConvergeEndEvent extends MigronautEventBase {
1057
1611
  trigger: ConvergeTrigger;
1058
1612
  success: boolean;
@@ -1085,12 +1639,55 @@ export interface MigronautEvents {
1085
1639
  'lock:lost': (event: LockEvent) => void;
1086
1640
  'converge:start': (event: ConvergeStartEvent) => void;
1087
1641
  'converge:action': (event: ConvergeActionEvent) => void;
1642
+ 'converge:wait': (event: ConvergeWaitEvent) => void;
1088
1643
  'converge:end': (event: ConvergeEndEvent) => void;
1644
+ /** @experimental New in 2.3 */
1645
+ 'background:registered': (event: BackgroundRegisteredEvent) => void;
1646
+ /** @experimental New in 2.3 */
1647
+ 'background:waiting': (event: BackgroundEvent) => void;
1648
+ /** @experimental New in 2.3 */
1649
+ 'background:drift': (event: BackgroundEvent) => void;
1650
+ /**
1651
+ * A collection's live drift watcher changed state
1652
+ * @experimental New in 2.3
1653
+ */
1654
+ 'background:watch': (event: {
1655
+ runId?: string;
1656
+ collection: string;
1657
+ state: BackgroundWatchState;
1658
+ }) => void;
1659
+ /** @experimental New in 2.3 */
1660
+ 'background:unblocked': (event: BackgroundEvent) => void;
1661
+ /** @experimental New in 2.3 */
1662
+ 'background:partitioned': (event: BackgroundEvent) => void;
1663
+ /** @experimental New in 2.3 */
1664
+ 'background:pass': (event: BackgroundEvent) => void;
1665
+ /** @experimental New in 2.3 */
1666
+ 'background:slice:start': (event: BackgroundEvent) => void;
1667
+ /** @experimental New in 2.3 */
1668
+ 'background:batch': (event: BackgroundEvent) => void;
1669
+ /** @experimental New in 2.3 */
1670
+ 'background:slice:end': (event: BackgroundEvent) => void;
1671
+ /** @experimental New in 2.3 */
1672
+ 'background:lease:lost': (event: BackgroundEvent) => void;
1673
+ /** @experimental New in 2.3 */
1674
+ 'background:throttle': (event: BackgroundEvent) => void;
1675
+ /** @experimental New in 2.3 */
1676
+ 'background:control': (event: BackgroundEvent) => void;
1677
+ /** @experimental New in 2.3 */
1678
+ 'background:completed': (event: BackgroundEvent) => void;
1679
+ /** @experimental New in 2.3 */
1680
+ 'background:failed': (event: BackgroundEvent) => void;
1089
1681
  }
1090
1682
 
1091
1683
  /** One check performed by {@link MigratorKit.audit} */
1092
1684
  export interface AuditCheck {
1093
- /** e.g. 'config', 'connection', 'transactions', 'indexes', 'lock', 'checksums' */
1685
+ /**
1686
+ * e.g. 'config', 'connection', 'transactions', 'indexes', 'lock', 'checksums',
1687
+ * 'pending', 'ordering', 'runtime' — 'search' when declared collections
1688
+ * hold search indexes, and 'background' when background migrations are
1689
+ * registered
1690
+ */
1094
1691
  name: string;
1095
1692
  status: 'pass' | 'warn' | 'fail';
1096
1693
  detail: string;
@@ -1130,6 +1727,12 @@ export interface CreateOptions {
1130
1727
  * `createExtension`. Leave unset to let the config decide (default: `'js'`).
1131
1728
  */
1132
1729
  js?: boolean;
1730
+ /**
1731
+ * Generate a background migration (`export const background`) instead of
1732
+ * `up`/`down`. Not combined with `template`.
1733
+ * @experimental New in 2.3
1734
+ */
1735
+ background?: boolean;
1133
1736
  }
1134
1737
 
1135
1738
  /** Options for {@link MigratorKit.init} */
@@ -1325,11 +1928,514 @@ export class MigratorKit extends EventEmitter {
1325
1928
  * on and something is declared. Resolves the config; does not connect.
1326
1929
  */
1327
1930
  convergesAfterUp(): Promise<boolean>;
1931
+ /**
1932
+ * How drift is watched — the `backgroundDrift` setting — which a runner or
1933
+ * a queue worker hosting this kit follows. Resolves the config; does not
1934
+ * connect. @experimental
1935
+ */
1936
+ driftMode(): Promise<'poll' | 'stream' | 'both'>;
1328
1937
  /**
1329
1938
  * The converge history, newest first (`limit` 1–1000, default 20): one entry
1330
1939
  * per converge that changed something or failed. Read-only.
1331
1940
  */
1332
1941
  convergeHistory(options?: { limit?: number }): Promise<ConvergeHistoryEntry[]>;
1942
+
1943
+ // ─── Background migrations (experimental, new in 2.3) ─────────────────────
1944
+ // Reentrant: none of these is a run — no migration lock, no run id; one kit
1945
+ // may drive many at once.
1946
+
1947
+ /** One coordinator step — see {@link BackgroundCoordinatorAnswer}. @experimental */
1948
+ coordinateBackground(
1949
+ name: string,
1950
+ options?: { signal?: AbortSignal; driver?: BackgroundDriver },
1951
+ ): Promise<BackgroundCoordinatorAnswer>;
1952
+ /** One slice of one lane: claim a partition and a slot, work it, release. @experimental */
1953
+ runBackgroundSlice(
1954
+ name: string,
1955
+ options?: { signal?: AbortSignal; sliceMs?: number },
1956
+ ): Promise<BackgroundSliceResult>;
1957
+ /**
1958
+ * Drive a background migration from this process until it is done (or one
1959
+ * round, `untilDone: false`) with up to `concurrency` lanes (≤ its
1960
+ * `maxParallel`). A failed one throws {@link BackgroundFailedError}; a stop
1961
+ * {@link RunAbortedError} — it goes on from there next time. @experimental
1962
+ */
1963
+ runBackground(
1964
+ name: string,
1965
+ options?: {
1966
+ signal?: AbortSignal;
1967
+ sliceMs?: number;
1968
+ untilDone?: boolean;
1969
+ concurrency?: number;
1970
+ },
1971
+ ): Promise<BackgroundStatus>;
1972
+ /** One background migration's status, or `null` when it is not registered. @experimental */
1973
+ backgroundStatus(name: string): Promise<BackgroundStatus | null>;
1974
+ /** Every background migration's status, oldest registration first. @experimental */
1975
+ backgroundStatus(): Promise<BackgroundStatus[]>;
1976
+ /** The partitions of a background migration's latest generation. @experimental */
1977
+ backgroundPartitions(name: string): Promise<BackgroundPartitionInfo[]>;
1978
+ /** The background migrations with work to do (blocked ones unblocked on the way). @experimental */
1979
+ runnableBackground(): Promise<RunnableBackground[]>;
1980
+ /** Pause; its lanes stop at the next batch (`wait` until they have). @experimental */
1981
+ pauseBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
1982
+ /** Resume a paused one. @experimental */
1983
+ resumeBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
1984
+ /** Cancel (`wait` until its lanes have stopped). @experimental */
1985
+ cancelBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
1986
+ /**
1987
+ * Retry a failed or cancelled one — the same generation, or `fromStart`;
1988
+ * `repin` pins the file on disk first. A completed one is reopened. @experimental
1989
+ */
1990
+ retryBackground(
1991
+ name: string,
1992
+ options?: BackgroundControlOptions & { fromStart?: boolean; repin?: boolean },
1993
+ ): Promise<BackgroundControlResult>;
1994
+ /** Pin the file on disk (checksum, spec — and the changelog's checksum). @experimental */
1995
+ repinBackground(
1996
+ name: string,
1997
+ options?: BackgroundControlOptions,
1998
+ ): Promise<BackgroundControlResult & { replan: boolean; checksum: string }>;
1999
+ /** Clear the coordinator lock and every lease of a stuck one. @experimental */
2000
+ unlockBackground(name: string): Promise<{ lock: boolean; leases: number }>;
2001
+ /**
2002
+ * Dry-run a background migration, registered or not, with nothing written:
2003
+ * on a sample, its transformation alone — or with `validate`, the real write
2004
+ * path in a transaction that is always aborted. @experimental
2005
+ */
2006
+ dryRunBackground(name: string, options?: BackgroundDryRunOptions): Promise<BackgroundDryRun>;
2007
+ /**
2008
+ * The drift watch, once: one indexed probe per completed background
2009
+ * migration for documents of its old shape that appeared since; a finding
2010
+ * reopens it (`onDrift: 'reopen'`, the `backgroundOnDrift` default) or is
2011
+ * only reported. No document id is returned. @experimental
2012
+ */
2013
+ verifyBackground(options?: {
2014
+ onDrift?: 'reopen' | 'report';
2015
+ collections?: string[];
2016
+ }): Promise<BackgroundVerifyResult>;
2017
+ /**
2018
+ * The live drift watcher: a change stream per collection with a completed
2019
+ * background migration — one leader per collection across every process —
2020
+ * that upgrades each old-shape write moments after it lands, through the
2021
+ * lanes' own write path. Resolves once started; rejects with
2022
+ * ConfigInvalidError on a standalone server (no change streams).
2023
+ * @experimental
2024
+ */
2025
+ watchBackground(options?: WatchBackgroundOptions): Promise<BackgroundWatcher>;
2026
+ /** What the live drift watchers recorded for a collection — `null` when it has none @experimental */
2027
+ backgroundWatchStatus(collection: string): Promise<BackgroundWatchStatus | null>;
2028
+ /** What the live drift watchers recorded, one row per watched collection @experimental */
2029
+ backgroundWatchStatus(): Promise<BackgroundWatchStatus[]>;
2030
+ }
2031
+
2032
+ /** What a collection's live drift watcher is doing */
2033
+ export type BackgroundWatchState =
2034
+ | 'following'
2035
+ | 'catching-up'
2036
+ | 'streaming'
2037
+ | 'history-lost'
2038
+ | 'overloaded'
2039
+ | 'restarting'
2040
+ | 'suspended'
2041
+ | 'fallback'
2042
+ | 'stopped';
2043
+
2044
+ /** Options of {@link MigratorKit.watchBackground} */
2045
+ export interface WatchBackgroundOptions {
2046
+ /** Only these collections. Default: every one with a completed background migration */
2047
+ collections?: string[];
2048
+ /** Stops the watcher when aborted */
2049
+ signal?: AbortSignal;
2050
+ /** `false`: only report old-shape writes, never upgrade them. Default `true` */
2051
+ upgrade?: boolean;
2052
+ /** How often the edges and the collections are read again (ms). Default 30000 */
2053
+ refreshMs?: number;
2054
+ /** The most often the resume token is saved (ms). Default 5000 */
2055
+ checkpointMs?: number;
2056
+ /** How often a follower tries to become the leader (ms, jittered). Default 10000 */
2057
+ leaderRetryMs?: number;
2058
+ /** Collections watched by this process at most; the rest stay with the poll. Default 16 */
2059
+ maxCollections?: number;
2060
+ /**
2061
+ * A stream this far behind (ms) gives up on its backlog: the background
2062
+ * migrations it serves are reopened, and it starts again from now. Default 60000
2063
+ */
2064
+ maxLagMs?: number;
2065
+ /** Hears every failure (the watcher itself never throws) */
2066
+ onError?(error: unknown, collection?: string): void;
2067
+ }
2068
+
2069
+ /** A running live drift watcher */
2070
+ export interface BackgroundWatcher {
2071
+ readonly running: boolean;
2072
+ /** What each followed collection's watcher is doing in this process */
2073
+ status(): {
2074
+ collection: string;
2075
+ state: BackgroundWatchState | 'starting';
2076
+ /** Whether this process leads the collection */
2077
+ leading: boolean;
2078
+ counters: { events: number; upgraded: number; failed: number; skipped: number };
2079
+ lastEventAt?: Date;
2080
+ }[];
2081
+ /** Close every stream, save its position, release its lock */
2082
+ stop(): Promise<void>;
2083
+ }
2084
+
2085
+ /** A collection's live drift watcher, as stored — never its resume token */
2086
+ export interface BackgroundWatchStatus {
2087
+ collection: string;
2088
+ state: BackgroundWatchState | 'starting';
2089
+ /** The version a document should have at least */
2090
+ target?: number;
2091
+ /** The background migrations it upgrades with */
2092
+ edges: string[];
2093
+ leader?: { host: string; pid: number; at: Date };
2094
+ counters: { events: number; upgraded: number; failed: number; skipped: number };
2095
+ lastEventAt?: Date;
2096
+ updatedAt: Date;
2097
+ }
2098
+
2099
+ // ─── Background migration results ─────────────────────────────────────────────
2100
+
2101
+ /** Where a background migration stands */
2102
+ export type BackgroundState =
2103
+ | 'blocked'
2104
+ | 'pending'
2105
+ | 'running'
2106
+ | 'paused'
2107
+ | 'completed'
2108
+ | 'failed'
2109
+ | 'cancelled';
2110
+
2111
+ /**
2112
+ * Who runs a coordinator step: `{ kind, ref?, round? }` — a BullMQ round lets the newest win
2113
+ * @experimental New in 2.3
2114
+ */
2115
+ export interface BackgroundDriver {
2116
+ kind: 'bullmq' | 'runner' | 'cli' | 'inline' | 'local';
2117
+ ref?: string;
2118
+ round?: number;
2119
+ }
2120
+
2121
+ /**
2122
+ * One of {@link MigratorKit.runnableBackground}: what a driver needs to pick it up, or to tell it stalled
2123
+ * @experimental New in 2.3
2124
+ */
2125
+ export interface RunnableBackground {
2126
+ migration: string;
2127
+ status: BackgroundState;
2128
+ maxParallel: number;
2129
+ /** Leases renewed within their TTL — lanes working right now */
2130
+ liveLeases: number;
2131
+ registeredAt: Date;
2132
+ startedAt?: Date;
2133
+ lastProgressAt?: Date;
2134
+ coordinator?: { kind: string; round?: number; at: Date };
2135
+ }
2136
+
2137
+ /**
2138
+ * What a coordinator step says to do next
2139
+ * @experimental New in 2.3
2140
+ */
2141
+ export interface BackgroundCoordinatorAnswer {
2142
+ next: 'process' | 'wait' | 'done' | 'busy' | 'superseded';
2143
+ /** `process`: lanes that could start now */
2144
+ lanes?: number;
2145
+ generation?: number;
2146
+ /** `process`: the registration the lanes work for (it names their jobs) */
2147
+ registration?: string;
2148
+ /** `process`: the current plan's partitions by status */
2149
+ counts?: BackgroundStatus['partitions'];
2150
+ /**
2151
+ * A `bullmq` driver's round — handed out by this step to a chain that
2152
+ * asked without one; a chain whose round is not the latest is `superseded`
2153
+ */
2154
+ round?: number;
2155
+ /** `done`: where it stands */
2156
+ status?: BackgroundState | 'unregistered';
2157
+ /** `wait`: why — `checksum`, `replan-draining`, `plan-race`, … */
2158
+ reason?: string;
2159
+ /** `done` + `failed`: why */
2160
+ error?: string;
2161
+ waitsFor?: string[];
2162
+ retryAfterMs?: number;
2163
+ }
2164
+
2165
+ /** Counters of a slice, a partition or a whole background migration */
2166
+ export interface BackgroundCounters {
2167
+ scanned?: number;
2168
+ migrated?: number;
2169
+ skipped?: number;
2170
+ conflicts?: number;
2171
+ failed?: number;
2172
+ retried?: number;
2173
+ batches?: number;
2174
+ processed?: number;
2175
+ txnRetries?: number;
2176
+ }
2177
+
2178
+ /** How a lane's slice ended */
2179
+ export interface BackgroundSliceResult {
2180
+ outcome:
2181
+ | 'yielded'
2182
+ | 'exhausted'
2183
+ | 'busy'
2184
+ | 'stale'
2185
+ | 'paused'
2186
+ | 'cancelled'
2187
+ | 'failed'
2188
+ | 'stopped'
2189
+ | 'lost';
2190
+ counters: BackgroundCounters;
2191
+ retryAfterMs?: number;
2192
+ error?: BackgroundFailedError;
2193
+ }
2194
+
2195
+ /** A background migration, as {@link MigratorKit.backgroundStatus} reports it */
2196
+ export interface BackgroundStatus {
2197
+ migration: string;
2198
+ status: BackgroundState;
2199
+ phase: 'partition' | 'process' | 'replan';
2200
+ direction: 'forward' | 'revert';
2201
+ /** Minted at every (re-)registration — `up --force`, `redo` and `down` get a new one */
2202
+ registration: string;
2203
+ mode: 'declarative' | 'step';
2204
+ collection?: string;
2205
+ from?: number;
2206
+ to?: number;
2207
+ generation: number;
2208
+ pass: number;
2209
+ maxParallel: number;
2210
+ transaction: boolean;
2211
+ totals: BackgroundCounters & { slices?: number; reclaims?: number };
2212
+ /** Distinct documents that failed (within `maxDocumentErrors`) */
2213
+ failedDocuments: number;
2214
+ requires: string[];
2215
+ waitsFor: string[];
2216
+ /** The current plan's partitions by status */
2217
+ partitions?: {
2218
+ total: number;
2219
+ pending: number;
2220
+ running: number;
2221
+ done: number;
2222
+ failed: number;
2223
+ cancelled: number;
2224
+ superseded: number;
2225
+ leased: number;
2226
+ };
2227
+ /** Leases renewed within their TTL — lanes working right now */
2228
+ liveLeases: number;
2229
+ /**
2230
+ * The driver of the latest coordinator step that said who it was — a queue's
2231
+ * coordinator chain carries its `round`, and an older round bows out
2232
+ */
2233
+ coordinator?: { kind: string; round?: number; at: Date };
2234
+ /**
2235
+ * The current plan. `estimate` is the documents it expects to rewrite —
2236
+ * with `atLeast`, a count that stopped at its limit (there are more)
2237
+ */
2238
+ plan?: {
2239
+ method: string;
2240
+ estimate: number;
2241
+ atLeast?: boolean;
2242
+ partitions: number;
2243
+ degraded?: string;
2244
+ };
2245
+ /**
2246
+ * On a sharded collection, how the plan used the shard key: `chunks` (a
2247
+ * partition per run of chunks on one shard), `sampled` (the key space
2248
+ * sampled — the chunks could not be read), or `untargeted` (partitions by
2249
+ * `_id`: the key could not be read, or the version index does not carry it)
2250
+ */
2251
+ sharding?: {
2252
+ mode: 'chunks' | 'sampled' | 'empty' | 'untargeted';
2253
+ shardKey?: Record<string, 1 | 'hashed'>;
2254
+ hashed?: boolean;
2255
+ /** Shards the partitions are grouped by */
2256
+ groups?: number;
2257
+ };
2258
+ registeredAt: Date;
2259
+ startedAt?: Date;
2260
+ completedAt?: Date;
2261
+ lastProgressAt?: Date;
2262
+ lastError?: string;
2263
+ description?: string;
2264
+ /** The registration this one replaced (`up --force`, `redo`, `down`), as it stood then */
2265
+ previous?: {
2266
+ registration: string;
2267
+ status: BackgroundState;
2268
+ direction: 'forward' | 'revert';
2269
+ pass: number;
2270
+ totals: BackgroundCounters & { slices?: number; reclaims?: number };
2271
+ registeredAt: Date;
2272
+ completedAt?: Date;
2273
+ };
2274
+ }
2275
+
2276
+ /** One partition of a background migration */
2277
+ export interface BackgroundPartitionInfo {
2278
+ id: string;
2279
+ generation: number;
2280
+ seq: number;
2281
+ status: 'pending' | 'running' | 'done' | 'failed' | 'cancelled' | 'superseded';
2282
+ scope: Record<string, unknown>;
2283
+ estimate: number;
2284
+ counters: BackgroundCounters;
2285
+ group?: string;
2286
+ lease?: { slot: number; owner: string; host: string; pid: number; renewedAt: Date };
2287
+ throttle?: { batchSize: number; pauseMs: number };
2288
+ claims: number;
2289
+ reclaims: number;
2290
+ failures: number;
2291
+ lastError?: string;
2292
+ }
2293
+
2294
+ /** Options every background control action takes */
2295
+ export interface BackgroundControlOptions {
2296
+ /** Who asked — recorded in its history */
2297
+ requestedBy?: string;
2298
+ /** Why — recorded in its history */
2299
+ reason?: string;
2300
+ /** pause / cancel: resolve once no lane holds a lease any more */
2301
+ wait?: boolean;
2302
+ signal?: AbortSignal;
2303
+ }
2304
+
2305
+ /** What a control action did */
2306
+ export interface BackgroundControlResult {
2307
+ applied: 'changed' | 'unchanged';
2308
+ status: BackgroundState;
2309
+ /** `wait`: whether every lane stopped in time */
2310
+ stopped?: boolean;
2311
+ }
2312
+
2313
+ /** What {@link MigratorKit.verifyBackground} found */
2314
+ export interface BackgroundVerifyResult {
2315
+ /** Completed background migrations probed */
2316
+ checked: number;
2317
+ /** Skipped: another one at work on the collection, the validator guards it, no version index */
2318
+ skipped: number;
2319
+ drift: { migration: string; collection: string; action: 'reopened' | 'reported' }[];
2320
+ }
2321
+
2322
+ /** Options of {@link MigratorKit.dryRunBackground} */
2323
+ export interface BackgroundDryRunOptions {
2324
+ /** A random sample of this many matching documents (1–1000, default 5) */
2325
+ sample?: number;
2326
+ /** The first n matching documents by `_id`, instead of a sample */
2327
+ first?: number;
2328
+ /** Through the real write path, in the always-aborted sandbox */
2329
+ validate?: boolean;
2330
+ /** Dry-run the way back */
2331
+ direction?: 'forward' | 'revert';
2332
+ /** Step migrations: how many steps (1–50, default 1) */
2333
+ steps?: number;
2334
+ /** Step migrations: document images kept (default 20, at most 1000) */
2335
+ maxDocuments?: number;
2336
+ /** Step migrations: from no checkpoint, not the pinned one */
2337
+ fromStart?: boolean;
2338
+ /** Stop the sandbox after this long (default 50 000 ms) — steps, or a `validate` sample */
2339
+ deadlineMs?: number;
2340
+ }
2341
+
2342
+ /** One document of a dry run, as relaxed EJSON */
2343
+ export interface BackgroundDryRunDocument {
2344
+ _id: unknown;
2345
+ before: Record<string, unknown>;
2346
+ after?: Record<string, unknown>;
2347
+ /** The operator update it would be written with (without `validate`) */
2348
+ change?: Record<string, unknown>;
2349
+ error?: string;
2350
+ /** With `validate`: what the server made of it */
2351
+ validation?: 'ok' | 'failed' | 'skipped';
2352
+ }
2353
+
2354
+ /** One operation the sandbox ran — the filter as relaxed EJSON, at most 2 KiB */
2355
+ export interface BackgroundSandboxOperation {
2356
+ seq: number;
2357
+ step: number;
2358
+ collection?: string;
2359
+ method: string;
2360
+ filter?: unknown;
2361
+ result?: unknown;
2362
+ durationMs?: number;
2363
+ error?: string;
2364
+ }
2365
+
2366
+ /** A document the sandbox saw change */
2367
+ export interface BackgroundSandboxDocument {
2368
+ collection: string;
2369
+ _id: unknown;
2370
+ op: 'insert' | 'update' | 'delete' | 'unknown';
2371
+ before?: Record<string, unknown>;
2372
+ after?: Record<string, unknown>;
2373
+ }
2374
+
2375
+ /** What {@link MigratorKit.dryRunBackground} found */
2376
+ export type BackgroundDryRun =
2377
+ | {
2378
+ mode: 'declarative';
2379
+ migration: string;
2380
+ direction: 'forward' | 'revert';
2381
+ method: 'sample' | 'first';
2382
+ requested: number;
2383
+ found: number;
2384
+ migrated: number;
2385
+ failed: number;
2386
+ documents: BackgroundDryRunDocument[];
2387
+ /** With `validate` */
2388
+ validated?: true;
2389
+ aborted?: true;
2390
+ ops?: BackgroundSandboxOperation[];
2391
+ refusals?: { method: string; reason: string; collection?: string }[];
2392
+ /** Documents of other collections the side writes touched */
2393
+ sideEffects?: BackgroundSandboxDocument[];
2394
+ attempts?: number;
2395
+ }
2396
+ | BackgroundStepDryRun;
2397
+
2398
+ /** A step migration's dry run: up to `steps` steps in one always-aborted transaction */
2399
+ export interface BackgroundStepDryRun {
2400
+ mode: 'step';
2401
+ migration: string;
2402
+ direction: 'forward' | 'revert';
2403
+ aborted: true;
2404
+ ok: boolean;
2405
+ attempts: number;
2406
+ stoppedBy?: 'deadline' | 'done' | 'steps';
2407
+ steps: {
2408
+ step: number;
2409
+ checkpointIn: unknown;
2410
+ checkpointOut?: unknown;
2411
+ done?: boolean;
2412
+ processed?: number;
2413
+ migrated?: number;
2414
+ error?: string;
2415
+ }[];
2416
+ ops: BackgroundSandboxOperation[];
2417
+ documents: BackgroundSandboxDocument[];
2418
+ refusals: { method: string; reason: string; collection?: string }[];
2419
+ leakedCursors: number;
2420
+ truncated: boolean;
2421
+ abortedBy?: string;
2422
+ error?: string;
2423
+ }
2424
+
2425
+ /** `background:registered` */
2426
+ export interface BackgroundRegisteredEvent {
2427
+ runId?: string;
2428
+ migration: string;
2429
+ status: BackgroundState | 'withdrawn';
2430
+ direction: 'forward' | 'revert';
2431
+ waitsFor?: string[];
2432
+ }
2433
+
2434
+ /** Any other `background:*` event: the migration it is about, and what happened */
2435
+ export interface BackgroundEvent {
2436
+ runId?: string;
2437
+ migration: string;
2438
+ [field: string]: unknown;
1333
2439
  }
1334
2440
 
1335
2441
  // ─── Programmatic entry points ─────────────────────────────────────────────────
@@ -1341,6 +2447,13 @@ export type OnLockHeld = 'throw' | 'wait';
1341
2447
  export interface RunMigrationsOptions extends MigratorKitOptions {
1342
2448
  /** Skip lock acquisition (dev only — never in production) */
1343
2449
  noLock?: boolean;
2450
+ /**
2451
+ * At a migration that requires an unfinished background migration: throw
2452
+ * (`'error'`, default) or stop the run there (`'stop'`, listed in
2453
+ * `waiting`) — `'stop'` lets an app boot while a background migration runs.
2454
+ * @experimental New in 2.3
2455
+ */
2456
+ onBackgroundPending?: 'error' | 'stop';
1344
2457
  /**
1345
2458
  * How to react when another process already holds the migration lock — the
1346
2459
  * typical case when several app instances boot at once.
@@ -1395,6 +2508,12 @@ export interface MigrationSummary {
1395
2508
  attempts: number;
1396
2509
  /** The converge that ended the run — present only when `convergeAfterUp` converged */
1397
2510
  converge?: ConvergeResult;
2511
+ /**
2512
+ * With `onBackgroundPending: 'stop'`: the migration the run stopped at and
2513
+ * the background migrations it waits for.
2514
+ * @experimental New in 2.3
2515
+ */
2516
+ waiting?: { migration: string; waitsFor: { migration: string; status: string }[] }[];
1398
2517
  }
1399
2518
 
1400
2519
  /**
@@ -1431,6 +2550,56 @@ export const EXIT_CODES: Readonly<
1431
2550
  >
1432
2551
  >;
1433
2552
 
2553
+ // ─── Background runner ────────────────────────────────────────────────────────
2554
+
2555
+ /** Options of {@link startBackgroundRunner} */
2556
+ export interface BackgroundRunnerOptions {
2557
+ /** The kit to drive — or `config` (and `kitOptions`) for one the runner makes and closes */
2558
+ kit?: MigratorKit;
2559
+ config?: Partial<MigronautConfig>;
2560
+ kitOptions?: MigratorKitOptions;
2561
+ /** Lane loops in this process, shared by every background migration (default 1, ≤ 64) */
2562
+ concurrency?: number;
2563
+ /** How often the runnable list is read again (default 5000 ms) */
2564
+ pollIntervalMs?: number;
2565
+ /** A slice's length (default: each background migration's `sliceMs`) */
2566
+ sliceMs?: number;
2567
+ /** The drift watch's period (default 600 000 ms — 10 minutes); `false`: off */
2568
+ verifyIntervalMs?: number | false;
2569
+ /**
2570
+ * Host the live drift watcher in this process — `true`, or its options.
2571
+ * Default: when `backgroundDrift` is `'stream'` or `'both'`
2572
+ */
2573
+ watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
2574
+ /** Stops the runner, as `stop()` does */
2575
+ signal?: AbortSignal;
2576
+ /** Hears every failed slice (the runner itself never throws) */
2577
+ onError?: (error: unknown, migration?: string) => void;
2578
+ }
2579
+
2580
+ /** A running {@link startBackgroundRunner} */
2581
+ export interface BackgroundRunner {
2582
+ readonly kit: MigratorKit;
2583
+ readonly running: boolean;
2584
+ /** The live drift watcher this runner hosts, once started — or undefined */
2585
+ readonly watcher: BackgroundWatcher | undefined;
2586
+ /**
2587
+ * Stop at the next batch, release every lease, and close the kit the runner
2588
+ * made. `timeoutMs`: stop waiting for a lane stuck in its transformation
2589
+ * (its lease expires; the work resumes from the last checkpoint)
2590
+ */
2591
+ stop(options?: { timeoutMs?: number }): Promise<void>;
2592
+ }
2593
+
2594
+ /**
2595
+ * Drive background migrations from inside the application — no queue:
2596
+ * `concurrency` lane loops shared by every runnable background migration,
2597
+ * round-robin, plus the drift watch every `verifyIntervalMs`. Several
2598
+ * application instances share the work through the leases.
2599
+ * @experimental New in 2.3
2600
+ */
2601
+ export function startBackgroundRunner(options?: BackgroundRunnerOptions): BackgroundRunner;
2602
+
1434
2603
  // ─── Logger factory ───────────────────────────────────────────────────────────
1435
2604
 
1436
2605
  /** Threshold accepted by {@link createLogger} — drops anything less severe */
@@ -1620,12 +2789,97 @@ export class QueueJobFailedError extends MigronautError {
1620
2789
 
1621
2790
  /**
1622
2791
  * Thrown by {@link MigratorKit.converge} when the database cannot be brought
1623
- * to the declared state. `context.phase` is `'plan'` for a refused plan
1624
- * (`context.conflicts` lists why; nothing was written) or `'apply'` for a
1625
- * failed step (`collection`, `target`, `name`, `action`, `cause`, and
1626
- * `mongoCode`, `hint` and — after a failed rebuild — `restored` when they
1627
- * apply). `context.converge` is the {@link ConvergeResult} so far.
2792
+ * to the declared state. `context.phase` is:
2793
+ * - `'plan'` for a refused plan (`context.conflicts` lists why, with a `hint`
2794
+ * when Atlas Search is missing; nothing was written) — or a search index
2795
+ * list that could not be read before the first write (`collection`,
2796
+ * `target: 'searchIndex'`, `cause`, `mongoCode`, `hint`);
2797
+ * - `'replan'` when a collection changed while the run was under way
2798
+ * (`collection`, `introduced`: the new conflicts or drops; nothing of that
2799
+ * collection was written) — or its search index list could not be read
2800
+ * again (as for `'plan'`);
2801
+ * - `'apply'` for a failed step (`collection`, `target`, `name`, `action`,
2802
+ * `cause`, and `mongoCode`, `hint` and — after a failed rebuild — `restored`
2803
+ * when they apply) — or a search index list that could not be read to check
2804
+ * the steps just applied (as for `'plan'`);
2805
+ * - `'wait'` when `waitForSearchIndexes` gave up: `reason` is `'failed'` (the
2806
+ * build of a search index this run created or changed FAILED) or `'timeout'`,
2807
+ * with `notReady` the indexes not serving their declaration, `waitedMs`,
2808
+ * `timeoutMs` — or `'unreadable'`: a search index list that could not be
2809
+ * read (as for `'plan'`), after up to three network or failover blips in a
2810
+ * row. Everything was applied — only the builds were not finished.
2811
+ *
2812
+ * `context.converge` is the {@link ConvergeResult} so far.
1628
2813
  */
1629
2814
  export class ConvergeFailedError extends MigronautError {
1630
2815
  constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
1631
2816
  }
2817
+
2818
+ /**
2819
+ * Thrown by the optimistic-concurrency helpers of `@alexify/migronaut/versioning`
2820
+ * when a revision-guarded write matched nothing. `context.reason` is
2821
+ * `'conflict'` (the document is at another revision — `context.actual`),
2822
+ * `'not-found'` (nothing matches the filter) or `'unknown'` (the follow-up read
2823
+ * was skipped or could not tell); `context.expected` is the revision the
2824
+ * caller held. The filter is never copied into the error. Experimental.
2825
+ */
2826
+ export class RevisionConflictError extends MigronautError {
2827
+ readonly context?: RevisionConflictContext;
2828
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2829
+ }
2830
+
2831
+ /** {@link RevisionConflictError}'s `context` — what a caller decides on */
2832
+ export interface RevisionConflictContext {
2833
+ reason: 'conflict' | 'not-found' | 'unknown';
2834
+ /** The revision the caller held */
2835
+ expected: number;
2836
+ /** `conflict`: the revision the document is at */
2837
+ actual?: number;
2838
+ /** The collection's name, when the collection object has one */
2839
+ collection?: string;
2840
+ [key: string]: unknown;
2841
+ }
2842
+
2843
+ /**
2844
+ * Thrown by an upcaster that cannot bring a document to the current shape:
2845
+ * `context.reason` is `'newer'`, `'below-min'` or `'invalid'`, with
2846
+ * `context.version` and `context.current`. Experimental.
2847
+ */
2848
+ export class ShapeVersionError extends MigronautError {
2849
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2850
+ }
2851
+
2852
+ /**
2853
+ * Thrown when a migration `requires` a background migration that has not
2854
+ * completed — or whose collection still holds old-shape documents. Nothing was
2855
+ * run; `context.waitsFor` lists what it waits for. Experimental.
2856
+ */
2857
+ export class BackgroundPendingError extends MigronautError {
2858
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2859
+ }
2860
+
2861
+ /**
2862
+ * Thrown when a background migration ended `failed`; `context.migration` names
2863
+ * it and `context.lastError` says what happened last. Experimental.
2864
+ */
2865
+ export class BackgroundFailedError extends MigronautError {
2866
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2867
+ }
2868
+
2869
+ /**
2870
+ * Thrown when a control action does not fit the background migration's state;
2871
+ * `context.status` is the state found and `context.action` what was asked.
2872
+ * Experimental.
2873
+ */
2874
+ export class BackgroundConflictError extends MigronautError {
2875
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2876
+ }
2877
+
2878
+ /**
2879
+ * Thrown by the dry-run sandbox when a step reaches for something it cannot
2880
+ * run inside an always-aborted transaction; `context.method` names the call
2881
+ * and `context.reason` the rule. Experimental.
2882
+ */
2883
+ export class SandboxRefusedError extends MigronautError {
2884
+ constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
2885
+ }