@alexify/migronaut 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/README.md +68 -10
- package/bullmq.d.ts +465 -7
- package/index.d.ts +1272 -18
- package/migronaut.schema.json +150 -2
- package/package.json +8 -2
- package/src/bullmq/background-processor.js +469 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +153 -15
- package/src/bullmq/producer.js +202 -27
- package/src/bullmq/service.js +480 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/converge.js +38 -10
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/cli/table.js +68 -9
- package/src/core/audit.js +98 -3
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +366 -0
- package/src/core/background-engine.js +818 -0
- package/src/core/background-kit.js +425 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +605 -0
- package/src/core/background.js +1121 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +125 -31
- package/src/core/config.js +133 -13
- package/src/core/converge-plan.js +343 -61
- package/src/core/converge-search-run.js +440 -0
- package/src/core/converge-search.js +404 -0
- package/src/core/converge.js +428 -183
- package/src/core/index-spec.js +27 -16
- package/src/core/lock.js +97 -32
- package/src/core/migrator.js +951 -26
- package/src/core/options.js +32 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +1 -1
- package/src/core/search-index-spec.js +758 -0
- package/src/core/server-info.js +70 -0
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +97 -5
- package/src/index.js +16 -0
- package/src/utils/canonical.js +34 -1
- package/src/utils/error.js +11 -2
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/telemetry.js +125 -1
- package/src/utils/template.js +69 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/index.d.ts
CHANGED
|
@@ -41,6 +41,12 @@ export interface MigrationContext {
|
|
|
41
41
|
export interface MigrationModule {
|
|
42
42
|
up: (ctx: MigrationContext) => Promise<void>;
|
|
43
43
|
down: (ctx: MigrationContext) => Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Background migrations (file names, each sorting before this file) that
|
|
46
|
+
* must have completed before this migration runs.
|
|
47
|
+
* @experimental New in 2.3
|
|
48
|
+
*/
|
|
49
|
+
requires?: readonly string[];
|
|
44
50
|
/** If true, wraps this migration in a MongoDB session + transaction */
|
|
45
51
|
useTransaction?: boolean;
|
|
46
52
|
/** Overrides `MigronautConfig.timeoutMs` for this migration only */
|
|
@@ -49,6 +55,215 @@ export interface MigrationModule {
|
|
|
49
55
|
description?: string;
|
|
50
56
|
}
|
|
51
57
|
|
|
58
|
+
// ─── Document shapes and background migrations ───────────────────────────────
|
|
59
|
+
|
|
60
|
+
/** The system field names of a versioned collection — `revisionField` is `null` without revisions */
|
|
61
|
+
export interface ShapeFieldNames {
|
|
62
|
+
field: string;
|
|
63
|
+
revisionField: string | null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The default system field names: `__v` and `__rev` */
|
|
67
|
+
export interface DefaultShapeFieldNames {
|
|
68
|
+
field: '__v';
|
|
69
|
+
revisionField: '__rev';
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* A document type without its system fields (the version and the revision),
|
|
74
|
+
* distributed over a union. What a shape body is declared as, and what a
|
|
75
|
+
* background transformation returns: migronaut writes the system fields.
|
|
76
|
+
* @experimental New in 2.3
|
|
77
|
+
*/
|
|
78
|
+
export type Body<T, N extends ShapeFieldNames = DefaultShapeFieldNames> = T extends unknown
|
|
79
|
+
? Omit<T, N['field'] | Extract<N['revisionField'], string>>
|
|
80
|
+
: never;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* What a background migration's transformation gets besides the document.
|
|
84
|
+
* `session`, `db` and `client` are there only in a `transaction` background
|
|
85
|
+
* migration — writes to other collections must pass `session` to commit
|
|
86
|
+
* with the batch.
|
|
87
|
+
* @experimental New in 2.3
|
|
88
|
+
*/
|
|
89
|
+
export interface BackgroundMigrationContext {
|
|
90
|
+
/** Aborted when the slice is stopping (lease lost, pause, shutdown) */
|
|
91
|
+
signal: AbortSignal;
|
|
92
|
+
logger: MigronautLogger;
|
|
93
|
+
direction: 'forward' | 'revert';
|
|
94
|
+
background: { name: string; generation: number; partition: string };
|
|
95
|
+
session?: ClientSession;
|
|
96
|
+
db?: Db;
|
|
97
|
+
client?: MongoClient;
|
|
98
|
+
/** True in a dry run — the writes are rolled back */
|
|
99
|
+
dryRun?: boolean;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** How a background migration splits its collection into partitions */
|
|
103
|
+
export interface BackgroundPartitionSettings {
|
|
104
|
+
/** Partitions per lane (default 4), so a slow partition does not hold the pass */
|
|
105
|
+
overPartition?: number;
|
|
106
|
+
/** Default 256 */
|
|
107
|
+
maxPartitions?: number;
|
|
108
|
+
/** No partition is planned smaller than this (default 4 × batchSize) */
|
|
109
|
+
minPartitionDocs?: number;
|
|
110
|
+
/** Ids sampled to place the boundaries (default min(10 000, 100 × partitions)) */
|
|
111
|
+
sampleSize?: number;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** A transactional background migration's budget */
|
|
115
|
+
export interface BackgroundTransactionSettings {
|
|
116
|
+
/** Per batch, ≤ 50 000 (default 10 000) */
|
|
117
|
+
timeoutMs?: number;
|
|
118
|
+
/** Retries of a batch on a transient transaction error (default 5) */
|
|
119
|
+
maxRetries?: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The latency-driven throttle (AIMD) */
|
|
123
|
+
export interface BackgroundAdaptiveSettings {
|
|
124
|
+
/** A batch write slower than this halves the batch (default 500) */
|
|
125
|
+
targetLatencyMs?: number;
|
|
126
|
+
/** Default 10 */
|
|
127
|
+
minBatchSize?: number;
|
|
128
|
+
/** Never above `batchSize` (the default) */
|
|
129
|
+
maxBatchSize?: number;
|
|
130
|
+
/** Default 30 000 */
|
|
131
|
+
maxPauseMs?: number;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** What a `throttle` hook is told before every batch */
|
|
135
|
+
export interface BackgroundThrottleContext {
|
|
136
|
+
name: string;
|
|
137
|
+
collection?: string;
|
|
138
|
+
generation: number;
|
|
139
|
+
partition: string;
|
|
140
|
+
batchSize: number;
|
|
141
|
+
signal: AbortSignal;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Settings every background migration may carry, with their defaults.
|
|
146
|
+
* @experimental New in 2.3
|
|
147
|
+
*/
|
|
148
|
+
export interface BackgroundMigrationSettings {
|
|
149
|
+
description?: string;
|
|
150
|
+
/** Documents per batch: 500 (100 with `transaction`) */
|
|
151
|
+
batchSize?: number;
|
|
152
|
+
/** Pause between batches: 100 ms */
|
|
153
|
+
pauseMs?: number;
|
|
154
|
+
/** How long a lane holds a partition before it yields: 30 000 ms */
|
|
155
|
+
sliceMs?: number;
|
|
156
|
+
/** Every batch write's, transactions included — default `{ w: 'majority' }` */
|
|
157
|
+
writeConcern?: { w?: number | 'majority'; j?: boolean; wtimeoutMS?: number };
|
|
158
|
+
/** Documents that may fail before the background migration does: 0 (at most 1000) */
|
|
159
|
+
maxDocumentErrors?: number;
|
|
160
|
+
/** Passes over the remaining old-shape documents before giving up: 10 */
|
|
161
|
+
maxPasses?: number;
|
|
162
|
+
/** Re-read rounds for documents a concurrent write changed under a batch: 3 */
|
|
163
|
+
maxConflictRetries?: number;
|
|
164
|
+
/** Failed slices of one partition in a row (no checkpoint between) before it fails: 3 */
|
|
165
|
+
maxSliceFailures?: number;
|
|
166
|
+
/** Wait while a secondary lags more than this: 10 000 ms (`false`: never) */
|
|
167
|
+
maxReplicationLagMs?: number | false;
|
|
168
|
+
/** Called before every batch; a number it returns is an extra pause (ms) */
|
|
169
|
+
throttle?(ctx: BackgroundThrottleContext): number | void | Promise<number | void>;
|
|
170
|
+
/** Partitions processed at once, across every process: 1 (at most 64) */
|
|
171
|
+
maxParallel?: number;
|
|
172
|
+
partitions?: BackgroundPartitionSettings;
|
|
173
|
+
/** Batch and checkpoint in one transaction (needs a replica set or mongos): false */
|
|
174
|
+
transaction?: boolean | BackgroundTransactionSettings;
|
|
175
|
+
/** The latency-driven throttle: true */
|
|
176
|
+
adaptive?: boolean | BackgroundAdaptiveSettings;
|
|
177
|
+
/** Lanes per shard on a sharded collection: 1 */
|
|
178
|
+
shardConcurrency?: number;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* A declarative background migration: every document of `collection` at
|
|
183
|
+
* version `from` (and matching `filter`) rewritten to version `to` by
|
|
184
|
+
* `migrate` (or `migrateBatch`), in partitions, behind an optimistic guard.
|
|
185
|
+
* `From` and `To` type the documents — see `BackgroundMigrationFor` in
|
|
186
|
+
* `@alexify/migronaut/versioning` for the shape-map form.
|
|
187
|
+
*
|
|
188
|
+
* The callbacks are declared as methods, so a transformation typed for the
|
|
189
|
+
* stored document (with its version) or for its body fits either way.
|
|
190
|
+
* @experimental New in 2.3
|
|
191
|
+
*/
|
|
192
|
+
export interface DeclarativeBackgroundMigration<
|
|
193
|
+
From extends object = Record<string, any>,
|
|
194
|
+
To extends object = Record<string, any>,
|
|
195
|
+
> extends BackgroundMigrationSettings {
|
|
196
|
+
collection: string;
|
|
197
|
+
/** The version rewritten — 0 for documents without a version field */
|
|
198
|
+
from: number;
|
|
199
|
+
to: number;
|
|
200
|
+
filter?: Record<string, unknown>;
|
|
201
|
+
/** The new document for one old one; the engine sets the version and bumps the revision */
|
|
202
|
+
migrate?(doc: From, ctx: BackgroundMigrationContext): To | Promise<To>;
|
|
203
|
+
/** The new documents for a batch, aligned — an `Error` fails just that document */
|
|
204
|
+
migrateBatch?(docs: From[], ctx: BackgroundMigrationContext): (To | Error)[] | Promise<(To | Error)[]>;
|
|
205
|
+
/** The way back, for `down` */
|
|
206
|
+
revert?(doc: To, ctx: BackgroundMigrationContext): From | Promise<From>;
|
|
207
|
+
revertBatch?(docs: To[], ctx: BackgroundMigrationContext): (From | Error)[] | Promise<(From | Error)[]>;
|
|
208
|
+
/** Default: the collection's `versioning.field`, else `'__v'` */
|
|
209
|
+
versionField?: string;
|
|
210
|
+
/** Default: the collection's `versioning.revisionField`, else `'__rev'` */
|
|
211
|
+
revisionField?: string;
|
|
212
|
+
/**
|
|
213
|
+
* `'revision'` (default) guards each write with the revision; a collection
|
|
214
|
+
* without revisions must say `'version-only'` — a concurrent write that
|
|
215
|
+
* leaves the version alone is then invisible to it.
|
|
216
|
+
*/
|
|
217
|
+
occ?: 'revision' | 'version-only';
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** What a `step` background migration gets */
|
|
221
|
+
export interface BackgroundStepContext extends BackgroundMigrationContext {
|
|
222
|
+
db: Db;
|
|
223
|
+
client: MongoClient;
|
|
224
|
+
/** What the previous step returned (`null` at the start) */
|
|
225
|
+
checkpoint: unknown;
|
|
226
|
+
/** Epoch ms the step should return by — the slice ends then */
|
|
227
|
+
deadline: number;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** What a `step` returns */
|
|
231
|
+
export interface BackgroundStepResult {
|
|
232
|
+
/** Saved (≤ 64 KiB of BSON) and handed to the next step */
|
|
233
|
+
checkpoint: unknown;
|
|
234
|
+
/** True once there is nothing left */
|
|
235
|
+
done: boolean;
|
|
236
|
+
processed?: number;
|
|
237
|
+
migrated?: number;
|
|
238
|
+
/** For progress, when known */
|
|
239
|
+
total?: number;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* A free-form background migration — the escape hatch: migronaut runs `step`
|
|
244
|
+
* again and again with its last checkpoint until it says `done`, owning the
|
|
245
|
+
* lease, the slices, the throttle and the controls. One partition only; the
|
|
246
|
+
* writes must be idempotent.
|
|
247
|
+
* @experimental New in 2.3
|
|
248
|
+
*/
|
|
249
|
+
export interface StepBackgroundMigration extends BackgroundMigrationSettings {
|
|
250
|
+
/** Shown in status — the collection it works on, if one */
|
|
251
|
+
collection?: string;
|
|
252
|
+
step(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
|
|
253
|
+
revertStep?(ctx: BackgroundStepContext): BackgroundStepResult | Promise<BackgroundStepResult>;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** A background migration, as a migration file exports it: `export const background = {…}` */
|
|
257
|
+
export type BackgroundMigration = DeclarativeBackgroundMigration | StepBackgroundMigration;
|
|
258
|
+
|
|
259
|
+
/** Shape of a background migration file module — no `up`/`down` */
|
|
260
|
+
export interface BackgroundMigrationModule {
|
|
261
|
+
background: BackgroundMigration;
|
|
262
|
+
/** Background migrations that must complete first (each sorting before this file) */
|
|
263
|
+
requires?: readonly string[];
|
|
264
|
+
description?: string;
|
|
265
|
+
}
|
|
266
|
+
|
|
52
267
|
// ─── Changelog ────────────────────────────────────────────────────────────────
|
|
53
268
|
|
|
54
269
|
export type MigrationStatus = 'applied' | 'reverted' | 'failed';
|
|
@@ -93,6 +308,12 @@ export interface MigrationRecord {
|
|
|
93
308
|
* `migronaut import` — these are not reversible by migronaut. Absent for native records.
|
|
94
309
|
*/
|
|
95
310
|
origin?: MigrationOrigin;
|
|
311
|
+
/**
|
|
312
|
+
* `'background'` for a background migration file — applying it registered
|
|
313
|
+
* the background migration; its documents are rewritten later.
|
|
314
|
+
* @experimental New in 2.3
|
|
315
|
+
*/
|
|
316
|
+
kind?: 'background';
|
|
96
317
|
}
|
|
97
318
|
|
|
98
319
|
// ─── Config ───────────────────────────────────────────────────────────────────
|
|
@@ -292,6 +513,61 @@ export interface MigronautConfig {
|
|
|
292
513
|
* Default: false
|
|
293
514
|
*/
|
|
294
515
|
convergeAfterUp?: boolean;
|
|
516
|
+
/**
|
|
517
|
+
* What converge does with declared search indexes on a server without
|
|
518
|
+
* Atlas Search: `'fail'` refuses the run before anything is written,
|
|
519
|
+
* `'skip'` converges everything else and reports them as `skip` rows.
|
|
520
|
+
* Default: 'fail'
|
|
521
|
+
* @experimental New in 2.2
|
|
522
|
+
*/
|
|
523
|
+
onSearchUnavailable?: 'fail' | 'skip';
|
|
524
|
+
/**
|
|
525
|
+
* Hold every converge — the after-up one included — until each declared
|
|
526
|
+
* search index is queryable with its declared definition. Search indexes
|
|
527
|
+
* build in the background, so without it a new one is not queryable yet when
|
|
528
|
+
* converge returns. An index that FAILED or went STALE before the run, its
|
|
529
|
+
* definition unchanged, does not hold it — it is warned about instead. The
|
|
530
|
+
* migration lock is released while it waits. Default: false
|
|
531
|
+
* @experimental New in 2.2
|
|
532
|
+
*/
|
|
533
|
+
waitForSearchIndexes?: boolean;
|
|
534
|
+
/**
|
|
535
|
+
* How long `waitForSearchIndexes` waits before the converge fails with
|
|
536
|
+
* `phase: 'wait'` (the server goes on building). Default: 600000 (10 minutes)
|
|
537
|
+
* @experimental New in 2.2
|
|
538
|
+
*/
|
|
539
|
+
searchIndexWaitTimeoutMs?: number;
|
|
540
|
+
/**
|
|
541
|
+
* Where background migrations keep their state — and, named after it,
|
|
542
|
+
* their partitions (`<name>_partitions`) and the drift watcher's resume
|
|
543
|
+
* tokens (`<name>_watch`). Default `'_migronaut_background'`.
|
|
544
|
+
* @experimental New in 2.3
|
|
545
|
+
*/
|
|
546
|
+
backgroundCollection?: string;
|
|
547
|
+
/**
|
|
548
|
+
* Run a background migration to the end inside the `up` that registers it,
|
|
549
|
+
* under the migration lock — for small collections and tests. Default false.
|
|
550
|
+
* @experimental New in 2.3
|
|
551
|
+
*/
|
|
552
|
+
backgroundInline?: boolean;
|
|
553
|
+
/**
|
|
554
|
+
* What the drift watch does with old-shape documents that appear after a
|
|
555
|
+
* background migration completed: `'reopen'` it (default) or only `'report'`.
|
|
556
|
+
* @experimental New in 2.3
|
|
557
|
+
*/
|
|
558
|
+
backgroundOnDrift?: 'reopen' | 'report';
|
|
559
|
+
/**
|
|
560
|
+
* How drift is watched: `'poll'` (default — a check every 10 minutes),
|
|
561
|
+
* `'stream'` (change streams, the check as a backstop) or `'both'`.
|
|
562
|
+
* @experimental New in 2.3
|
|
563
|
+
*/
|
|
564
|
+
backgroundDrift?: 'poll' | 'stream' | 'both';
|
|
565
|
+
/**
|
|
566
|
+
* Partition a sharded collection by its shard key and target each write at
|
|
567
|
+
* one shard (`'auto'`, default), or treat it like any other (`'off'`).
|
|
568
|
+
* @experimental New in 2.3
|
|
569
|
+
*/
|
|
570
|
+
backgroundShardAware?: 'auto' | 'off';
|
|
295
571
|
/** Mongoose instance — required only if your migrations use Mongoose models */
|
|
296
572
|
mongoose?: MongooseLike;
|
|
297
573
|
hooks?: MigrationHooks;
|
|
@@ -393,9 +669,77 @@ export interface IndexDefinition {
|
|
|
393
669
|
export type ValidationLevel = 'off' | 'strict' | 'moderate';
|
|
394
670
|
export type ValidationAction = 'error' | 'warn' | 'errorAndLog';
|
|
395
671
|
|
|
672
|
+
/** The two kinds of Atlas search index */
|
|
673
|
+
export type SearchIndexType = 'search' | 'vectorSearch';
|
|
674
|
+
|
|
675
|
+
/** `mappings` of an Atlas Search definition */
|
|
676
|
+
export interface SearchIndexMappings {
|
|
677
|
+
/** Default: false */
|
|
678
|
+
dynamic?: boolean | { typeSet: string };
|
|
679
|
+
fields?: Record<string, unknown>;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* An Atlas Search index definition, as Atlas defines it. Compared whole, with
|
|
684
|
+
* the documented defaults filled in; anything Atlas adds can be declared too.
|
|
685
|
+
*/
|
|
686
|
+
export interface SearchDefinition {
|
|
687
|
+
mappings: SearchIndexMappings;
|
|
688
|
+
/** Default: 'lucene.standard' */
|
|
689
|
+
analyzer?: string;
|
|
690
|
+
/** Default: the analyzer */
|
|
691
|
+
searchAnalyzer?: string;
|
|
692
|
+
analyzers?: Array<Record<string, unknown>>;
|
|
693
|
+
synonyms?: Array<Record<string, unknown>>;
|
|
694
|
+
/** Default: false */
|
|
695
|
+
storedSource?: boolean | { include?: string[]; exclude?: string[] };
|
|
696
|
+
/** Default: 1 */
|
|
697
|
+
numPartitions?: number;
|
|
698
|
+
[option: string]: unknown;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/** One field of a Vector Search definition */
|
|
702
|
+
export interface VectorSearchField {
|
|
703
|
+
/** `'autoEmbed'` is Atlas's automated embedding (in preview); one index holds vector or autoEmbed fields, not both */
|
|
704
|
+
type: 'vector' | 'filter' | 'autoEmbed' | (string & {});
|
|
705
|
+
path: string;
|
|
706
|
+
numDimensions?: number;
|
|
707
|
+
similarity?: 'euclidean' | 'cosine' | 'dotProduct';
|
|
708
|
+
/** Default: 'none' ('scalar' for autoEmbed) */
|
|
709
|
+
quantization?: string;
|
|
710
|
+
/** Default: 'hnsw' */
|
|
711
|
+
indexingMethod?: 'hnsw' | 'flat';
|
|
712
|
+
/** Default: { maxEdges: 16, numEdgeCandidates: 100 } */
|
|
713
|
+
hnswOptions?: { maxEdges?: number; numEdgeCandidates?: number };
|
|
714
|
+
/** autoEmbed: the embedding model */
|
|
715
|
+
model?: string;
|
|
716
|
+
/** autoEmbed: 'text' */
|
|
717
|
+
modality?: string;
|
|
718
|
+
[option: string]: unknown;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/** A Vector Search index definition */
|
|
722
|
+
export interface VectorSearchDefinition {
|
|
723
|
+
fields: VectorSearchField[];
|
|
724
|
+
[option: string]: unknown;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/**
|
|
728
|
+
* One declared Atlas Search or Vector Search index. The name defaults to
|
|
729
|
+
* `'default'` and the type to `'search'`, as on the server. A change of type
|
|
730
|
+
* — or of an autoEmbed field's path, model, size, quantization or modality —
|
|
731
|
+
* cannot be made in place: converge refuses it, and the way is a new index
|
|
732
|
+
* under a new name (converge, then remove the old declaration and converge
|
|
733
|
+
* with prune).
|
|
734
|
+
* @experimental New in 2.2 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
735
|
+
*/
|
|
736
|
+
export type SearchIndexDefinition =
|
|
737
|
+
| { name?: string; type?: 'search'; definition: SearchDefinition }
|
|
738
|
+
| { name?: string; type: 'vectorSearch'; definition: VectorSearchDefinition };
|
|
739
|
+
|
|
396
740
|
/**
|
|
397
741
|
* A declared collection: the end state `converge()` keeps it in. Leave
|
|
398
|
-
* `indexes` or `validator` out to leave that part unmanaged.
|
|
742
|
+
* `indexes`, `searchIndexes` or `validator` out to leave that part unmanaged.
|
|
399
743
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
400
744
|
*/
|
|
401
745
|
export interface CollectionDefinition {
|
|
@@ -404,15 +748,65 @@ export interface CollectionDefinition {
|
|
|
404
748
|
* Every index besides `_id`. Undeclared live indexes are kept (and reported)
|
|
405
749
|
* unless `prune` is on.
|
|
406
750
|
*/
|
|
407
|
-
indexes?: IndexDefinition[];
|
|
408
|
-
/**
|
|
751
|
+
indexes?: readonly IndexDefinition[];
|
|
752
|
+
/**
|
|
753
|
+
* Atlas Search and Vector Search indexes (Atlas, an Atlas CLI local
|
|
754
|
+
* deployment, or MongoDB 8.3+ with mongot). Undeclared live ones are kept
|
|
755
|
+
* unless `prune` is on; leave the key out and they are not managed at all.
|
|
756
|
+
* @experimental New in 2.2
|
|
757
|
+
*/
|
|
758
|
+
searchIndexes?: readonly SearchIndexDefinition[];
|
|
759
|
+
/**
|
|
760
|
+
* A query or `{ $jsonSchema }` document; `null` (or `{}`) for no validator.
|
|
761
|
+
* With `versioning`, its rules are merged in — and `null` is refused.
|
|
762
|
+
*/
|
|
409
763
|
validator?: Record<string, unknown> | null;
|
|
410
|
-
/**
|
|
764
|
+
/**
|
|
765
|
+
* Default: 'strict' — 'moderate' when the only rules are the ones
|
|
766
|
+
* `versioning` adds. Only with a validator (or `versioning`)
|
|
767
|
+
*/
|
|
411
768
|
validationLevel?: ValidationLevel;
|
|
412
|
-
/** Default: 'error'. Only with a validator */
|
|
769
|
+
/** Default: 'error'. Only with a validator (or `versioning`) */
|
|
413
770
|
validationAction?: ValidationAction;
|
|
414
|
-
/**
|
|
771
|
+
/**
|
|
772
|
+
* Drop live indexes (and search indexes, when `searchIndexes` is declared)
|
|
773
|
+
* this definition does not declare. Default: the call's `prune`, else false.
|
|
774
|
+
* With `versioning` and no `indexes`, only the version index is managed —
|
|
775
|
+
* prune leaves the others alone.
|
|
776
|
+
*/
|
|
415
777
|
prune?: boolean;
|
|
778
|
+
/**
|
|
779
|
+
* Document shape versioning: the version (and revision) field typed and
|
|
780
|
+
* required by the validator, and the version index background migrations
|
|
781
|
+
* scan. The source of truth `defineShapes` reads too.
|
|
782
|
+
* @experimental New in 2.3
|
|
783
|
+
*/
|
|
784
|
+
versioning?: CollectionVersioning;
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/**
|
|
788
|
+
* The `versioning` block of a collection definition.
|
|
789
|
+
* @experimental New in 2.3
|
|
790
|
+
*/
|
|
791
|
+
export interface CollectionVersioning {
|
|
792
|
+
/** The shape version new documents are written at (≥ 1) */
|
|
793
|
+
current: number;
|
|
794
|
+
/**
|
|
795
|
+
* The oldest shape still allowed (default 1, ≤ `current`). `0` types the
|
|
796
|
+
* fields without requiring them — for a collection that predates
|
|
797
|
+
* versioning. Converge refuses to raise it while documents below it remain.
|
|
798
|
+
* There is deliberately no maximum: a newer release may write ahead of the
|
|
799
|
+
* declaration during a rolling deploy.
|
|
800
|
+
*/
|
|
801
|
+
min?: number;
|
|
802
|
+
/** The version field. Default `'__v'` */
|
|
803
|
+
field?: string;
|
|
804
|
+
/** Also manage a revision field for optimistic concurrency. Default `true` */
|
|
805
|
+
revision?: boolean;
|
|
806
|
+
/** The revision field. Default `'__rev'` */
|
|
807
|
+
revisionField?: string;
|
|
808
|
+
/** Declare the `{ <field>: 1, _id: 1 }` index. Default `true` */
|
|
809
|
+
index?: boolean;
|
|
416
810
|
}
|
|
417
811
|
|
|
418
812
|
/** What a `collectionsDir` file exports: a definition whose name defaults to the file name */
|
|
@@ -443,20 +837,33 @@ export interface ConvergeOptions {
|
|
|
443
837
|
* CLI: `--rebuild-unique`.
|
|
444
838
|
*/
|
|
445
839
|
rebuildUnique?: boolean;
|
|
840
|
+
/**
|
|
841
|
+
* Hold the run until every declared search index serves its
|
|
842
|
+
* declaration — failing on a FAILED build of an index this run created or
|
|
843
|
+
* changed, or after `searchIndexWaitTimeoutMs`. The migration lock is released
|
|
844
|
+
* when the wait starts — it only reads. Overrides the config's `waitForSearchIndexes`;
|
|
845
|
+
* not with `dryRun`. CLI: `--wait-search` / `--no-wait-search`.
|
|
846
|
+
* @experimental New in 2.2
|
|
847
|
+
*/
|
|
848
|
+
waitForSearchIndexes?: boolean;
|
|
446
849
|
/** Who asked for this converge — recorded in the converge history */
|
|
447
850
|
requestedBy?: string;
|
|
448
851
|
/** Why — recorded in the converge history */
|
|
449
852
|
reason?: string;
|
|
450
853
|
}
|
|
451
854
|
|
|
452
|
-
export type ConvergeTarget = 'collection' | 'validator' | 'index';
|
|
855
|
+
export type ConvergeTarget = 'collection' | 'validator' | 'index' | 'searchIndex';
|
|
453
856
|
|
|
454
857
|
/**
|
|
455
858
|
* What converge does to one target. `keep` is an undeclared index left alone
|
|
456
859
|
* (prune off); `conflict` refuses the run — an undeclared index covers the
|
|
457
860
|
* declared one's key under another name, a unique index would be rebuilt
|
|
458
|
-
* without {@link ConvergeOptions.rebuildUnique},
|
|
459
|
-
*
|
|
861
|
+
* without {@link ConvergeOptions.rebuildUnique}, the collection is a view or a
|
|
862
|
+
* time-series collection, a search index would need a change no update can
|
|
863
|
+
* make (its type, an autoEmbed field's model or size), or the server has no
|
|
864
|
+
* Atlas Search; `skip` is a declared search index left alone on a server
|
|
865
|
+
* without Search (`onSearchUnavailable: 'skip'`). A search index is never
|
|
866
|
+
* `recreate`d: `modify` updates it in place.
|
|
460
867
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
461
868
|
*/
|
|
462
869
|
export type ConvergeActionKind =
|
|
@@ -466,7 +873,37 @@ export type ConvergeActionKind =
|
|
|
466
873
|
| 'drop'
|
|
467
874
|
| 'keep'
|
|
468
875
|
| 'unchanged'
|
|
469
|
-
| 'conflict'
|
|
876
|
+
| 'conflict'
|
|
877
|
+
| 'skip';
|
|
878
|
+
|
|
879
|
+
/**
|
|
880
|
+
* The status `$listSearchIndexes` reports for a search index — `'UNKNOWN'`
|
|
881
|
+
* when the server reports none
|
|
882
|
+
*/
|
|
883
|
+
export type SearchIndexStatus =
|
|
884
|
+
| 'PENDING'
|
|
885
|
+
| 'BUILDING'
|
|
886
|
+
| 'READY'
|
|
887
|
+
| 'FAILED'
|
|
888
|
+
| 'STALE'
|
|
889
|
+
| 'DELETING'
|
|
890
|
+
| 'DOES_NOT_EXIST'
|
|
891
|
+
| 'UNKNOWN'
|
|
892
|
+
| (string & {});
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* Where the server is with a search index: it builds in the background, so a
|
|
896
|
+
* created or updated one is not queryable (with its new definition) at once
|
|
897
|
+
* @experimental New in 2.2
|
|
898
|
+
*/
|
|
899
|
+
export interface SearchIndexBuild {
|
|
900
|
+
status: SearchIndexStatus;
|
|
901
|
+
queryable: boolean;
|
|
902
|
+
/** The server's message — why a build FAILED, typically */
|
|
903
|
+
message?: string;
|
|
904
|
+
/** A newer definition is being built next to the one served */
|
|
905
|
+
updating?: true;
|
|
906
|
+
}
|
|
470
907
|
|
|
471
908
|
/**
|
|
472
909
|
* `planned` in a dry run; otherwise `applied`, `failed`, or `skipped` — no
|
|
@@ -497,6 +934,21 @@ export interface ConvergeAction {
|
|
|
497
934
|
from?: Record<string, unknown>;
|
|
498
935
|
/** What the row puts there — the declared index or validator — on rows that create or change it */
|
|
499
936
|
to?: Record<string, unknown>;
|
|
937
|
+
/**
|
|
938
|
+
* A search index row's build state on the server — as read before the run,
|
|
939
|
+
* and after it for a row the run applied
|
|
940
|
+
* @experimental New in 2.2
|
|
941
|
+
*/
|
|
942
|
+
build?: SearchIndexBuild;
|
|
943
|
+
/**
|
|
944
|
+
* On a search index row: the options the server reports that the
|
|
945
|
+
* declaration does not set and migronaut knows no default for
|
|
946
|
+
* (`mappings.fields.title.similarity`) — left out of the comparison, so a
|
|
947
|
+
* new server default does not make every converge update the index.
|
|
948
|
+
* Declare one to manage it.
|
|
949
|
+
* @experimental New in 2.2
|
|
950
|
+
*/
|
|
951
|
+
ignored?: string[];
|
|
500
952
|
}
|
|
501
953
|
|
|
502
954
|
/**
|
|
@@ -532,6 +984,12 @@ export interface ConvergeHistoryEntry {
|
|
|
532
984
|
/** The rows that changed, failed or refused the run — each with its collection and `from` / `to` */
|
|
533
985
|
actions: Array<ConvergeAction & { collection: string }>;
|
|
534
986
|
unstable?: ConvergeUnstable[];
|
|
987
|
+
/**
|
|
988
|
+
* What the run saw of Atlas Search, and how a wait for its builds ended —
|
|
989
|
+
* when a definition declares `searchIndexes`
|
|
990
|
+
* @experimental New in 2.2
|
|
991
|
+
*/
|
|
992
|
+
search?: ConvergeSearchSummary;
|
|
535
993
|
}
|
|
536
994
|
|
|
537
995
|
/**
|
|
@@ -546,6 +1004,42 @@ export interface ConvergeUnstable {
|
|
|
546
1004
|
reason?: string;
|
|
547
1005
|
}
|
|
548
1006
|
|
|
1007
|
+
/**
|
|
1008
|
+
* A declared search index that exists but does not serve its declaration yet
|
|
1009
|
+
* @experimental New in 2.2
|
|
1010
|
+
*/
|
|
1011
|
+
export interface SearchIndexNotReady extends SearchIndexBuild {
|
|
1012
|
+
collection: string;
|
|
1013
|
+
name: string;
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
/**
|
|
1017
|
+
* What a converge saw of Atlas Search — present when a definition declares
|
|
1018
|
+
* `searchIndexes`
|
|
1019
|
+
* @experimental New in 2.2
|
|
1020
|
+
*/
|
|
1021
|
+
export interface ConvergeSearchSummary {
|
|
1022
|
+
/** Whether the server has Atlas Search */
|
|
1023
|
+
available: boolean;
|
|
1024
|
+
/**
|
|
1025
|
+
* How that was told: `'listed'` (the server listed search indexes),
|
|
1026
|
+
* `'parameter'` (its search index manager setting), `'error'` (it refused
|
|
1027
|
+
* a search command), `'version'` (older than 6.0, not asked) or
|
|
1028
|
+
* `'assumed'` (it would not say — a refusal at apply time reports it)
|
|
1029
|
+
*/
|
|
1030
|
+
evidence?: 'listed' | 'parameter' | 'error' | 'version' | 'assumed';
|
|
1031
|
+
/**
|
|
1032
|
+
* Declared search indexes still building, updating, stale or failed. Does
|
|
1033
|
+
* not count against `inSync`: a build is the server's work, not a difference.
|
|
1034
|
+
*/
|
|
1035
|
+
notReady: SearchIndexNotReady[];
|
|
1036
|
+
/** How a wait for the builds (`waitForSearchIndexes`) ended, when there was one */
|
|
1037
|
+
wait?: { outcome: ConvergeWaitOutcome; waitedMs: number };
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/** How a wait for search index builds ended */
|
|
1041
|
+
export type ConvergeWaitOutcome = 'ready' | 'failed' | 'timeout' | 'unreadable' | 'aborted';
|
|
1042
|
+
|
|
549
1043
|
/**
|
|
550
1044
|
* Outcome of {@link MigratorKit.converge}
|
|
551
1045
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
@@ -556,11 +1050,15 @@ export interface ConvergeResult {
|
|
|
556
1050
|
changed: number;
|
|
557
1051
|
/**
|
|
558
1052
|
* True when the database matches the declarations: nothing left to do and
|
|
559
|
-
* no conflict. Undeclared indexes kept with prune off
|
|
1053
|
+
* no conflict. Undeclared indexes kept with prune off, search indexes
|
|
1054
|
+
* skipped on a server without Search, and search index builds still under
|
|
1055
|
+
* way do not count against it.
|
|
560
1056
|
*/
|
|
561
1057
|
inSync: boolean;
|
|
562
1058
|
collections: CollectionConvergeResult[];
|
|
563
1059
|
unstable?: ConvergeUnstable[];
|
|
1060
|
+
/** @experimental New in 2.2 */
|
|
1061
|
+
search?: ConvergeSearchSummary;
|
|
564
1062
|
}
|
|
565
1063
|
|
|
566
1064
|
// ─── Logger ───────────────────────────────────────────────────────────────────
|
|
@@ -735,6 +1233,12 @@ export interface StatusRow {
|
|
|
735
1233
|
origin?: MigrationOrigin;
|
|
736
1234
|
/** Redacted message of the last failed attempt (status `'failed'` only) */
|
|
737
1235
|
error?: string;
|
|
1236
|
+
/**
|
|
1237
|
+
* `'background'` for a background migration file (applied = registered) —
|
|
1238
|
+
* in `status()` and `dryRun('up')` rows alike
|
|
1239
|
+
* @experimental New in 2.3
|
|
1240
|
+
*/
|
|
1241
|
+
kind?: 'background';
|
|
738
1242
|
/** When the last failed attempt was recorded (status `'failed'` only) */
|
|
739
1243
|
failedAt?: Date;
|
|
740
1244
|
/**
|
|
@@ -755,6 +1259,16 @@ export interface StatusRow {
|
|
|
755
1259
|
* (`up(file, { checksum })`).
|
|
756
1260
|
*/
|
|
757
1261
|
checksum?: string;
|
|
1262
|
+
/**
|
|
1263
|
+
* `dryRun('up')` rows: the background migrations the file requires
|
|
1264
|
+
* @experimental New in 2.3
|
|
1265
|
+
*/
|
|
1266
|
+
requires?: string[];
|
|
1267
|
+
/**
|
|
1268
|
+
* `dryRun('up')` rows: those of `requires` not completed yet
|
|
1269
|
+
* @experimental New in 2.3
|
|
1270
|
+
*/
|
|
1271
|
+
waitsFor?: string[];
|
|
758
1272
|
/** Who asked for the apply, and why — when the run said (`requestedBy` / `reason` options) */
|
|
759
1273
|
requestedBy?: string;
|
|
760
1274
|
reason?: string;
|
|
@@ -855,7 +1369,13 @@ export type MigronautErrorCode =
|
|
|
855
1369
|
| 'MIGRATION_BLOCKED'
|
|
856
1370
|
| 'QUEUE_JOB_INVALID'
|
|
857
1371
|
| 'QUEUE_JOB_FAILED'
|
|
858
|
-
| 'CONVERGE_FAILED'
|
|
1372
|
+
| 'CONVERGE_FAILED'
|
|
1373
|
+
| 'REVISION_CONFLICT'
|
|
1374
|
+
| 'SHAPE_VERSION_UNSUPPORTED'
|
|
1375
|
+
| 'BACKGROUND_PENDING'
|
|
1376
|
+
| 'BACKGROUND_FAILED'
|
|
1377
|
+
| 'BACKGROUND_CONFLICT'
|
|
1378
|
+
| 'SANDBOX_REFUSED';
|
|
859
1379
|
|
|
860
1380
|
// ─── Config file format ─────────────────────────────────────────────────────────
|
|
861
1381
|
|
|
@@ -917,6 +1437,14 @@ export interface UpOptions {
|
|
|
917
1437
|
* with a filename or `to`.
|
|
918
1438
|
*/
|
|
919
1439
|
converge?: boolean;
|
|
1440
|
+
/**
|
|
1441
|
+
* What the run does at a migration that `requires` a background migration
|
|
1442
|
+
* not completed yet: `'error'` (default) throws {@link BackgroundPendingError};
|
|
1443
|
+
* `'stop'` ends the run there, cleanly (`background:waiting`). Either way
|
|
1444
|
+
* that migration fires no hook and leaves no failed trace.
|
|
1445
|
+
* @experimental New in 2.3
|
|
1446
|
+
*/
|
|
1447
|
+
onBackgroundPending?: 'error' | 'stop';
|
|
920
1448
|
/**
|
|
921
1449
|
* Who asked for this run (≤ 128 characters) — stamped on the changelog
|
|
922
1450
|
* records it writes. `executedBy` is the OS user that ran it; on a queue
|
|
@@ -1014,6 +1542,12 @@ export interface LockEvent extends MigronautEventBase {
|
|
|
1014
1542
|
ttlMs?: number;
|
|
1015
1543
|
/** How long acquisition took in ms (on `lock:acquired`) */
|
|
1016
1544
|
acquireMs?: number;
|
|
1545
|
+
/**
|
|
1546
|
+
* True on a `lock:released` that came before the run ended: a converge gave
|
|
1547
|
+
* the lock up to wait for search index builds, which only reads.
|
|
1548
|
+
* @experimental New in 2.2
|
|
1549
|
+
*/
|
|
1550
|
+
early?: true;
|
|
1017
1551
|
}
|
|
1018
1552
|
|
|
1019
1553
|
/** Who started a converge: the `converge` call itself, or a bulk `up` (`convergeAfterUp`) */
|
|
@@ -1053,6 +1587,26 @@ export interface ConvergeActionEvent extends MigronautEventBase {
|
|
|
1053
1587
|
/**
|
|
1054
1588
|
* @experimental New in 2.1 — the shape may still change in a minor release (named in the CHANGELOG).
|
|
1055
1589
|
*/
|
|
1590
|
+
/**
|
|
1591
|
+
* A converge's wait for search index builds (`waitForSearchIndexes`):
|
|
1592
|
+
* `started` once, `progress` every 30 seconds, then how it ended — one of
|
|
1593
|
+
* {@link ConvergeWaitOutcome}.
|
|
1594
|
+
* @experimental New in 2.2
|
|
1595
|
+
*/
|
|
1596
|
+
export interface ConvergeWaitEvent extends MigronautEventBase {
|
|
1597
|
+
status: 'started' | 'progress' | ConvergeWaitOutcome;
|
|
1598
|
+
/** How many search indexes the wait is for */
|
|
1599
|
+
searchIndexes: number;
|
|
1600
|
+
/** `started` only — whether the migration lock was released for the wait */
|
|
1601
|
+
lockReleased?: boolean;
|
|
1602
|
+
/** `started` only — the budget (`searchIndexWaitTimeoutMs`) */
|
|
1603
|
+
timeoutMs?: number;
|
|
1604
|
+
/** Every status but `started` */
|
|
1605
|
+
waitedMs?: number;
|
|
1606
|
+
/** `failed` and `timeout` — the indexes that did not get there */
|
|
1607
|
+
notReady?: SearchIndexNotReady[];
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1056
1610
|
export interface ConvergeEndEvent extends MigronautEventBase {
|
|
1057
1611
|
trigger: ConvergeTrigger;
|
|
1058
1612
|
success: boolean;
|
|
@@ -1085,12 +1639,55 @@ export interface MigronautEvents {
|
|
|
1085
1639
|
'lock:lost': (event: LockEvent) => void;
|
|
1086
1640
|
'converge:start': (event: ConvergeStartEvent) => void;
|
|
1087
1641
|
'converge:action': (event: ConvergeActionEvent) => void;
|
|
1642
|
+
'converge:wait': (event: ConvergeWaitEvent) => void;
|
|
1088
1643
|
'converge:end': (event: ConvergeEndEvent) => void;
|
|
1644
|
+
/** @experimental New in 2.3 */
|
|
1645
|
+
'background:registered': (event: BackgroundRegisteredEvent) => void;
|
|
1646
|
+
/** @experimental New in 2.3 */
|
|
1647
|
+
'background:waiting': (event: BackgroundEvent) => void;
|
|
1648
|
+
/** @experimental New in 2.3 */
|
|
1649
|
+
'background:drift': (event: BackgroundEvent) => void;
|
|
1650
|
+
/**
|
|
1651
|
+
* A collection's live drift watcher changed state
|
|
1652
|
+
* @experimental New in 2.3
|
|
1653
|
+
*/
|
|
1654
|
+
'background:watch': (event: {
|
|
1655
|
+
runId?: string;
|
|
1656
|
+
collection: string;
|
|
1657
|
+
state: BackgroundWatchState;
|
|
1658
|
+
}) => void;
|
|
1659
|
+
/** @experimental New in 2.3 */
|
|
1660
|
+
'background:unblocked': (event: BackgroundEvent) => void;
|
|
1661
|
+
/** @experimental New in 2.3 */
|
|
1662
|
+
'background:partitioned': (event: BackgroundEvent) => void;
|
|
1663
|
+
/** @experimental New in 2.3 */
|
|
1664
|
+
'background:pass': (event: BackgroundEvent) => void;
|
|
1665
|
+
/** @experimental New in 2.3 */
|
|
1666
|
+
'background:slice:start': (event: BackgroundEvent) => void;
|
|
1667
|
+
/** @experimental New in 2.3 */
|
|
1668
|
+
'background:batch': (event: BackgroundEvent) => void;
|
|
1669
|
+
/** @experimental New in 2.3 */
|
|
1670
|
+
'background:slice:end': (event: BackgroundEvent) => void;
|
|
1671
|
+
/** @experimental New in 2.3 */
|
|
1672
|
+
'background:lease:lost': (event: BackgroundEvent) => void;
|
|
1673
|
+
/** @experimental New in 2.3 */
|
|
1674
|
+
'background:throttle': (event: BackgroundEvent) => void;
|
|
1675
|
+
/** @experimental New in 2.3 */
|
|
1676
|
+
'background:control': (event: BackgroundEvent) => void;
|
|
1677
|
+
/** @experimental New in 2.3 */
|
|
1678
|
+
'background:completed': (event: BackgroundEvent) => void;
|
|
1679
|
+
/** @experimental New in 2.3 */
|
|
1680
|
+
'background:failed': (event: BackgroundEvent) => void;
|
|
1089
1681
|
}
|
|
1090
1682
|
|
|
1091
1683
|
/** One check performed by {@link MigratorKit.audit} */
|
|
1092
1684
|
export interface AuditCheck {
|
|
1093
|
-
/**
|
|
1685
|
+
/**
|
|
1686
|
+
* e.g. 'config', 'connection', 'transactions', 'indexes', 'lock', 'checksums',
|
|
1687
|
+
* 'pending', 'ordering', 'runtime' — 'search' when declared collections
|
|
1688
|
+
* hold search indexes, and 'background' when background migrations are
|
|
1689
|
+
* registered
|
|
1690
|
+
*/
|
|
1094
1691
|
name: string;
|
|
1095
1692
|
status: 'pass' | 'warn' | 'fail';
|
|
1096
1693
|
detail: string;
|
|
@@ -1130,6 +1727,12 @@ export interface CreateOptions {
|
|
|
1130
1727
|
* `createExtension`. Leave unset to let the config decide (default: `'js'`).
|
|
1131
1728
|
*/
|
|
1132
1729
|
js?: boolean;
|
|
1730
|
+
/**
|
|
1731
|
+
* Generate a background migration (`export const background`) instead of
|
|
1732
|
+
* `up`/`down`. Not combined with `template`.
|
|
1733
|
+
* @experimental New in 2.3
|
|
1734
|
+
*/
|
|
1735
|
+
background?: boolean;
|
|
1133
1736
|
}
|
|
1134
1737
|
|
|
1135
1738
|
/** Options for {@link MigratorKit.init} */
|
|
@@ -1325,11 +1928,514 @@ export class MigratorKit extends EventEmitter {
|
|
|
1325
1928
|
* on and something is declared. Resolves the config; does not connect.
|
|
1326
1929
|
*/
|
|
1327
1930
|
convergesAfterUp(): Promise<boolean>;
|
|
1931
|
+
/**
|
|
1932
|
+
* How drift is watched — the `backgroundDrift` setting — which a runner or
|
|
1933
|
+
* a queue worker hosting this kit follows. Resolves the config; does not
|
|
1934
|
+
* connect. @experimental
|
|
1935
|
+
*/
|
|
1936
|
+
driftMode(): Promise<'poll' | 'stream' | 'both'>;
|
|
1328
1937
|
/**
|
|
1329
1938
|
* The converge history, newest first (`limit` 1–1000, default 20): one entry
|
|
1330
1939
|
* per converge that changed something or failed. Read-only.
|
|
1331
1940
|
*/
|
|
1332
1941
|
convergeHistory(options?: { limit?: number }): Promise<ConvergeHistoryEntry[]>;
|
|
1942
|
+
|
|
1943
|
+
// ─── Background migrations (experimental, new in 2.3) ─────────────────────
|
|
1944
|
+
// Reentrant: none of these is a run — no migration lock, no run id; one kit
|
|
1945
|
+
// may drive many at once.
|
|
1946
|
+
|
|
1947
|
+
/** One coordinator step — see {@link BackgroundCoordinatorAnswer}. @experimental */
|
|
1948
|
+
coordinateBackground(
|
|
1949
|
+
name: string,
|
|
1950
|
+
options?: { signal?: AbortSignal; driver?: BackgroundDriver },
|
|
1951
|
+
): Promise<BackgroundCoordinatorAnswer>;
|
|
1952
|
+
/** One slice of one lane: claim a partition and a slot, work it, release. @experimental */
|
|
1953
|
+
runBackgroundSlice(
|
|
1954
|
+
name: string,
|
|
1955
|
+
options?: { signal?: AbortSignal; sliceMs?: number },
|
|
1956
|
+
): Promise<BackgroundSliceResult>;
|
|
1957
|
+
/**
|
|
1958
|
+
* Drive a background migration from this process until it is done (or one
|
|
1959
|
+
* round, `untilDone: false`) with up to `concurrency` lanes (≤ its
|
|
1960
|
+
* `maxParallel`). A failed one throws {@link BackgroundFailedError}; a stop
|
|
1961
|
+
* {@link RunAbortedError} — it goes on from there next time. @experimental
|
|
1962
|
+
*/
|
|
1963
|
+
runBackground(
|
|
1964
|
+
name: string,
|
|
1965
|
+
options?: {
|
|
1966
|
+
signal?: AbortSignal;
|
|
1967
|
+
sliceMs?: number;
|
|
1968
|
+
untilDone?: boolean;
|
|
1969
|
+
concurrency?: number;
|
|
1970
|
+
},
|
|
1971
|
+
): Promise<BackgroundStatus>;
|
|
1972
|
+
/** One background migration's status, or `null` when it is not registered. @experimental */
|
|
1973
|
+
backgroundStatus(name: string): Promise<BackgroundStatus | null>;
|
|
1974
|
+
/** Every background migration's status, oldest registration first. @experimental */
|
|
1975
|
+
backgroundStatus(): Promise<BackgroundStatus[]>;
|
|
1976
|
+
/** The partitions of a background migration's latest generation. @experimental */
|
|
1977
|
+
backgroundPartitions(name: string): Promise<BackgroundPartitionInfo[]>;
|
|
1978
|
+
/** The background migrations with work to do (blocked ones unblocked on the way). @experimental */
|
|
1979
|
+
runnableBackground(): Promise<RunnableBackground[]>;
|
|
1980
|
+
/** Pause; its lanes stop at the next batch (`wait` until they have). @experimental */
|
|
1981
|
+
pauseBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
1982
|
+
/** Resume a paused one. @experimental */
|
|
1983
|
+
resumeBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
1984
|
+
/** Cancel (`wait` until its lanes have stopped). @experimental */
|
|
1985
|
+
cancelBackground(name: string, options?: BackgroundControlOptions): Promise<BackgroundControlResult>;
|
|
1986
|
+
/**
|
|
1987
|
+
* Retry a failed or cancelled one — the same generation, or `fromStart`;
|
|
1988
|
+
* `repin` pins the file on disk first. A completed one is reopened. @experimental
|
|
1989
|
+
*/
|
|
1990
|
+
retryBackground(
|
|
1991
|
+
name: string,
|
|
1992
|
+
options?: BackgroundControlOptions & { fromStart?: boolean; repin?: boolean },
|
|
1993
|
+
): Promise<BackgroundControlResult>;
|
|
1994
|
+
/** Pin the file on disk (checksum, spec — and the changelog's checksum). @experimental */
|
|
1995
|
+
repinBackground(
|
|
1996
|
+
name: string,
|
|
1997
|
+
options?: BackgroundControlOptions,
|
|
1998
|
+
): Promise<BackgroundControlResult & { replan: boolean; checksum: string }>;
|
|
1999
|
+
/** Clear the coordinator lock and every lease of a stuck one. @experimental */
|
|
2000
|
+
unlockBackground(name: string): Promise<{ lock: boolean; leases: number }>;
|
|
2001
|
+
/**
|
|
2002
|
+
* Dry-run a background migration, registered or not, with nothing written:
|
|
2003
|
+
* on a sample, its transformation alone — or with `validate`, the real write
|
|
2004
|
+
* path in a transaction that is always aborted. @experimental
|
|
2005
|
+
*/
|
|
2006
|
+
dryRunBackground(name: string, options?: BackgroundDryRunOptions): Promise<BackgroundDryRun>;
|
|
2007
|
+
/**
|
|
2008
|
+
* The drift watch, once: one indexed probe per completed background
|
|
2009
|
+
* migration for documents of its old shape that appeared since; a finding
|
|
2010
|
+
* reopens it (`onDrift: 'reopen'`, the `backgroundOnDrift` default) or is
|
|
2011
|
+
* only reported. No document id is returned. @experimental
|
|
2012
|
+
*/
|
|
2013
|
+
verifyBackground(options?: {
|
|
2014
|
+
onDrift?: 'reopen' | 'report';
|
|
2015
|
+
collections?: string[];
|
|
2016
|
+
}): Promise<BackgroundVerifyResult>;
|
|
2017
|
+
/**
|
|
2018
|
+
* The live drift watcher: a change stream per collection with a completed
|
|
2019
|
+
* background migration — one leader per collection across every process —
|
|
2020
|
+
* that upgrades each old-shape write moments after it lands, through the
|
|
2021
|
+
* lanes' own write path. Resolves once started; rejects with
|
|
2022
|
+
* ConfigInvalidError on a standalone server (no change streams).
|
|
2023
|
+
* @experimental
|
|
2024
|
+
*/
|
|
2025
|
+
watchBackground(options?: WatchBackgroundOptions): Promise<BackgroundWatcher>;
|
|
2026
|
+
/** What the live drift watchers recorded for a collection — `null` when it has none @experimental */
|
|
2027
|
+
backgroundWatchStatus(collection: string): Promise<BackgroundWatchStatus | null>;
|
|
2028
|
+
/** What the live drift watchers recorded, one row per watched collection @experimental */
|
|
2029
|
+
backgroundWatchStatus(): Promise<BackgroundWatchStatus[]>;
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
/** What a collection's live drift watcher is doing */
|
|
2033
|
+
export type BackgroundWatchState =
|
|
2034
|
+
| 'following'
|
|
2035
|
+
| 'catching-up'
|
|
2036
|
+
| 'streaming'
|
|
2037
|
+
| 'history-lost'
|
|
2038
|
+
| 'overloaded'
|
|
2039
|
+
| 'restarting'
|
|
2040
|
+
| 'suspended'
|
|
2041
|
+
| 'fallback'
|
|
2042
|
+
| 'stopped';
|
|
2043
|
+
|
|
2044
|
+
/** Options of {@link MigratorKit.watchBackground} */
|
|
2045
|
+
export interface WatchBackgroundOptions {
|
|
2046
|
+
/** Only these collections. Default: every one with a completed background migration */
|
|
2047
|
+
collections?: string[];
|
|
2048
|
+
/** Stops the watcher when aborted */
|
|
2049
|
+
signal?: AbortSignal;
|
|
2050
|
+
/** `false`: only report old-shape writes, never upgrade them. Default `true` */
|
|
2051
|
+
upgrade?: boolean;
|
|
2052
|
+
/** How often the edges and the collections are read again (ms). Default 30000 */
|
|
2053
|
+
refreshMs?: number;
|
|
2054
|
+
/** The most often the resume token is saved (ms). Default 5000 */
|
|
2055
|
+
checkpointMs?: number;
|
|
2056
|
+
/** How often a follower tries to become the leader (ms, jittered). Default 10000 */
|
|
2057
|
+
leaderRetryMs?: number;
|
|
2058
|
+
/** Collections watched by this process at most; the rest stay with the poll. Default 16 */
|
|
2059
|
+
maxCollections?: number;
|
|
2060
|
+
/**
|
|
2061
|
+
* A stream this far behind (ms) gives up on its backlog: the background
|
|
2062
|
+
* migrations it serves are reopened, and it starts again from now. Default 60000
|
|
2063
|
+
*/
|
|
2064
|
+
maxLagMs?: number;
|
|
2065
|
+
/** Hears every failure (the watcher itself never throws) */
|
|
2066
|
+
onError?(error: unknown, collection?: string): void;
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
/** A running live drift watcher */
|
|
2070
|
+
export interface BackgroundWatcher {
|
|
2071
|
+
readonly running: boolean;
|
|
2072
|
+
/** What each followed collection's watcher is doing in this process */
|
|
2073
|
+
status(): {
|
|
2074
|
+
collection: string;
|
|
2075
|
+
state: BackgroundWatchState | 'starting';
|
|
2076
|
+
/** Whether this process leads the collection */
|
|
2077
|
+
leading: boolean;
|
|
2078
|
+
counters: { events: number; upgraded: number; failed: number; skipped: number };
|
|
2079
|
+
lastEventAt?: Date;
|
|
2080
|
+
}[];
|
|
2081
|
+
/** Close every stream, save its position, release its lock */
|
|
2082
|
+
stop(): Promise<void>;
|
|
2083
|
+
}
|
|
2084
|
+
|
|
2085
|
+
/** A collection's live drift watcher, as stored — never its resume token */
|
|
2086
|
+
export interface BackgroundWatchStatus {
|
|
2087
|
+
collection: string;
|
|
2088
|
+
state: BackgroundWatchState | 'starting';
|
|
2089
|
+
/** The version a document should have at least */
|
|
2090
|
+
target?: number;
|
|
2091
|
+
/** The background migrations it upgrades with */
|
|
2092
|
+
edges: string[];
|
|
2093
|
+
leader?: { host: string; pid: number; at: Date };
|
|
2094
|
+
counters: { events: number; upgraded: number; failed: number; skipped: number };
|
|
2095
|
+
lastEventAt?: Date;
|
|
2096
|
+
updatedAt: Date;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// ─── Background migration results ─────────────────────────────────────────────
|
|
2100
|
+
|
|
2101
|
+
/** Where a background migration stands */
|
|
2102
|
+
export type BackgroundState =
|
|
2103
|
+
| 'blocked'
|
|
2104
|
+
| 'pending'
|
|
2105
|
+
| 'running'
|
|
2106
|
+
| 'paused'
|
|
2107
|
+
| 'completed'
|
|
2108
|
+
| 'failed'
|
|
2109
|
+
| 'cancelled';
|
|
2110
|
+
|
|
2111
|
+
/**
|
|
2112
|
+
* Who runs a coordinator step: `{ kind, ref?, round? }` — a BullMQ round lets the newest win
|
|
2113
|
+
* @experimental New in 2.3
|
|
2114
|
+
*/
|
|
2115
|
+
export interface BackgroundDriver {
|
|
2116
|
+
kind: 'bullmq' | 'runner' | 'cli' | 'inline' | 'local';
|
|
2117
|
+
ref?: string;
|
|
2118
|
+
round?: number;
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
/**
|
|
2122
|
+
* One of {@link MigratorKit.runnableBackground}: what a driver needs to pick it up, or to tell it stalled
|
|
2123
|
+
* @experimental New in 2.3
|
|
2124
|
+
*/
|
|
2125
|
+
export interface RunnableBackground {
|
|
2126
|
+
migration: string;
|
|
2127
|
+
status: BackgroundState;
|
|
2128
|
+
maxParallel: number;
|
|
2129
|
+
/** Leases renewed within their TTL — lanes working right now */
|
|
2130
|
+
liveLeases: number;
|
|
2131
|
+
registeredAt: Date;
|
|
2132
|
+
startedAt?: Date;
|
|
2133
|
+
lastProgressAt?: Date;
|
|
2134
|
+
coordinator?: { kind: string; round?: number; at: Date };
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
/**
|
|
2138
|
+
* What a coordinator step says to do next
|
|
2139
|
+
* @experimental New in 2.3
|
|
2140
|
+
*/
|
|
2141
|
+
export interface BackgroundCoordinatorAnswer {
|
|
2142
|
+
next: 'process' | 'wait' | 'done' | 'busy' | 'superseded';
|
|
2143
|
+
/** `process`: lanes that could start now */
|
|
2144
|
+
lanes?: number;
|
|
2145
|
+
generation?: number;
|
|
2146
|
+
/** `process`: the registration the lanes work for (it names their jobs) */
|
|
2147
|
+
registration?: string;
|
|
2148
|
+
/** `process`: the current plan's partitions by status */
|
|
2149
|
+
counts?: BackgroundStatus['partitions'];
|
|
2150
|
+
/**
|
|
2151
|
+
* A `bullmq` driver's round — handed out by this step to a chain that
|
|
2152
|
+
* asked without one; a chain whose round is not the latest is `superseded`
|
|
2153
|
+
*/
|
|
2154
|
+
round?: number;
|
|
2155
|
+
/** `done`: where it stands */
|
|
2156
|
+
status?: BackgroundState | 'unregistered';
|
|
2157
|
+
/** `wait`: why — `checksum`, `replan-draining`, `plan-race`, … */
|
|
2158
|
+
reason?: string;
|
|
2159
|
+
/** `done` + `failed`: why */
|
|
2160
|
+
error?: string;
|
|
2161
|
+
waitsFor?: string[];
|
|
2162
|
+
retryAfterMs?: number;
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
/** Counters of a slice, a partition or a whole background migration */
|
|
2166
|
+
export interface BackgroundCounters {
|
|
2167
|
+
scanned?: number;
|
|
2168
|
+
migrated?: number;
|
|
2169
|
+
skipped?: number;
|
|
2170
|
+
conflicts?: number;
|
|
2171
|
+
failed?: number;
|
|
2172
|
+
retried?: number;
|
|
2173
|
+
batches?: number;
|
|
2174
|
+
processed?: number;
|
|
2175
|
+
txnRetries?: number;
|
|
2176
|
+
}
|
|
2177
|
+
|
|
2178
|
+
/** How a lane's slice ended */
|
|
2179
|
+
export interface BackgroundSliceResult {
|
|
2180
|
+
outcome:
|
|
2181
|
+
| 'yielded'
|
|
2182
|
+
| 'exhausted'
|
|
2183
|
+
| 'busy'
|
|
2184
|
+
| 'stale'
|
|
2185
|
+
| 'paused'
|
|
2186
|
+
| 'cancelled'
|
|
2187
|
+
| 'failed'
|
|
2188
|
+
| 'stopped'
|
|
2189
|
+
| 'lost';
|
|
2190
|
+
counters: BackgroundCounters;
|
|
2191
|
+
retryAfterMs?: number;
|
|
2192
|
+
error?: BackgroundFailedError;
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
/** A background migration, as {@link MigratorKit.backgroundStatus} reports it */
|
|
2196
|
+
export interface BackgroundStatus {
|
|
2197
|
+
migration: string;
|
|
2198
|
+
status: BackgroundState;
|
|
2199
|
+
phase: 'partition' | 'process' | 'replan';
|
|
2200
|
+
direction: 'forward' | 'revert';
|
|
2201
|
+
/** Minted at every (re-)registration — `up --force`, `redo` and `down` get a new one */
|
|
2202
|
+
registration: string;
|
|
2203
|
+
mode: 'declarative' | 'step';
|
|
2204
|
+
collection?: string;
|
|
2205
|
+
from?: number;
|
|
2206
|
+
to?: number;
|
|
2207
|
+
generation: number;
|
|
2208
|
+
pass: number;
|
|
2209
|
+
maxParallel: number;
|
|
2210
|
+
transaction: boolean;
|
|
2211
|
+
totals: BackgroundCounters & { slices?: number; reclaims?: number };
|
|
2212
|
+
/** Distinct documents that failed (within `maxDocumentErrors`) */
|
|
2213
|
+
failedDocuments: number;
|
|
2214
|
+
requires: string[];
|
|
2215
|
+
waitsFor: string[];
|
|
2216
|
+
/** The current plan's partitions by status */
|
|
2217
|
+
partitions?: {
|
|
2218
|
+
total: number;
|
|
2219
|
+
pending: number;
|
|
2220
|
+
running: number;
|
|
2221
|
+
done: number;
|
|
2222
|
+
failed: number;
|
|
2223
|
+
cancelled: number;
|
|
2224
|
+
superseded: number;
|
|
2225
|
+
leased: number;
|
|
2226
|
+
};
|
|
2227
|
+
/** Leases renewed within their TTL — lanes working right now */
|
|
2228
|
+
liveLeases: number;
|
|
2229
|
+
/**
|
|
2230
|
+
* The driver of the latest coordinator step that said who it was — a queue's
|
|
2231
|
+
* coordinator chain carries its `round`, and an older round bows out
|
|
2232
|
+
*/
|
|
2233
|
+
coordinator?: { kind: string; round?: number; at: Date };
|
|
2234
|
+
/**
|
|
2235
|
+
* The current plan. `estimate` is the documents it expects to rewrite —
|
|
2236
|
+
* with `atLeast`, a count that stopped at its limit (there are more)
|
|
2237
|
+
*/
|
|
2238
|
+
plan?: {
|
|
2239
|
+
method: string;
|
|
2240
|
+
estimate: number;
|
|
2241
|
+
atLeast?: boolean;
|
|
2242
|
+
partitions: number;
|
|
2243
|
+
degraded?: string;
|
|
2244
|
+
};
|
|
2245
|
+
/**
|
|
2246
|
+
* On a sharded collection, how the plan used the shard key: `chunks` (a
|
|
2247
|
+
* partition per run of chunks on one shard), `sampled` (the key space
|
|
2248
|
+
* sampled — the chunks could not be read), or `untargeted` (partitions by
|
|
2249
|
+
* `_id`: the key could not be read, or the version index does not carry it)
|
|
2250
|
+
*/
|
|
2251
|
+
sharding?: {
|
|
2252
|
+
mode: 'chunks' | 'sampled' | 'empty' | 'untargeted';
|
|
2253
|
+
shardKey?: Record<string, 1 | 'hashed'>;
|
|
2254
|
+
hashed?: boolean;
|
|
2255
|
+
/** Shards the partitions are grouped by */
|
|
2256
|
+
groups?: number;
|
|
2257
|
+
};
|
|
2258
|
+
registeredAt: Date;
|
|
2259
|
+
startedAt?: Date;
|
|
2260
|
+
completedAt?: Date;
|
|
2261
|
+
lastProgressAt?: Date;
|
|
2262
|
+
lastError?: string;
|
|
2263
|
+
description?: string;
|
|
2264
|
+
/** The registration this one replaced (`up --force`, `redo`, `down`), as it stood then */
|
|
2265
|
+
previous?: {
|
|
2266
|
+
registration: string;
|
|
2267
|
+
status: BackgroundState;
|
|
2268
|
+
direction: 'forward' | 'revert';
|
|
2269
|
+
pass: number;
|
|
2270
|
+
totals: BackgroundCounters & { slices?: number; reclaims?: number };
|
|
2271
|
+
registeredAt: Date;
|
|
2272
|
+
completedAt?: Date;
|
|
2273
|
+
};
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
/** One partition of a background migration */
|
|
2277
|
+
export interface BackgroundPartitionInfo {
|
|
2278
|
+
id: string;
|
|
2279
|
+
generation: number;
|
|
2280
|
+
seq: number;
|
|
2281
|
+
status: 'pending' | 'running' | 'done' | 'failed' | 'cancelled' | 'superseded';
|
|
2282
|
+
scope: Record<string, unknown>;
|
|
2283
|
+
estimate: number;
|
|
2284
|
+
counters: BackgroundCounters;
|
|
2285
|
+
group?: string;
|
|
2286
|
+
lease?: { slot: number; owner: string; host: string; pid: number; renewedAt: Date };
|
|
2287
|
+
throttle?: { batchSize: number; pauseMs: number };
|
|
2288
|
+
claims: number;
|
|
2289
|
+
reclaims: number;
|
|
2290
|
+
failures: number;
|
|
2291
|
+
lastError?: string;
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
/** Options every background control action takes */
|
|
2295
|
+
export interface BackgroundControlOptions {
|
|
2296
|
+
/** Who asked — recorded in its history */
|
|
2297
|
+
requestedBy?: string;
|
|
2298
|
+
/** Why — recorded in its history */
|
|
2299
|
+
reason?: string;
|
|
2300
|
+
/** pause / cancel: resolve once no lane holds a lease any more */
|
|
2301
|
+
wait?: boolean;
|
|
2302
|
+
signal?: AbortSignal;
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
/** What a control action did */
|
|
2306
|
+
export interface BackgroundControlResult {
|
|
2307
|
+
applied: 'changed' | 'unchanged';
|
|
2308
|
+
status: BackgroundState;
|
|
2309
|
+
/** `wait`: whether every lane stopped in time */
|
|
2310
|
+
stopped?: boolean;
|
|
2311
|
+
}
|
|
2312
|
+
|
|
2313
|
+
/** What {@link MigratorKit.verifyBackground} found */
|
|
2314
|
+
export interface BackgroundVerifyResult {
|
|
2315
|
+
/** Completed background migrations probed */
|
|
2316
|
+
checked: number;
|
|
2317
|
+
/** Skipped: another one at work on the collection, the validator guards it, no version index */
|
|
2318
|
+
skipped: number;
|
|
2319
|
+
drift: { migration: string; collection: string; action: 'reopened' | 'reported' }[];
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
/** Options of {@link MigratorKit.dryRunBackground} */
|
|
2323
|
+
export interface BackgroundDryRunOptions {
|
|
2324
|
+
/** A random sample of this many matching documents (1–1000, default 5) */
|
|
2325
|
+
sample?: number;
|
|
2326
|
+
/** The first n matching documents by `_id`, instead of a sample */
|
|
2327
|
+
first?: number;
|
|
2328
|
+
/** Through the real write path, in the always-aborted sandbox */
|
|
2329
|
+
validate?: boolean;
|
|
2330
|
+
/** Dry-run the way back */
|
|
2331
|
+
direction?: 'forward' | 'revert';
|
|
2332
|
+
/** Step migrations: how many steps (1–50, default 1) */
|
|
2333
|
+
steps?: number;
|
|
2334
|
+
/** Step migrations: document images kept (default 20, at most 1000) */
|
|
2335
|
+
maxDocuments?: number;
|
|
2336
|
+
/** Step migrations: from no checkpoint, not the pinned one */
|
|
2337
|
+
fromStart?: boolean;
|
|
2338
|
+
/** Stop the sandbox after this long (default 50 000 ms) — steps, or a `validate` sample */
|
|
2339
|
+
deadlineMs?: number;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
/** One document of a dry run, as relaxed EJSON */
|
|
2343
|
+
export interface BackgroundDryRunDocument {
|
|
2344
|
+
_id: unknown;
|
|
2345
|
+
before: Record<string, unknown>;
|
|
2346
|
+
after?: Record<string, unknown>;
|
|
2347
|
+
/** The operator update it would be written with (without `validate`) */
|
|
2348
|
+
change?: Record<string, unknown>;
|
|
2349
|
+
error?: string;
|
|
2350
|
+
/** With `validate`: what the server made of it */
|
|
2351
|
+
validation?: 'ok' | 'failed' | 'skipped';
|
|
2352
|
+
}
|
|
2353
|
+
|
|
2354
|
+
/** One operation the sandbox ran — the filter as relaxed EJSON, at most 2 KiB */
|
|
2355
|
+
export interface BackgroundSandboxOperation {
|
|
2356
|
+
seq: number;
|
|
2357
|
+
step: number;
|
|
2358
|
+
collection?: string;
|
|
2359
|
+
method: string;
|
|
2360
|
+
filter?: unknown;
|
|
2361
|
+
result?: unknown;
|
|
2362
|
+
durationMs?: number;
|
|
2363
|
+
error?: string;
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
/** A document the sandbox saw change */
|
|
2367
|
+
export interface BackgroundSandboxDocument {
|
|
2368
|
+
collection: string;
|
|
2369
|
+
_id: unknown;
|
|
2370
|
+
op: 'insert' | 'update' | 'delete' | 'unknown';
|
|
2371
|
+
before?: Record<string, unknown>;
|
|
2372
|
+
after?: Record<string, unknown>;
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2375
|
+
/** What {@link MigratorKit.dryRunBackground} found */
|
|
2376
|
+
export type BackgroundDryRun =
|
|
2377
|
+
| {
|
|
2378
|
+
mode: 'declarative';
|
|
2379
|
+
migration: string;
|
|
2380
|
+
direction: 'forward' | 'revert';
|
|
2381
|
+
method: 'sample' | 'first';
|
|
2382
|
+
requested: number;
|
|
2383
|
+
found: number;
|
|
2384
|
+
migrated: number;
|
|
2385
|
+
failed: number;
|
|
2386
|
+
documents: BackgroundDryRunDocument[];
|
|
2387
|
+
/** With `validate` */
|
|
2388
|
+
validated?: true;
|
|
2389
|
+
aborted?: true;
|
|
2390
|
+
ops?: BackgroundSandboxOperation[];
|
|
2391
|
+
refusals?: { method: string; reason: string; collection?: string }[];
|
|
2392
|
+
/** Documents of other collections the side writes touched */
|
|
2393
|
+
sideEffects?: BackgroundSandboxDocument[];
|
|
2394
|
+
attempts?: number;
|
|
2395
|
+
}
|
|
2396
|
+
| BackgroundStepDryRun;
|
|
2397
|
+
|
|
2398
|
+
/** A step migration's dry run: up to `steps` steps in one always-aborted transaction */
|
|
2399
|
+
export interface BackgroundStepDryRun {
|
|
2400
|
+
mode: 'step';
|
|
2401
|
+
migration: string;
|
|
2402
|
+
direction: 'forward' | 'revert';
|
|
2403
|
+
aborted: true;
|
|
2404
|
+
ok: boolean;
|
|
2405
|
+
attempts: number;
|
|
2406
|
+
stoppedBy?: 'deadline' | 'done' | 'steps';
|
|
2407
|
+
steps: {
|
|
2408
|
+
step: number;
|
|
2409
|
+
checkpointIn: unknown;
|
|
2410
|
+
checkpointOut?: unknown;
|
|
2411
|
+
done?: boolean;
|
|
2412
|
+
processed?: number;
|
|
2413
|
+
migrated?: number;
|
|
2414
|
+
error?: string;
|
|
2415
|
+
}[];
|
|
2416
|
+
ops: BackgroundSandboxOperation[];
|
|
2417
|
+
documents: BackgroundSandboxDocument[];
|
|
2418
|
+
refusals: { method: string; reason: string; collection?: string }[];
|
|
2419
|
+
leakedCursors: number;
|
|
2420
|
+
truncated: boolean;
|
|
2421
|
+
abortedBy?: string;
|
|
2422
|
+
error?: string;
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
/** `background:registered` */
|
|
2426
|
+
export interface BackgroundRegisteredEvent {
|
|
2427
|
+
runId?: string;
|
|
2428
|
+
migration: string;
|
|
2429
|
+
status: BackgroundState | 'withdrawn';
|
|
2430
|
+
direction: 'forward' | 'revert';
|
|
2431
|
+
waitsFor?: string[];
|
|
2432
|
+
}
|
|
2433
|
+
|
|
2434
|
+
/** Any other `background:*` event: the migration it is about, and what happened */
|
|
2435
|
+
export interface BackgroundEvent {
|
|
2436
|
+
runId?: string;
|
|
2437
|
+
migration: string;
|
|
2438
|
+
[field: string]: unknown;
|
|
1333
2439
|
}
|
|
1334
2440
|
|
|
1335
2441
|
// ─── Programmatic entry points ─────────────────────────────────────────────────
|
|
@@ -1341,6 +2447,13 @@ export type OnLockHeld = 'throw' | 'wait';
|
|
|
1341
2447
|
export interface RunMigrationsOptions extends MigratorKitOptions {
|
|
1342
2448
|
/** Skip lock acquisition (dev only — never in production) */
|
|
1343
2449
|
noLock?: boolean;
|
|
2450
|
+
/**
|
|
2451
|
+
* At a migration that requires an unfinished background migration: throw
|
|
2452
|
+
* (`'error'`, default) or stop the run there (`'stop'`, listed in
|
|
2453
|
+
* `waiting`) — `'stop'` lets an app boot while a background migration runs.
|
|
2454
|
+
* @experimental New in 2.3
|
|
2455
|
+
*/
|
|
2456
|
+
onBackgroundPending?: 'error' | 'stop';
|
|
1344
2457
|
/**
|
|
1345
2458
|
* How to react when another process already holds the migration lock — the
|
|
1346
2459
|
* typical case when several app instances boot at once.
|
|
@@ -1395,6 +2508,12 @@ export interface MigrationSummary {
|
|
|
1395
2508
|
attempts: number;
|
|
1396
2509
|
/** The converge that ended the run — present only when `convergeAfterUp` converged */
|
|
1397
2510
|
converge?: ConvergeResult;
|
|
2511
|
+
/**
|
|
2512
|
+
* With `onBackgroundPending: 'stop'`: the migration the run stopped at and
|
|
2513
|
+
* the background migrations it waits for.
|
|
2514
|
+
* @experimental New in 2.3
|
|
2515
|
+
*/
|
|
2516
|
+
waiting?: { migration: string; waitsFor: { migration: string; status: string }[] }[];
|
|
1398
2517
|
}
|
|
1399
2518
|
|
|
1400
2519
|
/**
|
|
@@ -1431,6 +2550,56 @@ export const EXIT_CODES: Readonly<
|
|
|
1431
2550
|
>
|
|
1432
2551
|
>;
|
|
1433
2552
|
|
|
2553
|
+
// ─── Background runner ────────────────────────────────────────────────────────
|
|
2554
|
+
|
|
2555
|
+
/** Options of {@link startBackgroundRunner} */
|
|
2556
|
+
export interface BackgroundRunnerOptions {
|
|
2557
|
+
/** The kit to drive — or `config` (and `kitOptions`) for one the runner makes and closes */
|
|
2558
|
+
kit?: MigratorKit;
|
|
2559
|
+
config?: Partial<MigronautConfig>;
|
|
2560
|
+
kitOptions?: MigratorKitOptions;
|
|
2561
|
+
/** Lane loops in this process, shared by every background migration (default 1, ≤ 64) */
|
|
2562
|
+
concurrency?: number;
|
|
2563
|
+
/** How often the runnable list is read again (default 5000 ms) */
|
|
2564
|
+
pollIntervalMs?: number;
|
|
2565
|
+
/** A slice's length (default: each background migration's `sliceMs`) */
|
|
2566
|
+
sliceMs?: number;
|
|
2567
|
+
/** The drift watch's period (default 600 000 ms — 10 minutes); `false`: off */
|
|
2568
|
+
verifyIntervalMs?: number | false;
|
|
2569
|
+
/**
|
|
2570
|
+
* Host the live drift watcher in this process — `true`, or its options.
|
|
2571
|
+
* Default: when `backgroundDrift` is `'stream'` or `'both'`
|
|
2572
|
+
*/
|
|
2573
|
+
watch?: boolean | Omit<WatchBackgroundOptions, 'signal' | 'onError'>;
|
|
2574
|
+
/** Stops the runner, as `stop()` does */
|
|
2575
|
+
signal?: AbortSignal;
|
|
2576
|
+
/** Hears every failed slice (the runner itself never throws) */
|
|
2577
|
+
onError?: (error: unknown, migration?: string) => void;
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
/** A running {@link startBackgroundRunner} */
|
|
2581
|
+
export interface BackgroundRunner {
|
|
2582
|
+
readonly kit: MigratorKit;
|
|
2583
|
+
readonly running: boolean;
|
|
2584
|
+
/** The live drift watcher this runner hosts, once started — or undefined */
|
|
2585
|
+
readonly watcher: BackgroundWatcher | undefined;
|
|
2586
|
+
/**
|
|
2587
|
+
* Stop at the next batch, release every lease, and close the kit the runner
|
|
2588
|
+
* made. `timeoutMs`: stop waiting for a lane stuck in its transformation
|
|
2589
|
+
* (its lease expires; the work resumes from the last checkpoint)
|
|
2590
|
+
*/
|
|
2591
|
+
stop(options?: { timeoutMs?: number }): Promise<void>;
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
/**
|
|
2595
|
+
* Drive background migrations from inside the application — no queue:
|
|
2596
|
+
* `concurrency` lane loops shared by every runnable background migration,
|
|
2597
|
+
* round-robin, plus the drift watch every `verifyIntervalMs`. Several
|
|
2598
|
+
* application instances share the work through the leases.
|
|
2599
|
+
* @experimental New in 2.3
|
|
2600
|
+
*/
|
|
2601
|
+
export function startBackgroundRunner(options?: BackgroundRunnerOptions): BackgroundRunner;
|
|
2602
|
+
|
|
1434
2603
|
// ─── Logger factory ───────────────────────────────────────────────────────────
|
|
1435
2604
|
|
|
1436
2605
|
/** Threshold accepted by {@link createLogger} — drops anything less severe */
|
|
@@ -1620,12 +2789,97 @@ export class QueueJobFailedError extends MigronautError {
|
|
|
1620
2789
|
|
|
1621
2790
|
/**
|
|
1622
2791
|
* Thrown by {@link MigratorKit.converge} when the database cannot be brought
|
|
1623
|
-
* to the declared state. `context.phase` is
|
|
1624
|
-
* (`context.conflicts` lists why
|
|
1625
|
-
*
|
|
1626
|
-
*
|
|
1627
|
-
*
|
|
2792
|
+
* to the declared state. `context.phase` is:
|
|
2793
|
+
* - `'plan'` for a refused plan (`context.conflicts` lists why, with a `hint`
|
|
2794
|
+
* when Atlas Search is missing; nothing was written) — or a search index
|
|
2795
|
+
* list that could not be read before the first write (`collection`,
|
|
2796
|
+
* `target: 'searchIndex'`, `cause`, `mongoCode`, `hint`);
|
|
2797
|
+
* - `'replan'` when a collection changed while the run was under way
|
|
2798
|
+
* (`collection`, `introduced`: the new conflicts or drops; nothing of that
|
|
2799
|
+
* collection was written) — or its search index list could not be read
|
|
2800
|
+
* again (as for `'plan'`);
|
|
2801
|
+
* - `'apply'` for a failed step (`collection`, `target`, `name`, `action`,
|
|
2802
|
+
* `cause`, and `mongoCode`, `hint` and — after a failed rebuild — `restored`
|
|
2803
|
+
* when they apply) — or a search index list that could not be read to check
|
|
2804
|
+
* the steps just applied (as for `'plan'`);
|
|
2805
|
+
* - `'wait'` when `waitForSearchIndexes` gave up: `reason` is `'failed'` (the
|
|
2806
|
+
* build of a search index this run created or changed FAILED) or `'timeout'`,
|
|
2807
|
+
* with `notReady` the indexes not serving their declaration, `waitedMs`,
|
|
2808
|
+
* `timeoutMs` — or `'unreadable'`: a search index list that could not be
|
|
2809
|
+
* read (as for `'plan'`), after up to three network or failover blips in a
|
|
2810
|
+
* row. Everything was applied — only the builds were not finished.
|
|
2811
|
+
*
|
|
2812
|
+
* `context.converge` is the {@link ConvergeResult} so far.
|
|
1628
2813
|
*/
|
|
1629
2814
|
export class ConvergeFailedError extends MigronautError {
|
|
1630
2815
|
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
1631
2816
|
}
|
|
2817
|
+
|
|
2818
|
+
/**
|
|
2819
|
+
* Thrown by the optimistic-concurrency helpers of `@alexify/migronaut/versioning`
|
|
2820
|
+
* when a revision-guarded write matched nothing. `context.reason` is
|
|
2821
|
+
* `'conflict'` (the document is at another revision — `context.actual`),
|
|
2822
|
+
* `'not-found'` (nothing matches the filter) or `'unknown'` (the follow-up read
|
|
2823
|
+
* was skipped or could not tell); `context.expected` is the revision the
|
|
2824
|
+
* caller held. The filter is never copied into the error. Experimental.
|
|
2825
|
+
*/
|
|
2826
|
+
export class RevisionConflictError extends MigronautError {
|
|
2827
|
+
readonly context?: RevisionConflictContext;
|
|
2828
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2829
|
+
}
|
|
2830
|
+
|
|
2831
|
+
/** {@link RevisionConflictError}'s `context` — what a caller decides on */
|
|
2832
|
+
export interface RevisionConflictContext {
|
|
2833
|
+
reason: 'conflict' | 'not-found' | 'unknown';
|
|
2834
|
+
/** The revision the caller held */
|
|
2835
|
+
expected: number;
|
|
2836
|
+
/** `conflict`: the revision the document is at */
|
|
2837
|
+
actual?: number;
|
|
2838
|
+
/** The collection's name, when the collection object has one */
|
|
2839
|
+
collection?: string;
|
|
2840
|
+
[key: string]: unknown;
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2843
|
+
/**
|
|
2844
|
+
* Thrown by an upcaster that cannot bring a document to the current shape:
|
|
2845
|
+
* `context.reason` is `'newer'`, `'below-min'` or `'invalid'`, with
|
|
2846
|
+
* `context.version` and `context.current`. Experimental.
|
|
2847
|
+
*/
|
|
2848
|
+
export class ShapeVersionError extends MigronautError {
|
|
2849
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
/**
|
|
2853
|
+
* Thrown when a migration `requires` a background migration that has not
|
|
2854
|
+
* completed — or whose collection still holds old-shape documents. Nothing was
|
|
2855
|
+
* run; `context.waitsFor` lists what it waits for. Experimental.
|
|
2856
|
+
*/
|
|
2857
|
+
export class BackgroundPendingError extends MigronautError {
|
|
2858
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
/**
|
|
2862
|
+
* Thrown when a background migration ended `failed`; `context.migration` names
|
|
2863
|
+
* it and `context.lastError` says what happened last. Experimental.
|
|
2864
|
+
*/
|
|
2865
|
+
export class BackgroundFailedError extends MigronautError {
|
|
2866
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2867
|
+
}
|
|
2868
|
+
|
|
2869
|
+
/**
|
|
2870
|
+
* Thrown when a control action does not fit the background migration's state;
|
|
2871
|
+
* `context.status` is the state found and `context.action` what was asked.
|
|
2872
|
+
* Experimental.
|
|
2873
|
+
*/
|
|
2874
|
+
export class BackgroundConflictError extends MigronautError {
|
|
2875
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
/**
|
|
2879
|
+
* Thrown by the dry-run sandbox when a step reaches for something it cannot
|
|
2880
|
+
* run inside an always-aborted transaction; `context.method` names the call
|
|
2881
|
+
* and `context.reason` the rule. Experimental.
|
|
2882
|
+
*/
|
|
2883
|
+
export class SandboxRefusedError extends MigronautError {
|
|
2884
|
+
constructor(message: string, context?: Record<string, unknown>, options?: MigronautErrorOptions);
|
|
2885
|
+
}
|