@alexify/migronaut 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +436 -0
  2. package/README.md +235 -6
  3. package/bullmq.d.ts +860 -0
  4. package/bullmq.js +1 -0
  5. package/index.d.ts +888 -19
  6. package/migronaut.schema.json +238 -1
  7. package/package.json +21 -5
  8. package/src/bullmq/index.js +55 -0
  9. package/src/bullmq/jobs.js +454 -0
  10. package/src/bullmq/processor.js +632 -0
  11. package/src/bullmq/producer.js +427 -0
  12. package/src/bullmq/service.js +653 -0
  13. package/src/bullmq/wait.js +124 -0
  14. package/src/cli/args.js +12 -2
  15. package/src/cli/commands/converge.js +188 -0
  16. package/src/cli/commands/down.js +2 -0
  17. package/src/cli/commands/lock.js +2 -1
  18. package/src/cli/commands/redo.js +8 -1
  19. package/src/cli/commands/up.js +14 -1
  20. package/src/cli/exit-codes.js +9 -2
  21. package/src/cli/index.js +2 -0
  22. package/src/cli/shared.js +14 -4
  23. package/src/cli/table.js +164 -0
  24. package/src/core/audit.js +88 -3
  25. package/src/core/changelog.js +71 -6
  26. package/src/core/collections.js +396 -0
  27. package/src/core/config.js +130 -25
  28. package/src/core/converge-log.js +47 -0
  29. package/src/core/converge-plan.js +686 -0
  30. package/src/core/converge-search-run.js +440 -0
  31. package/src/core/converge-search.js +404 -0
  32. package/src/core/converge.js +1024 -0
  33. package/src/core/index-spec.js +507 -0
  34. package/src/core/lock-wait.js +260 -0
  35. package/src/core/lock.js +95 -28
  36. package/src/core/migrator.js +600 -287
  37. package/src/core/options.js +266 -0
  38. package/src/core/run-recorder.js +157 -0
  39. package/src/core/run.js +58 -90
  40. package/src/core/search-index-spec.js +758 -0
  41. package/src/core/sequence.js +134 -0
  42. package/src/core/server-info.js +63 -0
  43. package/src/errors/index.js +60 -0
  44. package/src/index.js +8 -0
  45. package/src/utils/actor.js +48 -0
  46. package/src/utils/canonical.js +212 -0
  47. package/src/utils/collection-name.js +21 -0
  48. package/src/utils/error.js +18 -1
  49. package/src/utils/id.js +77 -0
  50. package/src/utils/loader.js +39 -21
  51. package/src/utils/migration-name.js +32 -0
  52. package/src/utils/redact.js +21 -1
  53. package/src/utils/telemetry.js +410 -0
  54. package/src/utils/template.js +43 -2
@@ -1,6 +1,6 @@
1
- const { randomUUID } = require('node:crypto');
2
1
  const { EventEmitter } = require('node:events');
3
2
  const fs = require('node:fs/promises');
3
+ const os = require('node:os');
4
4
  const path = require('node:path');
5
5
  // `mongodb` is required lazily inside connect(): loading the driver costs ~60ms
6
6
  // and pulls in ~150 modules, which `--help`, `--version`, `init` and `create`
@@ -11,6 +11,7 @@ const {
11
11
  ConnectionFailedError,
12
12
  HookFailedError,
13
13
  IrreversibleMigrationError,
14
+ LockAlreadyHeldError,
14
15
  MigrationFileNotFoundError,
15
16
  MigrationInvalidNameError,
16
17
  MigronautError,
@@ -18,11 +19,15 @@ const {
18
19
  OutOfOrderMigrationError,
19
20
  RunAbortedError,
20
21
  } = require('../errors/index.js');
22
+ const { actorFields, pickActor } = require('../utils/actor.js');
21
23
  const { computeChecksum } = require('../utils/checksum.js');
22
24
  const { mapLimit } = require('../utils/concurrency.js');
23
- const { errorText } = require('../utils/error.js');
25
+ const { errorText, errorWithCause } = require('../utils/error.js');
26
+ const { createIdGenerator } = require('../utils/id.js');
24
27
  const { loadMigrationFile } = require('../utils/loader.js');
25
28
  const { resolveLogger } = require('../utils/logger.js');
29
+ const { assertMigrationName } = require('../utils/migration-name.js');
30
+ const { ATTRIBUTES, SPANS, createTelemetry } = require('../utils/telemetry.js');
26
31
  const {
27
32
  createConfigFile,
28
33
  createMigrationFile,
@@ -32,11 +37,35 @@ const { safeUsername } = require('../utils/user.js');
32
37
  const { runAudit } = require('./audit.js');
33
38
  const { runBaseline } = require('./baseline.js');
34
39
  const { Changelog } = require('./changelog.js');
35
- const { isCollectionName, loadConfig } = require('./config.js');
40
+ const { ConvergeLog } = require('./converge-log.js');
41
+ const { resolveDefinitions } = require('./collections.js');
42
+ const { loadConfig } = require('./config.js');
36
43
  const { buildContext } = require('./context.js');
44
+ const { runConverge } = require('./converge.js');
37
45
  const { runImport } = require('./import-runner.js');
38
46
  const { MigrationLock, runWithLock, toLockInfo } = require('./lock.js');
47
+ const {
48
+ assertConvergeOptions,
49
+ assertDownOptions,
50
+ assertDryRunOptions,
51
+ assertFilename,
52
+ assertHistoryLimit,
53
+ assertImportOptions,
54
+ assertListOptions,
55
+ assertRedoOptions,
56
+ assertUpOptions,
57
+ } = require('./options.js');
58
+ const { RunRecorder } = require('./run-recorder.js');
39
59
  const { runMigration } = require('./runner.js');
60
+ const {
61
+ blockedError,
62
+ lateArrivals,
63
+ listMigrationFiles,
64
+ newestOf,
65
+ pendingIn,
66
+ revertOrder,
67
+ truncateAtTarget,
68
+ } = require('./sequence.js');
40
69
 
41
70
  /** Simultaneous file reads — keeps a large migrations dir clear of EMFILE */
42
71
  const FS_CONCURRENCY = 16;
@@ -51,6 +80,13 @@ const FS_CONCURRENCY = 16;
51
80
  * logic in the migration's flow; listeners attach from outside, may be several,
52
81
  * and a listener that throws is contained rather than failing the run.
53
82
  */
83
+ /**
84
+ * How a lock-wait loop outside the kit (`runMigrations`, the queue processor)
85
+ * reports a finished wait to the kit's telemetry. A symbol, and not exported
86
+ * from the package: the loop is migronaut's own, and so is this channel.
87
+ */
88
+ const RECORD_LOCK_WAIT = Symbol('migronaut.recordLockWait');
89
+
54
90
  class MigratorKit extends EventEmitter {
55
91
  #partialConfig;
56
92
  #configPath;
@@ -67,6 +103,10 @@ class MigratorKit extends EventEmitter {
67
103
  #runSetupDepth = 0;
68
104
  /** Correlation id for the run in flight — ties logs, lock and changelog together */
69
105
  #runId;
106
+ /** Mints an id in the configured format (`generateId`, else a UUID); set with the config */
107
+ #newId;
108
+ /** Spans and metrics through the injected `telemetry` (a no-op without one); set with the config */
109
+ #telemetry;
70
110
  /** Whether changelog indexes have already been ensured on this instance */
71
111
  #indexesEnsured = false;
72
112
  /** Memoized resolved logger — resolveLogger allocates on every call otherwise */
@@ -75,6 +115,13 @@ class MigratorKit extends EventEmitter {
75
115
  #fallbackLogger;
76
116
  /** False when the client was injected by the caller, who keeps ownership of it */
77
117
  #ownsClient = true;
118
+ /** The connect() in flight, so overlapping callers share one client instead of racing */
119
+ #connecting;
120
+ /**
121
+ * The config load in flight, so overlapping first callers share one — a
122
+ * config factory may fetch secrets, and must not run twice.
123
+ */
124
+ #configLoading;
78
125
  /**
79
126
  * Project root this instance resolves against — config discovery, the .env
80
127
  * file and a relative migrationsDir. Defaults to process.cwd(); an explicit
@@ -83,6 +130,16 @@ class MigratorKit extends EventEmitter {
83
130
  #cwd;
84
131
  /** filepath → {mtimeMs, size, checksum} — spares repeat status()/audit() calls a full re-hash */
85
132
  #checksumCache = new Map();
133
+ /** The converge history store, created on first use */
134
+ #convergeLogStore;
135
+ /**
136
+ * Definitions an after-up converge resolved, kept only while `up()` is being
137
+ * refused for a held lock: a caller polling for the lock retries `up()`
138
+ * every few hundred milliseconds, and each retry would otherwise re-import
139
+ * every definition file — under `reloadMigrations`, a module per file per
140
+ * poll that the module cache never frees. Cleared by any other outcome.
141
+ */
142
+ #upDefinitions;
86
143
 
87
144
  constructor(config = {}, options = {}) {
88
145
  super();
@@ -155,19 +212,28 @@ class MigratorKit extends EventEmitter {
155
212
  }
156
213
  }
157
214
 
158
- /** Resolve and cache the full configuration */
215
+ /** Resolve and cache the full configuration — once, however many callers ask at once */
159
216
  async #ensureConfig(requireDb = true, lenient = false) {
160
- if (!this.#config) {
161
- this.#config = await loadConfig({
162
- flags: this.#partialConfig,
163
- requireDb,
164
- ...(lenient ? { lenient: true } : {}),
165
- ...(this.#configPath ? { configPath: this.#configPath } : {}),
166
- ...(this.#cwd ? { cwd: this.#cwd } : {}),
167
- ...(this.#fallbackLogger !== undefined ? { fallbackLogger: this.#fallbackLogger } : {}),
168
- });
169
- }
170
- return this.#config;
217
+ if (this.#config) return this.#config;
218
+ this.#configLoading ??= this.#loadConfig(requireDb, lenient).finally(() => {
219
+ this.#configLoading = undefined;
220
+ });
221
+ return this.#configLoading;
222
+ }
223
+
224
+ async #loadConfig(requireDb, lenient) {
225
+ const config = await loadConfig({
226
+ flags: this.#partialConfig,
227
+ requireDb,
228
+ ...(lenient ? { lenient: true } : {}),
229
+ ...(this.#configPath ? { configPath: this.#configPath } : {}),
230
+ ...(this.#cwd ? { cwd: this.#cwd } : {}),
231
+ ...(this.#fallbackLogger !== undefined ? { fallbackLogger: this.#fallbackLogger } : {}),
232
+ });
233
+ this.#newId = createIdGenerator(config.generateId);
234
+ this.#telemetry = createTelemetry(config.telemetry, { dbName: config.dbName });
235
+ this.#config = config;
236
+ return config;
171
237
  }
172
238
 
173
239
  get #logger() {
@@ -215,12 +281,27 @@ class MigratorKit extends EventEmitter {
215
281
  return this.#runId ? { runId: this.#runId, ...extra } : { ...extra };
216
282
  }
217
283
 
284
+ /** Record a finished wait for the lock — see {@link RECORD_LOCK_WAIT} */
285
+ [RECORD_LOCK_WAIT](wait) {
286
+ this.#telemetry?.lockWaited(wait);
287
+ }
288
+
218
289
  /** Connect to MongoDB and ensure changelog indexes exist */
219
290
  async connect() {
220
291
  const config = await this.#ensureConfig();
221
292
  if (this.#client && this.#db) {
222
293
  return;
223
294
  }
295
+ // A long-lived kit serves overlapping callers (a status probe while a
296
+ // queue job starts). Without this, each would open its own MongoClient
297
+ // and all but the last would leak their pools.
298
+ this.#connecting ??= this.#openConnection(config).finally(() => {
299
+ this.#connecting = undefined;
300
+ });
301
+ return this.#connecting;
302
+ }
303
+
304
+ async #openConnection(config) {
224
305
  const startedAt = Date.now();
225
306
  try {
226
307
  if (config.client) {
@@ -327,10 +408,12 @@ class MigratorKit extends EventEmitter {
327
408
  }
328
409
 
329
410
  /**
330
- * Run `fn` under the migration lock. The single place that pairs a lock with
331
- * a unit of work, so `redo` can hold one lock across both directions instead
332
- * of releasing between them. `info` names the run (`{command, direction?}`)
333
- * for the `run:start`/`run:end` events.
411
+ * Run `fn(signal, lock)` under the migration lock. The single place that
412
+ * pairs a lock with a unit of work, so `redo` can hold one lock across both
413
+ * directions instead of releasing between them. `info` names the run
414
+ * (`{command, direction?}`) for the `run:start`/`run:end` events.
415
+ * `lock.release()` gives the lock up before `fn` returns, for a tail that
416
+ * only reads (see runWithLock) — the run, its id and its span go on.
334
417
  */
335
418
  async #withLock(options, info, fn) {
336
419
  // Not reentrant: a second overlapping run on this instance would clobber
@@ -343,8 +426,10 @@ class MigratorKit extends EventEmitter {
343
426
  }
344
427
  // One id per run, reused as the lock's owner token and stamped on every
345
428
  // changelog record and log line, so the three can be correlated after the
346
- // fact ("which run left this lock?", "what did run X apply?").
347
- this.#runId = randomUUID();
429
+ // fact ("which run left this lock?", "what did run X apply?"). Minted
430
+ // before any other run state exists: a `generateId` that throws or returns
431
+ // a non-id rejects here, leaving nothing to unwind and no event emitted.
432
+ this.#runId = this.#newId();
348
433
  // A second controller layered over the lock's own signal, so stop() and a
349
434
  // lost lock abort through the same path the run loops already watch.
350
435
  const stopper = new AbortController();
@@ -359,8 +444,15 @@ class MigratorKit extends EventEmitter {
359
444
  this.#stopRequested = undefined;
360
445
  this.#abort(pending);
361
446
  }
362
- const startedAt = Date.now();
363
- this.#emit('run:start', { ...info });
447
+ const recorder = new RunRecorder({
448
+ info,
449
+ runId: this.#runId,
450
+ telemetry: this.#telemetry,
451
+ emit: (event, payload) => this.#emit(event, payload),
452
+ logger: this.#logger,
453
+ fields: (extra) => this.#fields(extra),
454
+ });
455
+ recorder.start();
364
456
  let failure;
365
457
  let result;
366
458
  try {
@@ -373,62 +465,28 @@ class MigratorKit extends EventEmitter {
373
465
  logger: this.#lockLogger(),
374
466
  onLockLost: this.#config.onLockLost,
375
467
  owner: this.#runId,
376
- onLockAcquired: (extra) => this.#emit('lock:acquired', { owner: this.#runId, ...extra }),
377
- onLockReleased: (extra) => this.#emit('lock:released', { owner: this.#runId, ...extra }),
378
- onLockLostEvent: (reason) => this.#emit('lock:lost', { owner: this.#runId, reason }),
468
+ onLockAcquired: (extra) => recorder.lockAcquired(extra),
469
+ onLockReleased: (extra) => recorder.lockReleased(extra),
470
+ onLockLostEvent: (reason) => recorder.lockLost(reason),
379
471
  ...(options.noLock ? { noLock: true } : {}),
380
472
  },
381
- (lockSignal) => fn(AbortSignal.any([lockSignal, stopper.signal])),
473
+ (lockSignal, lockControl) =>
474
+ // The run's span exists only once the lock is held: a caller polling
475
+ // for a busy lock retries the whole run every few hundred
476
+ // milliseconds, and a span per refusal would bury the one run that
477
+ // did the work. Active around the unit of work only, and ended by the
478
+ // recorder after the release, so the span's outcome is the run's.
479
+ this.#telemetry.open(SPANS.RUN, recorder.spanAttributes(), (span) => {
480
+ recorder.spanOpened(span);
481
+ return fn(AbortSignal.any([lockSignal, stopper.signal]), lockControl);
482
+ }),
382
483
  );
383
484
  return result;
384
485
  } catch (error) {
385
486
  failure = error;
386
487
  throw error;
387
488
  } finally {
388
- // Result counts, so a metrics subscriber gets "3 applied in 812ms"
389
- // without reconstructing it from per-migration events. On the failure
390
- // path the partial rows live on the error's context — exactly the case
391
- // where "how far did it get?" is the question, so they count too. One
392
- // pass fills both counters.
393
- const rows = Array.isArray(result)
394
- ? result
395
- : failure instanceof MigronautError && Array.isArray(failure.context?.results)
396
- ? failure.context.results
397
- : null;
398
- let summary = {};
399
- if (rows) {
400
- let applied = 0;
401
- let reverted = 0;
402
- for (const row of rows) {
403
- if (row.status === 'applied') applied += 1;
404
- else if (row.status === 'reverted') reverted += 1;
405
- }
406
- summary = { applied, reverted, total: rows.length };
407
- }
408
- const durationMs = Date.now() - startedAt;
409
- this.#emit('run:end', {
410
- ...info,
411
- success: failure === undefined,
412
- durationMs,
413
- ...summary,
414
- // A raw Error here would hand subscribers an unredacted driver message
415
- // (which can echo the credentialed URI) — errorText is the same
416
- // chokepoint every log line and result row already goes through.
417
- ...(failure ? { error: errorText(failure) } : {}),
418
- });
419
- // One human rollup after the per-migration lines: total wall-clock time
420
- // (lock wait and hooks included) is otherwise unobtainable from the
421
- // output — per-file durations exclude all overhead. Success path only;
422
- // a failure already ends with its own error line.
423
- if (failure === undefined && (summary.applied || summary.reverted)) {
424
- const parts = [];
425
- if (summary.applied) parts.push(`${summary.applied} applied`);
426
- if (summary.reverted) parts.push(`${summary.reverted} reverted`);
427
- this.#logger.info(
428
- `✔ Done ${parts.join(', ')} in ${durationMs}ms`,
429
- this.#fields({ ...info, ...summary, durationMs }),
430
- );
431
- }
489
+ recorder.finish(result, failure);
432
490
  this.#abort = undefined;
433
491
  this.#runId = undefined;
434
492
  }
@@ -477,6 +535,30 @@ class MigratorKit extends EventEmitter {
477
535
  return toLockInfo(await this.#buildLock().forceRelease());
478
536
  }
479
537
 
538
+ /**
539
+ * The batch number the next `up` would use — a peek, not a reservation: two
540
+ * callers asking before either applies get the same number. Counts reverted
541
+ * and failed records too, so a rolled-back number is never handed out again.
542
+ * Pair it with `up(name, { batch })` to stamp several single-file runs as one
543
+ * batch.
544
+ */
545
+ async nextBatch() {
546
+ await this.#ensureConfig();
547
+ await this.connect();
548
+ return this.#nextBatch();
549
+ }
550
+
551
+ /**
552
+ * A new id in the format this kit is configured with — the `generateId`
553
+ * option, else a random UUID. It is what every run id comes from; exposed so
554
+ * a layer above the kit (the queue adapter's group ids) mints its own ids in
555
+ * the same format without being configured a second time. Does not connect.
556
+ */
557
+ async generateId() {
558
+ await this.#ensureConfig();
559
+ return this.#newId();
560
+ }
561
+
480
562
  /** Internal accessors that assume a successful connect() */
481
563
  #requireDb() {
482
564
  if (!this.#db) {
@@ -498,19 +580,6 @@ class MigratorKit extends EventEmitter {
498
580
  return path.resolve(this.#cwd ?? process.cwd(), this.#config.migrationsDir);
499
581
  }
500
582
 
501
- /**
502
- * Reject non-string filenames before they reach a changelog query or a path
503
- * join. A programmatic caller passing e.g. `{ $ne: null }` would otherwise
504
- * become a query-operator injection in `findOne({ name })`.
505
- */
506
- #assertFilename(filename) {
507
- if (filename !== undefined && typeof filename !== 'string') {
508
- throw new MigrationInvalidNameError('Migration name must be a string', {
509
- name: filename,
510
- });
511
- }
512
- }
513
-
514
583
  /**
515
584
  * Resolve a migration name to an absolute path inside the migrations dir.
516
585
  *
@@ -522,20 +591,7 @@ class MigratorKit extends EventEmitter {
522
591
  */
523
592
  #filepath(name) {
524
593
  const dir = this.#migrationsPath();
525
- if (
526
- typeof name !== 'string' ||
527
- name.length === 0 ||
528
- name === '.' ||
529
- name === '..' ||
530
- name.includes('/') ||
531
- name.includes('\\') ||
532
- name.includes('\0')
533
- ) {
534
- throw new MigrationInvalidNameError(
535
- 'Invalid migration name — must be a bare filename with no path segments',
536
- { name },
537
- );
538
- }
594
+ assertMigrationName(name);
539
595
  const resolved = path.join(dir, name);
540
596
  const relative = path.relative(dir, resolved);
541
597
  if (relative.startsWith('..') || path.isAbsolute(relative)) {
@@ -548,33 +604,8 @@ class MigratorKit extends EventEmitter {
548
604
 
549
605
  /** List migration files on disk, sorted ascending */
550
606
  async #listMigrationFiles() {
551
- const dir = this.#migrationsPath();
552
607
  // No value fallback — DEFAULT_CONFIG always supplies fileExtensions.
553
- const extensions = this.#config.fileExtensions;
554
- let entries;
555
- try {
556
- entries = await fs.readdir(dir, { withFileTypes: true });
557
- } catch (error) {
558
- if (error.code === 'ENOENT') return [];
559
- throw error;
560
- }
561
- const matches = [];
562
- for (const entry of entries) {
563
- // A directory named `foo.js`, a dotfile, or a `types.d.ts` sitting next
564
- // to the migrations is not a migration — including it would hard-fail
565
- // the whole run with MigrationInvalidExportError.
566
- if (!entry.isFile()) continue;
567
- const file = entry.name;
568
- if (file.startsWith('.')) continue;
569
- if (file.endsWith('.d.ts') || file.endsWith('.d.mts') || file.endsWith('.d.cts')) continue;
570
- for (const ext of extensions) {
571
- if (file.endsWith(ext)) {
572
- matches.push(file);
573
- break;
574
- }
575
- }
576
- }
577
- return matches.sort();
608
+ return listMigrationFiles(this.#migrationsPath(), this.#config.fileExtensions);
578
609
  }
579
610
 
580
611
  /** Compute the next batch number (monotonic across the full history) */
@@ -583,78 +614,52 @@ class MigratorKit extends EventEmitter {
583
614
  }
584
615
 
585
616
  /**
586
- * Validate the `--steps` option for `down`/`dry-run down`: a positive integer,
587
- * mutually exclusive with a filename and `--batch`. No-op when steps is unset.
617
+ * `ordered` guard for a single-file `up`: refuse while an earlier file on
618
+ * disk is still pending. Pending means "no applied record" — a `'failed'`
619
+ * trace counts, so a migration that failed stops the line exactly like one
620
+ * that never ran. Checked inside the lock and before `beforeAll`, so a
621
+ * blocked run fires no hooks and consumes no batch number.
588
622
  */
589
- #assertStepsValid(steps, filename, batch) {
590
- if (steps === undefined) {
591
- return;
592
- }
593
- if (filename) {
594
- throw new ConfigInvalidError('Cannot combine a filename with --steps', { filename });
595
- }
596
- if (batch !== undefined) {
597
- throw new ConfigInvalidError('Cannot combine --batch with --steps', { batch, steps });
598
- }
599
- if (!Number.isInteger(steps) || steps < 1) {
600
- throw new ConfigInvalidError('--steps must be a positive integer', { steps });
601
- }
623
+ async #assertUpNotBlocked(name, appliedNames, sequence) {
624
+ const files = sequence ?? (await this.#listMigrationFiles());
625
+ const blockedBy = pendingIn(files, appliedNames, name);
626
+ if (blockedBy.length === 0) return;
627
+ throw blockedError(name, 'earlier migration(s) still pending', {
628
+ name,
629
+ direction: 'up',
630
+ blockedBy,
631
+ failed: await this.#failedAmong(blockedBy),
632
+ });
602
633
  }
603
634
 
604
635
  /**
605
- * Keep only the pending migrations up to and including `to`.
606
- *
607
- * `to` must name a migration that exists; it may already be applied (then
608
- * nothing before it is pending either, and the result is empty), which is
609
- * what makes `up --to X` idempotent — running it twice is a no-op rather
610
- * than an error.
636
+ * The blockers that failed — a stopped line — as opposed to ones that have
637
+ * simply not run yet: with several queue workers, an earlier job may still be
638
+ * in flight elsewhere, and waiting for it is the right reaction.
611
639
  */
612
- #truncateAtTarget(pending, allFiles, to) {
613
- if (!allFiles.includes(to)) {
614
- throw new MigrationFileNotFoundError('Migration file not found', { to });
615
- }
616
- const kept = [];
617
- for (const file of pending) {
618
- if (file > to) break;
619
- kept.push(file);
620
- }
621
- return kept;
640
+ async #failedAmong(names) {
641
+ return this.#requireChangelog().getFailedNames(this.#requireDb(), names);
622
642
  }
623
643
 
624
644
  /**
625
- * `--to` names a point in the sequence, so it cannot be combined with the
626
- * other ways of choosing targets.
645
+ * `ordered` guard for a single-file `down`: refuse while a migration applied
646
+ * *after* this one is still applied. "After" is chronological (`appliedAt`),
647
+ * not alphabetical — the order `down --steps` reverts in, and the only one
648
+ * that undoes effects in reverse of how they were made. Name order would
649
+ * deadlock a batch holding a file merged late from a parallel branch.
627
650
  */
628
- #assertToValid(to, filename, options = {}) {
629
- if (to === undefined) return;
630
- this.#assertFilename(to);
631
- if (filename) {
632
- throw new ConfigInvalidError('Cannot combine a filename with --to', { filename, to });
633
- }
634
- if (options.steps !== undefined) {
635
- throw new ConfigInvalidError('Cannot combine --steps with --to', {
636
- steps: options.steps,
637
- to,
638
- });
639
- }
640
- if (options.batch !== undefined) {
641
- throw new ConfigInvalidError('Cannot combine --batch with --to', {
642
- batch: options.batch,
643
- to,
644
- });
645
- }
646
- }
647
-
648
- /**
649
- * Validate `--batch`. Without this a typo (`--batch abc` → NaN) matches no
650
- * records, so the run prints "Nothing to rollback" and exits 0 — the worst
651
- * possible answer to a mistyped rollback.
652
- */
653
- #assertBatchValid(batch) {
654
- if (batch === undefined) return;
655
- if (!Number.isInteger(batch) || batch < 1) {
656
- throw new ConfigInvalidError('--batch must be a positive integer', { batch });
657
- }
651
+ async #assertDownNotBlocked(record) {
652
+ const newer = await this.#requireChangelog().getAppliedNewerThan(this.#requireDb(), record);
653
+ if (newer.length === 0) return;
654
+ const blockedBy = [];
655
+ for (const later of newer) blockedBy.push(later.name);
656
+ throw blockedError(record.name, 'later migration(s) still applied', {
657
+ name: record.name,
658
+ direction: 'down',
659
+ blockedBy,
660
+ // A rollback has no failed trace to tell a stopped line apart.
661
+ failed: [],
662
+ });
658
663
  }
659
664
 
660
665
  /**
@@ -675,11 +680,8 @@ class MigratorKit extends EventEmitter {
675
680
  return [filename];
676
681
  }
677
682
  const files = await this.#listMigrationFiles();
678
- const targets = [];
679
- for (const file of files) {
680
- if (!appliedNames.has(file)) targets.push(file);
681
- }
682
- return options.to !== undefined ? this.#truncateAtTarget(targets, files, options.to) : targets;
683
+ const targets = pendingIn(files, appliedNames);
684
+ return options.to !== undefined ? truncateAtTarget(targets, files, options.to) : targets;
683
685
  }
684
686
 
685
687
  /**
@@ -695,16 +697,10 @@ class MigratorKit extends EventEmitter {
695
697
  */
696
698
  #assertOrderIntact(targets, appliedNames) {
697
699
  const policy = this.#config?.onOutOfOrder ?? 'warn';
698
- if (policy === 'allow' || targets.length === 0 || appliedNames.size === 0) return;
699
- let newestApplied = '';
700
- for (const name of appliedNames) {
701
- if (name > newestApplied) newestApplied = name;
702
- }
703
- const late = [];
704
- for (const target of targets) {
705
- if (!appliedNames.has(target) && target < newestApplied) late.push(target);
706
- }
707
- if (late.length === 0) return;
700
+ if (policy === 'allow') return;
701
+ const arrivals = lateArrivals(targets, appliedNames);
702
+ if (arrivals === null) return;
703
+ const { late, newestApplied } = arrivals;
708
704
  if (policy === 'error') {
709
705
  throw new OutOfOrderMigrationError(
710
706
  `${late.length} pending migration(s) sort before the newest applied one ` +
@@ -769,13 +765,57 @@ class MigratorKit extends EventEmitter {
769
765
  return results;
770
766
  }
771
767
 
768
+ /**
769
+ * Execute one migration under its own span, and time it for the meter.
770
+ *
771
+ * The span is the *active* one for everything the migration does — hooks,
772
+ * the file's own `up`/`down`, the changelog write — which is what lets an
773
+ * instrumented driver hang its command spans under the migration that issued
774
+ * them. A listener on `migration:start` could open a span but never make it
775
+ * active, so this is the one thing telemetry needs from inside the kit.
776
+ */
777
+ async #executeMigration(step) {
778
+ const { name, direction, index, total, batch } = step;
779
+ const telemetry = this.#telemetry;
780
+ const startedAt = Date.now();
781
+ try {
782
+ const duration = await telemetry.wrap(
783
+ SPANS.MIGRATION,
784
+ {
785
+ [ATTRIBUTES.MIGRATION_NAME]: name,
786
+ [ATTRIBUTES.MIGRATION_DIRECTION]: direction,
787
+ [ATTRIBUTES.MIGRATION_BATCH]: batch,
788
+ [ATTRIBUTES.MIGRATION_INDEX]: index,
789
+ [ATTRIBUTES.MIGRATION_TOTAL]: total,
790
+ [ATTRIBUTES.RUN_ID]: this.#runId,
791
+ },
792
+ (span) => this.#executeMigrationSteps(step, span),
793
+ );
794
+ telemetry.migrationEnded({ direction, durationMs: duration });
795
+ return duration;
796
+ } catch (error) {
797
+ // The runner's own measurement when it got as far as the body; the
798
+ // elapsed time here otherwise (a failing beforeEach, a file that does
799
+ // not load) — a failure deserves a data point as much as a success.
800
+ const durationMs =
801
+ error instanceof MigronautError && typeof error.context?.durationMs === 'number'
802
+ ? error.context.durationMs
803
+ : Date.now() - startedAt;
804
+ telemetry.migrationEnded({ direction, durationMs, error });
805
+ throw error;
806
+ }
807
+ }
808
+
772
809
  /**
773
810
  * Execute one migration end to end: beforeEach → load → run (with the
774
811
  * changelog write inside the transaction via `onSuccess`) → events, logs,
775
812
  * result row, afterEach — and the mirrored error path. Shared verbatim by
776
813
  * `up` and `down`, so a fix to one direction cannot silently miss the other.
777
814
  */
778
- async #executeMigration({ name, direction, context, index, total, results, batch, onSuccess }) {
815
+ async #executeMigrationSteps(
816
+ { name, direction, context, index, total, results, batch, onSuccess, failureFields },
817
+ span,
818
+ ) {
779
819
  const config = this.#config;
780
820
  const logger = this.#logger;
781
821
  const batchField = batch !== undefined ? { batch } : {};
@@ -789,6 +829,7 @@ class MigratorKit extends EventEmitter {
789
829
  reload: config.reloadMigrations,
790
830
  });
791
831
  const useTransaction = migration.useTransaction ?? config.useTransaction;
832
+ span.set({ [ATTRIBUTES.MIGRATION_TRANSACTION]: useTransaction });
792
833
 
793
834
  this.#progress?.onStart(name, direction);
794
835
  this.#emit('migration:start', { migration: name, direction, ...batchField });
@@ -873,22 +914,17 @@ class MigratorKit extends EventEmitter {
873
914
  // Swallowed on its own failure: the changelog may be the thing that is
874
915
  // down, and this trace must never mask the migration's real error.
875
916
  if (direction === 'up') {
876
- // The wrapper says WHICH migration failed; the cause says WHY — the
877
- // half forensics actually needs. Redacted like every string that
878
- // leaves the process (the cause is the raw thrown message).
879
- const cause =
880
- error instanceof MigronautError && typeof error.context?.cause === 'string'
881
- ? errorText(error.context.cause)
882
- : undefined;
883
917
  try {
884
918
  await this.#requireChangelog().markFailed(this.#requireDb(), {
885
919
  name,
886
- error: cause ? `${errorText(error)} — ${cause}` : errorText(error),
920
+ // Which migration failed, and why.
921
+ error: errorWithCause(error),
887
922
  environment: this.#environment(),
888
923
  executedBy: safeUsername(),
889
924
  ...batchField,
890
925
  ...(durationMs !== undefined ? { duration: durationMs } : {}),
891
926
  ...(this.#runId ? { runId: this.#runId } : {}),
927
+ ...failureFields,
892
928
  });
893
929
  } catch {
894
930
  // Duplicate key when an 'applied' record exists (forced re-run), or
@@ -904,17 +940,70 @@ class MigratorKit extends EventEmitter {
904
940
 
905
941
  /** Run all pending migrations, or a specific named file */
906
942
  async up(filename, options = {}) {
907
- this.#assertFilename(filename);
908
- this.#assertToValid(options.to, filename, options);
943
+ assertUpOptions(filename, options);
909
944
  return this.#runWindow(async () => {
910
- await this.#ensureConfig();
911
- await this.connect();
912
- return this.#withLock(options, { command: 'up', direction: 'up' }, (signal) =>
913
- this.#runUp(filename, options, signal),
914
- );
945
+ const config = await this.#ensureConfig();
946
+ // Converge only after a run that brings the database to the head: the
947
+ // declared end state describes the newest schema, and a unique index
948
+ // may well depend on a dedupe migration a `--to` run stops short of.
949
+ // A single-file run is one step of a sequence (a queue job), never its end.
950
+ const converge = filename === undefined && (options.converge ?? config.convergeAfterUp);
951
+ if (converge && options.to !== undefined) {
952
+ this.#logger.info(
953
+ 'Converge after up skipped: --to stops short of the newest migration',
954
+ this.#fields({ command: 'up', to: options.to }),
955
+ );
956
+ }
957
+ // Resolved before the lock: a definition file that does not load fails
958
+ // the run before any migration is applied, not after.
959
+ const definitions =
960
+ converge && options.to === undefined
961
+ ? (this.#upDefinitions ??= await this.#resolveCollections())
962
+ : [];
963
+ return this.#keepDefinitionsWhileRefused(async () => {
964
+ await this.connect();
965
+ return this.#withLock(options, { command: 'up', direction: 'up' }, async (signal, lock) => {
966
+ const results = await this.#runUp(filename, options, signal);
967
+ // Even when nothing was pending: a converge that failed last time is
968
+ // retried by the next `up` instead of waiting for the next migration.
969
+ if (definitions.length > 0) {
970
+ this.#assertNotAborted(signal, results);
971
+ try {
972
+ await runConverge(
973
+ this.#convergeDeps(lock),
974
+ {
975
+ definitions,
976
+ trigger: 'up',
977
+ search: this.#convergeSearchOptions(),
978
+ ...pickActor(options),
979
+ },
980
+ signal,
981
+ );
982
+ } catch (error) {
983
+ // The migrations are applied and recorded either way — they must
984
+ // survive into what a `--json` consumer sees about the failure.
985
+ this.#attachResults(error, results);
986
+ throw error;
987
+ }
988
+ }
989
+ return results;
990
+ });
991
+ });
915
992
  });
916
993
  }
917
994
 
995
+ /** Run `fn`, keeping the cached after-up definitions only when it is refused for the lock */
996
+ async #keepDefinitionsWhileRefused(fn) {
997
+ try {
998
+ const result = await fn();
999
+ this.#upDefinitions = undefined;
1000
+ return result;
1001
+ } catch (error) {
1002
+ if (!(error instanceof LockAlreadyHeldError)) this.#upDefinitions = undefined;
1003
+ throw error;
1004
+ }
1005
+ }
1006
+
918
1007
  async #runUp(filename, options = {}, signal) {
919
1008
  const force = options.force ?? false;
920
1009
  const config = this.#config;
@@ -922,29 +1011,70 @@ class MigratorKit extends EventEmitter {
922
1011
  const changelog = this.#requireChangelog();
923
1012
  const logger = this.#logger;
924
1013
 
925
- // A strict bulk run needs the applied records' checksums anyway, so fetch
926
- // full records once and derive the name set from them; a single-file run
927
- // needs only that file's record, so one getByName is both the
928
- // applied-check and the checksum source; otherwise the cheaper covered
1014
+ // An `ordered` single-file run is one step of a sequence someone else is
1015
+ // driving (a queue job), so it must uphold what a bulk run would: it reads
1016
+ // the full applied set, and with it the strict drift check and the
1017
+ // out-of-order policy apply — otherwise splitting a run into jobs would
1018
+ // silently drop both.
1019
+ const ordered = filename !== undefined && options.ordered === true;
1020
+ const fullSet = !filename || ordered;
1021
+ // A strict full-set run needs the applied records' checksums anyway, so
1022
+ // fetch full records once and derive the name set from them; a plain
1023
+ // single-file run needs only that file's record, so one getByName is both
1024
+ // the applied-check and the checksum source; otherwise the cheaper covered
929
1025
  // name query suffices.
930
- const strictBulk = !filename && config.strict && !force;
1026
+ const strictBulk = fullSet && config.strict && !force;
931
1027
  const appliedRecords = strictBulk ? await changelog.getApplied(db) : undefined;
932
1028
  const appliedNames = new Set();
933
1029
  let singleRecord = null;
934
1030
  if (filename) {
935
1031
  singleRecord = await changelog.getByName(db, filename);
936
1032
  if (singleRecord?.status === 'applied') appliedNames.add(filename);
937
- } else if (appliedRecords) {
938
- for (const record of appliedRecords) appliedNames.add(record.name);
939
- } else {
940
- for (const name of await changelog.getAppliedNames(db)) appliedNames.add(name);
1033
+ }
1034
+ if (fullSet) {
1035
+ if (appliedRecords) {
1036
+ for (const record of appliedRecords) appliedNames.add(record.name);
1037
+ } else {
1038
+ for (const name of await changelog.getAppliedNames(db)) appliedNames.add(name);
1039
+ }
941
1040
  }
942
1041
 
943
1042
  const targets = await this.#selectUpTargets(filename, options, appliedNames);
944
1043
  // Pending files are the only bulk targets, so the per-target checksum
945
1044
  // check below can never see an applied one. Verify them up front instead,
946
1045
  // otherwise `up --strict` over a bulk run would police nothing.
947
- if (!filename && strictBulk) await this.#assertNoChecksumDrift(appliedRecords);
1046
+ if (strictBulk) await this.#assertNoChecksumDrift(appliedRecords);
1047
+ // An ordered run is a step of the sequence, so its target must be a file
1048
+ // of the sequence: an existing dotfile, declaration file or helper module
1049
+ // next to the migrations is never one — and importing it would run its
1050
+ // top-level code. The name may come from a queue payload; a plain
1051
+ // single-file `up` keeps accepting any file it is pointed at.
1052
+ const sequence = ordered ? await this.#listMigrationFiles() : undefined;
1053
+ if (ordered && !sequence.includes(filename)) {
1054
+ throw new MigrationFileNotFoundError(
1055
+ 'Not a migration of the sequence — a dotfile, a declaration file or an extension ' +
1056
+ 'outside fileExtensions',
1057
+ { filename },
1058
+ );
1059
+ }
1060
+ // An already-applied target is exempt (unless forced): a duplicate job
1061
+ // for it must report the usual "skipped", not a failure.
1062
+ if (ordered && (force || !appliedNames.has(filename))) {
1063
+ await this.#assertUpNotBlocked(filename, appliedNames, sequence);
1064
+ }
1065
+ // The file the caller planned with (a queue job's checksum) must be the
1066
+ // one on this disk: a worker from an older deploy would otherwise apply an
1067
+ // older version of an edited pending file, and record its checksum.
1068
+ if (options.checksum !== undefined && (force || !appliedNames.has(filename))) {
1069
+ const actual = await computeChecksum(this.#filepath(filename));
1070
+ if (actual !== options.checksum) {
1071
+ throw new ChecksumMismatchError(
1072
+ `${filename} is not the file this run was planned with — this process has another ` +
1073
+ 'version of it (roll workers out before the producers that enqueue)',
1074
+ { name: filename, expected: options.checksum, actual, planned: true },
1075
+ );
1076
+ }
1077
+ }
948
1078
  this.#assertOrderIntact(targets, appliedNames);
949
1079
 
950
1080
  if (targets.length === 0) {
@@ -956,8 +1086,10 @@ class MigratorKit extends EventEmitter {
956
1086
  // Without --step every file in this run shares one batch. With --step each
957
1087
  // applied file gets its own sequential batch (base, base+1, …) so a later
958
1088
  // `down` can revert them individually. Only successful applies advance the
959
- // counter, so --step never leaves gaps.
960
- const baseBatch = await this.#nextBatch();
1089
+ // counter, so --step never leaves gaps. An explicit `batch` is a label the
1090
+ // caller chose — it may equal one already in use, which is how several
1091
+ // single-file runs end up as one rollback unit.
1092
+ const baseBatch = options.batch ?? (await this.#nextBatch());
961
1093
  let appliedCount = 0;
962
1094
 
963
1095
  return this.#runSequence({
@@ -1031,9 +1163,14 @@ class MigratorKit extends EventEmitter {
1031
1163
  duration: elapsed,
1032
1164
  ...(this.#runId ? { runId: this.#runId } : {}),
1033
1165
  ...(migration.description ? { description: migration.description } : {}),
1166
+ ...actorFields(options),
1034
1167
  },
1035
1168
  session,
1036
1169
  ),
1170
+ // What a failed attempt's trace records besides the failure: the
1171
+ // version of the file that failed (a breaker compares it), and who
1172
+ // asked for the run.
1173
+ failureFields: { checksum, ...actorFields(options) },
1037
1174
  });
1038
1175
  appliedCount += 1;
1039
1176
  return 'done';
@@ -1094,10 +1231,7 @@ class MigratorKit extends EventEmitter {
1094
1231
 
1095
1232
  /** Rollback the last batch, a specific batch, a specific file, or the last N steps */
1096
1233
  async down(filename, options = {}) {
1097
- this.#assertFilename(filename);
1098
- this.#assertStepsValid(options.steps, filename, options.batch);
1099
- this.#assertBatchValid(options.batch);
1100
- this.#assertToValid(options.to, filename, options);
1234
+ assertDownOptions(filename, options);
1101
1235
  return this.#runWindow(async () => {
1102
1236
  await this.#ensureConfig();
1103
1237
  await this.connect();
@@ -1167,17 +1301,6 @@ class MigratorKit extends EventEmitter {
1167
1301
  return { records, preserveOrder };
1168
1302
  }
1169
1303
 
1170
- /** Order the selected records for execution (newest first unless pre-ordered) */
1171
- #downNames(records, preserveOrder) {
1172
- const names = [];
1173
- for (const record of records) names.push(record.name);
1174
- if (!preserveOrder) {
1175
- names.sort();
1176
- names.reverse();
1177
- }
1178
- return names;
1179
- }
1180
-
1181
1304
  async #runDown(filename, options = {}, signal) {
1182
1305
  const config = this.#config;
1183
1306
  const db = this.#requireDb();
@@ -1190,8 +1313,9 @@ class MigratorKit extends EventEmitter {
1190
1313
  logger.info('Nothing to rollback', this.#fields({ direction: 'down' }));
1191
1314
  return [];
1192
1315
  }
1316
+ if (filename && options.ordered === true) await this.#assertDownNotBlocked(toRevert[0]);
1193
1317
 
1194
- const names = this.#downNames(toRevert, preserveOrder);
1318
+ const names = revertOrder(toRevert, preserveOrder);
1195
1319
 
1196
1320
  // The signal must reach the rollback context too: a long-running down()
1197
1321
  // under SIGTERM or a lost lock is exactly the case ctx.signal exists for.
@@ -1211,7 +1335,7 @@ class MigratorKit extends EventEmitter {
1211
1335
  total: names.length,
1212
1336
  results,
1213
1337
  onSuccess: async (_migration, _elapsed, session) => {
1214
- const result = await changelog.markReverted(db, name, session);
1338
+ const result = await changelog.markReverted(db, name, session, pickActor(options));
1215
1339
  // Under --no-lock or onLockLost:'warn' a peer may have flipped the
1216
1340
  // record first: the down() body already ran against the data, but
1217
1341
  // the changelog still claims the migration is applied. Silence
@@ -1273,7 +1397,8 @@ class MigratorKit extends EventEmitter {
1273
1397
  * show for it.
1274
1398
  */
1275
1399
  async redo(filename, options = {}) {
1276
- this.#assertFilename(filename);
1400
+ assertRedoOptions(filename, options);
1401
+ const actor = pickActor(options);
1277
1402
  return this.#runWindow(async () => {
1278
1403
  await this.#ensureConfig();
1279
1404
  await this.connect();
@@ -1295,10 +1420,10 @@ class MigratorKit extends EventEmitter {
1295
1420
  target = newest.name;
1296
1421
  }
1297
1422
 
1298
- const downResults = await this.#runDown(target, {}, signal);
1423
+ const downResults = await this.#runDown(target, actor, signal);
1299
1424
  let upResults;
1300
1425
  try {
1301
- upResults = await this.#runUp(target, {}, signal);
1426
+ upResults = await this.#runUp(target, actor, signal);
1302
1427
  } catch (error) {
1303
1428
  // The revert already happened — after a failed re-apply that is the
1304
1429
  // single most important fact, so the down rows must survive into the
@@ -1316,12 +1441,7 @@ class MigratorKit extends EventEmitter {
1316
1441
 
1317
1442
  /** Preview what would run — never writes to the database */
1318
1443
  async dryRun(direction, filename, options = {}) {
1319
- this.#assertFilename(filename);
1320
- // `batch`/`to` must be passed too, or a conflict that `down` rejects would
1321
- // be silently allowed in its own preview.
1322
- this.#assertStepsValid(options.steps, filename, options.batch);
1323
- this.#assertBatchValid(options.batch);
1324
- this.#assertToValid(options.to, filename, options);
1444
+ assertDryRunOptions(filename, options);
1325
1445
  await this.#ensureConfig();
1326
1446
  await this.connect();
1327
1447
  const db = this.#requireDb();
@@ -1349,6 +1469,10 @@ class MigratorKit extends EventEmitter {
1349
1469
  // The same selection (and preflight) the real `up` executes, so a
1350
1470
  // preview never invents a pending row for a file that does not exist.
1351
1471
  names = await this.#selectUpTargets(filename, options, applied);
1472
+ // …and the same order policy: under onOutOfOrder: 'error' the real run
1473
+ // refuses, so the preview must too instead of listing rows it would
1474
+ // never apply. A single file is exempt, exactly as in `up`.
1475
+ if (!filename) this.#assertOrderIntact(names, applied);
1352
1476
  } else {
1353
1477
  // The same selection the real `down` executes — including the
1354
1478
  // irreversible-import refusal, so a preview can never show a rollback
@@ -1357,21 +1481,41 @@ class MigratorKit extends EventEmitter {
1357
1481
  for (const record of records) {
1358
1482
  recordByName.set(record.name, record);
1359
1483
  }
1360
- names = this.#downNames(records, preserveOrder);
1484
+ names = revertOrder(records, preserveOrder);
1361
1485
  }
1362
1486
 
1363
- const rows = await mapLimit(names, FS_CONCURRENCY, (name) =>
1364
- this.#buildStatusRow(name, recordByName.get(name)),
1365
- );
1487
+ const rows = await mapLimit(names, FS_CONCURRENCY, async (name) => {
1488
+ const row = await this.#buildStatusRow(name, recordByName.get(name));
1489
+ // What an `up` would apply, by content: a caller that applies the rows
1490
+ // later (a queue job) can insist on exactly this version of the file.
1491
+ if (direction === 'up' && !row.invalid) {
1492
+ row.checksum = await this.#cachedChecksum(this.#filepath(name));
1493
+ }
1494
+ return row;
1495
+ });
1366
1496
  logger.info(
1367
1497
  `◎ Dry-run Would ${direction === 'up' ? 'apply' : 'revert'}: ${rows.length}`,
1368
1498
  this.#fields({ direction, count: rows.length, dryRun: true }),
1369
1499
  );
1500
+ // A converge plan is only meaningful against the database the migrations
1501
+ // leave behind — previewing it now would compare with the wrong state.
1502
+ if (
1503
+ direction === 'up' &&
1504
+ !filename &&
1505
+ options.to === undefined &&
1506
+ (await this.convergesAfterUp())
1507
+ ) {
1508
+ logger.info(
1509
+ '◎ Dry-run Converge after up is not previewed — it is planned against the database ' +
1510
+ 'the migrations leave behind (`converge --dry-run` once they are applied)',
1511
+ this.#fields({ direction, dryRun: true }),
1512
+ );
1513
+ }
1370
1514
  return rows;
1371
1515
  }
1372
1516
 
1373
1517
  /** Full migration status for all known files and records */
1374
- async status() {
1518
+ async status(options = {}) {
1375
1519
  await this.#ensureConfig();
1376
1520
  await this.connect();
1377
1521
  const records = await this.#requireChangelog().getAll(this.#requireDb());
@@ -1384,15 +1528,16 @@ class MigratorKit extends EventEmitter {
1384
1528
  // Each row may read and hash a file; unbounded fan-out over thousands of
1385
1529
  // migrations exhausts the descriptor limit.
1386
1530
  const rows = await mapLimit(sortedNames, FS_CONCURRENCY, (name) =>
1387
- this.#buildStatusRow(name, recordByName.get(name)),
1531
+ this.#buildStatusRow(name, recordByName.get(name), options),
1388
1532
  );
1389
1533
  // Mark late arrivals: a not-yet-applied row sorting before the newest
1390
1534
  // applied name will run after migrations authored later — the same signal
1391
1535
  // #assertOrderIntact acts on, surfaced here as data.
1392
- let newestApplied = '';
1536
+ const applied = [];
1393
1537
  for (const row of rows) {
1394
- if (row.status === 'applied' && row.file > newestApplied) newestApplied = row.file;
1538
+ if (row.status === 'applied') applied.push(row.file);
1395
1539
  }
1540
+ const newestApplied = newestOf(applied);
1396
1541
  if (newestApplied !== '') {
1397
1542
  for (const row of rows) {
1398
1543
  if (row.status !== 'applied' && row.file < newestApplied) row.outOfOrder = true;
@@ -1412,20 +1557,21 @@ class MigratorKit extends EventEmitter {
1412
1557
  getDb: () => this.#requireDb(),
1413
1558
  inspectLock: () => this.#buildLock().inspect(),
1414
1559
  status: () => this.status(),
1560
+ definitions: () => this.#resolveCollections(),
1415
1561
  });
1416
1562
  }
1417
1563
 
1418
- /** Filtered list of migrations */
1419
- async list(filter = 'all') {
1420
- // An unknown filter silently returning [] reads as "nothing to report" —
1421
- // the worst possible answer to a typo.
1422
- if (filter !== 'all' && filter !== 'pending' && filter !== 'applied') {
1423
- throw new ConfigInvalidError("list filter must be 'all', 'pending' or 'applied'", { filter });
1424
- }
1564
+ /**
1565
+ * Filtered list of migrations. `checksums: false` skips hashing the applied
1566
+ * files (`checksumOk` stays null) — for a caller that only needs names and
1567
+ * dates, where a full re-hash of the history would be the whole cost.
1568
+ */
1569
+ async list(filter = 'all', options = {}) {
1570
+ assertListOptions(filter, options);
1425
1571
  if (filter === 'pending') {
1426
1572
  return this.#listPending();
1427
1573
  }
1428
- const rows = await this.status();
1574
+ const rows = await this.status(options.checksums === false ? { checksums: false } : {});
1429
1575
  if (filter === 'all') {
1430
1576
  return rows;
1431
1577
  }
@@ -1450,8 +1596,7 @@ class MigratorKit extends EventEmitter {
1450
1596
  await this.connect();
1451
1597
  const applied = new Set(await this.#requireChangelog().getAppliedNames(this.#requireDb()));
1452
1598
  const rows = [];
1453
- for (const file of await this.#listMigrationFiles()) {
1454
- if (applied.has(file)) continue;
1599
+ for (const file of pendingIn(await this.#listMigrationFiles(), applied)) {
1455
1600
  rows.push({
1456
1601
  file,
1457
1602
  status: 'pending',
@@ -1498,6 +1643,12 @@ class MigratorKit extends EventEmitter {
1498
1643
  ...(record.origin ? { origin: record.origin } : {}),
1499
1644
  ...(record.status === 'failed' && record.error ? { error: record.error } : {}),
1500
1645
  ...(record.status === 'failed' && record.failedAt ? { failedAt: record.failedAt } : {}),
1646
+ // The version of the file that failed — what tells "failed and unchanged since" apart.
1647
+ ...(record.status === 'failed' && record.checksum ? { failedChecksum: record.checksum } : {}),
1648
+ ...(record.requestedBy ? { requestedBy: record.requestedBy } : {}),
1649
+ ...(record.reason ? { reason: record.reason } : {}),
1650
+ ...(record.revertRequestedBy ? { revertRequestedBy: record.revertRequestedBy } : {}),
1651
+ ...(record.revertReason ? { revertReason: record.revertReason } : {}),
1501
1652
  };
1502
1653
  }
1503
1654
 
@@ -1514,7 +1665,7 @@ class MigratorKit extends EventEmitter {
1514
1665
  }
1515
1666
 
1516
1667
  /** Build a StatusRow for a migration, verifying checksum when possible */
1517
- async #buildStatusRow(name, record) {
1668
+ async #buildStatusRow(name, record, { checksums = true } = {}) {
1518
1669
  const isApplied = record?.status === 'applied';
1519
1670
  const status = MigratorKit.#rowStatus(record);
1520
1671
  let filepath;
@@ -1539,7 +1690,7 @@ class MigratorKit extends EventEmitter {
1539
1690
  // access()-first probe cost an extra syscall per row, for pending rows
1540
1691
  // whose result was never even used.
1541
1692
  let checksumOk = null;
1542
- if (isApplied && record) {
1693
+ if (isApplied && record && checksums) {
1543
1694
  try {
1544
1695
  checksumOk = (await this.#cachedChecksum(filepath)) === record.checksum;
1545
1696
  } catch (error) {
@@ -1626,7 +1777,7 @@ class MigratorKit extends EventEmitter {
1626
1777
  * baseline racing an `up`) must serialize like any other mutation.
1627
1778
  */
1628
1779
  async baseline(options = {}) {
1629
- this.#assertFilename(options.to);
1780
+ assertFilename(options.to);
1630
1781
  return this.#runWindow(async () => {
1631
1782
  await this.#ensureConfig();
1632
1783
  await this.connect();
@@ -1640,7 +1791,7 @@ class MigratorKit extends EventEmitter {
1640
1791
  filepath: (name) => this.#filepath(name),
1641
1792
  listMigrationFiles: () => this.#listMigrationFiles(),
1642
1793
  nextBatch: () => this.#nextBatch(),
1643
- truncateAtTarget: (pending, all, to) => this.#truncateAtTarget(pending, all, to),
1794
+ truncateAtTarget,
1644
1795
  environment: () => this.#environment(),
1645
1796
  executedBy: () => safeUsername(),
1646
1797
  runId: () => this.#runId,
@@ -1661,14 +1812,7 @@ class MigratorKit extends EventEmitter {
1661
1812
  * file signatures, so `down`/`redo` on imported files is unsupported.
1662
1813
  */
1663
1814
  async import(options = {}) {
1664
- // Validate before connecting: a bad --from/--to must not cost a round trip
1665
- // or take the lock. Defaults come from the already-validated config.
1666
- if (options.from !== undefined && !isCollectionName(options.from)) {
1667
- throw new ConfigInvalidError('Invalid source collection name', { from: options.from });
1668
- }
1669
- if (options.to !== undefined && !isCollectionName(options.to)) {
1670
- throw new ConfigInvalidError('Invalid target collection name', { to: options.to });
1671
- }
1815
+ assertImportOptions(options);
1672
1816
  return this.#runWindow(async () => {
1673
1817
  await this.#ensureConfig();
1674
1818
  await this.connect();
@@ -1691,6 +1835,175 @@ class MigratorKit extends EventEmitter {
1691
1835
  );
1692
1836
  });
1693
1837
  }
1838
+
1839
+ /**
1840
+ * Bring the declared collections (`collections`, `collectionsDir`) to their
1841
+ * declared indexes and validators — see {@link runConverge} in converge.js
1842
+ * for the mechanics. Stateless: the live database is read and compared on
1843
+ * every call. What a run changed is appended to the converge history
1844
+ * (`convergeHistory()`), which no run reads back.
1845
+ *
1846
+ * `dryRun` plans without writing — no lock, no events. A real run holds the
1847
+ * migration lock, like every other mutation. `prune` drops undeclared
1848
+ * indexes in collections whose definition does not decide for itself.
1849
+ * `ordered` refuses while any migration is still pending — checked under
1850
+ * the lock, which is what lets a queue run it as the tail of a deploy.
1851
+ * `rebuildUnique` lets a rebuild drop a unique index it builds back; without
1852
+ * it such a rebuild is a conflict (the constraint would be gone until the
1853
+ * build ends), which is why the after-up hook and a queue job never pass it.
1854
+ */
1855
+ async converge(options = {}) {
1856
+ assertConvergeOptions(options);
1857
+ const actor = pickActor(options);
1858
+ const empty = (dryRun) => ({ dryRun, changed: 0, inSync: true, collections: [] });
1859
+ if (options.dryRun) {
1860
+ await this.#ensureConfig();
1861
+ const definitions = await this.#resolveCollections();
1862
+ if (definitions.length === 0) return empty(true);
1863
+ await this.connect();
1864
+ return runConverge(
1865
+ this.#convergeDeps(),
1866
+ {
1867
+ definitions,
1868
+ prune: options.prune,
1869
+ rebuildUnique: options.rebuildUnique,
1870
+ dryRun: true,
1871
+ search: this.#convergeSearchOptions(),
1872
+ },
1873
+ undefined,
1874
+ );
1875
+ }
1876
+ return this.#runWindow(async () => {
1877
+ await this.#ensureConfig();
1878
+ // Before connecting or locking: a broken definition file must not cost
1879
+ // a round trip, and must not hold the lock while it is reported.
1880
+ const definitions = await this.#resolveCollections();
1881
+ if (definitions.length === 0) {
1882
+ this.#logger.info(
1883
+ 'No collections declared — set collections or collectionsDir',
1884
+ this.#fields({ command: 'converge' }),
1885
+ );
1886
+ return empty(false);
1887
+ }
1888
+ await this.connect();
1889
+ return this.#withLock(options, { command: 'converge' }, async (signal, lock) => {
1890
+ if (options.ordered) await this.#assertNothingPending();
1891
+ return runConverge(
1892
+ this.#convergeDeps(lock),
1893
+ {
1894
+ definitions,
1895
+ prune: options.prune,
1896
+ rebuildUnique: options.rebuildUnique,
1897
+ search: this.#convergeSearchOptions(options),
1898
+ ...actor,
1899
+ },
1900
+ signal,
1901
+ );
1902
+ });
1903
+ });
1904
+ }
1905
+
1906
+ /**
1907
+ * Whether a bulk `up` on this kit ends by converging: `convergeAfterUp` is
1908
+ * on and there is something declared to converge. Resolves the config; does
1909
+ * not connect, and does not load definition files. A layer above the kit
1910
+ * (the queue adapter) uses it to mirror that behaviour across single-file
1911
+ * jobs, where the kit's own after-up hook never fires.
1912
+ */
1913
+ async convergesAfterUp() {
1914
+ const config = await this.#ensureConfig();
1915
+ return (
1916
+ config.convergeAfterUp === true &&
1917
+ ((config.collections?.length ?? 0) > 0 || config.collectionsDir !== undefined)
1918
+ );
1919
+ }
1920
+
1921
+ /** Every declared collection, normalized — the config key first, then `collectionsDir` */
1922
+ async #resolveCollections() {
1923
+ const config = this.#config;
1924
+ return resolveDefinitions({
1925
+ inline: config.collections,
1926
+ ...(config.collectionsDir !== undefined
1927
+ ? { dir: path.resolve(this.#cwd ?? process.cwd(), config.collectionsDir) }
1928
+ : {}),
1929
+ extensions: config.fileExtensions,
1930
+ reload: config.reloadMigrations,
1931
+ reserved: [config.migrationsCollection, config.lockCollection, config.convergeLogCollection],
1932
+ });
1933
+ }
1934
+
1935
+ /** How converge treats search indexes: the config, and a call's own `waitForSearchIndexes` */
1936
+ #convergeSearchOptions(options = {}) {
1937
+ const config = this.#config;
1938
+ return {
1939
+ onUnavailable: config.onSearchUnavailable,
1940
+ wait: options.waitForSearchIndexes ?? config.waitForSearchIndexes,
1941
+ waitTimeoutMs: config.searchIndexWaitTimeoutMs,
1942
+ };
1943
+ }
1944
+
1945
+ /** What runConverge works with; `lock` (a run's) lets it give the lock up before waiting */
1946
+ #convergeDeps(lock) {
1947
+ const db = this.#requireDb();
1948
+ return {
1949
+ db,
1950
+ ...(lock ? { releaseLock: () => lock.release() } : {}),
1951
+ recordSearchWait: (waitedMs, outcome) => this.#telemetry.searchWaited({ waitedMs, outcome }),
1952
+ logger: this.#logger,
1953
+ fields: (extra) => this.#fields(extra),
1954
+ emit: (event, payload) => this.#emit(event, payload),
1955
+ assertNotAborted: (abortSignal) => this.#assertNotAborted(abortSignal),
1956
+ // The history entry's who-and-where, like a changelog record's.
1957
+ audit: () => ({
1958
+ ...(this.#runId ? { runId: this.#runId } : {}),
1959
+ executedBy: safeUsername(),
1960
+ host: os.hostname(),
1961
+ environment: this.#environment(),
1962
+ }),
1963
+ record: (entry) => this.#convergeLog().append(db, entry),
1964
+ // Behind a mongos only: the shard key, so prune never tries to drop its index.
1965
+ shardKeyOf: async (name) =>
1966
+ (
1967
+ await this.#client
1968
+ .db('config')
1969
+ .collection('collections')
1970
+ .findOne({ _id: `${db.databaseName}.${name}` }, { projection: { key: 1 } })
1971
+ )?.key,
1972
+ };
1973
+ }
1974
+
1975
+ #convergeLog() {
1976
+ this.#convergeLogStore ??= new ConvergeLog(this.#config.convergeLogCollection);
1977
+ return this.#convergeLogStore;
1978
+ }
1979
+
1980
+ /**
1981
+ * The converge history, newest first: one entry per converge that changed
1982
+ * something or failed — when, triggered how, by whom and why, and every
1983
+ * index or validator it touched, with its before and after. Read-only.
1984
+ */
1985
+ async convergeHistory(options = {}) {
1986
+ const { limit = 20 } = options;
1987
+ assertHistoryLimit(limit);
1988
+ await this.#ensureConfig();
1989
+ await this.connect();
1990
+ return this.#convergeLog().list(this.#requireDb(), limit);
1991
+ }
1992
+
1993
+ /**
1994
+ * `converge({ ordered })`: refuse while a migration on disk has no applied
1995
+ * record — a failed one included, exactly as for an ordered `up` job.
1996
+ */
1997
+ async #assertNothingPending() {
1998
+ const applied = new Set(await this.#requireChangelog().getAppliedNames(this.#requireDb()));
1999
+ const blockedBy = pendingIn(await this.#listMigrationFiles(), applied);
2000
+ if (blockedBy.length === 0) return;
2001
+ throw blockedError('converge', 'migration(s) still pending', {
2002
+ command: 'converge',
2003
+ blockedBy,
2004
+ failed: await this.#failedAmong(blockedBy),
2005
+ });
2006
+ }
1694
2007
  }
1695
2008
 
1696
- module.exports = { MigratorKit };
2009
+ module.exports = { MigratorKit, RECORD_LOCK_WAIT };