@alexify/migronaut 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +41 -3
  3. package/bullmq.d.ts +484 -8
  4. package/index.d.ts +1264 -9
  5. package/migronaut.schema.json +93 -1
  6. package/package.json +9 -2
  7. package/src/bullmq/background-processor.js +541 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +348 -21
  11. package/src/bullmq/producer.js +185 -13
  12. package/src/bullmq/service.js +484 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/create.js +6 -0
  15. package/src/cli/exit-codes.js +6 -0
  16. package/src/cli/index.js +2 -0
  17. package/src/core/audit.js +11 -1
  18. package/src/core/background-audit.js +139 -0
  19. package/src/core/background-drift.js +126 -0
  20. package/src/core/background-dry-run.js +375 -0
  21. package/src/core/background-engine.js +849 -0
  22. package/src/core/background-kit.js +432 -0
  23. package/src/core/background-partition.js +298 -0
  24. package/src/core/background-runner.js +305 -0
  25. package/src/core/background-sandbox.js +701 -0
  26. package/src/core/background-shard.js +542 -0
  27. package/src/core/background-spec.js +597 -0
  28. package/src/core/background-store.js +951 -0
  29. package/src/core/background-throttle.js +269 -0
  30. package/src/core/background-watch-plan.js +164 -0
  31. package/src/core/background-watch-store.js +78 -0
  32. package/src/core/background-watch.js +610 -0
  33. package/src/core/background.js +1127 -0
  34. package/src/core/bson-peer.js +23 -0
  35. package/src/core/changelog.js +32 -0
  36. package/src/core/collections.js +78 -8
  37. package/src/core/config.js +102 -12
  38. package/src/core/converge-plan.js +86 -7
  39. package/src/core/converge.js +88 -0
  40. package/src/core/lock.js +48 -21
  41. package/src/core/migration-logger.js +279 -0
  42. package/src/core/migrator.js +1027 -22
  43. package/src/core/options.js +36 -0
  44. package/src/core/run-recorder.js +6 -1
  45. package/src/core/run.js +26 -12
  46. package/src/core/runner.js +34 -8
  47. package/src/core/server-info.js +9 -2
  48. package/src/core/shard-info.js +76 -0
  49. package/src/core/versioning-spec.js +181 -0
  50. package/src/errors/index.js +88 -0
  51. package/src/index.js +16 -0
  52. package/src/utils/error.js +11 -2
  53. package/src/utils/job-ref.js +44 -0
  54. package/src/utils/loader.js +77 -9
  55. package/src/utils/migration-name.js +33 -1
  56. package/src/utils/redact.js +140 -3
  57. package/src/utils/telemetry.js +110 -0
  58. package/src/utils/template.js +62 -1
  59. package/src/versioning/config.js +155 -0
  60. package/src/versioning/document.js +326 -0
  61. package/src/versioning/index.js +50 -0
  62. package/src/versioning/internal.js +279 -0
  63. package/src/versioning/mongoose.js +151 -0
  64. package/src/versioning/occ.js +318 -0
  65. package/src/versioning/registry.js +187 -0
  66. package/src/versioning/upcaster.js +213 -0
  67. package/versioning.d.ts +666 -0
  68. package/versioning.js +1 -0
@@ -6,6 +6,7 @@ const path = require('node:path');
6
6
  // and pulls in ~150 modules, which `--help`, `--version`, `init` and `create`
7
7
  // have no use for.
8
8
  const {
9
+ BackgroundPendingError,
9
10
  ChecksumMismatchError,
10
11
  ConfigInvalidError,
11
12
  ConnectionFailedError,
@@ -13,6 +14,7 @@ const {
13
14
  IrreversibleMigrationError,
14
15
  LockAlreadyHeldError,
15
16
  MigrationFileNotFoundError,
17
+ MigrationInvalidExportError,
16
18
  MigrationInvalidNameError,
17
19
  MigronautError,
18
20
  NotAppliedError,
@@ -24,6 +26,7 @@ const { computeChecksum } = require('../utils/checksum.js');
24
26
  const { mapLimit } = require('../utils/concurrency.js');
25
27
  const { errorText, errorWithCause } = require('../utils/error.js');
26
28
  const { createIdGenerator } = require('../utils/id.js');
29
+ const { jobFields } = require('../utils/job-ref.js');
27
30
  const { loadMigrationFile } = require('../utils/loader.js');
28
31
  const { resolveLogger } = require('../utils/logger.js');
29
32
  const { assertMigrationName } = require('../utils/migration-name.js');
@@ -35,22 +38,56 @@ const {
35
38
  } = require('../utils/template.js');
36
39
  const { safeUsername } = require('../utils/user.js');
37
40
  const { runAudit } = require('./audit.js');
41
+ const {
42
+ control: controlBackground,
43
+ coordinate,
44
+ repin: repinBackgroundState,
45
+ runSlice,
46
+ tryUnblock,
47
+ waitForLanes,
48
+ waitingFor,
49
+ } = require('./background.js');
50
+ const { auditFindings } = require('./background-audit.js');
51
+ const {
52
+ STREAMING_FRESH_MS,
53
+ drive,
54
+ hasRewritten,
55
+ irreversibleBackground,
56
+ partitionView,
57
+ runnable,
58
+ stateView,
59
+ unsatisfied,
60
+ watchRow,
61
+ withdraw,
62
+ } = require('./background-kit.js');
63
+ const { verify: verifyDrift } = require('./background-drift.js');
64
+ const { assertSliceMs, resolveBackgroundSpec } = require('./background-spec.js');
65
+ const { previewSample, previewSteps } = require('./background-dry-run.js');
66
+ const { BackgroundStore } = require('./background-store.js');
67
+ const { startWatch, watchOptions } = require('./background-watch.js');
68
+ const { BackgroundWatchStore } = require('./background-watch-store.js');
38
69
  const { runBaseline } = require('./baseline.js');
39
70
  const { Changelog } = require('./changelog.js');
40
71
  const { ConvergeLog } = require('./converge-log.js');
41
72
  const { resolveDefinitions } = require('./collections.js');
42
- const { loadConfig } = require('./config.js');
73
+ const { backgroundCollectionNames, loadConfig } = require('./config.js');
43
74
  const { buildContext } = require('./context.js');
44
75
  const { runConverge } = require('./converge.js');
76
+ const { readServer } = require('./server-info.js');
77
+ const { readChunks, readShardKey } = require('./shard-info.js');
78
+
45
79
  const { runImport } = require('./import-runner.js');
46
80
  const { MigrationLock, runWithLock, toLockInfo } = require('./lock.js');
81
+ const { MIGRATION_LOG_EVENT, backgroundLogs, createRunLog } = require('./migration-logger.js');
47
82
  const {
83
+ assertActorValid,
48
84
  assertConvergeOptions,
49
85
  assertDownOptions,
50
86
  assertDryRunOptions,
51
87
  assertFilename,
52
88
  assertHistoryLimit,
53
89
  assertImportOptions,
90
+ assertJobValid,
54
91
  assertListOptions,
55
92
  assertRedoOptions,
56
93
  assertUpOptions,
@@ -103,6 +140,14 @@ class MigratorKit extends EventEmitter {
103
140
  #runSetupDepth = 0;
104
141
  /** Correlation id for the run in flight — ties logs, lock and changelog together */
105
142
  #runId;
143
+ /**
144
+ * The log side of the run in flight (migration-logger's `createRunLog`): its
145
+ * job and actor, `ctx.run`, `ctx.logger`, the `migration:log` counter. Set
146
+ * and cleared with #runId; a logger keeps what it was made with.
147
+ */
148
+ #runLog;
149
+ /** The `migration:log` channel handed to migration loggers — made once */
150
+ #logEmitterInstance;
106
151
  /** Mints an id in the configured format (`generateId`, else a UUID); set with the config */
107
152
  #newId;
108
153
  /** Spans and metrics through the injected `telemetry` (a no-op without one); set with the config */
@@ -140,9 +185,28 @@ class MigratorKit extends EventEmitter {
140
185
  * poll that the module cache never frees. Cleared by any other outcome.
141
186
  */
142
187
  #upDefinitions;
188
+ /** The background migrations' store — made on first use (see #backgroundStore) */
189
+ #backgroundStoreInstance;
190
+ #watchStoreInstance;
191
+ /** Warnings about background migrations said once per kit (a missing index, no lag rights) */
192
+ #backgroundWarned = new Set();
193
+ /** The adaptive throttles of this process, per background migration and group */
194
+ #adaptiveCache = new Map();
195
+ /** Index keys per collection, read at most every few seconds — every slice asks */
196
+ #indexCache = new Map();
197
+ /** Declared collections, for background specs — resolved once unless migrations reload */
198
+ #backgroundDefinitions;
199
+ /** The server's topology, read once — transactional background migrations need it */
200
+ #topology;
201
+ /** Migration modules read ahead of a run (requires, kind), by checksum */
202
+ #moduleCache = new Map();
143
203
 
144
204
  constructor(config = {}, options = {}) {
145
- super();
205
+ // An async listener whose promise rejects (a subscriber's insert that
206
+ // failed) would otherwise surface as an unhandledRejection, which ends the
207
+ // process — captureRejections routes it to the method below instead, so it
208
+ // is contained the way #emit contains a listener that throws.
209
+ super({ captureRejections: true });
146
210
  this.#partialConfig = config;
147
211
  this.#configPath = options.configPath;
148
212
  this.#progress = options.progress;
@@ -159,8 +223,13 @@ class MigratorKit extends EventEmitter {
159
223
  * failure into a second, unrelated one.
160
224
  */
161
225
  #emit(event, payload) {
226
+ this.#deliver(event, { ...(this.#runId ? { runId: this.#runId } : {}), ...payload });
227
+ }
228
+
229
+ /** {@link #emit} without the run id stamp: for a payload that carries its own */
230
+ #deliver(event, payload) {
162
231
  try {
163
- this.emit(event, { ...(this.#runId ? { runId: this.#runId } : {}), ...payload });
232
+ this.emit(event, payload);
164
233
  } catch (error) {
165
234
  // A listener's failure is its own problem — but an invisible one is
166
235
  // undebuggable, so leave a trace at debug level.
@@ -171,6 +240,23 @@ class MigratorKit extends EventEmitter {
171
240
  }
172
241
  }
173
242
 
243
+ /**
244
+ * Where a listener's rejected promise lands (see the constructor): left at
245
+ * debug level like a listener that throws, never re-emitted as `error` —
246
+ * with no `error` listener that would throw, from a tick no one awaits.
247
+ */
248
+ [EventEmitter.captureRejectionSymbol](error, event) {
249
+ try {
250
+ const name = String(event);
251
+ this.#logger.debug(
252
+ `Event listener for '${name}' rejected: ${errorText(error)}`,
253
+ this.#fields({ event: name, error: errorText(error) }),
254
+ );
255
+ } catch {
256
+ // Reporting a listener's failure must not become a failure of its own.
257
+ }
258
+ }
259
+
174
260
  /**
175
261
  * Stop the run in progress: the migration currently executing is allowed to
176
262
  * finish (interrupting it mid-write is what leaves a database half-migrated),
@@ -275,10 +361,11 @@ class MigratorKit extends EventEmitter {
275
361
  * Structured fields for a log line. Passed as the logger's second argument
276
362
  * (first, for pino-style loggers) so a machine-readable sink gets
277
363
  * `{migration, direction, durationMs, …}` instead of having to parse the
278
- * emoji-prefixed human string.
364
+ * emoji-prefixed human string. A run's lines carry its id and, when the
365
+ * caller named one, its queue job (`jobId`, `groupId`).
279
366
  */
280
367
  #fields(extra) {
281
- return this.#runId ? { runId: this.#runId, ...extra } : { ...extra };
368
+ return this.#runId ? { runId: this.#runId, ...this.#runLog?.fields, ...extra } : { ...extra };
282
369
  }
283
370
 
284
371
  /** Record a finished wait for the lock — see {@link RECORD_LOCK_WAIT} */
@@ -369,6 +456,12 @@ class MigratorKit extends EventEmitter {
369
456
  this.#client = undefined;
370
457
  this.#db = undefined;
371
458
  this.#changelog = undefined;
459
+ // The background stores bind the Db they were made with, and the topology
460
+ // and definitions were read through it: a later connect() reads them anew.
461
+ this.#backgroundStoreInstance = undefined;
462
+ this.#watchStoreInstance = undefined;
463
+ this.#topology = undefined;
464
+ this.#backgroundDefinitions = undefined;
372
465
  this.#ownsClient = true;
373
466
  if (owned) await client.close();
374
467
  }
@@ -430,6 +523,13 @@ class MigratorKit extends EventEmitter {
430
523
  // before any other run state exists: a `generateId` that throws or returns
431
524
  // a non-id rejects here, leaving nothing to unwind and no event emitted.
432
525
  this.#runId = this.#newId();
526
+ this.#runLog = createRunLog({
527
+ id: this.#runId,
528
+ job: options.job,
529
+ actor: pickActor(options),
530
+ sink: this.#logger,
531
+ emitter: this.#logEmitter(),
532
+ });
433
533
  // A second controller layered over the lock's own signal, so stop() and a
434
534
  // lost lock abort through the same path the run loops already watch.
435
535
  const stopper = new AbortController();
@@ -447,6 +547,7 @@ class MigratorKit extends EventEmitter {
447
547
  const recorder = new RunRecorder({
448
548
  info,
449
549
  runId: this.#runId,
550
+ job: this.#runLog.fields,
450
551
  telemetry: this.#telemetry,
451
552
  emit: (event, payload) => this.#emit(event, payload),
452
553
  logger: this.#logger,
@@ -489,9 +590,23 @@ class MigratorKit extends EventEmitter {
489
590
  recorder.finish(result, failure);
490
591
  this.#abort = undefined;
491
592
  this.#runId = undefined;
593
+ this.#runLog = undefined;
492
594
  }
493
595
  }
494
596
 
597
+ /**
598
+ * The kit's side of `migration:log` for migration loggers: `wanted` spares
599
+ * them the copy of a payload no one listens to. A payload names its own run
600
+ * (a late call's is not the run in flight), so it is delivered as it is.
601
+ */
602
+ #logEmitter() {
603
+ this.#logEmitterInstance ??= {
604
+ wanted: () => this.listenerCount(MIGRATION_LOG_EVENT) > 0,
605
+ emit: (payload) => this.#deliver(MIGRATION_LOG_EVENT, payload),
606
+ };
607
+ return this.#logEmitterInstance;
608
+ }
609
+
495
610
  /**
496
611
  * Attach partial results to an error's context. Copy-on-write: the error may
497
612
  * live on a shared abort signal (see #assertNotAborted) or already carry
@@ -726,14 +841,20 @@ class MigratorKit extends EventEmitter {
726
841
  const results = [];
727
842
  let doneCount = 0;
728
843
  let failure;
844
+ // The hooks around the whole run see the run's correlation, not one
845
+ // migration's: no migration name, no attempt.
846
+ const run = this.#runLog.info(direction);
847
+ const runContext = { ...context, run, logger: this.#runLog.logger(run) };
729
848
  try {
730
- await this.#runHook(config.hooks?.beforeAll, 'beforeAll', [context]);
849
+ await this.#runHook(config.hooks?.beforeAll, 'beforeAll', [runContext]);
731
850
  for (const [index, name] of names.entries()) {
732
851
  // Between migrations is the only safe place to stop: the one in flight
733
852
  // has committed, and the next has not started.
734
853
  this.#assertNotAborted(signal, results);
735
854
  const outcome = await execute(name, index, results);
736
855
  if (outcome === 'done') doneCount += 1;
856
+ // A clean stop: what follows waits for a background migration.
857
+ if (outcome === 'stop') break;
737
858
  }
738
859
  } catch (error) {
739
860
  failure = error;
@@ -751,7 +872,7 @@ class MigratorKit extends EventEmitter {
751
872
  // the diagnosis the caller needs, not the notification hook's own trouble.
752
873
  try {
753
874
  await this.#runHook(config.hooks?.afterAll, 'afterAll', [
754
- context,
875
+ runContext,
755
876
  { success: succeeded, applied: doneCount, direction },
756
877
  ]);
757
878
  } catch (hookError) {
@@ -819,15 +940,26 @@ class MigratorKit extends EventEmitter {
819
940
  const config = this.#config;
820
941
  const logger = this.#logger;
821
942
  const batchField = batch !== undefined ? { batch } : {};
943
+ // What each attempt adds to the context — ctx.run and ctx.logger — made
944
+ // anew per attempt, so a retried transaction's body knows it is one.
945
+ const runLog = this.#runLog;
946
+ const attemptContext = (attempt) =>
947
+ runLog.attempt(direction, { migration: name, batch, attempt });
822
948
 
823
949
  await this.#runHook(config.hooks?.beforeEach, 'beforeEach', [
824
950
  name,
825
- context,
951
+ { ...context, ...attemptContext(1) },
826
952
  { direction, index, total },
827
953
  ]);
828
- const migration = await loadMigrationFile(this.#filepath(name), {
954
+ const loaded = await loadMigrationFile(this.#filepath(name), {
829
955
  reload: config.reloadMigrations,
830
956
  });
957
+ let migration = loaded;
958
+ if (loaded.kind === 'background') {
959
+ // Indexes cannot be created inside a transaction — the registration may run in one.
960
+ await this.#backgroundIndexes();
961
+ migration = await this.#asBackground(name, loaded, direction);
962
+ }
831
963
  const useTransaction = migration.useTransaction ?? config.useTransaction;
832
964
  span.set({ [ATTRIBUTES.MIGRATION_TRANSACTION]: useTransaction });
833
965
 
@@ -836,29 +968,50 @@ class MigratorKit extends EventEmitter {
836
968
  try {
837
969
  // The changelog write happens inside runMigration so that, under
838
970
  // useTransaction, it commits atomically with the migration itself.
839
- const { duration } = await runMigration({
971
+ const { duration, attempts } = await runMigration({
840
972
  name,
841
973
  migration,
842
974
  direction,
843
975
  context,
844
976
  useTransaction,
845
977
  logger,
978
+ attemptContext,
846
979
  ...(config.timeoutMs ? { timeoutMs: config.timeoutMs } : {}),
847
980
  onSuccess: (elapsed, session) => onSuccess(migration, elapsed, session),
848
981
  ...(config.hooks ? { hooks: config.hooks } : {}),
849
982
  });
850
983
  this.#progress?.onStop('success');
984
+ span.set({ [ATTRIBUTES.MIGRATION_ATTEMPTS]: attempts });
985
+ // A transaction the driver retried ran the body again: say how often.
986
+ const retried = attempts > 1 ? { attempts } : {};
851
987
  this.#emit('migration:success', {
852
988
  migration: name,
853
989
  direction,
854
990
  ...batchField,
855
991
  durationMs: duration,
992
+ ...retried,
856
993
  });
857
- const label = direction === 'up' ? '✔ Applied ' : '↩ Reverted';
994
+ const label =
995
+ migration.kind === 'background'
996
+ ? direction === 'up'
997
+ ? '⧗ Registered'
998
+ : '⧗ Reverting'
999
+ : direction === 'up'
1000
+ ? '✔ Applied '
1001
+ : '↩ Reverted';
858
1002
  logger.info(
859
1003
  `${label} ${name} [${duration}ms]`,
860
- this.#fields({ migration: name, direction, ...batchField, durationMs: duration }),
1004
+ this.#fields({
1005
+ migration: name,
1006
+ direction,
1007
+ ...batchField,
1008
+ durationMs: duration,
1009
+ ...retried,
1010
+ }),
861
1011
  );
1012
+ if (migration.registered) {
1013
+ this.#emit('background:registered', { migration: name, ...migration.registered });
1014
+ }
862
1015
  results.push({
863
1016
  file: name,
864
1017
  status: direction === 'up' ? 'applied' : 'reverted',
@@ -868,7 +1021,7 @@ class MigratorKit extends EventEmitter {
868
1021
  await this.#runHook(config.hooks?.afterEach, 'afterEach', [
869
1022
  name,
870
1023
  duration,
871
- context,
1024
+ { ...context, ...attemptContext(attempts) },
872
1025
  { direction, index, total },
873
1026
  ]);
874
1027
  return duration;
@@ -883,6 +1036,13 @@ class MigratorKit extends EventEmitter {
883
1036
  ? error.context.durationMs
884
1037
  : undefined;
885
1038
  const durationField = durationMs !== undefined ? { durationMs } : {};
1039
+ // And how many times the body ran, when it did.
1040
+ const attempts =
1041
+ error instanceof MigronautError && typeof error.context?.attempts === 'number'
1042
+ ? error.context.attempts
1043
+ : undefined;
1044
+ if (attempts !== undefined) span.set({ [ATTRIBUTES.MIGRATION_ATTEMPTS]: attempts });
1045
+ const retried = attempts > 1 ? { attempts } : {};
886
1046
  // errorText, not the raw Error: a driver message can echo the
887
1047
  // credentialed URI, and event subscribers (Sentry, JSON logs) would
888
1048
  // ship it — the same redaction the log line below already gets.
@@ -891,6 +1051,7 @@ class MigratorKit extends EventEmitter {
891
1051
  direction,
892
1052
  ...batchField,
893
1053
  ...durationField,
1054
+ ...retried,
894
1055
  error: errorText(error),
895
1056
  });
896
1057
  logger.error(
@@ -900,6 +1061,7 @@ class MigratorKit extends EventEmitter {
900
1061
  direction,
901
1062
  ...batchField,
902
1063
  ...durationField,
1064
+ ...retried,
903
1065
  error: errorText(error),
904
1066
  }),
905
1067
  );
@@ -1139,6 +1301,25 @@ class MigratorKit extends EventEmitter {
1139
1301
  );
1140
1302
  }
1141
1303
 
1304
+ // Before beforeEach: a migration that cannot run yet fires no hook and
1305
+ // leaves no failed trace.
1306
+ const waiting = await this.#requiresGuard(name, signal);
1307
+ if (waiting.length > 0) {
1308
+ const list = waiting.map((entry) => `${entry.migration} (${entry.status})`).join(', ');
1309
+ if ((options.onBackgroundPending ?? 'error') === 'stop') {
1310
+ logger.info(
1311
+ `⧗ Waiting ${name} — requires ${list}`,
1312
+ this.#fields({ migration: name, direction: 'up', waitsFor: waiting.length }),
1313
+ );
1314
+ this.#emit('background:waiting', { migration: name, waitsFor: waiting });
1315
+ return 'stop';
1316
+ }
1317
+ throw new BackgroundPendingError(
1318
+ `${name} requires background migration(s) that have not completed: ${list}`,
1319
+ { migration: name, waitsFor: waiting },
1320
+ );
1321
+ }
1322
+
1142
1323
  const batch = options.step ? baseBatch + appliedCount : baseBatch;
1143
1324
  await this.#executeMigration({
1144
1325
  name,
@@ -1163,6 +1344,7 @@ class MigratorKit extends EventEmitter {
1163
1344
  duration: elapsed,
1164
1345
  ...(this.#runId ? { runId: this.#runId } : {}),
1165
1346
  ...(migration.description ? { description: migration.description } : {}),
1347
+ ...(migration.kind === 'background' ? { kind: 'background' } : {}),
1166
1348
  ...actorFields(options),
1167
1349
  },
1168
1350
  session,
@@ -1173,6 +1355,9 @@ class MigratorKit extends EventEmitter {
1173
1355
  failureFields: { checksum, ...actorFields(options) },
1174
1356
  });
1175
1357
  appliedCount += 1;
1358
+ if (this.#config.backgroundInline && (await this.#peek(name))?.kind === 'background') {
1359
+ await this.#runInline(name, signal);
1360
+ }
1176
1361
  return 'done';
1177
1362
  },
1178
1363
  });
@@ -1480,6 +1665,8 @@ class MigratorKit extends EventEmitter {
1480
1665
  const { records, preserveOrder } = await this.#selectDownTargets(filename, options);
1481
1666
  for (const record of records) {
1482
1667
  recordByName.set(record.name, record);
1668
+ // The refusal a real `down` of a one-way background migration meets.
1669
+ if (record.kind === 'background') await this.#assertBackgroundRevertible(record.name);
1483
1670
  }
1484
1671
  names = revertOrder(records, preserveOrder);
1485
1672
  }
@@ -1490,6 +1677,7 @@ class MigratorKit extends EventEmitter {
1490
1677
  // later (a queue job) can insist on exactly this version of the file.
1491
1678
  if (direction === 'up' && !row.invalid) {
1492
1679
  row.checksum = await this.#cachedChecksum(this.#filepath(name));
1680
+ await this.#annotateBackground(row, name);
1493
1681
  }
1494
1682
  return row;
1495
1683
  });
@@ -1514,6 +1702,28 @@ class MigratorKit extends EventEmitter {
1514
1702
  return rows;
1515
1703
  }
1516
1704
 
1705
+ /**
1706
+ * A preview row's background facts: a background file, what it requires,
1707
+ * and what of that is not done yet — read without writing anything. A file
1708
+ * that does not load is left for the real run to report.
1709
+ */
1710
+ async #annotateBackground(row, name) {
1711
+ let loaded;
1712
+ try {
1713
+ loaded = await this.#peek(name);
1714
+ } catch {
1715
+ return;
1716
+ }
1717
+ if (loaded === null) return;
1718
+ // One fact, one field: `kind`, as a status row says it.
1719
+ if (loaded.kind === 'background') row.kind = 'background';
1720
+ const requires = loaded.requires ?? [];
1721
+ if (requires.length === 0) return;
1722
+ row.requires = requires;
1723
+ const waiting = await this.#unsatisfied(requires, { reopen: false }).catch(() => []);
1724
+ if (waiting.length > 0) row.waitsFor = waiting.map((entry) => entry.migration);
1725
+ }
1726
+
1517
1727
  /** Full migration status for all known files and records */
1518
1728
  async status(options = {}) {
1519
1729
  await this.#ensureConfig();
@@ -1558,6 +1768,22 @@ class MigratorKit extends EventEmitter {
1558
1768
  inspectLock: () => this.#buildLock().inspect(),
1559
1769
  status: () => this.status(),
1560
1770
  definitions: () => this.#resolveCollections(),
1771
+ background: async () => {
1772
+ const deps = this.#backgroundDeps();
1773
+ return auditFindings({
1774
+ ...deps,
1775
+ checksumOf: (name) => computeChecksum(this.#filepath(name)),
1776
+ backgroundRecords: () =>
1777
+ this.#requireDb()
1778
+ .collection(this.#config.migrationsCollection)
1779
+ .find({ kind: 'background', status: 'applied' }, { projection: { name: 1 } })
1780
+ .toArray(),
1781
+ // Only where drift is streamed: a watcher's record is not a finding otherwise.
1782
+ ...(this.#config.backgroundDrift !== 'poll'
1783
+ ? { watchRows: () => this.#watchStore().status() }
1784
+ : {}),
1785
+ });
1786
+ },
1561
1787
  });
1562
1788
  }
1563
1789
 
@@ -1707,6 +1933,7 @@ class MigratorKit extends EventEmitter {
1707
1933
  duration: isApplied && record ? record.duration : null,
1708
1934
  checksumOk,
1709
1935
  ...(record?.description ? { description: record.description } : {}),
1936
+ ...(record?.kind === 'background' ? { kind: 'background' } : {}),
1710
1937
  ...this.#auditFields(record),
1711
1938
  };
1712
1939
  }
@@ -1719,6 +1946,12 @@ class MigratorKit extends EventEmitter {
1719
1946
  * fail (or reach the network at all) when that manager is unreachable.
1720
1947
  */
1721
1948
  async create(name, options = {}) {
1949
+ if (options.background && options.template !== undefined) {
1950
+ throw new ConfigInvalidError(
1951
+ 'create: background and template do not combine — a background migration is ' +
1952
+ 'scaffolded from its own template',
1953
+ );
1954
+ }
1722
1955
  const config = await this.#ensureConfig(false, true);
1723
1956
  const dir = this.#migrationsPath();
1724
1957
  await fs.mkdir(dir, { recursive: true });
@@ -1731,6 +1964,7 @@ class MigratorKit extends EventEmitter {
1731
1964
  js,
1732
1965
  fileExtensions: config.fileExtensions,
1733
1966
  ...(templatePath ? { templatePath } : {}),
1967
+ ...(options.background ? { background: true } : {}),
1734
1968
  });
1735
1969
  this.#logger.info(
1736
1970
  `✔ Created ${path.basename(filepath)}`,
@@ -1928,10 +2162,782 @@ class MigratorKit extends EventEmitter {
1928
2162
  : {}),
1929
2163
  extensions: config.fileExtensions,
1930
2164
  reload: config.reloadMigrations,
1931
- reserved: [config.migrationsCollection, config.lockCollection, config.convergeLogCollection],
2165
+ reserved: this.#bookkeepingNames(),
2166
+ });
2167
+ }
2168
+
2169
+ /** Every collection migronaut keeps its own records in — never a declared one */
2170
+ #bookkeepingNames() {
2171
+ const config = this.#config;
2172
+ const names = [
2173
+ config.migrationsCollection,
2174
+ config.lockCollection,
2175
+ config.convergeLogCollection,
2176
+ ];
2177
+ if (config.backgroundCollection !== undefined) {
2178
+ const background = backgroundCollectionNames(config.backgroundCollection);
2179
+ names.push(background.state, background.partitions, background.watch);
2180
+ }
2181
+ return names;
2182
+ }
2183
+
2184
+ // ─── Background migrations ──────────────────────────────────────────────────
2185
+
2186
+ /** The store of background migration state, over this kit's database */
2187
+ #backgroundStore() {
2188
+ this.#backgroundStoreInstance ??= new BackgroundStore(
2189
+ this.#requireDb(),
2190
+ this.#config.backgroundCollection,
2191
+ );
2192
+ return this.#backgroundStoreInstance;
2193
+ }
2194
+
2195
+ /**
2196
+ * A background migration file as the run loop sees a migration: its `up`
2197
+ * registers the background migration (the documents are rewritten later,
2198
+ * in partitions, without the migration lock); its `down` sets off the way
2199
+ * back — or, when it has none, withdraws it while nothing was rewritten yet.
2200
+ * Everything around them is the regular path: hooks, span, transaction,
2201
+ * changelog (with `kind: 'background'`).
2202
+ */
2203
+ async #asBackground(name, loaded, direction) {
2204
+ // Validated before it runs: an invalid file is reported as itself, not as
2205
+ // a failed execution — and nothing is written. A down goes by the spec it
2206
+ // was registered with: a definition file broken since must not block it.
2207
+ const { spec } =
2208
+ direction === 'down'
2209
+ ? await this.#registeredSpec(name, loaded)
2210
+ : await this.#backgroundSpec(name, loaded);
2211
+ if (direction === 'up') await this.#assertRequiresAreBackground(name, loaded.requires ?? []);
2212
+ const migration = {
2213
+ kind: 'background',
2214
+ ...(loaded.description !== undefined ? { description: loaded.description } : {}),
2215
+ up: async (ctx) => {
2216
+ migration.registered = await this.#registerBackground(name, loaded, spec, ctx.session);
2217
+ },
2218
+ down: async (ctx) => {
2219
+ migration.registered = await this.#revertBackground(name, spec, ctx.session);
2220
+ },
2221
+ };
2222
+ return migration;
2223
+ }
2224
+
2225
+ /**
2226
+ * The spec a registered background migration runs by — the one stored at
2227
+ * its registration — with the file's functions. Not registered: the file's.
2228
+ */
2229
+ async #registeredSpec(name, loaded) {
2230
+ const state = await this.#backgroundStore().get(name);
2231
+ if (state?.spec === undefined) return this.#backgroundSpec(name, loaded);
2232
+ const { fns } = await this.#backgroundSpec(name, loaded, { tolerant: true });
2233
+ return { spec: state.spec, fns };
2234
+ }
2235
+
2236
+ /**
2237
+ * The spec of a background migration file, resolved against the
2238
+ * collection's declared versioning (when it has one). `tolerant`: for a
2239
+ * registered one, whose stored spec stands — a definition file that does
2240
+ * not load is warned about and passed over (only its functions are used).
2241
+ */
2242
+ async #backgroundSpec(name, loaded, { tolerant = false } = {}) {
2243
+ const raw = loaded.background;
2244
+ let versioning;
2245
+ const collection = raw?.collection;
2246
+ if (typeof collection === 'string') {
2247
+ if (this.#bookkeepingNames().includes(collection)) {
2248
+ throw new MigrationInvalidExportError(
2249
+ `Background migration ${name} targets "${collection}", one of migronaut's own collections`,
2250
+ { name, collection },
2251
+ );
2252
+ }
2253
+ const config = this.#config;
2254
+ if (config.collections !== undefined || config.collectionsDir !== undefined) {
2255
+ try {
2256
+ versioning = await this.#versioningOf(collection, { strict: true });
2257
+ } catch (error) {
2258
+ if (!tolerant) throw error;
2259
+ this.#warnOnce(
2260
+ `definitions:${collection}`,
2261
+ `⚠ The collection definitions do not load (${errorText(error)}) — ${name} goes by ` +
2262
+ 'the spec it was registered with',
2263
+ { background: name },
2264
+ );
2265
+ }
2266
+ }
2267
+ }
2268
+ return resolveBackgroundSpec(raw, { name, versioning });
2269
+ }
2270
+
2271
+ /**
2272
+ * What a file's `requires` names must be: background migration files of
2273
+ * the sequence. The order is the loader's to check (each sorts before the
2274
+ * file); whether they are done is the caller's.
2275
+ */
2276
+ async #assertRequiresAreBackground(name, requires) {
2277
+ if (requires.length === 0) return;
2278
+ const sequence = new Set(await this.#listMigrationFiles());
2279
+ for (const required of requires) {
2280
+ if (!sequence.has(required)) {
2281
+ throw new MigrationInvalidExportError(
2282
+ `${name} requires ${required}, which is not a migration of the sequence`,
2283
+ { name, requires: required },
2284
+ );
2285
+ }
2286
+ // Through the module cache: this runs for every file that requires one.
2287
+ const loaded =
2288
+ (await this.#peek(required)) ??
2289
+ (await loadMigrationFile(this.#filepath(required), {
2290
+ reload: this.#config.reloadMigrations,
2291
+ }));
2292
+ if (loaded.kind !== 'background') {
2293
+ throw new MigrationInvalidExportError(
2294
+ `${name} requires ${required}, which is not a background migration — requires ` +
2295
+ 'waits for background migrations only (regular ones already run in order)',
2296
+ { name, requires: required },
2297
+ );
2298
+ }
2299
+ }
2300
+ }
2301
+
2302
+ /** The required background migrations not done yet, in order */
2303
+ #waitsFor(requires) {
2304
+ return waitingFor(this.#backgroundDeps(), requires);
2305
+ }
2306
+
2307
+ /** Register a background migration (again): `blocked` while what it requires is not done */
2308
+ async #registerBackground(name, loaded, spec, session) {
2309
+ const requires = loaded.requires ?? [];
2310
+ const waitsFor = await this.#waitsFor(requires);
2311
+ const status = waitsFor.length > 0 ? 'blocked' : 'pending';
2312
+ await this.#backgroundStore().register(
2313
+ name,
2314
+ {
2315
+ status,
2316
+ mode: spec.mode,
2317
+ direction: 'forward',
2318
+ spec,
2319
+ checksum: await computeChecksum(this.#filepath(name)),
2320
+ requires,
2321
+ waitsFor,
2322
+ ...(spec.collection !== undefined ? { collection: spec.collection } : {}),
2323
+ ...(loaded.description !== undefined ? { description: loaded.description } : {}),
2324
+ },
2325
+ { session },
2326
+ );
2327
+ return { status, direction: 'forward', ...(waitsFor.length > 0 ? { waitsFor } : {}) };
2328
+ }
2329
+
2330
+ /** What a `down` of a background migration would refuse — for a preview to refuse it too */
2331
+ async #assertBackgroundRevertible(name) {
2332
+ const loaded = await this.#peek(name);
2333
+ if (loaded === null || loaded.kind !== 'background') return;
2334
+ const { spec } = await this.#registeredSpec(name, loaded);
2335
+ if (spec.reversible) return;
2336
+ if (await hasRewritten(this.#backgroundStore(), name, spec)) {
2337
+ throw irreversibleBackground(name);
2338
+ }
2339
+ }
2340
+
2341
+ /**
2342
+ * `down` of a background migration: with a `revert`, the forward one is
2343
+ * replaced by the way back (registered to run like any other); without one,
2344
+ * it is withdrawn while nothing has been rewritten (background-kit.js).
2345
+ */
2346
+ async #revertBackground(name, spec, session) {
2347
+ const store = this.#backgroundStore();
2348
+ if (spec.reversible) {
2349
+ await store.register(
2350
+ name,
2351
+ {
2352
+ status: 'pending',
2353
+ mode: spec.mode,
2354
+ direction: 'revert',
2355
+ spec,
2356
+ checksum: await computeChecksum(this.#filepath(name)),
2357
+ requires: [],
2358
+ waitsFor: [],
2359
+ ...(spec.collection !== undefined ? { collection: spec.collection } : {}),
2360
+ },
2361
+ { session },
2362
+ );
2363
+ return { status: 'pending', direction: 'revert' };
2364
+ }
2365
+ return withdraw(this.#backgroundHost(), name, spec, session);
2366
+ }
2367
+
2368
+ /** What background-kit.js runs with */
2369
+ #backgroundHost() {
2370
+ return {
2371
+ store: this.#backgroundStore(),
2372
+ deps: (owner, options) => this.#backgroundDeps(owner, options),
2373
+ newId: () => this.#newId(),
2374
+ logger: this.#logger,
2375
+ emit: (event, payload) => this.#emit(event, payload),
2376
+ registered: (name) => this.#registered(name),
2377
+ status: (name) => this.backgroundStatus(name),
2378
+ };
2379
+ }
2380
+
2381
+ /**
2382
+ * What background.js runs with — `owner` names a lane: its lease's owner, its
2383
+ * events' runId. `job` is the queue job working the lane, when there is one
2384
+ * (`{ id }` for a background queue's lane, `{ id, groupId? }` for a run that
2385
+ * drives it inline): it goes on the lane's log lines and `migration:log`
2386
+ * events.
2387
+ */
2388
+ #backgroundDeps(owner, { job } = {}) {
2389
+ const config = this.#config;
2390
+ const db = this.#requireDb();
2391
+ const stamp = owner ? { runId: owner } : {};
2392
+ const lineStamp = job ? { ...stamp, ...jobFields(job) } : stamp;
2393
+ return {
2394
+ db,
2395
+ client: this.#client,
2396
+ store: this.#backgroundStore(),
2397
+ logger: this.#logger,
2398
+ // The lane's (or watcher's) run id, for ctx.background — `owner` itself
2399
+ // is a function here and a string in the watcher's deps.
2400
+ ...(owner ? { runId: owner } : {}),
2401
+ // One counter per lane slice (or watcher): orders its migration:log events.
2402
+ logs: backgroundLogs({ sink: this.#logger, emitter: this.#logEmitter() }),
2403
+ ...(job ? { job } : {}),
2404
+ fields: (extra) => ({ ...lineStamp, ...extra }),
2405
+ emit: (event, payload) => this.#emit(event, { ...stamp, ...payload }),
2406
+ lockFor: (name) =>
2407
+ new MigrationLock(db, config.lockCollection, config.lockTTLSeconds, {
2408
+ id: `background:${name}`,
2409
+ label: 'background coordinator lock',
2410
+ }),
2411
+ load: (name, options) => this.#loadBackground(name, options),
2412
+ ttlMs: config.lockTTLSeconds * 1000,
2413
+ owner: () => owner,
2414
+ warned: this.#backgroundWarned,
2415
+ adaptiveCache: this.#adaptiveCache,
2416
+ indexCache: this.#indexCache,
2417
+ telemetry: this.#telemetry,
2418
+ // The third wrap site: a span per lease held (slice) and per coordinator
2419
+ // step — each opened only once the lease or the lock is held.
2420
+ span: (kind, name, fn) =>
2421
+ this.#telemetry.wrap(
2422
+ kind === 'slice' ? SPANS.BACKGROUND_SLICE : SPANS.BACKGROUND_COORDINATE,
2423
+ { [ATTRIBUTES.BACKGROUND_NAME]: name },
2424
+ async (span) => {
2425
+ const result = await fn();
2426
+ span.set({ [ATTRIBUTES.BACKGROUND_OUTCOME]: result?.outcome ?? result?.next });
2427
+ return result;
2428
+ },
2429
+ ),
2430
+ onCompleted: (name) => this.#unblockDependents(name),
2431
+ adoptedOf: (names) => this.#requireChangelog().getAdoptedNames(db, names),
2432
+ topology: () => this.#serverTopology(db),
2433
+ shardAware: config.backgroundShardAware,
2434
+ shardKeyOf: (collection) => readShardKey(this.#client, db.databaseName, collection),
2435
+ chunksOf: (collection, sharding) =>
2436
+ readChunks(this.#client, db.databaseName, collection, sharding),
2437
+ versioningOf: (collection) => this.#versioningOf(collection),
2438
+ };
2439
+ }
2440
+
2441
+ /**
2442
+ * The server's topology, read once it is known. A `hello` that failed (an
2443
+ * election, a blip) is not remembered as "not sharded": it is asked again.
2444
+ */
2445
+ async #serverTopology(db) {
2446
+ if (this.#topology !== undefined) return this.#topology;
2447
+ const { topology } = await readServer(db);
2448
+ if (topology !== undefined) this.#topology = topology;
2449
+ return topology;
2450
+ }
2451
+
2452
+ /**
2453
+ * A declared collection's versioning, or `undefined`. Definitions that do
2454
+ * not load are converge's to report — `undefined` too, unless `strict`.
2455
+ */
2456
+ async #versioningOf(collection, { strict = false } = {}) {
2457
+ const config = this.#config;
2458
+ if (config.collections === undefined && config.collectionsDir === undefined) return undefined;
2459
+ let definitions;
2460
+ try {
2461
+ definitions = config.reloadMigrations
2462
+ ? await this.#resolveCollections()
2463
+ : (this.#backgroundDefinitions ??= await this.#resolveCollections());
2464
+ } catch (error) {
2465
+ if (strict) throw error;
2466
+ return undefined;
2467
+ }
2468
+ for (const definition of definitions) {
2469
+ if (definition.name === collection) return definition.versioning;
2470
+ }
2471
+ return undefined;
2472
+ }
2473
+
2474
+ /** Say something about background migrations once per kit */
2475
+ #warnOnce(id, message, fields) {
2476
+ if (this.#backgroundWarned.has(id)) return;
2477
+ this.#backgroundWarned.add(id);
2478
+ this.#logger.warn(message, this.#fields(fields));
2479
+ }
2480
+
2481
+ /**
2482
+ * The live drift watcher: a change stream per collection with a completed
2483
+ * background migration, one leader per collection across every process,
2484
+ * upgrading each old-shape write moments after it lands. Resolves to
2485
+ * `{ running, status(), stop() }` once it has started; on a standalone
2486
+ * server (no change streams) it rejects with ConfigInvalidError.
2487
+ * @experimental
2488
+ */
2489
+ async watchBackground(options = {}) {
2490
+ // Checked before anything connects: a typo should not wait for a server.
2491
+ watchOptions(options);
2492
+ await this.#backgroundReady();
2493
+ const db = this.#requireDb();
2494
+ if ((await readServer(db)).topology === 'standalone') {
2495
+ throw new ConfigInvalidError(
2496
+ 'The live drift watcher needs change streams — a replica set or a sharded cluster; ' +
2497
+ "keep backgroundDrift: 'poll' on a standalone server",
2498
+ { key: 'backgroundDrift' },
2499
+ );
2500
+ }
2501
+ const config = this.#config;
2502
+ const owner = this.#newId();
2503
+ return startWatch(
2504
+ {
2505
+ ...this.#backgroundDeps(owner),
2506
+ watchStore: this.#watchStore(),
2507
+ watchLockFor: (collection) =>
2508
+ new MigrationLock(db, config.lockCollection, config.lockTTLSeconds, {
2509
+ id: `watch:${collection}`,
2510
+ label: 'drift watcher lock',
2511
+ }),
2512
+ owner,
2513
+ onDrift: config.backgroundOnDrift,
2514
+ },
2515
+ options,
2516
+ );
2517
+ }
2518
+
2519
+ /**
2520
+ * What the live drift watchers recorded — one row per watched collection
2521
+ * (or the one asked for, `null` when it has none): state, leader, counters,
2522
+ * last event. Never the resume token.
2523
+ * @experimental
2524
+ */
2525
+ async backgroundWatchStatus(collection) {
2526
+ await this.#backgroundReady();
2527
+ const rows = await this.#watchStore().status(collection);
2528
+ if (collection !== undefined) return rows === null ? null : watchRow(rows);
2529
+ const views = [];
2530
+ for (const row of rows) views.push(watchRow(row));
2531
+ return views;
2532
+ }
2533
+
2534
+ #watchStore() {
2535
+ this.#watchStoreInstance ??= new BackgroundWatchStore(
2536
+ this.#requireDb(),
2537
+ this.#config.backgroundCollection,
2538
+ );
2539
+ return this.#watchStoreInstance;
2540
+ }
2541
+
2542
+ /**
2543
+ * The drift watch, once: old-shape documents that appeared after a
2544
+ * background migration completed are found with one indexed probe each,
2545
+ * and reopen it (`onDrift: 'reopen'`, the `backgroundOnDrift` default) or
2546
+ * are only reported (`'report'`). `collections` narrows it.
2547
+ * @experimental
2548
+ */
2549
+ async verifyBackground(options = {}) {
2550
+ await this.#backgroundReady();
2551
+ const onDrift = options.onDrift ?? this.#config.backgroundOnDrift;
2552
+ if (onDrift !== 'reopen' && onDrift !== 'report') {
2553
+ throw new ConfigInvalidError("onDrift must be 'reopen' or 'report'", { onDrift });
2554
+ }
2555
+ // With `backgroundDrift: 'stream'` the poll is the safety net: it leaves
2556
+ // alone a collection whose watcher is streaming and alive.
2557
+ const streaming =
2558
+ this.#config.backgroundDrift === 'stream' ? await this.#streamingCollections() : undefined;
2559
+ return verifyDrift(this.#backgroundDeps(), {
2560
+ onDrift,
2561
+ ...(options.collections !== undefined ? { collections: options.collections } : {}),
2562
+ ...(streaming !== undefined ? { streaming } : {}),
2563
+ });
2564
+ }
2565
+
2566
+ /** Collections a live watcher leads right now — streaming, its record fresh */
2567
+ async #streamingCollections() {
2568
+ const fresh = Date.now() - STREAMING_FRESH_MS;
2569
+ const names = [];
2570
+ for (const row of await this.#watchStore().status()) {
2571
+ if (row.state === 'streaming' && row.updatedAt?.getTime() > fresh) names.push(row._id);
2572
+ }
2573
+ return names;
2574
+ }
2575
+
2576
+ /**
2577
+ * How drift is watched (`backgroundDrift`): `'poll'`, `'stream'` or
2578
+ * `'both'` — what a runner or a queue worker hosting this kit follows.
2579
+ * Resolves the config; does not connect.
2580
+ */
2581
+ async driftMode() {
2582
+ return (await this.#ensureConfig()).backgroundDrift;
2583
+ }
2584
+
2585
+ /** A background migration file, loaded and resolved: `{ spec, fns, checksum }` */
2586
+ async #loadBackground(name, { tolerant = false } = {}) {
2587
+ assertMigrationName(name);
2588
+ const filepath = this.#filepath(name);
2589
+ // Once per version of the file: lanes ask on every slice, and a re-import
2590
+ // per slice (under reloadMigrations) is a module the cache never frees.
2591
+ // A file that cannot be read is loaded for its error.
2592
+ const loaded =
2593
+ (await this.#peek(name)) ??
2594
+ (await loadMigrationFile(filepath, {
2595
+ reload: this.#config.reloadMigrations,
2596
+ }));
2597
+ if (loaded.kind !== 'background') {
2598
+ throw new MigrationInvalidExportError(`${name} is not a background migration`, { name });
2599
+ }
2600
+ const { spec, fns } = await this.#backgroundSpec(name, loaded, { tolerant });
2601
+ return { spec, fns, checksum: await this.#cachedChecksum(filepath) };
2602
+ }
2603
+
2604
+ /** After one completes: the blocked ones that required it, unblocked if nothing else holds them */
2605
+ async #unblockDependents(name) {
2606
+ const store = this.#backgroundStore();
2607
+ const deps = this.#backgroundDeps();
2608
+ for (const state of await store.list({ requires: name, status: 'blocked' })) {
2609
+ await tryUnblock(deps, state);
2610
+ }
2611
+ }
2612
+
2613
+ /**
2614
+ * A migration module read ahead of its run — for its `requires` and its
2615
+ * kind — cached by checksum, so a long-lived kit imports an edited file
2616
+ * again and an unchanged one never twice. `null` when it cannot be read.
2617
+ */
2618
+ async #peek(name) {
2619
+ let checksum;
2620
+ try {
2621
+ checksum = await this.#cachedChecksum(this.#filepath(name));
2622
+ } catch {
2623
+ return null;
2624
+ }
2625
+ const cached = this.#moduleCache.get(name);
2626
+ if (cached?.checksum === checksum) return cached.loaded;
2627
+ const loaded = await loadMigrationFile(this.#filepath(name), {
2628
+ reload: this.#config.reloadMigrations,
2629
+ });
2630
+ this.#moduleCache.set(name, { checksum, loaded });
2631
+ return loaded;
2632
+ }
2633
+
2634
+ /**
2635
+ * The background migrations of `requires` not done yet:
2636
+ * `[{ migration, status }]` — background-kit.js.
2637
+ */
2638
+ #unsatisfied(requires, options) {
2639
+ return unsatisfied(this.#backgroundHost(), requires, options);
2640
+ }
2641
+
2642
+ /**
2643
+ * The background migrations a pending regular migration waits for — run
2644
+ * first, under this run's lock, in inline mode. Empty when it may run.
2645
+ */
2646
+ async #requiresGuard(name, signal) {
2647
+ const loaded = await this.#peek(name);
2648
+ if (loaded === null || loaded.kind === 'background') return [];
2649
+ const requires = loaded.requires ?? [];
2650
+ if (requires.length === 0) return [];
2651
+ await this.#assertRequiresAreBackground(name, requires);
2652
+ await this.#backgroundIndexes();
2653
+ let waiting = await this.#unsatisfied(requires);
2654
+ if (waiting.length > 0 && this.#config.backgroundInline) {
2655
+ for (const entry of waiting) {
2656
+ if (entry.status !== 'unregistered') await this.#runInline(entry.migration, signal);
2657
+ }
2658
+ waiting = await this.#unsatisfied(requires);
2659
+ }
2660
+ return waiting;
2661
+ }
2662
+
2663
+ /**
2664
+ * Inline mode: drive a background migration to the end right here, under
2665
+ * the run's lock — what it requires first.
2666
+ *
2667
+ * @throws {BackgroundFailedError} when it fails; RunAbortedError on stop()
2668
+ */
2669
+ async #runInline(name, signal) {
2670
+ const state = await this.#backgroundStore().get(name);
2671
+ if (state === null) return;
2672
+ for (const required of state.status === 'blocked' ? (state.waitsFor ?? []) : []) {
2673
+ await this.#runInline(required, signal);
2674
+ }
2675
+ this.#logger.info(
2676
+ `⧗ Running ${name} inline`,
2677
+ this.#fields({ background: name, inline: true }),
2678
+ );
2679
+ await this.#backgroundReady(name, { lanes: true });
2680
+ // The lanes work for the run's queue job: their lines and events name it,
2681
+ // with its group — which a background queue's own lanes never have.
2682
+ const job = this.#runLog?.job;
2683
+ const status = await drive(this.#backgroundHost(), name, {
2684
+ signal,
2685
+ concurrency: state.spec?.maxParallel ?? 1,
2686
+ inline: true,
2687
+ ...(job ? { job } : {}),
2688
+ });
2689
+ if (status.status === 'blocked') {
2690
+ throw new BackgroundPendingError(
2691
+ `Background migration ${name} cannot run inline: it waits for ${status.waitsFor.join(', ')}`,
2692
+ { migration: name, waitsFor: status.waitsFor.map((migration) => ({ migration })) },
2693
+ );
2694
+ }
2695
+ }
2696
+
2697
+ /**
2698
+ * Config and connection for a background method — no run, no migration
2699
+ * lock. `lanes`: it claims partitions, and needs the slot indexes.
2700
+ */
2701
+ async #backgroundReady(name, { lanes = false } = {}) {
2702
+ if (name !== undefined) assertMigrationName(name);
2703
+ await this.#ensureConfig();
2704
+ await this.connect();
2705
+ await this.#backgroundIndexes({ lanes });
2706
+ }
2707
+
2708
+ /**
2709
+ * The background collections' indexes, created on first use — unless
2710
+ * `ensureIndexes: false` (a user who cannot create indexes): then they are
2711
+ * expected to exist, and only what claims partitions checks they do.
2712
+ */
2713
+ async #backgroundIndexes({ lanes = false } = {}) {
2714
+ const store = this.#backgroundStore();
2715
+ if (this.#config.ensureIndexes) await store.ensureIndexes();
2716
+ else if (lanes) await store.assertIndexes();
2717
+ }
2718
+
2719
+ /** The state, or NotAppliedError — a control action needs a registered background migration */
2720
+ async #registered(name) {
2721
+ const state = await this.#backgroundStore().get(name);
2722
+ if (state === null) {
2723
+ throw new NotAppliedError(`Background migration ${name} is not registered — run up first`, {
2724
+ migration: name,
2725
+ });
2726
+ }
2727
+ return state;
2728
+ }
2729
+
2730
+ /**
2731
+ * One coordinator step for a background migration — see background.js.
2732
+ * Reentrant: no run, no migration lock; serialized by its own lock.
2733
+ * @experimental
2734
+ */
2735
+ async coordinateBackground(name, { signal, driver } = {}) {
2736
+ await this.#backgroundReady(name);
2737
+ return coordinate(this.#backgroundDeps(this.#newId()), name, {
2738
+ signal,
2739
+ ...(driver ? { driver } : {}),
1932
2740
  });
1933
2741
  }
1934
2742
 
2743
+ /**
2744
+ * One slice of one lane of a background migration: claim a partition and a
2745
+ * slot, work it until `sliceMs` (the spec's by default) runs out, release.
2746
+ * `job: { id }` names the queue job working the lane, for its log lines.
2747
+ * @experimental
2748
+ */
2749
+ async runBackgroundSlice(name, { signal, sliceMs, job } = {}) {
2750
+ if (sliceMs !== undefined) assertSliceMs(sliceMs);
2751
+ // A lane's job has no group: `{ id }` only.
2752
+ assertJobValid(job, { groupId: false });
2753
+ await this.#backgroundReady(name, { lanes: true });
2754
+ const owner = this.#newId();
2755
+ return runSlice(this.#backgroundDeps(owner, job ? { job } : {}), name, {
2756
+ signal,
2757
+ sliceMs,
2758
+ owner,
2759
+ });
2760
+ }
2761
+
2762
+ /**
2763
+ * Drive a background migration from this process — the coordinator and up
2764
+ * to `concurrency` lanes (at most its `maxParallel`) — until it is done
2765
+ * (`untilDone`, the default) or for one round. Resolves to its status; a
2766
+ * failed one throws BackgroundFailedError, a stop RunAbortedError (the
2767
+ * background migration itself goes on from where it was).
2768
+ * @experimental
2769
+ */
2770
+ async runBackground(name, { signal, sliceMs, untilDone = true, concurrency = 1 } = {}) {
2771
+ await this.#backgroundReady(name, { lanes: true });
2772
+ if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
2773
+ throw new ConfigInvalidError('concurrency must be a positive integer', { concurrency });
2774
+ }
2775
+ if (sliceMs !== undefined) assertSliceMs(sliceMs);
2776
+ return drive(this.#backgroundHost(), name, { signal, sliceMs, untilDone, concurrency });
2777
+ }
2778
+
2779
+ /**
2780
+ * The status of one background migration (`null` when not registered), or
2781
+ * of every one when no name is given.
2782
+ * @experimental
2783
+ */
2784
+ async backgroundStatus(name) {
2785
+ await this.#backgroundReady(name);
2786
+ const store = this.#backgroundStore();
2787
+ if (name !== undefined) {
2788
+ const state = await store.get(name);
2789
+ return state === null ? null : stateView(store, state);
2790
+ }
2791
+ const views = [];
2792
+ for (const state of await store.list()) views.push(await stateView(store, state));
2793
+ return views;
2794
+ }
2795
+
2796
+ /**
2797
+ * The partitions of a background migration's latest generation — scope,
2798
+ * cursor, counters and lease holder (never its token).
2799
+ * @experimental
2800
+ */
2801
+ async backgroundPartitions(name) {
2802
+ await this.#backgroundReady(name);
2803
+ const state = await this.#registered(name);
2804
+ const partitions = await this.#backgroundStore().partitions(name, {
2805
+ generation: state.generation,
2806
+ });
2807
+ const views = [];
2808
+ for (const partition of partitions) views.push(partitionView(partition));
2809
+ return views;
2810
+ }
2811
+
2812
+ /**
2813
+ * The background migrations with work to do — blocked ones whose requires
2814
+ * are met are unblocked on the way. `[{ migration, status }]`, oldest first.
2815
+ * @experimental
2816
+ */
2817
+ async runnableBackground() {
2818
+ await this.#backgroundReady();
2819
+ return runnable(this.#backgroundHost());
2820
+ }
2821
+
2822
+ /** A control action, after the checks every one shares */
2823
+ async #controlBackground(name, action, options = {}) {
2824
+ // Who and why go into the state's history: held to the changelog's limits.
2825
+ assertActorValid(options);
2826
+ await this.#backgroundReady(name);
2827
+ await this.#registered(name);
2828
+ const deps = this.#backgroundDeps();
2829
+ const result = await controlBackground(deps, name, action, options);
2830
+ if (options.wait === true && (action === 'pause' || action === 'cancel')) {
2831
+ result.stopped = await waitForLanes(deps, name, { signal: options.signal });
2832
+ }
2833
+ return result;
2834
+ }
2835
+
2836
+ /** Pause a background migration; its lanes stop at the next batch. `wait` until they have. @experimental */
2837
+ pauseBackground(name, options = {}) {
2838
+ return this.#controlBackground(name, 'pause', options);
2839
+ }
2840
+
2841
+ /** Resume a paused background migration. @experimental */
2842
+ resumeBackground(name, options = {}) {
2843
+ return this.#controlBackground(name, 'resume', options);
2844
+ }
2845
+
2846
+ /** Cancel a background migration; `wait` until its lanes have stopped. @experimental */
2847
+ cancelBackground(name, options = {}) {
2848
+ return this.#controlBackground(name, 'cancel', options);
2849
+ }
2850
+
2851
+ /**
2852
+ * Retry a failed or cancelled background migration — the same generation,
2853
+ * or `fromStart`; `repin` pins the file on disk first. A completed one is
2854
+ * reopened over whatever is left.
2855
+ * @experimental
2856
+ */
2857
+ async retryBackground(name, options = {}) {
2858
+ if (options.repin === true) await this.repinBackground(name, options);
2859
+ return this.#controlBackground(name, 'retry', options);
2860
+ }
2861
+
2862
+ /**
2863
+ * Pin the file on disk as the background migration's version — its
2864
+ * checksum (in the changelog too, so a strict drift check agrees) and its
2865
+ * spec. A change to what it matches or how it splits means a new plan.
2866
+ * @experimental
2867
+ */
2868
+ async repinBackground(name, options = {}) {
2869
+ assertActorValid(options);
2870
+ await this.#backgroundReady(name);
2871
+ await this.#registered(name);
2872
+ const result = await repinBackgroundState(this.#backgroundDeps(), name, options);
2873
+ await this.#requireChangelog().setChecksum(this.#requireDb(), name, result.checksum);
2874
+ return result;
2875
+ }
2876
+
2877
+ /**
2878
+ * Dry-run a background migration — registered or not — with nothing
2879
+ * written: on a sample (`sample` random documents, or the `first` n), its
2880
+ * transformation alone; with `validate`, the real write path in a
2881
+ * transaction that is always aborted. A step migration runs `steps` steps
2882
+ * in that transaction instead.
2883
+ * @experimental
2884
+ */
2885
+ async dryRunBackground(name, options = {}) {
2886
+ await this.#ensureConfig();
2887
+ await this.connect();
2888
+ const loaded = await this.#loadBackground(name);
2889
+ const db = this.#requireDb();
2890
+ const deps = {
2891
+ db,
2892
+ client: this.#client,
2893
+ logger: this.#logger,
2894
+ forbidden: this.#bookkeepingNames(),
2895
+ topology: () => this.#serverTopology(db),
2896
+ // The sandbox refuses a sharded collection's distinct (a mongos cannot run it in a transaction).
2897
+ shardKeyOf: (collection) => readShardKey(this.#client, db.databaseName, collection),
2898
+ };
2899
+ if (loaded.spec.mode === 'step' || options.steps !== undefined) {
2900
+ if (loaded.spec.mode !== 'step') {
2901
+ throw new ConfigInvalidError(
2902
+ `${name} is declarative — dry-run it on a sample (sample, first), not by steps`,
2903
+ { migration: name },
2904
+ );
2905
+ }
2906
+ // From where the background migration is, unless asked to start over.
2907
+ let checkpoint = null;
2908
+ const state = await this.#backgroundStore().get(name);
2909
+ if (state !== null && !options.fromStart) {
2910
+ const [partition] = await this.#backgroundStore().partitions(name, {
2911
+ generation: state.generation,
2912
+ });
2913
+ checkpoint = partition?.cursor?.checkpoint ?? null;
2914
+ }
2915
+ return previewSteps(deps, name, loaded, { ...options, checkpoint });
2916
+ }
2917
+ return previewSample(deps, name, loaded, options);
2918
+ }
2919
+
2920
+ /**
2921
+ * Clear a background migration's coordinator lock and every partition
2922
+ * lease — for a stuck one; live lanes are fenced off at their next write.
2923
+ * @experimental
2924
+ */
2925
+ async unlockBackground(name) {
2926
+ await this.#backgroundReady(name);
2927
+ const deps = this.#backgroundDeps();
2928
+ const lock = (await deps.lockFor(name).forceRelease()) !== null;
2929
+ const leases = await deps.store.unlockAll(name);
2930
+ // It fences every live lane: an operator's act, recorded like any control.
2931
+ await deps.store.note(name, { action: 'unlock', lock, leases });
2932
+ this.#emit('background:control', { migration: name, action: 'unlock', lock, leases });
2933
+ this.#logger.warn(
2934
+ `⚠ ${name}: unlocked — coordinator lock ${lock ? 'cleared' : 'not held'}, ${leases} ` +
2935
+ 'lease(s) dropped (their lanes stop at their next write)',
2936
+ this.#fields({ background: name, lock, leases }),
2937
+ );
2938
+ return { lock, leases };
2939
+ }
2940
+
1935
2941
  /** How converge treats search indexes: the config, and a call's own `waitForSearchIndexes` */
1936
2942
  #convergeSearchOptions(options = {}) {
1937
2943
  const config = this.#config;
@@ -1961,14 +2967,13 @@ class MigratorKit extends EventEmitter {
1961
2967
  environment: this.#environment(),
1962
2968
  }),
1963
2969
  record: (entry) => this.#convergeLog().append(db, entry),
1964
- // Behind a mongos only: the shard key, so prune never tries to drop its index.
1965
- shardKeyOf: async (name) =>
1966
- (
1967
- await this.#client
1968
- .db('config')
1969
- .collection('collections')
1970
- .findOne({ _id: `${db.databaseName}.${name}` }, { projection: { key: 1 } })
1971
- )?.key,
2970
+ // Behind a mongos only: the shard key — so prune never drops its index,
2971
+ // and the version index takes it as a prefix. An 8.0 `unsplittable`
2972
+ // collection is not sharded; `undefined` when config may not be read.
2973
+ shardKeyOf: async (name) => {
2974
+ const sharding = await readShardKey(this.#client, db.databaseName, name);
2975
+ return sharding === undefined ? undefined : (sharding?.key ?? null);
2976
+ },
1972
2977
  };
1973
2978
  }
1974
2979