@alexify/migronaut 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +41 -3
  3. package/bullmq.d.ts +484 -8
  4. package/index.d.ts +1264 -9
  5. package/migronaut.schema.json +93 -1
  6. package/package.json +9 -2
  7. package/src/bullmq/background-processor.js +541 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +348 -21
  11. package/src/bullmq/producer.js +185 -13
  12. package/src/bullmq/service.js +484 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/create.js +6 -0
  15. package/src/cli/exit-codes.js +6 -0
  16. package/src/cli/index.js +2 -0
  17. package/src/core/audit.js +11 -1
  18. package/src/core/background-audit.js +139 -0
  19. package/src/core/background-drift.js +126 -0
  20. package/src/core/background-dry-run.js +375 -0
  21. package/src/core/background-engine.js +849 -0
  22. package/src/core/background-kit.js +432 -0
  23. package/src/core/background-partition.js +298 -0
  24. package/src/core/background-runner.js +305 -0
  25. package/src/core/background-sandbox.js +701 -0
  26. package/src/core/background-shard.js +542 -0
  27. package/src/core/background-spec.js +597 -0
  28. package/src/core/background-store.js +951 -0
  29. package/src/core/background-throttle.js +269 -0
  30. package/src/core/background-watch-plan.js +164 -0
  31. package/src/core/background-watch-store.js +78 -0
  32. package/src/core/background-watch.js +610 -0
  33. package/src/core/background.js +1127 -0
  34. package/src/core/bson-peer.js +23 -0
  35. package/src/core/changelog.js +32 -0
  36. package/src/core/collections.js +78 -8
  37. package/src/core/config.js +102 -12
  38. package/src/core/converge-plan.js +86 -7
  39. package/src/core/converge.js +88 -0
  40. package/src/core/lock.js +48 -21
  41. package/src/core/migration-logger.js +279 -0
  42. package/src/core/migrator.js +1027 -22
  43. package/src/core/options.js +36 -0
  44. package/src/core/run-recorder.js +6 -1
  45. package/src/core/run.js +26 -12
  46. package/src/core/runner.js +34 -8
  47. package/src/core/server-info.js +9 -2
  48. package/src/core/shard-info.js +76 -0
  49. package/src/core/versioning-spec.js +181 -0
  50. package/src/errors/index.js +88 -0
  51. package/src/index.js +16 -0
  52. package/src/utils/error.js +11 -2
  53. package/src/utils/job-ref.js +44 -0
  54. package/src/utils/loader.js +77 -9
  55. package/src/utils/migration-name.js +33 -1
  56. package/src/utils/redact.js +140 -3
  57. package/src/utils/telemetry.js +110 -0
  58. package/src/utils/template.js +62 -1
  59. package/src/versioning/config.js +155 -0
  60. package/src/versioning/document.js +326 -0
  61. package/src/versioning/index.js +50 -0
  62. package/src/versioning/internal.js +279 -0
  63. package/src/versioning/mongoose.js +151 -0
  64. package/src/versioning/occ.js +318 -0
  65. package/src/versioning/registry.js +187 -0
  66. package/src/versioning/upcaster.js +213 -0
  67. package/versioning.d.ts +666 -0
  68. package/versioning.js +1 -0
package/src/index.js CHANGED
@@ -1,8 +1,12 @@
1
1
  const { EXIT_CODES } = require('./cli/exit-codes.js');
2
+ const { startBackgroundRunner } = require('./core/background-runner.js');
2
3
  const { MigratorKit } = require('./core/migrator.js');
3
4
  const { pendingMigrations, runMigrations } = require('./core/run.js');
4
5
  const { createLogger } = require('./utils/logger.js');
5
6
  const {
7
+ BackgroundConflictError,
8
+ BackgroundFailedError,
9
+ BackgroundPendingError,
6
10
  ChecksumMismatchError,
7
11
  ConfigFileExistsError,
8
12
  ConfigInvalidError,
@@ -27,7 +31,10 @@ const {
27
31
  OutOfOrderMigrationError,
28
32
  QueueJobFailedError,
29
33
  QueueJobInvalidError,
34
+ RevisionConflictError,
30
35
  RunAbortedError,
36
+ SandboxRefusedError,
37
+ ShapeVersionError,
31
38
  } = require('./errors/index.js');
32
39
 
33
40
  module.exports = {
@@ -46,7 +53,13 @@ module.exports = {
46
53
  // The CLI's exit-code map, for wrappers that mirror its semantics
47
54
  EXIT_CODES,
48
55
 
56
+ // Background migrations driven from inside the application (experimental)
57
+ startBackgroundRunner,
58
+
49
59
  // Error classes
60
+ BackgroundConflictError,
61
+ BackgroundFailedError,
62
+ BackgroundPendingError,
50
63
  ChecksumMismatchError,
51
64
  ConfigFileExistsError,
52
65
  ConfigInvalidError,
@@ -71,5 +84,8 @@ module.exports = {
71
84
  OutOfOrderMigrationError,
72
85
  QueueJobFailedError,
73
86
  QueueJobInvalidError,
87
+ RevisionConflictError,
74
88
  RunAbortedError,
89
+ SandboxRefusedError,
90
+ ShapeVersionError,
75
91
  };
@@ -1,5 +1,5 @@
1
1
  const { MigronautError } = require('../errors/index.js');
2
- const { redactUris } = require('./redact.js');
2
+ const { redactOutbound, redactUris } = require('./redact.js');
3
3
 
4
4
  /**
5
5
  * Human-readable message from any thrown value, with URI credentials masked.
@@ -25,4 +25,13 @@ function errorWithCause(error) {
25
25
  return cause ? `${message} — ${cause}` : message;
26
26
  }
27
27
 
28
- module.exports = { errorText, errorWithCause };
28
+ /**
29
+ * {@link errorText} for an error about the application's data — a background
30
+ * migration's document errors and failed slices, kept in its state and
31
+ * logged: the values a server error quotes (an E11000's duplicate key — an
32
+ * email, a phone number) are masked too. Migronaut never logs a document's
33
+ * contents; the index name still says which constraint was violated.
34
+ */
35
+ const documentErrorText = (error) => redactOutbound(errorText(error));
36
+
37
+ module.exports = { documentErrorText, errorText, errorWithCause };
@@ -0,0 +1,44 @@
1
+ const { isPlainObject } = require('./canonical.js');
2
+ const { MAX_ID_LENGTH } = require('./id.js');
3
+
4
+ /**
5
+ * The queue job a run works for: `job: { id, groupId? }` on `up`, `down` and
6
+ * `redo` (a background lane's slice takes `{ id }`). It is bound into
7
+ * `ctx.run`, the run's log lines and every `migration:log` event, so what a
8
+ * migration logs can be joined to the job a dashboard shows — nothing is
9
+ * stored. The one definition of its limits, shared by the kit's options and
10
+ * the queue adapter, which must never pass what the kit refuses.
11
+ *
12
+ * `id` allows a custom BullMQ job id (they can be long); `groupId` is minted
13
+ * by the kit's own id generator, so it has that generator's bound.
14
+ */
15
+ const JOB_REF_LIMITS = Object.freeze({ id: 1024, groupId: MAX_ID_LENGTH });
16
+
17
+ /**
18
+ * The problem with a job reference, or null when it is absent or valid.
19
+ * `groupId: false` refuses that key (a lane has no group).
20
+ */
21
+ function jobRefIssue(job, { groupId = true } = {}) {
22
+ if (job === undefined) return null;
23
+ if (!isPlainObject(job)) return 'job must be an object: { id, groupId? }';
24
+ for (const key of Object.keys(job)) {
25
+ // Own keys only: `constructor` or `toString` must not pass for a limit.
26
+ if (!Object.hasOwn(JOB_REF_LIMITS, key) || (key === 'groupId' && !groupId)) {
27
+ return `job.${key} is not an option`;
28
+ }
29
+ const max = JOB_REF_LIMITS[key];
30
+ const value = job[key];
31
+ if (typeof value !== 'string' || value.length === 0 || value.length > max) {
32
+ return `job.${key} must be a non-empty string of at most ${max} characters`;
33
+ }
34
+ }
35
+ return job.id === undefined ? 'job.id is required' : null;
36
+ }
37
+
38
+ /** What a valid job reference adds to a run's correlation: `{ jobId, groupId? }` */
39
+ function jobFields(job) {
40
+ if (job === undefined) return {};
41
+ return { jobId: job.id, ...(job.groupId !== undefined ? { groupId: job.groupId } : {}) };
42
+ }
43
+
44
+ module.exports = { JOB_REF_LIMITS, jobFields, jobRefIssue };
@@ -3,6 +3,7 @@ const path = require('node:path');
3
3
  const { pathToFileURL } = require('node:url');
4
4
  const { MigrationFileNotFoundError, MigrationInvalidExportError } = require('../errors/index.js');
5
5
  const { errorText } = require('./error.js');
6
+ const { requiresIssues } = require('./migration-name.js');
6
7
 
7
8
  /** TypeScript source extensions that require a TS-capable runtime to import */
8
9
  const TS_EXTENSIONS = new Set(['.ts', '.mts', '.cts']);
@@ -96,16 +97,13 @@ function importUserFile(filepath, options = {}) {
96
97
  }
97
98
 
98
99
  /**
99
- * Dynamically load a migration file and validate its exports.
100
- *
101
- * Handles all three supported formats:
102
- * - TypeScript / JavaScript ESM named exports (`export async function up/down`)
103
- * - CommonJS default export (`module.exports = { up, down }`)
100
+ * Import a migration file: its module, resolved — the default export of a
101
+ * CommonJS file, the namespace of an ES module with named exports.
104
102
  *
105
103
  * @throws {MigrationFileNotFoundError} when the file does not exist
106
- * @throws {MigrationInvalidExportError} when up/down are not both functions
104
+ * @throws {MigrationInvalidExportError} when TypeScript cannot be loaded
107
105
  */
108
- async function loadMigrationFile(filepath, options = {}) {
106
+ async function importMigrationModule(filepath, options = {}) {
109
107
  try {
110
108
  await fs.access(filepath);
111
109
  } catch {
@@ -123,7 +121,48 @@ async function loadMigrationFile(filepath, options = {}) {
123
121
  throw error;
124
122
  }
125
123
  // `mod.default ?? mod` handles the CommonJS default-export case
126
- const resolved = imported.default ?? imported;
124
+ return imported.default ?? imported;
125
+ }
126
+
127
+ /** The `requires` export, validated against the file's own name */
128
+ function readRequires(resolved, filepath) {
129
+ if (resolved.requires === undefined) return {};
130
+ const issues = requiresIssues(resolved.requires, path.basename(filepath));
131
+ if (issues.length > 0) {
132
+ throw new MigrationInvalidExportError(`Invalid ${issues[0].path}: ${issues[0].message}`, {
133
+ filepath,
134
+ issues,
135
+ });
136
+ }
137
+ return { requires: [...resolved.requires] };
138
+ }
139
+
140
+ /**
141
+ * What a migration module exports, validated: a regular migration
142
+ * (`{ up, down, useTransaction?, timeoutMs?, description?, requires? }`) or a
143
+ * background one (`{ kind: 'background', background, description?,
144
+ * requires? }` — its spec is validated where the collection's versioning is
145
+ * known). A file with both `background` and `up`/`down` is refused: the
146
+ * expand steps belong in a migration of their own.
147
+ *
148
+ * @throws {MigrationInvalidExportError}
149
+ */
150
+ function resolveMigrationExports(resolved, filepath) {
151
+ if (resolved.background !== undefined) {
152
+ if (resolved.up !== undefined || resolved.down !== undefined) {
153
+ throw new MigrationInvalidExportError(
154
+ 'A background migration exports no up() or down() — put the expand steps in a ' +
155
+ 'migration of their own',
156
+ { filepath },
157
+ );
158
+ }
159
+ return {
160
+ kind: 'background',
161
+ background: resolved.background,
162
+ ...(typeof resolved.description === 'string' ? { description: resolved.description } : {}),
163
+ ...readRequires(resolved, filepath),
164
+ };
165
+ }
127
166
 
128
167
  if (!isFunction(resolved.up) || !isFunction(resolved.down)) {
129
168
  throw new MigrationInvalidExportError('Migration must export async up() and down() functions', {
@@ -142,8 +181,37 @@ async function loadMigrationFile(filepath, options = {}) {
142
181
  if (typeof resolved.description === 'string') {
143
182
  migration.description = resolved.description;
144
183
  }
184
+ Object.assign(migration, readRequires(resolved, filepath));
145
185
 
146
186
  return migration;
147
187
  }
148
188
 
149
- module.exports = { importUserFile, loadMigrationFile, tsLoadErrorOrNull, tsLoadMessageOrNull };
189
+ /**
190
+ * Dynamically load a migration file and validate its exports.
191
+ *
192
+ * Handles all three supported formats:
193
+ * - TypeScript / JavaScript ESM named exports (`export async function up/down`)
194
+ * - CommonJS default export (`module.exports = { up, down }`)
195
+ *
196
+ * A background migration (`kind: 'background'`) is returned like any other:
197
+ * the caller tells it apart by its kind.
198
+ *
199
+ * @throws {MigrationFileNotFoundError} when the file does not exist
200
+ * @throws {MigrationInvalidExportError} when up/down are not both functions
201
+ */
202
+ async function loadMigrationFile(filepath, options = {}) {
203
+ const migration = resolveMigrationExports(
204
+ await importMigrationModule(filepath, options),
205
+ filepath,
206
+ );
207
+ return migration;
208
+ }
209
+
210
+ module.exports = {
211
+ importMigrationModule,
212
+ importUserFile,
213
+ loadMigrationFile,
214
+ resolveMigrationExports,
215
+ tsLoadErrorOrNull,
216
+ tsLoadMessageOrNull,
217
+ };
@@ -29,4 +29,36 @@ function assertMigrationName(name, context = {}) {
29
29
  }
30
30
  }
31
31
 
32
- module.exports = { assertMigrationName, isBareFilename };
32
+ /**
33
+ * Why a file's `requires` export is not valid: an array of bare migration
34
+ * file names, no duplicates, each sorting strictly before the file itself
35
+ * (`name`). Files run in name order, so an edge that only ever points
36
+ * backwards can never close a cycle — the whole "is it a DAG?" question,
37
+ * answered by the name. Returns `{ path, message }` issues.
38
+ */
39
+ function requiresIssues(requires, name) {
40
+ if (requires === undefined) return [];
41
+ if (!Array.isArray(requires)) {
42
+ return [{ path: 'requires', message: 'must be an array of migration file names' }];
43
+ }
44
+ const issues = [];
45
+ const seen = new Set();
46
+ for (const [position, required] of requires.entries()) {
47
+ const path = `requires[${position}]`;
48
+ if (!isBareFilename(required)) {
49
+ issues.push({ path, message: 'must be a bare migration file name' });
50
+ } else if (seen.has(required)) {
51
+ issues.push({ path, message: `names "${required}" twice` });
52
+ } else if (name !== undefined && required >= name) {
53
+ issues.push({
54
+ path,
55
+ message: `must name an earlier migration ("${required}" does not sort before "${name}")`,
56
+ });
57
+ } else {
58
+ seen.add(required);
59
+ }
60
+ }
61
+ return issues;
62
+ }
63
+
64
+ module.exports = { assertMigrationName, isBareFilename, requiresIssues };
@@ -1,3 +1,5 @@
1
+ const { isPlainObject } = require('./canonical.js');
2
+
1
3
  /**
2
4
  * Credential redaction for anything that leaves the process — error messages,
3
5
  * stacks, `--json` payloads, log lines. The MongoDB driver echoes the raw
@@ -69,6 +71,24 @@ function redactOutbound(text) {
69
71
  return redactUris(text).replace(DUPLICATE_KEY_VALUES, '$1{ <redacted> }');
70
72
  }
71
73
 
74
+ /**
75
+ * Set `key` on a copy as an own property. A plain assignment of `__proto__` —
76
+ * a key `JSON.parse` happily makes — would replace the copy's prototype and
77
+ * lose the key instead.
78
+ */
79
+ function put(target, key, value) {
80
+ if (key === '__proto__') {
81
+ Object.defineProperty(target, key, {
82
+ value,
83
+ enumerable: true,
84
+ writable: true,
85
+ configurable: true,
86
+ });
87
+ } else {
88
+ target[key] = value;
89
+ }
90
+ }
91
+
72
92
  /**
73
93
  * Redact every string reachable from `value` (plain objects and arrays only —
74
94
  * class instances are left alone rather than cloned into broken shapes).
@@ -81,12 +101,129 @@ function redactDeep(value) {
81
101
  for (let index = 0; index < value.length; index++) copy[index] = redactDeep(value[index]);
82
102
  return copy;
83
103
  }
84
- if (value !== null && typeof value === 'object' && value.constructor === Object) {
104
+ if (isPlainObject(value)) {
85
105
  const copy = {};
86
- for (const key of Object.keys(value)) copy[key] = redactDeep(value[key]);
106
+ for (const key of Object.keys(value)) put(copy, key, redactDeep(value[key]));
87
107
  return copy;
88
108
  }
89
109
  return value;
90
110
  }
91
111
 
92
- module.exports = { redactDeep, redactOutbound, redactUris };
112
+ /**
113
+ * How much of a value {@link redactBounded} copies: nesting, entries in all,
114
+ * the length of one string and of one piece of binary data. What a migration
115
+ * hands to `ctx.logger` is the application's own data, of any size and shape —
116
+ * a cycle included — and a subscriber stores it; these keep one call's copy
117
+ * small and finite.
118
+ */
119
+ const BOUNDS = Object.freeze({ depth: 8, entries: 1000, string: 4096, bytes: 4096 });
120
+
121
+ /** Stands in for what {@link redactBounded} left out */
122
+ const TRUNCATED = '[truncated]';
123
+
124
+ /** The size of binary data — a Buffer or another Uint8Array, a BSON Binary — or undefined */
125
+ function byteLength(item) {
126
+ if (item instanceof Uint8Array) return item.byteLength;
127
+ if (item._bsontype === 'Binary') return item.position;
128
+ return undefined;
129
+ }
130
+
131
+ /**
132
+ * An Error as data: its name, its message — credentials and the values a
133
+ * server error quotes masked, as for anything that leaves the process — and
134
+ * its code. Not the stack, and not what else it carries: a driver error's raw
135
+ * server response repeats the offending document's values (`keyValue`,
136
+ * `errmsg`), and its `message` is not even enumerable, so the error itself
137
+ * would be stored as an empty document.
138
+ */
139
+ function errorData(error) {
140
+ const data = {
141
+ name: String(error.name),
142
+ message: redactOutbound(typeof error.message === 'string' ? error.message : ''),
143
+ };
144
+ if (typeof error.code === 'number' || typeof error.code === 'string') data.code = error.code;
145
+ if (typeof error.codeName === 'string') data.codeName = error.codeName;
146
+ return data;
147
+ }
148
+
149
+ /**
150
+ * {@link redactDeep} within {@link BOUNDS}, for data that leaves the process
151
+ * as a document: every string reachable redacted ({@link redactOutbound}: the
152
+ * values a server error quotes too) and clipped, nesting past
153
+ * the depth (which is what ends a cycle) and entries past the budget replaced
154
+ * or dropped. What the driver stores as a value of its own is kept as is — a
155
+ * Date, a RegExp, an ObjectId or another BSON value, and binary data within
156
+ * its bound. An Error becomes `{ name, message, code?, codeName? }`; a Map is
157
+ * copied as an object and a Set as an array; any other instance goes by its
158
+ * `toJSON()`, or its own fields, the way `JSON.stringify` would see it.
159
+ * `omit` names a key left out of the top level only. Returns
160
+ * `{ value, truncated }`; never mutates the input.
161
+ */
162
+ function redactBounded(value, { omit } = {}) {
163
+ let entries = 0;
164
+ let truncated = false;
165
+ /** One more entry, or false once the budget is spent */
166
+ const take = () => {
167
+ if (entries >= BOUNDS.entries) {
168
+ truncated = true;
169
+ return false;
170
+ }
171
+ entries += 1;
172
+ return true;
173
+ };
174
+ const copy = (item, depth) => {
175
+ if (typeof item === 'string') {
176
+ // Outbound, like the event's message: an error's text copied into a
177
+ // field (`{ error: err.message }`) loses the values it quotes too.
178
+ const text = redactOutbound(item);
179
+ if (text.length <= BOUNDS.string) return text;
180
+ truncated = true;
181
+ return `${text.slice(0, BOUNDS.string)}…`;
182
+ }
183
+ if (item === null || typeof item !== 'object') return item;
184
+ if (item instanceof Date || item instanceof RegExp) return item;
185
+ const bytes = byteLength(item);
186
+ if (bytes !== undefined) {
187
+ if (bytes <= BOUNDS.bytes) return item;
188
+ truncated = true;
189
+ return TRUNCATED;
190
+ }
191
+ if (typeof item._bsontype === 'string') return item;
192
+ if (item instanceof Error) return copy(errorData(item), depth);
193
+ if (depth >= BOUNDS.depth) {
194
+ truncated = true;
195
+ return TRUNCATED;
196
+ }
197
+ if (Array.isArray(item) || item instanceof Set) {
198
+ const out = [];
199
+ for (const element of item) {
200
+ if (!take()) break;
201
+ out.push(copy(element, depth + 1));
202
+ }
203
+ return out;
204
+ }
205
+ const out = {};
206
+ if (item instanceof Map) {
207
+ for (const [key, element] of item) {
208
+ if (!take()) break;
209
+ put(out, String(key), copy(element, depth + 1));
210
+ }
211
+ return out;
212
+ }
213
+ // What JSON.stringify would see: a toJSON() result one level down (so a
214
+ // chain of them ends at the depth bound), else the instance's own fields.
215
+ if (!isPlainObject(item) && typeof item.toJSON === 'function') {
216
+ return copy(item.toJSON(), depth + 1);
217
+ }
218
+ for (const key of Object.keys(item)) {
219
+ if (depth === 0 && key === omit) continue;
220
+ if (!take()) break;
221
+ put(out, key, copy(item[key], depth + 1));
222
+ }
223
+ return out;
224
+ };
225
+ const result = copy(value, 0);
226
+ return { value: result, truncated };
227
+ }
228
+
229
+ module.exports = { BOUNDS, redactBounded, redactDeep, redactOutbound, redactUris };
@@ -13,6 +13,8 @@ const SPAN_STATUS_ERROR = 2;
13
13
  const SPANS = {
14
14
  RUN: 'migronaut.run',
15
15
  MIGRATION: 'migronaut.migration',
16
+ BACKGROUND_SLICE: 'migronaut.background.slice',
17
+ BACKGROUND_COORDINATE: 'migronaut.background.coordinate',
16
18
  };
17
19
 
18
20
  /** Every attribute key migronaut sets, on spans and on metric points */
@@ -35,6 +37,14 @@ const ATTRIBUTES = {
35
37
  MIGRATION_INDEX: 'migronaut.migration.index',
36
38
  MIGRATION_TOTAL: 'migronaut.migration.total',
37
39
  MIGRATION_TRANSACTION: 'migronaut.migration.transaction',
40
+ MIGRATION_ATTEMPTS: 'migronaut.migration.attempts',
41
+ JOB_ID: 'migronaut.job.id',
42
+ JOB_GROUP_ID: 'migronaut.job.group_id',
43
+ BACKGROUND_NAME: 'migronaut.background.name',
44
+ BACKGROUND_OUTCOME: 'migronaut.background.outcome',
45
+ BACKGROUND_RESULT: 'migronaut.background.result',
46
+ BACKGROUND_REASON: 'migronaut.background.reason',
47
+ BACKGROUND_SHARD: 'migronaut.background.shard',
38
48
  ERROR_TYPE: 'error.type',
39
49
  /** The database a run is against — OpenTelemetry's database semantic convention */
40
50
  DB_NAMESPACE: 'db.namespace',
@@ -48,6 +58,14 @@ const METRICS = {
48
58
  LOCK_REFUSED: 'migronaut.lock.refused',
49
59
  LOCK_LOST: 'migronaut.lock.lost',
50
60
  SEARCH_WAIT_DURATION: 'migronaut.converge.search.wait.duration',
61
+ BACKGROUND_DOCUMENTS: 'migronaut.background.documents',
62
+ BACKGROUND_SLICE_DURATION: 'migronaut.background.slice.duration',
63
+ BACKGROUND_BATCH_WRITE_DURATION: 'migronaut.background.batch.write.duration',
64
+ BACKGROUND_THROTTLE: 'migronaut.background.throttled',
65
+ BACKGROUND_DRIFT: 'migronaut.background.drift.detected',
66
+ BACKGROUND_TRANSACTION_RETRIES: 'migronaut.background.transaction.retried',
67
+ BACKGROUND_LEASES_RECLAIMED: 'migronaut.background.leases.reclaimed',
68
+ BACKGROUND_WATCH_DELAY: 'migronaut.background.watch.delay',
51
69
  };
52
70
 
53
71
  /**
@@ -339,6 +357,47 @@ function createTelemetry(telemetry, { dbName } = {}) {
339
357
  'Time a converge waited for its search index builds, by how the wait ended',
340
358
  );
341
359
 
360
+ const backgroundDocuments = counter(
361
+ METRICS.BACKGROUND_DOCUMENTS,
362
+ 'Documents a background migration handled, by result (migrated, skipped, conflict, failed)',
363
+ '{document}',
364
+ );
365
+ const backgroundSliceDuration = histogram(
366
+ METRICS.BACKGROUND_SLICE_DURATION,
367
+ 'Duration of one background migration slice, by outcome',
368
+ );
369
+ const backgroundBatchWrite = histogram(
370
+ METRICS.BACKGROUND_BATCH_WRITE_DURATION,
371
+ 'Time to write one background migration batch',
372
+ );
373
+ const backgroundThrottle = counter(
374
+ METRICS.BACKGROUND_THROTTLE,
375
+ 'Adaptive throttle changes of background migrations, by reason',
376
+ '{change}',
377
+ );
378
+ const backgroundDrift = counter(
379
+ METRICS.BACKGROUND_DRIFT,
380
+ 'Old-shape documents found after a background migration completed',
381
+ '{finding}',
382
+ );
383
+ const backgroundTransactionRetries = counter(
384
+ METRICS.BACKGROUND_TRANSACTION_RETRIES,
385
+ 'Transactional background batches retried, by reason',
386
+ '{retry}',
387
+ );
388
+ const backgroundLeasesReclaimed = counter(
389
+ METRICS.BACKGROUND_LEASES_RECLAIMED,
390
+ 'Partition leases reclaimed from a lane that stopped renewing',
391
+ '{lease}',
392
+ );
393
+ const backgroundWatchDelay = histogram(
394
+ METRICS.BACKGROUND_WATCH_DELAY,
395
+ 'Time from an old-shape write to its upgrade by the live drift watcher',
396
+ );
397
+ const add = (instrument, value, attributes) => {
398
+ if (instrument && value > 0) safe(() => instrument.add(value, withBase(attributes)));
399
+ };
400
+
342
401
  // Durations are measured in milliseconds everywhere in migronaut and
343
402
  // reported in seconds, the unit OpenTelemetry's conventions settle on.
344
403
  const record = (instrument, durationMs, attributes) => {
@@ -391,6 +450,57 @@ function createTelemetry(telemetry, { dbName } = {}) {
391
450
  searchWaited({ waitedMs, outcome }) {
392
451
  record(searchWaitDuration, waitedMs, { [ATTRIBUTES.SEARCH_WAIT_OUTCOME]: outcome });
393
452
  },
453
+ /**
454
+ * A background migration slice ended: its duration by outcome, and the
455
+ * documents it handled by result. The partition is never an attribute —
456
+ * dimensions stay low-cardinality.
457
+ */
458
+ backgroundSliceEnded({ name, durationMs, outcome, counters = {}, error }) {
459
+ const at = { [ATTRIBUTES.BACKGROUND_NAME]: name };
460
+ record(backgroundSliceDuration, durationMs, {
461
+ ...at,
462
+ [ATTRIBUTES.BACKGROUND_OUTCOME]: outcome,
463
+ ...failure(error),
464
+ });
465
+ for (const [key, result] of [
466
+ ['migrated', 'migrated'],
467
+ ['skipped', 'skipped'],
468
+ ['conflicts', 'conflict'],
469
+ ['failed', 'failed'],
470
+ ]) {
471
+ add(backgroundDocuments, counters[key] ?? 0, {
472
+ ...at,
473
+ [ATTRIBUTES.BACKGROUND_RESULT]: result,
474
+ });
475
+ }
476
+ },
477
+ backgroundBatchWritten({ name, durationMs, shard }) {
478
+ record(backgroundBatchWrite, durationMs, {
479
+ [ATTRIBUTES.BACKGROUND_NAME]: name,
480
+ [ATTRIBUTES.BACKGROUND_SHARD]: shard,
481
+ });
482
+ },
483
+ backgroundThrottled({ name, reason }) {
484
+ add(backgroundThrottle, 1, {
485
+ [ATTRIBUTES.BACKGROUND_NAME]: name,
486
+ [ATTRIBUTES.BACKGROUND_REASON]: reason,
487
+ });
488
+ },
489
+ backgroundDrift({ name, count = 1 }) {
490
+ add(backgroundDrift, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
491
+ },
492
+ backgroundTransactionRetried({ name, reason, count = 1 }) {
493
+ add(backgroundTransactionRetries, count, {
494
+ [ATTRIBUTES.BACKGROUND_NAME]: name,
495
+ [ATTRIBUTES.BACKGROUND_REASON]: reason,
496
+ });
497
+ },
498
+ backgroundLeasesReclaimed({ name, count }) {
499
+ add(backgroundLeasesReclaimed, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
500
+ },
501
+ backgroundWatchDelay({ name, delayMs }) {
502
+ record(backgroundWatchDelay, delayMs, { [ATTRIBUTES.BACKGROUND_NAME]: name });
503
+ },
394
504
  };
395
505
  }
396
506
 
@@ -145,6 +145,51 @@ module.exports = { description, up, down };
145
145
  `;
146
146
  }
147
147
 
148
+ /**
149
+ * The built-in background migration template (`create --background`): the
150
+ * declarative form, with the knobs most worth knowing about spelled out.
151
+ */
152
+ function defaultBackgroundTemplate(js, esm = false) {
153
+ const typed = js
154
+ ? "/** @type {import('@alexify/migronaut').DeclarativeBackgroundMigration} */\n"
155
+ : '';
156
+ const typeImport = js
157
+ ? ''
158
+ : "import type { DeclarativeBackgroundMigration } from '@alexify/migronaut';\n\n";
159
+ const annotation = js ? '' : ': DeclarativeBackgroundMigration';
160
+ const body = `{
161
+ // The collection to rewrite (\`up\` refuses this placeholder).
162
+ collection: 'TODO',
163
+ // Documents at version \`from\` (0: no version field yet) become version \`to\`.
164
+ from: 1,
165
+ to: 2,
166
+ // The new document for one old one — return it reshaped; migronaut sets the
167
+ // version, bumps the revision and writes only the fields that changed.
168
+ migrate: (doc) => {
169
+ // TODO: reshape doc, and return it
170
+ throw new Error('migrate is not written yet');
171
+ },
172
+ // The way back, for \`down\`. Without one, \`down\` refuses once documents
173
+ // were rewritten. Never \`(doc) => doc\`: that would stamp the old version
174
+ // on documents still in the new shape.
175
+ // revert: ({ shipping, ...doc }) => ({ ...doc, address: shipping.address }),
176
+ // Partitions worked at once, across every process (default 1).
177
+ // maxParallel: 4,
178
+ }`;
179
+ if (esm) {
180
+ return `${typeImport}export const description = '';
181
+
182
+ ${typed}export const background${annotation} = ${body};
183
+ `;
184
+ }
185
+ return `${typeImport}const description = '';
186
+
187
+ ${typed}const background${annotation} = ${body};
188
+
189
+ module.exports = { description, background };
190
+ `;
191
+ }
192
+
148
193
  /** Extensions a custom `--template` file may have — anything else is refused */
149
194
  const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
150
195
 
@@ -152,7 +197,8 @@ const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
152
197
  const MAX_TEMPLATE_BYTES = 1024 * 1024;
153
198
 
154
199
  /** Resolve template file contents — a custom template if provided, else the built-in */
155
- async function resolveTemplateContent(templatePath, js, esm = false) {
200
+ async function resolveTemplateContent(templatePath, js, esm = false, background = false) {
201
+ if (background) return defaultBackgroundTemplate(js, esm);
156
202
  if (templatePath) {
157
203
  const ext = path.extname(templatePath);
158
204
  if (!TEMPLATE_EXTENSIONS.includes(ext)) {
@@ -198,6 +244,7 @@ async function createMigrationFile(options) {
198
244
  // Match the project's module system, so a generated migration never makes
199
245
  // Node reparse it and warn.
200
246
  await isEsmProject(options.dir),
247
+ options.background === true,
201
248
  );
202
249
  try {
203
250
  // 'wx' fails if the path exists — creating a migration must never silently
@@ -366,6 +413,20 @@ function configBody(values, createExtension) {
366
413
  // waitForSearchIndexes: false,
367
414
  // searchIndexWaitTimeoutMs: 600000,
368
415
 
416
+ // ── Background migrations (experimental) ────────────────────
417
+ // A migration file with \`export const background = {…}\` is registered by
418
+ // \`up\` and runs in partitions (BullMQ, \`migronaut background run\`, or an
419
+ // in-process runner) without holding the migration lock.
420
+ // backgroundCollection: '_migronaut_background',
421
+ // Run it to the end inside the \`up\` that registers it instead.
422
+ // backgroundInline: false,
423
+ // Old-shape documents after it completed: reopen it ('reopen') or report.
424
+ // backgroundOnDrift: 'reopen',
425
+ // How drift is watched: 'poll' (every 10 minutes), 'stream', or 'both'.
426
+ // backgroundDrift: 'poll',
427
+ // Partition a sharded collection by its shard key ('auto') or not ('off').
428
+ // backgroundShardAware: 'auto',
429
+
369
430
  // ── Lifecycle hooks (code only — not available in JSON config) ──
370
431
  // hooks: {
371
432
  // beforeAll: async (ctx) => {},