@alexify/migronaut 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +41 -3
- package/bullmq.d.ts +484 -8
- package/index.d.ts +1264 -9
- package/migronaut.schema.json +93 -1
- package/package.json +9 -2
- package/src/bullmq/background-processor.js +541 -0
- package/src/bullmq/index.js +12 -0
- package/src/bullmq/jobs.js +254 -7
- package/src/bullmq/processor.js +348 -21
- package/src/bullmq/producer.js +185 -13
- package/src/bullmq/service.js +484 -45
- package/src/cli/commands/background.js +500 -0
- package/src/cli/commands/create.js +6 -0
- package/src/cli/exit-codes.js +6 -0
- package/src/cli/index.js +2 -0
- package/src/core/audit.js +11 -1
- package/src/core/background-audit.js +139 -0
- package/src/core/background-drift.js +126 -0
- package/src/core/background-dry-run.js +375 -0
- package/src/core/background-engine.js +849 -0
- package/src/core/background-kit.js +432 -0
- package/src/core/background-partition.js +298 -0
- package/src/core/background-runner.js +305 -0
- package/src/core/background-sandbox.js +701 -0
- package/src/core/background-shard.js +542 -0
- package/src/core/background-spec.js +597 -0
- package/src/core/background-store.js +951 -0
- package/src/core/background-throttle.js +269 -0
- package/src/core/background-watch-plan.js +164 -0
- package/src/core/background-watch-store.js +78 -0
- package/src/core/background-watch.js +610 -0
- package/src/core/background.js +1127 -0
- package/src/core/bson-peer.js +23 -0
- package/src/core/changelog.js +32 -0
- package/src/core/collections.js +78 -8
- package/src/core/config.js +102 -12
- package/src/core/converge-plan.js +86 -7
- package/src/core/converge.js +88 -0
- package/src/core/lock.js +48 -21
- package/src/core/migration-logger.js +279 -0
- package/src/core/migrator.js +1027 -22
- package/src/core/options.js +36 -0
- package/src/core/run-recorder.js +6 -1
- package/src/core/run.js +26 -12
- package/src/core/runner.js +34 -8
- package/src/core/server-info.js +9 -2
- package/src/core/shard-info.js +76 -0
- package/src/core/versioning-spec.js +181 -0
- package/src/errors/index.js +88 -0
- package/src/index.js +16 -0
- package/src/utils/error.js +11 -2
- package/src/utils/job-ref.js +44 -0
- package/src/utils/loader.js +77 -9
- package/src/utils/migration-name.js +33 -1
- package/src/utils/redact.js +140 -3
- package/src/utils/telemetry.js +110 -0
- package/src/utils/template.js +62 -1
- package/src/versioning/config.js +155 -0
- package/src/versioning/document.js +326 -0
- package/src/versioning/index.js +50 -0
- package/src/versioning/internal.js +279 -0
- package/src/versioning/mongoose.js +151 -0
- package/src/versioning/occ.js +318 -0
- package/src/versioning/registry.js +187 -0
- package/src/versioning/upcaster.js +213 -0
- package/versioning.d.ts +666 -0
- package/versioning.js +1 -0
package/src/index.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
const { EXIT_CODES } = require('./cli/exit-codes.js');
|
|
2
|
+
const { startBackgroundRunner } = require('./core/background-runner.js');
|
|
2
3
|
const { MigratorKit } = require('./core/migrator.js');
|
|
3
4
|
const { pendingMigrations, runMigrations } = require('./core/run.js');
|
|
4
5
|
const { createLogger } = require('./utils/logger.js');
|
|
5
6
|
const {
|
|
7
|
+
BackgroundConflictError,
|
|
8
|
+
BackgroundFailedError,
|
|
9
|
+
BackgroundPendingError,
|
|
6
10
|
ChecksumMismatchError,
|
|
7
11
|
ConfigFileExistsError,
|
|
8
12
|
ConfigInvalidError,
|
|
@@ -27,7 +31,10 @@ const {
|
|
|
27
31
|
OutOfOrderMigrationError,
|
|
28
32
|
QueueJobFailedError,
|
|
29
33
|
QueueJobInvalidError,
|
|
34
|
+
RevisionConflictError,
|
|
30
35
|
RunAbortedError,
|
|
36
|
+
SandboxRefusedError,
|
|
37
|
+
ShapeVersionError,
|
|
31
38
|
} = require('./errors/index.js');
|
|
32
39
|
|
|
33
40
|
module.exports = {
|
|
@@ -46,7 +53,13 @@ module.exports = {
|
|
|
46
53
|
// The CLI's exit-code map, for wrappers that mirror its semantics
|
|
47
54
|
EXIT_CODES,
|
|
48
55
|
|
|
56
|
+
// Background migrations driven from inside the application (experimental)
|
|
57
|
+
startBackgroundRunner,
|
|
58
|
+
|
|
49
59
|
// Error classes
|
|
60
|
+
BackgroundConflictError,
|
|
61
|
+
BackgroundFailedError,
|
|
62
|
+
BackgroundPendingError,
|
|
50
63
|
ChecksumMismatchError,
|
|
51
64
|
ConfigFileExistsError,
|
|
52
65
|
ConfigInvalidError,
|
|
@@ -71,5 +84,8 @@ module.exports = {
|
|
|
71
84
|
OutOfOrderMigrationError,
|
|
72
85
|
QueueJobFailedError,
|
|
73
86
|
QueueJobInvalidError,
|
|
87
|
+
RevisionConflictError,
|
|
74
88
|
RunAbortedError,
|
|
89
|
+
SandboxRefusedError,
|
|
90
|
+
ShapeVersionError,
|
|
75
91
|
};
|
package/src/utils/error.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const { MigronautError } = require('../errors/index.js');
|
|
2
|
-
const { redactUris } = require('./redact.js');
|
|
2
|
+
const { redactOutbound, redactUris } = require('./redact.js');
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Human-readable message from any thrown value, with URI credentials masked.
|
|
@@ -25,4 +25,13 @@ function errorWithCause(error) {
|
|
|
25
25
|
return cause ? `${message} — ${cause}` : message;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
/**
|
|
29
|
+
* {@link errorText} for an error about the application's data — a background
|
|
30
|
+
* migration's document errors and failed slices, kept in its state and
|
|
31
|
+
* logged: the values a server error quotes (an E11000's duplicate key — an
|
|
32
|
+
* email, a phone number) are masked too. Migronaut never logs a document's
|
|
33
|
+
* contents; the index name still says which constraint was violated.
|
|
34
|
+
*/
|
|
35
|
+
const documentErrorText = (error) => redactOutbound(errorText(error));
|
|
36
|
+
|
|
37
|
+
module.exports = { documentErrorText, errorText, errorWithCause };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
const { isPlainObject } = require('./canonical.js');
|
|
2
|
+
const { MAX_ID_LENGTH } = require('./id.js');
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The queue job a run works for: `job: { id, groupId? }` on `up`, `down` and
|
|
6
|
+
* `redo` (a background lane's slice takes `{ id }`). It is bound into
|
|
7
|
+
* `ctx.run`, the run's log lines and every `migration:log` event, so what a
|
|
8
|
+
* migration logs can be joined to the job a dashboard shows — nothing is
|
|
9
|
+
* stored. The one definition of its limits, shared by the kit's options and
|
|
10
|
+
* the queue adapter, which must never pass what the kit refuses.
|
|
11
|
+
*
|
|
12
|
+
* `id` allows a custom BullMQ job id (they can be long); `groupId` is minted
|
|
13
|
+
* by the kit's own id generator, so it has that generator's bound.
|
|
14
|
+
*/
|
|
15
|
+
const JOB_REF_LIMITS = Object.freeze({ id: 1024, groupId: MAX_ID_LENGTH });
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The problem with a job reference, or null when it is absent or valid.
|
|
19
|
+
* `groupId: false` refuses that key (a lane has no group).
|
|
20
|
+
*/
|
|
21
|
+
function jobRefIssue(job, { groupId = true } = {}) {
|
|
22
|
+
if (job === undefined) return null;
|
|
23
|
+
if (!isPlainObject(job)) return 'job must be an object: { id, groupId? }';
|
|
24
|
+
for (const key of Object.keys(job)) {
|
|
25
|
+
// Own keys only: `constructor` or `toString` must not pass for a limit.
|
|
26
|
+
if (!Object.hasOwn(JOB_REF_LIMITS, key) || (key === 'groupId' && !groupId)) {
|
|
27
|
+
return `job.${key} is not an option`;
|
|
28
|
+
}
|
|
29
|
+
const max = JOB_REF_LIMITS[key];
|
|
30
|
+
const value = job[key];
|
|
31
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > max) {
|
|
32
|
+
return `job.${key} must be a non-empty string of at most ${max} characters`;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return job.id === undefined ? 'job.id is required' : null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** What a valid job reference adds to a run's correlation: `{ jobId, groupId? }` */
|
|
39
|
+
function jobFields(job) {
|
|
40
|
+
if (job === undefined) return {};
|
|
41
|
+
return { jobId: job.id, ...(job.groupId !== undefined ? { groupId: job.groupId } : {}) };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
module.exports = { JOB_REF_LIMITS, jobFields, jobRefIssue };
|
package/src/utils/loader.js
CHANGED
|
@@ -3,6 +3,7 @@ const path = require('node:path');
|
|
|
3
3
|
const { pathToFileURL } = require('node:url');
|
|
4
4
|
const { MigrationFileNotFoundError, MigrationInvalidExportError } = require('../errors/index.js');
|
|
5
5
|
const { errorText } = require('./error.js');
|
|
6
|
+
const { requiresIssues } = require('./migration-name.js');
|
|
6
7
|
|
|
7
8
|
/** TypeScript source extensions that require a TS-capable runtime to import */
|
|
8
9
|
const TS_EXTENSIONS = new Set(['.ts', '.mts', '.cts']);
|
|
@@ -96,16 +97,13 @@ function importUserFile(filepath, options = {}) {
|
|
|
96
97
|
}
|
|
97
98
|
|
|
98
99
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* Handles all three supported formats:
|
|
102
|
-
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
103
|
-
* - CommonJS default export (`module.exports = { up, down }`)
|
|
100
|
+
* Import a migration file: its module, resolved — the default export of a
|
|
101
|
+
* CommonJS file, the namespace of an ES module with named exports.
|
|
104
102
|
*
|
|
105
103
|
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
106
|
-
* @throws {MigrationInvalidExportError} when
|
|
104
|
+
* @throws {MigrationInvalidExportError} when TypeScript cannot be loaded
|
|
107
105
|
*/
|
|
108
|
-
async function
|
|
106
|
+
async function importMigrationModule(filepath, options = {}) {
|
|
109
107
|
try {
|
|
110
108
|
await fs.access(filepath);
|
|
111
109
|
} catch {
|
|
@@ -123,7 +121,48 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
123
121
|
throw error;
|
|
124
122
|
}
|
|
125
123
|
// `mod.default ?? mod` handles the CommonJS default-export case
|
|
126
|
-
|
|
124
|
+
return imported.default ?? imported;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The `requires` export, validated against the file's own name */
|
|
128
|
+
function readRequires(resolved, filepath) {
|
|
129
|
+
if (resolved.requires === undefined) return {};
|
|
130
|
+
const issues = requiresIssues(resolved.requires, path.basename(filepath));
|
|
131
|
+
if (issues.length > 0) {
|
|
132
|
+
throw new MigrationInvalidExportError(`Invalid ${issues[0].path}: ${issues[0].message}`, {
|
|
133
|
+
filepath,
|
|
134
|
+
issues,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return { requires: [...resolved.requires] };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* What a migration module exports, validated: a regular migration
|
|
142
|
+
* (`{ up, down, useTransaction?, timeoutMs?, description?, requires? }`) or a
|
|
143
|
+
* background one (`{ kind: 'background', background, description?,
|
|
144
|
+
* requires? }` — its spec is validated where the collection's versioning is
|
|
145
|
+
* known). A file with both `background` and `up`/`down` is refused: the
|
|
146
|
+
* expand steps belong in a migration of their own.
|
|
147
|
+
*
|
|
148
|
+
* @throws {MigrationInvalidExportError}
|
|
149
|
+
*/
|
|
150
|
+
function resolveMigrationExports(resolved, filepath) {
|
|
151
|
+
if (resolved.background !== undefined) {
|
|
152
|
+
if (resolved.up !== undefined || resolved.down !== undefined) {
|
|
153
|
+
throw new MigrationInvalidExportError(
|
|
154
|
+
'A background migration exports no up() or down() — put the expand steps in a ' +
|
|
155
|
+
'migration of their own',
|
|
156
|
+
{ filepath },
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
kind: 'background',
|
|
161
|
+
background: resolved.background,
|
|
162
|
+
...(typeof resolved.description === 'string' ? { description: resolved.description } : {}),
|
|
163
|
+
...readRequires(resolved, filepath),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
127
166
|
|
|
128
167
|
if (!isFunction(resolved.up) || !isFunction(resolved.down)) {
|
|
129
168
|
throw new MigrationInvalidExportError('Migration must export async up() and down() functions', {
|
|
@@ -142,8 +181,37 @@ async function loadMigrationFile(filepath, options = {}) {
|
|
|
142
181
|
if (typeof resolved.description === 'string') {
|
|
143
182
|
migration.description = resolved.description;
|
|
144
183
|
}
|
|
184
|
+
Object.assign(migration, readRequires(resolved, filepath));
|
|
145
185
|
|
|
146
186
|
return migration;
|
|
147
187
|
}
|
|
148
188
|
|
|
149
|
-
|
|
189
|
+
/**
|
|
190
|
+
* Dynamically load a migration file and validate its exports.
|
|
191
|
+
*
|
|
192
|
+
* Handles all three supported formats:
|
|
193
|
+
* - TypeScript / JavaScript ESM named exports (`export async function up/down`)
|
|
194
|
+
* - CommonJS default export (`module.exports = { up, down }`)
|
|
195
|
+
*
|
|
196
|
+
* A background migration (`kind: 'background'`) is returned like any other:
|
|
197
|
+
* the caller tells it apart by its kind.
|
|
198
|
+
*
|
|
199
|
+
* @throws {MigrationFileNotFoundError} when the file does not exist
|
|
200
|
+
* @throws {MigrationInvalidExportError} when up/down are not both functions
|
|
201
|
+
*/
|
|
202
|
+
async function loadMigrationFile(filepath, options = {}) {
|
|
203
|
+
const migration = resolveMigrationExports(
|
|
204
|
+
await importMigrationModule(filepath, options),
|
|
205
|
+
filepath,
|
|
206
|
+
);
|
|
207
|
+
return migration;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
module.exports = {
|
|
211
|
+
importMigrationModule,
|
|
212
|
+
importUserFile,
|
|
213
|
+
loadMigrationFile,
|
|
214
|
+
resolveMigrationExports,
|
|
215
|
+
tsLoadErrorOrNull,
|
|
216
|
+
tsLoadMessageOrNull,
|
|
217
|
+
};
|
|
@@ -29,4 +29,36 @@ function assertMigrationName(name, context = {}) {
|
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Why a file's `requires` export is not valid: an array of bare migration
|
|
34
|
+
* file names, no duplicates, each sorting strictly before the file itself
|
|
35
|
+
* (`name`). Files run in name order, so an edge that only ever points
|
|
36
|
+
* backwards can never close a cycle — the whole "is it a DAG?" question,
|
|
37
|
+
* answered by the name. Returns `{ path, message }` issues.
|
|
38
|
+
*/
|
|
39
|
+
function requiresIssues(requires, name) {
|
|
40
|
+
if (requires === undefined) return [];
|
|
41
|
+
if (!Array.isArray(requires)) {
|
|
42
|
+
return [{ path: 'requires', message: 'must be an array of migration file names' }];
|
|
43
|
+
}
|
|
44
|
+
const issues = [];
|
|
45
|
+
const seen = new Set();
|
|
46
|
+
for (const [position, required] of requires.entries()) {
|
|
47
|
+
const path = `requires[${position}]`;
|
|
48
|
+
if (!isBareFilename(required)) {
|
|
49
|
+
issues.push({ path, message: 'must be a bare migration file name' });
|
|
50
|
+
} else if (seen.has(required)) {
|
|
51
|
+
issues.push({ path, message: `names "${required}" twice` });
|
|
52
|
+
} else if (name !== undefined && required >= name) {
|
|
53
|
+
issues.push({
|
|
54
|
+
path,
|
|
55
|
+
message: `must name an earlier migration ("${required}" does not sort before "${name}")`,
|
|
56
|
+
});
|
|
57
|
+
} else {
|
|
58
|
+
seen.add(required);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return issues;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { assertMigrationName, isBareFilename, requiresIssues };
|
package/src/utils/redact.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
const { isPlainObject } = require('./canonical.js');
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Credential redaction for anything that leaves the process — error messages,
|
|
3
5
|
* stacks, `--json` payloads, log lines. The MongoDB driver echoes the raw
|
|
@@ -69,6 +71,24 @@ function redactOutbound(text) {
|
|
|
69
71
|
return redactUris(text).replace(DUPLICATE_KEY_VALUES, '$1{ <redacted> }');
|
|
70
72
|
}
|
|
71
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Set `key` on a copy as an own property. A plain assignment of `__proto__` —
|
|
76
|
+
* a key `JSON.parse` happily makes — would replace the copy's prototype and
|
|
77
|
+
* lose the key instead.
|
|
78
|
+
*/
|
|
79
|
+
function put(target, key, value) {
|
|
80
|
+
if (key === '__proto__') {
|
|
81
|
+
Object.defineProperty(target, key, {
|
|
82
|
+
value,
|
|
83
|
+
enumerable: true,
|
|
84
|
+
writable: true,
|
|
85
|
+
configurable: true,
|
|
86
|
+
});
|
|
87
|
+
} else {
|
|
88
|
+
target[key] = value;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
72
92
|
/**
|
|
73
93
|
* Redact every string reachable from `value` (plain objects and arrays only —
|
|
74
94
|
* class instances are left alone rather than cloned into broken shapes).
|
|
@@ -81,12 +101,129 @@ function redactDeep(value) {
|
|
|
81
101
|
for (let index = 0; index < value.length; index++) copy[index] = redactDeep(value[index]);
|
|
82
102
|
return copy;
|
|
83
103
|
}
|
|
84
|
-
if (value
|
|
104
|
+
if (isPlainObject(value)) {
|
|
85
105
|
const copy = {};
|
|
86
|
-
for (const key of Object.keys(value)) copy
|
|
106
|
+
for (const key of Object.keys(value)) put(copy, key, redactDeep(value[key]));
|
|
87
107
|
return copy;
|
|
88
108
|
}
|
|
89
109
|
return value;
|
|
90
110
|
}
|
|
91
111
|
|
|
92
|
-
|
|
112
|
+
/**
|
|
113
|
+
* How much of a value {@link redactBounded} copies: nesting, entries in all,
|
|
114
|
+
* the length of one string and of one piece of binary data. What a migration
|
|
115
|
+
* hands to `ctx.logger` is the application's own data, of any size and shape —
|
|
116
|
+
* a cycle included — and a subscriber stores it; these keep one call's copy
|
|
117
|
+
* small and finite.
|
|
118
|
+
*/
|
|
119
|
+
const BOUNDS = Object.freeze({ depth: 8, entries: 1000, string: 4096, bytes: 4096 });
|
|
120
|
+
|
|
121
|
+
/** Stands in for what {@link redactBounded} left out */
|
|
122
|
+
const TRUNCATED = '[truncated]';
|
|
123
|
+
|
|
124
|
+
/** The size of binary data — a Buffer or another Uint8Array, a BSON Binary — or undefined */
|
|
125
|
+
function byteLength(item) {
|
|
126
|
+
if (item instanceof Uint8Array) return item.byteLength;
|
|
127
|
+
if (item._bsontype === 'Binary') return item.position;
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* An Error as data: its name, its message — credentials and the values a
|
|
133
|
+
* server error quotes masked, as for anything that leaves the process — and
|
|
134
|
+
* its code. Not the stack, and not what else it carries: a driver error's raw
|
|
135
|
+
* server response repeats the offending document's values (`keyValue`,
|
|
136
|
+
* `errmsg`), and its `message` is not even enumerable, so the error itself
|
|
137
|
+
* would be stored as an empty document.
|
|
138
|
+
*/
|
|
139
|
+
function errorData(error) {
|
|
140
|
+
const data = {
|
|
141
|
+
name: String(error.name),
|
|
142
|
+
message: redactOutbound(typeof error.message === 'string' ? error.message : ''),
|
|
143
|
+
};
|
|
144
|
+
if (typeof error.code === 'number' || typeof error.code === 'string') data.code = error.code;
|
|
145
|
+
if (typeof error.codeName === 'string') data.codeName = error.codeName;
|
|
146
|
+
return data;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* {@link redactDeep} within {@link BOUNDS}, for data that leaves the process
|
|
151
|
+
* as a document: every string reachable redacted ({@link redactOutbound}: the
|
|
152
|
+
* values a server error quotes too) and clipped, nesting past
|
|
153
|
+
* the depth (which is what ends a cycle) and entries past the budget replaced
|
|
154
|
+
* or dropped. What the driver stores as a value of its own is kept as is — a
|
|
155
|
+
* Date, a RegExp, an ObjectId or another BSON value, and binary data within
|
|
156
|
+
* its bound. An Error becomes `{ name, message, code?, codeName? }`; a Map is
|
|
157
|
+
* copied as an object and a Set as an array; any other instance goes by its
|
|
158
|
+
* `toJSON()`, or its own fields, the way `JSON.stringify` would see it.
|
|
159
|
+
* `omit` names a key left out of the top level only. Returns
|
|
160
|
+
* `{ value, truncated }`; never mutates the input.
|
|
161
|
+
*/
|
|
162
|
+
function redactBounded(value, { omit } = {}) {
|
|
163
|
+
let entries = 0;
|
|
164
|
+
let truncated = false;
|
|
165
|
+
/** One more entry, or false once the budget is spent */
|
|
166
|
+
const take = () => {
|
|
167
|
+
if (entries >= BOUNDS.entries) {
|
|
168
|
+
truncated = true;
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
entries += 1;
|
|
172
|
+
return true;
|
|
173
|
+
};
|
|
174
|
+
const copy = (item, depth) => {
|
|
175
|
+
if (typeof item === 'string') {
|
|
176
|
+
// Outbound, like the event's message: an error's text copied into a
|
|
177
|
+
// field (`{ error: err.message }`) loses the values it quotes too.
|
|
178
|
+
const text = redactOutbound(item);
|
|
179
|
+
if (text.length <= BOUNDS.string) return text;
|
|
180
|
+
truncated = true;
|
|
181
|
+
return `${text.slice(0, BOUNDS.string)}…`;
|
|
182
|
+
}
|
|
183
|
+
if (item === null || typeof item !== 'object') return item;
|
|
184
|
+
if (item instanceof Date || item instanceof RegExp) return item;
|
|
185
|
+
const bytes = byteLength(item);
|
|
186
|
+
if (bytes !== undefined) {
|
|
187
|
+
if (bytes <= BOUNDS.bytes) return item;
|
|
188
|
+
truncated = true;
|
|
189
|
+
return TRUNCATED;
|
|
190
|
+
}
|
|
191
|
+
if (typeof item._bsontype === 'string') return item;
|
|
192
|
+
if (item instanceof Error) return copy(errorData(item), depth);
|
|
193
|
+
if (depth >= BOUNDS.depth) {
|
|
194
|
+
truncated = true;
|
|
195
|
+
return TRUNCATED;
|
|
196
|
+
}
|
|
197
|
+
if (Array.isArray(item) || item instanceof Set) {
|
|
198
|
+
const out = [];
|
|
199
|
+
for (const element of item) {
|
|
200
|
+
if (!take()) break;
|
|
201
|
+
out.push(copy(element, depth + 1));
|
|
202
|
+
}
|
|
203
|
+
return out;
|
|
204
|
+
}
|
|
205
|
+
const out = {};
|
|
206
|
+
if (item instanceof Map) {
|
|
207
|
+
for (const [key, element] of item) {
|
|
208
|
+
if (!take()) break;
|
|
209
|
+
put(out, String(key), copy(element, depth + 1));
|
|
210
|
+
}
|
|
211
|
+
return out;
|
|
212
|
+
}
|
|
213
|
+
// What JSON.stringify would see: a toJSON() result one level down (so a
|
|
214
|
+
// chain of them ends at the depth bound), else the instance's own fields.
|
|
215
|
+
if (!isPlainObject(item) && typeof item.toJSON === 'function') {
|
|
216
|
+
return copy(item.toJSON(), depth + 1);
|
|
217
|
+
}
|
|
218
|
+
for (const key of Object.keys(item)) {
|
|
219
|
+
if (depth === 0 && key === omit) continue;
|
|
220
|
+
if (!take()) break;
|
|
221
|
+
put(out, key, copy(item[key], depth + 1));
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
};
|
|
225
|
+
const result = copy(value, 0);
|
|
226
|
+
return { value: result, truncated };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
module.exports = { BOUNDS, redactBounded, redactDeep, redactOutbound, redactUris };
|
package/src/utils/telemetry.js
CHANGED
|
@@ -13,6 +13,8 @@ const SPAN_STATUS_ERROR = 2;
|
|
|
13
13
|
const SPANS = {
|
|
14
14
|
RUN: 'migronaut.run',
|
|
15
15
|
MIGRATION: 'migronaut.migration',
|
|
16
|
+
BACKGROUND_SLICE: 'migronaut.background.slice',
|
|
17
|
+
BACKGROUND_COORDINATE: 'migronaut.background.coordinate',
|
|
16
18
|
};
|
|
17
19
|
|
|
18
20
|
/** Every attribute key migronaut sets, on spans and on metric points */
|
|
@@ -35,6 +37,14 @@ const ATTRIBUTES = {
|
|
|
35
37
|
MIGRATION_INDEX: 'migronaut.migration.index',
|
|
36
38
|
MIGRATION_TOTAL: 'migronaut.migration.total',
|
|
37
39
|
MIGRATION_TRANSACTION: 'migronaut.migration.transaction',
|
|
40
|
+
MIGRATION_ATTEMPTS: 'migronaut.migration.attempts',
|
|
41
|
+
JOB_ID: 'migronaut.job.id',
|
|
42
|
+
JOB_GROUP_ID: 'migronaut.job.group_id',
|
|
43
|
+
BACKGROUND_NAME: 'migronaut.background.name',
|
|
44
|
+
BACKGROUND_OUTCOME: 'migronaut.background.outcome',
|
|
45
|
+
BACKGROUND_RESULT: 'migronaut.background.result',
|
|
46
|
+
BACKGROUND_REASON: 'migronaut.background.reason',
|
|
47
|
+
BACKGROUND_SHARD: 'migronaut.background.shard',
|
|
38
48
|
ERROR_TYPE: 'error.type',
|
|
39
49
|
/** The database a run is against — OpenTelemetry's database semantic convention */
|
|
40
50
|
DB_NAMESPACE: 'db.namespace',
|
|
@@ -48,6 +58,14 @@ const METRICS = {
|
|
|
48
58
|
LOCK_REFUSED: 'migronaut.lock.refused',
|
|
49
59
|
LOCK_LOST: 'migronaut.lock.lost',
|
|
50
60
|
SEARCH_WAIT_DURATION: 'migronaut.converge.search.wait.duration',
|
|
61
|
+
BACKGROUND_DOCUMENTS: 'migronaut.background.documents',
|
|
62
|
+
BACKGROUND_SLICE_DURATION: 'migronaut.background.slice.duration',
|
|
63
|
+
BACKGROUND_BATCH_WRITE_DURATION: 'migronaut.background.batch.write.duration',
|
|
64
|
+
BACKGROUND_THROTTLE: 'migronaut.background.throttled',
|
|
65
|
+
BACKGROUND_DRIFT: 'migronaut.background.drift.detected',
|
|
66
|
+
BACKGROUND_TRANSACTION_RETRIES: 'migronaut.background.transaction.retried',
|
|
67
|
+
BACKGROUND_LEASES_RECLAIMED: 'migronaut.background.leases.reclaimed',
|
|
68
|
+
BACKGROUND_WATCH_DELAY: 'migronaut.background.watch.delay',
|
|
51
69
|
};
|
|
52
70
|
|
|
53
71
|
/**
|
|
@@ -339,6 +357,47 @@ function createTelemetry(telemetry, { dbName } = {}) {
|
|
|
339
357
|
'Time a converge waited for its search index builds, by how the wait ended',
|
|
340
358
|
);
|
|
341
359
|
|
|
360
|
+
const backgroundDocuments = counter(
|
|
361
|
+
METRICS.BACKGROUND_DOCUMENTS,
|
|
362
|
+
'Documents a background migration handled, by result (migrated, skipped, conflict, failed)',
|
|
363
|
+
'{document}',
|
|
364
|
+
);
|
|
365
|
+
const backgroundSliceDuration = histogram(
|
|
366
|
+
METRICS.BACKGROUND_SLICE_DURATION,
|
|
367
|
+
'Duration of one background migration slice, by outcome',
|
|
368
|
+
);
|
|
369
|
+
const backgroundBatchWrite = histogram(
|
|
370
|
+
METRICS.BACKGROUND_BATCH_WRITE_DURATION,
|
|
371
|
+
'Time to write one background migration batch',
|
|
372
|
+
);
|
|
373
|
+
const backgroundThrottle = counter(
|
|
374
|
+
METRICS.BACKGROUND_THROTTLE,
|
|
375
|
+
'Adaptive throttle changes of background migrations, by reason',
|
|
376
|
+
'{change}',
|
|
377
|
+
);
|
|
378
|
+
const backgroundDrift = counter(
|
|
379
|
+
METRICS.BACKGROUND_DRIFT,
|
|
380
|
+
'Old-shape documents found after a background migration completed',
|
|
381
|
+
'{finding}',
|
|
382
|
+
);
|
|
383
|
+
const backgroundTransactionRetries = counter(
|
|
384
|
+
METRICS.BACKGROUND_TRANSACTION_RETRIES,
|
|
385
|
+
'Transactional background batches retried, by reason',
|
|
386
|
+
'{retry}',
|
|
387
|
+
);
|
|
388
|
+
const backgroundLeasesReclaimed = counter(
|
|
389
|
+
METRICS.BACKGROUND_LEASES_RECLAIMED,
|
|
390
|
+
'Partition leases reclaimed from a lane that stopped renewing',
|
|
391
|
+
'{lease}',
|
|
392
|
+
);
|
|
393
|
+
const backgroundWatchDelay = histogram(
|
|
394
|
+
METRICS.BACKGROUND_WATCH_DELAY,
|
|
395
|
+
'Time from an old-shape write to its upgrade by the live drift watcher',
|
|
396
|
+
);
|
|
397
|
+
const add = (instrument, value, attributes) => {
|
|
398
|
+
if (instrument && value > 0) safe(() => instrument.add(value, withBase(attributes)));
|
|
399
|
+
};
|
|
400
|
+
|
|
342
401
|
// Durations are measured in milliseconds everywhere in migronaut and
|
|
343
402
|
// reported in seconds, the unit OpenTelemetry's conventions settle on.
|
|
344
403
|
const record = (instrument, durationMs, attributes) => {
|
|
@@ -391,6 +450,57 @@ function createTelemetry(telemetry, { dbName } = {}) {
|
|
|
391
450
|
searchWaited({ waitedMs, outcome }) {
|
|
392
451
|
record(searchWaitDuration, waitedMs, { [ATTRIBUTES.SEARCH_WAIT_OUTCOME]: outcome });
|
|
393
452
|
},
|
|
453
|
+
/**
|
|
454
|
+
* A background migration slice ended: its duration by outcome, and the
|
|
455
|
+
* documents it handled by result. The partition is never an attribute —
|
|
456
|
+
* dimensions stay low-cardinality.
|
|
457
|
+
*/
|
|
458
|
+
backgroundSliceEnded({ name, durationMs, outcome, counters = {}, error }) {
|
|
459
|
+
const at = { [ATTRIBUTES.BACKGROUND_NAME]: name };
|
|
460
|
+
record(backgroundSliceDuration, durationMs, {
|
|
461
|
+
...at,
|
|
462
|
+
[ATTRIBUTES.BACKGROUND_OUTCOME]: outcome,
|
|
463
|
+
...failure(error),
|
|
464
|
+
});
|
|
465
|
+
for (const [key, result] of [
|
|
466
|
+
['migrated', 'migrated'],
|
|
467
|
+
['skipped', 'skipped'],
|
|
468
|
+
['conflicts', 'conflict'],
|
|
469
|
+
['failed', 'failed'],
|
|
470
|
+
]) {
|
|
471
|
+
add(backgroundDocuments, counters[key] ?? 0, {
|
|
472
|
+
...at,
|
|
473
|
+
[ATTRIBUTES.BACKGROUND_RESULT]: result,
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
},
|
|
477
|
+
backgroundBatchWritten({ name, durationMs, shard }) {
|
|
478
|
+
record(backgroundBatchWrite, durationMs, {
|
|
479
|
+
[ATTRIBUTES.BACKGROUND_NAME]: name,
|
|
480
|
+
[ATTRIBUTES.BACKGROUND_SHARD]: shard,
|
|
481
|
+
});
|
|
482
|
+
},
|
|
483
|
+
backgroundThrottled({ name, reason }) {
|
|
484
|
+
add(backgroundThrottle, 1, {
|
|
485
|
+
[ATTRIBUTES.BACKGROUND_NAME]: name,
|
|
486
|
+
[ATTRIBUTES.BACKGROUND_REASON]: reason,
|
|
487
|
+
});
|
|
488
|
+
},
|
|
489
|
+
backgroundDrift({ name, count = 1 }) {
|
|
490
|
+
add(backgroundDrift, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
|
|
491
|
+
},
|
|
492
|
+
backgroundTransactionRetried({ name, reason, count = 1 }) {
|
|
493
|
+
add(backgroundTransactionRetries, count, {
|
|
494
|
+
[ATTRIBUTES.BACKGROUND_NAME]: name,
|
|
495
|
+
[ATTRIBUTES.BACKGROUND_REASON]: reason,
|
|
496
|
+
});
|
|
497
|
+
},
|
|
498
|
+
backgroundLeasesReclaimed({ name, count }) {
|
|
499
|
+
add(backgroundLeasesReclaimed, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
|
|
500
|
+
},
|
|
501
|
+
backgroundWatchDelay({ name, delayMs }) {
|
|
502
|
+
record(backgroundWatchDelay, delayMs, { [ATTRIBUTES.BACKGROUND_NAME]: name });
|
|
503
|
+
},
|
|
394
504
|
};
|
|
395
505
|
}
|
|
396
506
|
|
package/src/utils/template.js
CHANGED
|
@@ -145,6 +145,51 @@ module.exports = { description, up, down };
|
|
|
145
145
|
`;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
/**
|
|
149
|
+
* The built-in background migration template (`create --background`): the
|
|
150
|
+
* declarative form, with the knobs most worth knowing about spelled out.
|
|
151
|
+
*/
|
|
152
|
+
function defaultBackgroundTemplate(js, esm = false) {
|
|
153
|
+
const typed = js
|
|
154
|
+
? "/** @type {import('@alexify/migronaut').DeclarativeBackgroundMigration} */\n"
|
|
155
|
+
: '';
|
|
156
|
+
const typeImport = js
|
|
157
|
+
? ''
|
|
158
|
+
: "import type { DeclarativeBackgroundMigration } from '@alexify/migronaut';\n\n";
|
|
159
|
+
const annotation = js ? '' : ': DeclarativeBackgroundMigration';
|
|
160
|
+
const body = `{
|
|
161
|
+
// The collection to rewrite (\`up\` refuses this placeholder).
|
|
162
|
+
collection: 'TODO',
|
|
163
|
+
// Documents at version \`from\` (0: no version field yet) become version \`to\`.
|
|
164
|
+
from: 1,
|
|
165
|
+
to: 2,
|
|
166
|
+
// The new document for one old one — return it reshaped; migronaut sets the
|
|
167
|
+
// version, bumps the revision and writes only the fields that changed.
|
|
168
|
+
migrate: (doc) => {
|
|
169
|
+
// TODO: reshape doc, and return it
|
|
170
|
+
throw new Error('migrate is not written yet');
|
|
171
|
+
},
|
|
172
|
+
// The way back, for \`down\`. Without one, \`down\` refuses once documents
|
|
173
|
+
// were rewritten. Never \`(doc) => doc\`: that would stamp the old version
|
|
174
|
+
// on documents still in the new shape.
|
|
175
|
+
// revert: ({ shipping, ...doc }) => ({ ...doc, address: shipping.address }),
|
|
176
|
+
// Partitions worked at once, across every process (default 1).
|
|
177
|
+
// maxParallel: 4,
|
|
178
|
+
}`;
|
|
179
|
+
if (esm) {
|
|
180
|
+
return `${typeImport}export const description = '';
|
|
181
|
+
|
|
182
|
+
${typed}export const background${annotation} = ${body};
|
|
183
|
+
`;
|
|
184
|
+
}
|
|
185
|
+
return `${typeImport}const description = '';
|
|
186
|
+
|
|
187
|
+
${typed}const background${annotation} = ${body};
|
|
188
|
+
|
|
189
|
+
module.exports = { description, background };
|
|
190
|
+
`;
|
|
191
|
+
}
|
|
192
|
+
|
|
148
193
|
/** Extensions a custom `--template` file may have — anything else is refused */
|
|
149
194
|
const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
|
|
150
195
|
|
|
@@ -152,7 +197,8 @@ const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
|
|
|
152
197
|
const MAX_TEMPLATE_BYTES = 1024 * 1024;
|
|
153
198
|
|
|
154
199
|
/** Resolve template file contents — a custom template if provided, else the built-in */
|
|
155
|
-
async function resolveTemplateContent(templatePath, js, esm = false) {
|
|
200
|
+
async function resolveTemplateContent(templatePath, js, esm = false, background = false) {
|
|
201
|
+
if (background) return defaultBackgroundTemplate(js, esm);
|
|
156
202
|
if (templatePath) {
|
|
157
203
|
const ext = path.extname(templatePath);
|
|
158
204
|
if (!TEMPLATE_EXTENSIONS.includes(ext)) {
|
|
@@ -198,6 +244,7 @@ async function createMigrationFile(options) {
|
|
|
198
244
|
// Match the project's module system, so a generated migration never makes
|
|
199
245
|
// Node reparse it and warn.
|
|
200
246
|
await isEsmProject(options.dir),
|
|
247
|
+
options.background === true,
|
|
201
248
|
);
|
|
202
249
|
try {
|
|
203
250
|
// 'wx' fails if the path exists — creating a migration must never silently
|
|
@@ -366,6 +413,20 @@ function configBody(values, createExtension) {
|
|
|
366
413
|
// waitForSearchIndexes: false,
|
|
367
414
|
// searchIndexWaitTimeoutMs: 600000,
|
|
368
415
|
|
|
416
|
+
// ── Background migrations (experimental) ────────────────────
|
|
417
|
+
// A migration file with \`export const background = {…}\` is registered by
|
|
418
|
+
// \`up\` and runs in partitions (BullMQ, \`migronaut background run\`, or an
|
|
419
|
+
// in-process runner) without holding the migration lock.
|
|
420
|
+
// backgroundCollection: '_migronaut_background',
|
|
421
|
+
// Run it to the end inside the \`up\` that registers it instead.
|
|
422
|
+
// backgroundInline: false,
|
|
423
|
+
// Old-shape documents after it completed: reopen it ('reopen') or report.
|
|
424
|
+
// backgroundOnDrift: 'reopen',
|
|
425
|
+
// How drift is watched: 'poll' (every 10 minutes), 'stream', or 'both'.
|
|
426
|
+
// backgroundDrift: 'poll',
|
|
427
|
+
// Partition a sharded collection by its shard key ('auto') or not ('off').
|
|
428
|
+
// backgroundShardAware: 'auto',
|
|
429
|
+
|
|
369
430
|
// ── Lifecycle hooks (code only — not available in JSON config) ──
|
|
370
431
|
// hooks: {
|
|
371
432
|
// beforeAll: async (ctx) => {},
|