@alexify/migronaut 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +409 -1
- package/README.md +248 -24
- package/bin/migronaut.js +11 -3
- package/bullmq.d.ts +845 -0
- package/bullmq.js +1 -0
- package/index.d.ts +757 -29
- package/migronaut.schema.json +191 -1
- package/package.json +27 -6
- package/src/bullmq/index.js +55 -0
- package/src/bullmq/jobs.js +454 -0
- package/src/bullmq/processor.js +608 -0
- package/src/bullmq/producer.js +424 -0
- package/src/bullmq/service.js +653 -0
- package/src/bullmq/wait.js +124 -0
- package/src/cli/args.js +12 -2
- package/src/cli/commands/baseline.js +45 -0
- package/src/cli/commands/converge.js +160 -0
- package/src/cli/commands/down.js +2 -0
- package/src/cli/commands/lock.js +2 -1
- package/src/cli/commands/redo.js +8 -1
- package/src/cli/commands/unlock.js +12 -2
- package/src/cli/commands/up.js +14 -1
- package/src/cli/exit-codes.js +10 -2
- package/src/cli/index.js +4 -0
- package/src/cli/shared.js +29 -7
- package/src/cli/table.js +105 -0
- package/src/core/audit.js +17 -3
- package/src/core/baseline.js +80 -0
- package/src/core/changelog.js +140 -24
- package/src/core/collections.js +372 -0
- package/src/core/config.js +125 -27
- package/src/core/converge-log.js +47 -0
- package/src/core/converge-plan.js +483 -0
- package/src/core/converge.js +867 -0
- package/src/core/import-runner.js +34 -6
- package/src/core/import.js +14 -7
- package/src/core/index-spec.js +496 -0
- package/src/core/lock-wait.js +260 -0
- package/src/core/lock.js +71 -20
- package/src/core/migrator.js +805 -304
- package/src/core/options.js +251 -0
- package/src/core/run-recorder.js +157 -0
- package/src/core/run.js +71 -71
- package/src/core/runner.js +70 -20
- package/src/core/sequence.js +134 -0
- package/src/errors/index.js +71 -1
- package/src/index.js +16 -0
- package/src/utils/actor.js +48 -0
- package/src/utils/canonical.js +179 -0
- package/src/utils/collection-name.js +21 -0
- package/src/utils/error.js +18 -1
- package/src/utils/id.js +77 -0
- package/src/utils/loader.js +39 -21
- package/src/utils/logger.js +30 -12
- package/src/utils/migration-name.js +32 -0
- package/src/utils/redact.js +57 -4
- package/src/utils/sanitize.js +8 -3
- package/src/utils/telemetry.js +393 -0
- package/src/utils/template.js +60 -12
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
const fs = require('node:fs/promises');
|
|
2
|
+
const path = require('node:path');
|
|
3
|
+
const { ConfigInvalidError } = require('../errors/index.js');
|
|
4
|
+
const { isPlainObject, regExpIssue, toWire } = require('../utils/canonical.js');
|
|
5
|
+
const { isCollectionName } = require('../utils/collection-name.js');
|
|
6
|
+
const { mapLimit } = require('../utils/concurrency.js');
|
|
7
|
+
const { errorText } = require('../utils/error.js');
|
|
8
|
+
const { importUserFile, tsLoadMessageOrNull } = require('../utils/loader.js');
|
|
9
|
+
const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('./index-spec.js');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Declared collections: validating a definition, normalizing it for the
|
|
13
|
+
* planner, and gathering definitions from their two sources — the
|
|
14
|
+
* `collections` config key and the files in `collectionsDir`.
|
|
15
|
+
*
|
|
16
|
+
* Knows nothing about the database. Definitions are validated strictly: an
|
|
17
|
+
* unknown key is an error rather than ignored, because every typo here
|
|
18
|
+
* (`indexs`, `validtor`) would otherwise read as "not managed" and silently
|
|
19
|
+
* leave the database alone.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** Every key a collection definition may carry */
|
|
23
|
+
const DEFINITION_KEYS = [
|
|
24
|
+
'name',
|
|
25
|
+
'indexes',
|
|
26
|
+
'validator',
|
|
27
|
+
'validationLevel',
|
|
28
|
+
'validationAction',
|
|
29
|
+
'prune',
|
|
30
|
+
];
|
|
31
|
+
const DEFINITION_KEY_SET = new Set(DEFINITION_KEYS);
|
|
32
|
+
const VALIDATION_LEVELS = ['off', 'strict', 'moderate'];
|
|
33
|
+
const VALIDATION_ACTIONS = ['error', 'warn', 'errorAndLog'];
|
|
34
|
+
|
|
35
|
+
/** Simultaneous definition-file loads — the same EMFILE bound as every other multi-file path */
|
|
36
|
+
const FS_CONCURRENCY = 16;
|
|
37
|
+
|
|
38
|
+
/** `collections[2]` + `indexes` → `collections[2].indexes`; `users.ts:` + `indexes` → `users.ts: indexes` */
|
|
39
|
+
function join(base, key) {
|
|
40
|
+
return base.endsWith(':') ? `${base} ${key}` : `${base}.${key}`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** What makes a validator unsendable — a function, a symbol, a cycle — or null */
|
|
44
|
+
function unsendable(value, seen = new Set()) {
|
|
45
|
+
if (seen.size === 0) {
|
|
46
|
+
const issue = regExpIssue(value);
|
|
47
|
+
if (issue) return issue;
|
|
48
|
+
}
|
|
49
|
+
const type = typeof value;
|
|
50
|
+
if (type === 'function') return 'must not contain functions';
|
|
51
|
+
if (type === 'symbol') return 'must not contain symbols';
|
|
52
|
+
if (value === null || type !== 'object') return null;
|
|
53
|
+
if (seen.has(value)) return 'must not contain circular references';
|
|
54
|
+
seen.add(value);
|
|
55
|
+
const items = Array.isArray(value) ? value : isPlainObject(value) ? Object.values(value) : [];
|
|
56
|
+
for (const item of items) {
|
|
57
|
+
const reason = unsendable(item, seen);
|
|
58
|
+
if (reason) return reason;
|
|
59
|
+
}
|
|
60
|
+
seen.delete(value);
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const isEmptyObject = (value) => isPlainObject(value) && Object.keys(value).length === 0;
|
|
65
|
+
|
|
66
|
+
function indexListIssues(indexes, base, issues) {
|
|
67
|
+
if (!Array.isArray(indexes)) {
|
|
68
|
+
issues.push({ path: join(base, 'indexes'), message: 'must be an array of index definitions' });
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const valid = [];
|
|
72
|
+
for (const [position, index] of indexes.entries()) {
|
|
73
|
+
const indexPath = `${join(base, 'indexes')}[${position}]`;
|
|
74
|
+
const found = indexIssues(index, indexPath);
|
|
75
|
+
issues.push(...found);
|
|
76
|
+
if (found.length === 0) {
|
|
77
|
+
valid.push({ position, path: indexPath, index: normalizeDeclaredIndex(index) });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (let a = 1; a < valid.length; a++) {
|
|
81
|
+
for (let b = 0; b < a; b++) {
|
|
82
|
+
const later = valid[a];
|
|
83
|
+
const earlier = valid[b];
|
|
84
|
+
let message;
|
|
85
|
+
if (later.index.name === earlier.index.name) {
|
|
86
|
+
message = `has the same name as indexes[${earlier.position}] ("${later.index.name}")`;
|
|
87
|
+
} else if (later.index.isText && earlier.index.isText) {
|
|
88
|
+
message = `is a second text index (after indexes[${earlier.position}]) — a collection has at most one`;
|
|
89
|
+
} else if (sameDeclaredSignature(later.index, earlier.index)) {
|
|
90
|
+
message =
|
|
91
|
+
`has the same key, partialFilterExpression and collation as indexes[${earlier.position}] ` +
|
|
92
|
+
'— the server keeps only one of them';
|
|
93
|
+
}
|
|
94
|
+
if (message) {
|
|
95
|
+
issues.push({ path: later.path, message });
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Validate one collection definition, returning `{ path, message }` issues
|
|
104
|
+
* (empty when valid). `path` prefixes every issue (`collections[2]`, or
|
|
105
|
+
* `users.ts:` for a file); `reserved` are migronaut's own collection names;
|
|
106
|
+
* `fallbackName` is the name a file-based definition gets when it sets none.
|
|
107
|
+
*/
|
|
108
|
+
function definitionIssues(definition, { path: base, reserved = [], fallbackName } = {}) {
|
|
109
|
+
const self = base.endsWith(':') ? base.slice(0, -1) : base;
|
|
110
|
+
if (!isPlainObject(definition)) {
|
|
111
|
+
return [{ path: self, message: 'must be a collection definition object' }];
|
|
112
|
+
}
|
|
113
|
+
const issues = [];
|
|
114
|
+
const report = (key, message) => issues.push({ path: join(base, key), message });
|
|
115
|
+
for (const key of Object.keys(definition)) {
|
|
116
|
+
if (!DEFINITION_KEY_SET.has(key)) {
|
|
117
|
+
report(
|
|
118
|
+
key,
|
|
119
|
+
`is not a collection definition key (expected one of: ${DEFINITION_KEYS.join(', ')})`,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const name = definition.name ?? fallbackName;
|
|
125
|
+
const nameSource =
|
|
126
|
+
definition.name === undefined && fallbackName !== undefined ? 'file name' : 'name';
|
|
127
|
+
if (name === undefined) {
|
|
128
|
+
report('name', 'is required');
|
|
129
|
+
} else if (!isCollectionName(name)) {
|
|
130
|
+
report(
|
|
131
|
+
'name',
|
|
132
|
+
nameSource === 'name'
|
|
133
|
+
? "must be a valid collection name (no '$'/NUL, not system.*)"
|
|
134
|
+
: `the file name gives "${name}", which is not a valid collection name — set name explicitly`,
|
|
135
|
+
);
|
|
136
|
+
} else if (reserved.includes(name)) {
|
|
137
|
+
report('name', `"${name}" is one of migronaut's own collections`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (definition.indexes === undefined && definition.validator === undefined) {
|
|
141
|
+
issues.push({
|
|
142
|
+
path: self,
|
|
143
|
+
message: 'declares neither indexes nor a validator — nothing to manage',
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
if (definition.indexes !== undefined) indexListIssues(definition.indexes, base, issues);
|
|
147
|
+
|
|
148
|
+
const { validator } = definition;
|
|
149
|
+
if (validator !== undefined && validator !== null) {
|
|
150
|
+
if (!isPlainObject(validator)) {
|
|
151
|
+
report('validator', 'must be an object (a query or { $jsonSchema }), or null for none');
|
|
152
|
+
} else {
|
|
153
|
+
const reason = unsendable(validator);
|
|
154
|
+
if (reason) report('validator', reason);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const hasValidator = isPlainObject(validator) && !isEmptyObject(validator);
|
|
158
|
+
for (const [key, allowed] of [
|
|
159
|
+
['validationLevel', VALIDATION_LEVELS],
|
|
160
|
+
['validationAction', VALIDATION_ACTIONS],
|
|
161
|
+
]) {
|
|
162
|
+
const value = definition[key];
|
|
163
|
+
if (value === undefined) continue;
|
|
164
|
+
if (!allowed.includes(value)) {
|
|
165
|
+
report(key, `must be ${allowed.map((item) => `'${item}'`).join(', ')}`);
|
|
166
|
+
} else if (!hasValidator) {
|
|
167
|
+
report(key, 'has no effect without a validator');
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (definition.prune !== undefined && typeof definition.prune !== 'boolean') {
|
|
171
|
+
report('prune', 'must be a boolean');
|
|
172
|
+
}
|
|
173
|
+
return issues;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Validate the `collections` config key: each definition, plus no collection
|
|
178
|
+
* declared twice. `reserved` are the changelog and lock collection names.
|
|
179
|
+
*/
|
|
180
|
+
function collectionsIssues(list, { reserved = [] } = {}) {
|
|
181
|
+
if (list === undefined) return [];
|
|
182
|
+
if (!Array.isArray(list)) {
|
|
183
|
+
return [{ path: 'collections', message: 'must be an array of collection definitions' }];
|
|
184
|
+
}
|
|
185
|
+
const issues = [];
|
|
186
|
+
const seen = new Map();
|
|
187
|
+
for (const [position, definition] of list.entries()) {
|
|
188
|
+
const base = `collections[${position}]`;
|
|
189
|
+
issues.push(...definitionIssues(definition, { path: base, reserved }));
|
|
190
|
+
const name = isPlainObject(definition) ? definition.name : undefined;
|
|
191
|
+
if (typeof name !== 'string') continue;
|
|
192
|
+
if (seen.has(name)) {
|
|
193
|
+
issues.push({
|
|
194
|
+
path: `${base}.name`,
|
|
195
|
+
message: `declares "${name}" again (already collections[${seen.get(name)}])`,
|
|
196
|
+
});
|
|
197
|
+
} else {
|
|
198
|
+
seen.set(name, position);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return issues;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* A valid definition in the planner's shape: indexes normalized (effective
|
|
206
|
+
* names, server-form keys, the spec to send), the validator cleaned for the
|
|
207
|
+
* wire. `indexes`/`validator` stay `undefined` when not managed.
|
|
208
|
+
*/
|
|
209
|
+
function normalizeDefinition(definition, { name, source } = {}) {
|
|
210
|
+
const { validator } = definition;
|
|
211
|
+
return {
|
|
212
|
+
name: definition.name ?? name,
|
|
213
|
+
source,
|
|
214
|
+
indexes: definition.indexes?.map((index) => normalizeDeclaredIndex(index)),
|
|
215
|
+
validator: validator === undefined || validator === null ? validator : toWire(validator),
|
|
216
|
+
...(definition.validationLevel !== undefined
|
|
217
|
+
? { validationLevel: definition.validationLevel }
|
|
218
|
+
: {}),
|
|
219
|
+
...(definition.validationAction !== undefined
|
|
220
|
+
? { validationAction: definition.validationAction }
|
|
221
|
+
: {}),
|
|
222
|
+
...(definition.prune !== undefined ? { prune: definition.prune } : {}),
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** The extensions a definition file may have: `fileExtensions`, dotted, plus `.json` — longest first */
|
|
227
|
+
function definitionExtensions(extensions) {
|
|
228
|
+
const set = new Set(['.json']);
|
|
229
|
+
for (const ext of extensions) set.add(ext.startsWith('.') ? ext : `.${ext}`);
|
|
230
|
+
return [...set].sort((a, b) => b.length - a.length);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Load one definition file. JSON is parsed; anything else is imported, and
|
|
235
|
+
* its default export (or, for an ES module with only named exports, the
|
|
236
|
+
* exports themselves) is the definition. A function is refused rather than
|
|
237
|
+
* called — a Mongoose model is a function, and a definitions directory is
|
|
238
|
+
* exactly where one might be left by mistake.
|
|
239
|
+
*/
|
|
240
|
+
async function loadDefinitionFile(filepath, options) {
|
|
241
|
+
if (filepath.endsWith('.json')) {
|
|
242
|
+
const raw = await fs.readFile(filepath, 'utf8');
|
|
243
|
+
try {
|
|
244
|
+
return JSON.parse(raw);
|
|
245
|
+
} catch (error) {
|
|
246
|
+
throw new ConfigInvalidError(
|
|
247
|
+
'Collection definition file is not valid JSON',
|
|
248
|
+
{ path: filepath, cause: errorText(error) },
|
|
249
|
+
{ cause: error },
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
let mod;
|
|
254
|
+
try {
|
|
255
|
+
mod = await importUserFile(filepath, { reload: options.reload });
|
|
256
|
+
} catch (error) {
|
|
257
|
+
throw new ConfigInvalidError(
|
|
258
|
+
tsLoadMessageOrNull(filepath, error, 'collection definition') ??
|
|
259
|
+
'Collection definition file failed to load',
|
|
260
|
+
{ path: filepath, cause: errorText(error) },
|
|
261
|
+
{ cause: error },
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
const exported = mod.default ?? mod;
|
|
265
|
+
if (typeof exported === 'function') {
|
|
266
|
+
throw new ConfigInvalidError(
|
|
267
|
+
'A collection definition file must export one definition object, not a function',
|
|
268
|
+
{ path: filepath },
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
// A shallow copy: an ES module namespace is an exotic object, not a plain one.
|
|
272
|
+
return isPlainObject(exported) || exported === mod ? { ...exported } : exported;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Load every definition file in `dir`: one collection per file, non-recursive,
|
|
277
|
+
* sorted by file name. Dotfiles and declaration files (`.d.ts`) are skipped,
|
|
278
|
+
* like in the migrations directory. An explicitly configured directory that
|
|
279
|
+
* does not exist is an error, not "no definitions".
|
|
280
|
+
*/
|
|
281
|
+
async function loadCollectionsDir(dir, { extensions, reload = false }) {
|
|
282
|
+
let entries;
|
|
283
|
+
try {
|
|
284
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
285
|
+
} catch (error) {
|
|
286
|
+
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
|
|
287
|
+
throw new ConfigInvalidError('collectionsDir not found', { path: dir });
|
|
288
|
+
}
|
|
289
|
+
throw new ConfigInvalidError(
|
|
290
|
+
'collectionsDir could not be read',
|
|
291
|
+
{ path: dir, cause: errorText(error) },
|
|
292
|
+
{ cause: error },
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
const accepted = definitionExtensions(extensions);
|
|
296
|
+
const files = [];
|
|
297
|
+
for (const entry of entries) {
|
|
298
|
+
if (!entry.isFile()) continue;
|
|
299
|
+
const file = entry.name;
|
|
300
|
+
if (file.startsWith('.')) continue;
|
|
301
|
+
if (file.endsWith('.d.ts') || file.endsWith('.d.mts') || file.endsWith('.d.cts')) continue;
|
|
302
|
+
const ext = accepted.find(
|
|
303
|
+
(candidate) => file.endsWith(candidate) && file.length > candidate.length,
|
|
304
|
+
);
|
|
305
|
+
if (ext) files.push({ file, name: file.slice(0, -ext.length) });
|
|
306
|
+
}
|
|
307
|
+
files.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
|
|
308
|
+
return mapLimit(files, FS_CONCURRENCY, async ({ file, name }) => {
|
|
309
|
+
const filepath = path.join(dir, file);
|
|
310
|
+
return { file, name, definition: await loadDefinitionFile(filepath, { reload }) };
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* Every declared collection, normalized: the `collections` key first (already
|
|
316
|
+
* validated with the rest of the config), then the files in `dir`, if one is
|
|
317
|
+
* configured. A collection declared by both sources — or by two files — is
|
|
318
|
+
* an error: which declaration wins would otherwise depend on load order.
|
|
319
|
+
*
|
|
320
|
+
* @throws {ConfigInvalidError} on a missing directory, a file that does not
|
|
321
|
+
* load, or invalid definitions (all issues at once, in `context.issues`)
|
|
322
|
+
*/
|
|
323
|
+
async function resolveDefinitions({
|
|
324
|
+
inline,
|
|
325
|
+
dir,
|
|
326
|
+
extensions = ['.ts', '.js'],
|
|
327
|
+
reload,
|
|
328
|
+
reserved = [],
|
|
329
|
+
}) {
|
|
330
|
+
const definitions = [];
|
|
331
|
+
const declaredBy = new Map();
|
|
332
|
+
for (const [position, definition] of (inline ?? []).entries()) {
|
|
333
|
+
const source = `collections[${position}]`;
|
|
334
|
+
definitions.push(normalizeDefinition(definition, { source }));
|
|
335
|
+
declaredBy.set(definition.name, source);
|
|
336
|
+
}
|
|
337
|
+
if (dir === undefined) return definitions;
|
|
338
|
+
|
|
339
|
+
const files = await loadCollectionsDir(dir, { extensions, reload });
|
|
340
|
+
const issues = [];
|
|
341
|
+
for (const { file, name: fallbackName, definition } of files) {
|
|
342
|
+
const base = `${file}:`;
|
|
343
|
+
const found = definitionIssues(definition, { path: base, reserved, fallbackName });
|
|
344
|
+
if (found.length > 0) {
|
|
345
|
+
issues.push(...found);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const name = definition.name ?? fallbackName;
|
|
349
|
+
if (declaredBy.has(name)) {
|
|
350
|
+
issues.push({
|
|
351
|
+
path: join(base, 'name'),
|
|
352
|
+
message: `declares "${name}" again (already ${declaredBy.get(name)})`,
|
|
353
|
+
});
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
declaredBy.set(name, file);
|
|
357
|
+
definitions.push(normalizeDefinition(definition, { name, source: file }));
|
|
358
|
+
}
|
|
359
|
+
if (issues.length > 0) {
|
|
360
|
+
throw new ConfigInvalidError('Invalid collection definition(s)', { path: dir, issues });
|
|
361
|
+
}
|
|
362
|
+
return definitions;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
module.exports = {
|
|
366
|
+
DEFINITION_KEYS,
|
|
367
|
+
collectionsIssues,
|
|
368
|
+
definitionIssues,
|
|
369
|
+
loadCollectionsDir,
|
|
370
|
+
normalizeDefinition,
|
|
371
|
+
resolveDefinitions,
|
|
372
|
+
};
|
package/src/core/config.js
CHANGED
|
@@ -2,22 +2,36 @@ const fs = require('node:fs/promises');
|
|
|
2
2
|
const path = require('node:path');
|
|
3
3
|
const { pathToFileURL } = require('node:url');
|
|
4
4
|
const { ConfigInvalidError } = require('../errors/index.js');
|
|
5
|
+
const { isCollectionName } = require('../utils/collection-name.js');
|
|
6
|
+
const { TELEMETRY_KEYS, telemetryIssues } = require('../utils/telemetry.js');
|
|
5
7
|
const { applyEnvFile } = require('../utils/env.js');
|
|
6
8
|
const { errorText } = require('../utils/error.js');
|
|
7
9
|
const { resolveLogger } = require('../utils/logger.js');
|
|
8
10
|
const { redactDeep } = require('../utils/redact.js');
|
|
11
|
+
const { collectionsIssues } = require('./collections.js');
|
|
9
12
|
|
|
10
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Default values applied when no flag, env var, or config-file value is
|
|
15
|
+
* present. The single canonical home for every effective default — a use-site
|
|
16
|
+
* `??` fallback would hide these from the schema/template sync tests and let
|
|
17
|
+
* the same fact drift across hand-written copies.
|
|
18
|
+
*/
|
|
11
19
|
const DEFAULT_CONFIG = {
|
|
12
20
|
migrationsDir: './migrations',
|
|
13
21
|
migrationsCollection: '_migronaut_migrations',
|
|
14
22
|
lockCollection: '_migronaut_locks',
|
|
23
|
+
convergeLogCollection: '_migronaut_converge',
|
|
15
24
|
lockTTLSeconds: 60,
|
|
16
25
|
strict: false,
|
|
17
26
|
useTransaction: false,
|
|
18
27
|
fileExtensions: ['.ts', '.js'],
|
|
19
28
|
createExtension: 'js',
|
|
20
29
|
sequential: false,
|
|
30
|
+
ensureIndexes: true,
|
|
31
|
+
onLockLost: 'abort',
|
|
32
|
+
onOutOfOrder: 'warn',
|
|
33
|
+
reloadMigrations: false,
|
|
34
|
+
convergeAfterUp: false,
|
|
21
35
|
};
|
|
22
36
|
|
|
23
37
|
/** Candidate config file names, checked in priority order within the cwd */
|
|
@@ -28,20 +42,6 @@ const isBoolean = (value) => typeof value === 'boolean';
|
|
|
28
42
|
const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
|
|
29
43
|
const isExtension = (value) => value === 'ts' || value === 'js';
|
|
30
44
|
|
|
31
|
-
/**
|
|
32
|
-
* Collection names we accept for the changelog/lock collections and
|
|
33
|
-
* `import --from/--to`: non-empty, no `$` or NUL (invalid server-side), and
|
|
34
|
-
* outside the reserved `system.` namespace — so a flag can never point a
|
|
35
|
-
* read or write at a system collection.
|
|
36
|
-
*/
|
|
37
|
-
function isCollectionName(value) {
|
|
38
|
-
return (
|
|
39
|
-
isNonEmptyString(value) &&
|
|
40
|
-
!value.includes('$') &&
|
|
41
|
-
!value.includes('\0') &&
|
|
42
|
-
!value.startsWith('system.')
|
|
43
|
-
);
|
|
44
|
-
}
|
|
45
45
|
function isStringList(value) {
|
|
46
46
|
if (!Array.isArray(value) || value.length === 0) return false;
|
|
47
47
|
for (const item of value) {
|
|
@@ -53,8 +53,13 @@ function isStringList(value) {
|
|
|
53
53
|
/**
|
|
54
54
|
* Validation spec for every checked config key: predicate + failure message.
|
|
55
55
|
* `mongoose`, `hooks`, `logger` and `client` are deliberately unchecked —
|
|
56
|
-
* they hold live instances the validator has nothing to say about.
|
|
57
|
-
*
|
|
56
|
+
* they hold live instances the validator has nothing to say about.
|
|
57
|
+
* `generateId` and `telemetry` are code-only too, but each has exactly one
|
|
58
|
+
* valid shape, so validateConfig checks them on their own rather than through
|
|
59
|
+
* this table (which a test pins against the JSON schema — and a function or a
|
|
60
|
+
* tracer has no place there).
|
|
61
|
+
* Unknown keys are allowed, matching the previous zod (non-strict object)
|
|
62
|
+
* behavior.
|
|
58
63
|
*/
|
|
59
64
|
const CONFIG_KEYS = [
|
|
60
65
|
{ path: 'uri', check: isNonEmptyString, message: 'uri is required' },
|
|
@@ -70,6 +75,11 @@ const CONFIG_KEYS = [
|
|
|
70
75
|
check: isCollectionName,
|
|
71
76
|
message: "must be a valid collection name (no '$'/NUL, not system.*)",
|
|
72
77
|
},
|
|
78
|
+
{
|
|
79
|
+
path: 'convergeLogCollection',
|
|
80
|
+
check: isCollectionName,
|
|
81
|
+
message: "must be a valid collection name (no '$'/NUL, not system.*)",
|
|
82
|
+
},
|
|
73
83
|
{ path: 'lockTTLSeconds', check: isPositiveInteger, message: 'must be a positive integer' },
|
|
74
84
|
{ path: 'strict', check: isBoolean, message: 'must be a boolean' },
|
|
75
85
|
{ path: 'useTransaction', check: isBoolean, message: 'must be a boolean' },
|
|
@@ -98,6 +108,12 @@ const CONFIG_KEYS = [
|
|
|
98
108
|
message: "must be 'abort' or 'warn'",
|
|
99
109
|
optional: true,
|
|
100
110
|
},
|
|
111
|
+
{
|
|
112
|
+
path: 'onOutOfOrder',
|
|
113
|
+
check: (value) => value === 'warn' || value === 'error' || value === 'allow',
|
|
114
|
+
message: "must be 'warn', 'error' or 'allow'",
|
|
115
|
+
optional: true,
|
|
116
|
+
},
|
|
101
117
|
{
|
|
102
118
|
path: 'envFile',
|
|
103
119
|
check: (value) => value === false || isNonEmptyString(value),
|
|
@@ -118,15 +134,39 @@ const CONFIG_KEYS = [
|
|
|
118
134
|
optional: true,
|
|
119
135
|
},
|
|
120
136
|
{ path: 'reloadMigrations', check: isBoolean, message: 'must be a boolean', optional: true },
|
|
137
|
+
// Only the outer shape here — each definition is checked by
|
|
138
|
+
// collectionsIssues (core/collections.js), which reports nested paths
|
|
139
|
+
// (`collections[2].indexes[0].key`) instead of one opaque message.
|
|
140
|
+
{
|
|
141
|
+
path: 'collections',
|
|
142
|
+
check: Array.isArray,
|
|
143
|
+
message: 'must be an array of collection definitions',
|
|
144
|
+
optional: true,
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
path: 'collectionsDir',
|
|
148
|
+
check: isNonEmptyString,
|
|
149
|
+
message: 'must be a non-empty string',
|
|
150
|
+
optional: true,
|
|
151
|
+
},
|
|
152
|
+
{ path: 'convergeAfterUp', check: isBoolean, message: 'must be a boolean', optional: true },
|
|
121
153
|
];
|
|
122
154
|
|
|
123
155
|
/**
|
|
124
156
|
* Every key the merged config legitimately carries: the validated ones plus
|
|
125
|
-
* the
|
|
126
|
-
* (`migrationsDirectory`, `useTransactions`) at debug
|
|
127
|
-
* stay allowed, matching the documented non-strict
|
|
157
|
+
* the code-only ones (the live instances, `generateId` and `telemetry`). Used
|
|
158
|
+
* only to *mention* typos (`migrationsDirectory`, `useTransactions`) at debug
|
|
159
|
+
* level — unknown keys stay allowed, matching the documented non-strict
|
|
160
|
+
* contract.
|
|
128
161
|
*/
|
|
129
|
-
const KNOWN_CONFIG_KEYS = new Set([
|
|
162
|
+
const KNOWN_CONFIG_KEYS = new Set([
|
|
163
|
+
'logger',
|
|
164
|
+
'hooks',
|
|
165
|
+
'mongoose',
|
|
166
|
+
'client',
|
|
167
|
+
'generateId',
|
|
168
|
+
'telemetry',
|
|
169
|
+
]);
|
|
130
170
|
for (const spec of CONFIG_KEYS) KNOWN_CONFIG_KEYS.add(spec.path);
|
|
131
171
|
|
|
132
172
|
/**
|
|
@@ -150,6 +190,34 @@ function validateConfig(config, options = {}) {
|
|
|
150
190
|
}
|
|
151
191
|
if (!spec.check(value)) issues.push({ path: spec.path, message: spec.message });
|
|
152
192
|
}
|
|
193
|
+
// What it returns is checked on every call (utils/id.js); that it is callable
|
|
194
|
+
// at all is a config mistake — a JSON config's `"generateId": "ulid"` — and
|
|
195
|
+
// belongs with the others, before a run is started.
|
|
196
|
+
if (config.generateId !== undefined && typeof config.generateId !== 'function') {
|
|
197
|
+
issues.push({ path: 'generateId', message: 'must be a function' });
|
|
198
|
+
}
|
|
199
|
+
for (const issue of telemetryIssues(config.telemetry)) issues.push(issue);
|
|
200
|
+
// Inline definitions are checked with the rest of the config — they are
|
|
201
|
+
// pure data, so this costs nothing. Definition *files* are loaded only when
|
|
202
|
+
// a converge runs: importing them here would make one broken file block
|
|
203
|
+
// every command, an emergency `down` included.
|
|
204
|
+
// Three bookkeeping collections, three jobs: sharing one would mix records.
|
|
205
|
+
const bookkeeping = ['migrationsCollection', 'lockCollection', 'convergeLogCollection'];
|
|
206
|
+
for (const [position, key] of bookkeeping.entries()) {
|
|
207
|
+
for (const other of bookkeeping.slice(0, position)) {
|
|
208
|
+
if (config[key] !== undefined && config[key] === config[other]) {
|
|
209
|
+
issues.push({ path: key, message: `must differ from ${other}` });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (Array.isArray(config.collections)) {
|
|
214
|
+
const reserved = [
|
|
215
|
+
config.migrationsCollection,
|
|
216
|
+
config.lockCollection,
|
|
217
|
+
config.convergeLogCollection,
|
|
218
|
+
];
|
|
219
|
+
for (const issue of collectionsIssues(config.collections, { reserved })) issues.push(issue);
|
|
220
|
+
}
|
|
153
221
|
return issues;
|
|
154
222
|
}
|
|
155
223
|
|
|
@@ -227,8 +295,9 @@ const parseString = (value) => value;
|
|
|
227
295
|
*
|
|
228
296
|
* Every *scalar* config option has an entry here, which is what makes the
|
|
229
297
|
* documented "a config file is never required" promise literally true. Options
|
|
230
|
-
* holding non-scalars — `fileExtensions`, `clientOptions`, `
|
|
231
|
-
* `hooks`, `logger` — are
|
|
298
|
+
* holding non-scalars — `fileExtensions`, `clientOptions`, `collections`,
|
|
299
|
+
* `client`, `mongoose`, `hooks`, `logger`, `generateId`, `telemetry` — are
|
|
300
|
+
* config-file/API only; an env var cannot express them.
|
|
232
301
|
*
|
|
233
302
|
* MIGRONAUT_ENV_FILE is deliberately absent: it selects which .env file to load,
|
|
234
303
|
* so it has to be read before this table can run (see loadConfig).
|
|
@@ -239,6 +308,11 @@ const ENV_KEYS = [
|
|
|
239
308
|
{ env: 'MIGRONAUT_MIGRATIONS_DIR', path: 'migrationsDir', parse: parseString },
|
|
240
309
|
{ env: 'MIGRONAUT_COLLECTION', path: 'migrationsCollection', parse: parseString },
|
|
241
310
|
{ env: 'MIGRONAUT_LOCK_COLLECTION', path: 'lockCollection', parse: parseString },
|
|
311
|
+
{
|
|
312
|
+
env: 'MIGRONAUT_CONVERGE_LOG_COLLECTION',
|
|
313
|
+
path: 'convergeLogCollection',
|
|
314
|
+
parse: parseString,
|
|
315
|
+
},
|
|
242
316
|
{ env: 'MIGRONAUT_LOCK_TTL', path: 'lockTTLSeconds', parse: parsePositiveInteger },
|
|
243
317
|
{ env: 'MIGRONAUT_STRICT', path: 'strict', parse: parseBoolean },
|
|
244
318
|
{ env: 'MIGRONAUT_USE_TRANSACTION', path: 'useTransaction', parse: parseBoolean },
|
|
@@ -248,8 +322,15 @@ const ENV_KEYS = [
|
|
|
248
322
|
{ env: 'MIGRONAUT_TEMPLATE_PATH', path: 'templatePath', parse: parseString },
|
|
249
323
|
{ env: 'MIGRONAUT_TIMEOUT_MS', path: 'timeoutMs', parse: parsePositiveInteger },
|
|
250
324
|
{ env: 'MIGRONAUT_ON_LOCK_LOST', path: 'onLockLost', parse: parseEnum(['abort', 'warn']) },
|
|
325
|
+
{
|
|
326
|
+
env: 'MIGRONAUT_ON_OUT_OF_ORDER',
|
|
327
|
+
path: 'onOutOfOrder',
|
|
328
|
+
parse: parseEnum(['warn', 'error', 'allow']),
|
|
329
|
+
},
|
|
251
330
|
{ env: 'MIGRONAUT_ENSURE_INDEXES', path: 'ensureIndexes', parse: parseBoolean },
|
|
252
331
|
{ env: 'MIGRONAUT_RELOAD_MIGRATIONS', path: 'reloadMigrations', parse: parseBoolean },
|
|
332
|
+
{ env: 'MIGRONAUT_COLLECTIONS_DIR', path: 'collectionsDir', parse: parseString },
|
|
333
|
+
{ env: 'MIGRONAUT_CONVERGE_AFTER_UP', path: 'convergeAfterUp', parse: parseBoolean },
|
|
253
334
|
];
|
|
254
335
|
|
|
255
336
|
/** Build a partial config from the MIGRONAUT_* environment variables */
|
|
@@ -394,7 +475,10 @@ async function loadConfig(options = {}) {
|
|
|
394
475
|
: await discoverConfigFile(cwd);
|
|
395
476
|
|
|
396
477
|
if (configFilePath) {
|
|
397
|
-
|
|
478
|
+
// Only an explicit --config path needs the probe (a typo deserves a clear
|
|
479
|
+
// "not found") — discovery already proved existence, and re-checking it
|
|
480
|
+
// would pay a redundant fs.access on every invocation.
|
|
481
|
+
if (options.configPath && !(await pathExists(configFilePath))) {
|
|
398
482
|
throw new ConfigInvalidError('Config file not found', { path: configFilePath });
|
|
399
483
|
}
|
|
400
484
|
const fileConfig = await loadConfigFile(configFilePath, options.lenient ?? false);
|
|
@@ -435,6 +519,13 @@ async function loadConfig(options = {}) {
|
|
|
435
519
|
for (const key in config) {
|
|
436
520
|
if (!KNOWN_CONFIG_KEYS.has(key)) (unknown ??= []).push(key);
|
|
437
521
|
}
|
|
522
|
+
// Same for `telemetry`: handing over the whole `@opentelemetry/api` module
|
|
523
|
+
// (`trace`, `metrics`) instead of a tracer and a meter turns it off silently.
|
|
524
|
+
if (config.telemetry) {
|
|
525
|
+
for (const key in config.telemetry) {
|
|
526
|
+
if (!TELEMETRY_KEYS.includes(key)) (unknown ??= []).push(`telemetry.${key}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
438
529
|
if (unknown) {
|
|
439
530
|
effectiveLogger(config.logger).debug(
|
|
440
531
|
`Unrecognized config key(s), ignored: ${unknown.join(', ')}`,
|
|
@@ -447,18 +538,23 @@ async function loadConfig(options = {}) {
|
|
|
447
538
|
}
|
|
448
539
|
|
|
449
540
|
// "Which config did it actually pick up?" — the merged result, once, at
|
|
450
|
-
// debug level. Live instances (client, mongoose, hooks, logger)
|
|
451
|
-
//
|
|
541
|
+
// debug level. Live instances (client, mongoose, hooks, logger, telemetry)
|
|
542
|
+
// and the generateId function are elided: they are not serializable and
|
|
543
|
+
// redactDeep rightly refuses to clone them. Declared collections show as
|
|
544
|
+
// names only — validators can run to hundreds of lines.
|
|
452
545
|
{
|
|
453
|
-
const { client, mongoose, hooks, logger, ...rest } = config;
|
|
546
|
+
const { client, mongoose, hooks, logger, generateId, telemetry, collections, ...rest } = config;
|
|
454
547
|
effectiveLogger(config.logger).debug(
|
|
455
548
|
`Resolved config (source: ${configFilePath ? path.basename(configFilePath) : 'env/flags/defaults'})`,
|
|
456
549
|
redactDeep({
|
|
457
550
|
...rest,
|
|
551
|
+
...(collections ? { collections: collections.map((definition) => definition.name) } : {}),
|
|
458
552
|
...(client ? { client: '[injected]' } : {}),
|
|
459
553
|
...(mongoose ? { mongoose: '[injected]' } : {}),
|
|
460
554
|
...(hooks ? { hooks: Object.keys(hooks) } : {}),
|
|
461
555
|
...(logger !== undefined ? { logger: logger === null ? null : '[injected]' } : {}),
|
|
556
|
+
...(generateId ? { generateId: '[injected]' } : {}),
|
|
557
|
+
...(telemetry ? { telemetry: '[injected]' } : {}),
|
|
462
558
|
}),
|
|
463
559
|
);
|
|
464
560
|
}
|
|
@@ -473,6 +569,8 @@ module.exports = {
|
|
|
473
569
|
CONFIG_KEYS,
|
|
474
570
|
DEFAULT_CONFIG,
|
|
475
571
|
ENV_KEYS,
|
|
572
|
+
// Re-exported from utils/collection-name.js, where it moved so that
|
|
573
|
+
// core/collections.js can use it without a require cycle through here.
|
|
476
574
|
isCollectionName,
|
|
477
575
|
loadConfig,
|
|
478
576
|
validateConfig,
|