@alexify/migronaut 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/README.md +68 -10
  3. package/bullmq.d.ts +465 -7
  4. package/index.d.ts +1272 -18
  5. package/migronaut.schema.json +150 -2
  6. package/package.json +8 -2
  7. package/src/bullmq/background-processor.js +469 -0
  8. package/src/bullmq/index.js +12 -0
  9. package/src/bullmq/jobs.js +254 -7
  10. package/src/bullmq/processor.js +153 -15
  11. package/src/bullmq/producer.js +202 -27
  12. package/src/bullmq/service.js +480 -45
  13. package/src/cli/commands/background.js +500 -0
  14. package/src/cli/commands/converge.js +38 -10
  15. package/src/cli/commands/create.js +6 -0
  16. package/src/cli/exit-codes.js +6 -0
  17. package/src/cli/index.js +2 -0
  18. package/src/cli/table.js +68 -9
  19. package/src/core/audit.js +98 -3
  20. package/src/core/background-audit.js +139 -0
  21. package/src/core/background-drift.js +126 -0
  22. package/src/core/background-dry-run.js +366 -0
  23. package/src/core/background-engine.js +818 -0
  24. package/src/core/background-kit.js +425 -0
  25. package/src/core/background-partition.js +298 -0
  26. package/src/core/background-runner.js +305 -0
  27. package/src/core/background-sandbox.js +701 -0
  28. package/src/core/background-shard.js +542 -0
  29. package/src/core/background-spec.js +597 -0
  30. package/src/core/background-store.js +951 -0
  31. package/src/core/background-throttle.js +269 -0
  32. package/src/core/background-watch-plan.js +164 -0
  33. package/src/core/background-watch-store.js +78 -0
  34. package/src/core/background-watch.js +605 -0
  35. package/src/core/background.js +1121 -0
  36. package/src/core/bson-peer.js +23 -0
  37. package/src/core/changelog.js +32 -0
  38. package/src/core/collections.js +125 -31
  39. package/src/core/config.js +133 -13
  40. package/src/core/converge-plan.js +343 -61
  41. package/src/core/converge-search-run.js +440 -0
  42. package/src/core/converge-search.js +404 -0
  43. package/src/core/converge.js +428 -183
  44. package/src/core/index-spec.js +27 -16
  45. package/src/core/lock.js +97 -32
  46. package/src/core/migrator.js +951 -26
  47. package/src/core/options.js +32 -1
  48. package/src/core/run.js +26 -12
  49. package/src/core/runner.js +1 -1
  50. package/src/core/search-index-spec.js +758 -0
  51. package/src/core/server-info.js +70 -0
  52. package/src/core/shard-info.js +76 -0
  53. package/src/core/versioning-spec.js +181 -0
  54. package/src/errors/index.js +97 -5
  55. package/src/index.js +16 -0
  56. package/src/utils/canonical.js +34 -1
  57. package/src/utils/error.js +11 -2
  58. package/src/utils/loader.js +77 -9
  59. package/src/utils/migration-name.js +33 -1
  60. package/src/utils/telemetry.js +125 -1
  61. package/src/utils/template.js +69 -1
  62. package/src/versioning/config.js +155 -0
  63. package/src/versioning/document.js +326 -0
  64. package/src/versioning/index.js +50 -0
  65. package/src/versioning/internal.js +279 -0
  66. package/src/versioning/mongoose.js +151 -0
  67. package/src/versioning/occ.js +318 -0
  68. package/src/versioning/registry.js +187 -0
  69. package/src/versioning/upcaster.js +213 -0
  70. package/versioning.d.ts +666 -0
  71. package/versioning.js +1 -0
@@ -163,6 +163,62 @@
163
163
  "type": "boolean",
164
164
  "default": false,
165
165
  "description": "End every bulk `up` (no file, no --to) by converging the declared collections under the same lock"
166
+ },
167
+ "onSearchUnavailable": {
168
+ "enum": [
169
+ "fail",
170
+ "skip"
171
+ ],
172
+ "default": "fail",
173
+ "description": "What converge does with declared search indexes on a server without Atlas Search: refuse the run (fail), or converge everything else and skip them (skip)"
174
+ },
175
+ "waitForSearchIndexes": {
176
+ "type": "boolean",
177
+ "default": false,
178
+ "description": "Hold converge until every declared search index is queryable (search indexes build in the background)"
179
+ },
180
+ "searchIndexWaitTimeoutMs": {
181
+ "type": "integer",
182
+ "minimum": 1,
183
+ "default": 600000,
184
+ "description": "How long waitForSearchIndexes waits, in milliseconds, before the converge fails (the build goes on)"
185
+ },
186
+ "backgroundCollection": {
187
+ "type": "string",
188
+ "minLength": 1,
189
+ "pattern": "^(?!system\\.)[^$\\u0000]+$",
190
+ "default": "_migronaut_background",
191
+ "description": "Where background migrations keep their state — and, named after it, their partitions (<name>_partitions) and drift-watch resume tokens (<name>_watch). Experimental"
192
+ },
193
+ "backgroundInline": {
194
+ "type": "boolean",
195
+ "default": false,
196
+ "description": "Run a background migration to the end inside the up that registers it, under the migration lock (small collections, tests). Experimental"
197
+ },
198
+ "backgroundOnDrift": {
199
+ "enum": [
200
+ "reopen",
201
+ "report"
202
+ ],
203
+ "default": "reopen",
204
+ "description": "What the drift watch does with old-shape documents after a background migration completed: reopen it, or only report. Experimental"
205
+ },
206
+ "backgroundDrift": {
207
+ "enum": [
208
+ "poll",
209
+ "stream",
210
+ "both"
211
+ ],
212
+ "default": "poll",
213
+ "description": "How drift is watched: a periodic check (poll), change streams (stream, with the poll as a backstop), or both in full. Experimental"
214
+ },
215
+ "backgroundShardAware": {
216
+ "enum": [
217
+ "auto",
218
+ "off"
219
+ ],
220
+ "default": "auto",
221
+ "description": "Partition a sharded collection by its shard key and target each write at one shard (auto), or treat it like any other (off). Experimental"
166
222
  }
167
223
  },
168
224
  "definitions": {
@@ -182,6 +238,16 @@
182
238
  "required": [
183
239
  "validator"
184
240
  ]
241
+ },
242
+ {
243
+ "required": [
244
+ "searchIndexes"
245
+ ]
246
+ },
247
+ {
248
+ "required": [
249
+ "versioning"
250
+ ]
185
251
  }
186
252
  ],
187
253
  "properties": {
@@ -198,6 +264,13 @@
198
264
  },
199
265
  "description": "Every index besides _id. Omit to leave the collection's indexes unmanaged"
200
266
  },
267
+ "searchIndexes": {
268
+ "type": "array",
269
+ "items": {
270
+ "$ref": "#/definitions/searchIndex"
271
+ },
272
+ "description": "Atlas Search and Vector Search indexes. Omit to leave the collection's search indexes unmanaged (Atlas, an Atlas CLI local deployment, or MongoDB 8.3+ with mongot)"
273
+ },
201
274
  "validator": {
202
275
  "oneOf": [
203
276
  {
@@ -215,7 +288,7 @@
215
288
  "strict",
216
289
  "moderate"
217
290
  ],
218
- "description": "How the validator applies to updates. Defaults to 'strict'"
291
+ "description": "How the validator applies to updates. Defaults to 'strict' — 'moderate' when the only rules are the ones versioning adds"
219
292
  },
220
293
  "validationAction": {
221
294
  "enum": [
@@ -227,7 +300,57 @@
227
300
  },
228
301
  "prune": {
229
302
  "type": "boolean",
230
- "description": "Drop indexes this definition does not declare (otherwise they are kept and reported)"
303
+ "description": "Drop indexes (and search indexes, when declared) this definition does not declare — otherwise they are kept and reported"
304
+ },
305
+ "versioning": {
306
+ "$ref": "#/definitions/versioning"
307
+ }
308
+ }
309
+ },
310
+ "versioning": {
311
+ "type": "object",
312
+ "required": [
313
+ "current"
314
+ ],
315
+ "additionalProperties": false,
316
+ "description": "Document shape versioning: a validator rule and an index for the version field (and the revision field, for optimistic concurrency). Experimental",
317
+ "properties": {
318
+ "current": {
319
+ "type": "integer",
320
+ "minimum": 1,
321
+ "description": "The shape version new documents are written at"
322
+ },
323
+ "min": {
324
+ "type": "integer",
325
+ "minimum": 0,
326
+ "default": 1,
327
+ "description": "The oldest shape version still allowed — 0 types the fields without requiring them. Converge refuses to raise it while documents below it remain"
328
+ },
329
+ "field": {
330
+ "type": "string",
331
+ "minLength": 1,
332
+ "maxLength": 64,
333
+ "pattern": "^(?!\\$)(?!_id$)[^.\\u0000]+$",
334
+ "default": "__v",
335
+ "description": "The version field"
336
+ },
337
+ "revision": {
338
+ "type": "boolean",
339
+ "default": true,
340
+ "description": "Also manage a revision field for optimistic concurrency"
341
+ },
342
+ "revisionField": {
343
+ "type": "string",
344
+ "minLength": 1,
345
+ "maxLength": 64,
346
+ "pattern": "^(?!\\$)(?!_id$)[^.\\u0000]+$",
347
+ "default": "__rev",
348
+ "description": "The revision field"
349
+ },
350
+ "index": {
351
+ "type": "boolean",
352
+ "default": true,
353
+ "description": "Declare the { <field>: 1, _id: 1 } index background migrations scan"
231
354
  }
232
355
  }
233
356
  },
@@ -320,6 +443,31 @@
320
443
  "description": "Accepted and ignored — a no-op since MongoDB 4.2"
321
444
  }
322
445
  }
446
+ },
447
+ "searchIndex": {
448
+ "type": "object",
449
+ "required": [
450
+ "definition"
451
+ ],
452
+ "additionalProperties": false,
453
+ "properties": {
454
+ "name": {
455
+ "type": "string",
456
+ "minLength": 1,
457
+ "description": "Index name. Defaults to \"default\", as on the server"
458
+ },
459
+ "type": {
460
+ "enum": [
461
+ "search",
462
+ "vectorSearch"
463
+ ],
464
+ "description": "Defaults to 'search'. Cannot change in place — declare a new index under a new name instead"
465
+ },
466
+ "definition": {
467
+ "type": "object",
468
+ "description": "The index definition, as Atlas defines it: { mappings, analyzer, … } for search, { fields: [...] } for vectorSearch"
469
+ }
470
+ }
323
471
  }
324
472
  }
325
473
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alexify/migronaut",
3
- "version": "2.1.0",
3
+ "version": "2.3.0",
4
4
  "description": "Elegant, fast, fully-typed, zero-dependency MongoDB migrations for Node.js — adopts an existing migrate-mongo changelog in one command",
5
5
  "license": "MIT",
6
6
  "author": "Alex Dolid <dolid.sasha@gmail.com>",
@@ -28,6 +28,10 @@
28
28
  "./bullmq": {
29
29
  "types": "./bullmq.d.ts",
30
30
  "default": "./bullmq.js"
31
+ },
32
+ "./versioning": {
33
+ "types": "./versioning.d.ts",
34
+ "default": "./versioning.js"
31
35
  }
32
36
  },
33
37
  "directories": {
@@ -38,6 +42,8 @@
38
42
  "index.d.ts",
39
43
  "bullmq.js",
40
44
  "bullmq.d.ts",
45
+ "versioning.js",
46
+ "versioning.d.ts",
41
47
  "migronaut.schema.json",
42
48
  "bin",
43
49
  "src",
@@ -112,7 +118,7 @@
112
118
  "test:integration": "node scripts/node-test.js --test-concurrency=1 \"tests/integration/**/*.test.js\"",
113
119
  "test:coverage": "c8 --all --include 'src/**' --check-coverage --lines 90 --branches 90 --functions 90 --reporter text --reporter lcov node scripts/node-test.js --test-concurrency=1 \"tests/unit/**/*.test.js\" \"tests/integration/**/*.test.js\"",
114
120
  "test:types": "tsd",
115
- "check:dts": "tsc --noEmit --strict --skipLibCheck false index.d.ts bullmq.d.ts",
121
+ "check:dts": "tsc --noEmit --strict --skipLibCheck false index.d.ts bullmq.d.ts versioning.d.ts",
116
122
  "lint": "oxlint src bin scripts tests bench examples",
117
123
  "format": "oxfmt src bin scripts tests bench examples",
118
124
  "format:check": "oxfmt --check src bin scripts tests bench examples",
@@ -0,0 +1,469 @@
1
+ const { MigratorKit } = require('../core/migrator.js');
2
+ const { ConfigInvalidError, MigronautError, RunAbortedError } = require('../errors/index.js');
3
+ const { errorText } = require('../utils/error.js');
4
+ const { redactOutbound } = require('../utils/redact.js');
5
+ const { JOB_NAMES, buildLaneJob, isObjectLike, parseBackgroundJobData } = require('./jobs.js');
6
+ const {
7
+ UNRECOVERABLE_ERROR_NAME,
8
+ isRetryableError,
9
+ prepareErrorForQueue,
10
+ } = require('./processor.js');
11
+ const {
12
+ DEFAULT_STALL_MS,
13
+ assertBackgroundJobOptions,
14
+ enqueueBackground,
15
+ } = require('./producer.js');
16
+
17
+ /**
18
+ * Background migrations on a queue of their own. A coordinator job per
19
+ * background migration plans its partitions and spawns lanes as its children
20
+ * (`parent` + `moveToWaitingChildren`); each lane works slices, continuing
21
+ * itself with `moveToDelayed` between them, until nothing is left to claim.
22
+ * The coordinator wakes once its last lane is done and decides — from
23
+ * MongoDB, never from how its lanes ended — whether to spawn more, plan
24
+ * another pass or finish. Everything that matters (plans, cursors, leases,
25
+ * counters) is in MongoDB: a lost job or a lost Redis costs a heal, not work.
26
+ *
27
+ * Unlike the migration processor, jobs run side by side here — the kit's
28
+ * background methods are reentrant, and the leases cap the lanes.
29
+ */
30
+
31
+ const DEFAULTS = Object.freeze({
32
+ pollIntervalMs: 5_000,
33
+ maxLaneRetries: 8,
34
+ });
35
+
36
+ /** What BullMQ checks (by name) for a job the processor moved to delayed itself */
37
+ const DELAYED_ERROR_NAME = 'DelayedError';
38
+ /** What BullMQ checks (by name) for a job the processor moved to wait for its children */
39
+ const WAITING_CHILDREN_ERROR_NAME = 'WaitingChildrenError';
40
+ /** The longest a failing lane backs off for */
41
+ const MAX_LANE_BACKOFF_MS = 5 * 60_000;
42
+ /**
43
+ * Coordinator steps one job takes in a row when its lanes all finished before
44
+ * it could wait for them — then it yields the worker and comes back.
45
+ */
46
+ const MAX_INLINE_STEPS = 5;
47
+
48
+ /** The longest slice a caller may ask for — the kit's limit, mirrored (a unit test pins it) */
49
+ const MAX_SLICE_MS = 3_600_000;
50
+
51
+ /**
52
+ * The error that tells BullMQ a job was moved (delayed, waiting for its
53
+ * children) — a typed one, renamed: BullMQ matches the name, and the adapter
54
+ * cannot import its classes.
55
+ */
56
+ const moved = (name) => {
57
+ const error = new RunAbortedError(`Moved: ${name}`, { reason: name, moved: true });
58
+ error.name = name;
59
+ return error;
60
+ };
61
+
62
+ /**
63
+ * Validate the background processor's options. Pure, like the migration
64
+ * processor's: nothing is constructed.
65
+ */
66
+ /** Every option createBackgroundProcessor takes — a typo is refused, not ignored */
67
+ const PROCESSOR_KEYS = new Set([
68
+ 'kit',
69
+ 'config',
70
+ 'kitOptions',
71
+ 'queue',
72
+ 'jobOptions',
73
+ 'sliceMs',
74
+ 'children',
75
+ 'pollIntervalMs',
76
+ 'stallMs',
77
+ 'maxLaneRetries',
78
+ ]);
79
+
80
+ function resolveBackgroundProcessorOptions(options) {
81
+ if (!isObjectLike(options)) {
82
+ throw new ConfigInvalidError('createBackgroundProcessor options must be an object');
83
+ }
84
+ for (const key of Object.keys(options)) {
85
+ if (!PROCESSOR_KEYS.has(key)) {
86
+ throw new ConfigInvalidError(`createBackgroundProcessor: "${key}" is not an option`, { key });
87
+ }
88
+ }
89
+ const {
90
+ kit,
91
+ config,
92
+ queue,
93
+ jobOptions,
94
+ sliceMs,
95
+ children = 'auto',
96
+ pollIntervalMs = DEFAULTS.pollIntervalMs,
97
+ stallMs = DEFAULT_STALL_MS,
98
+ maxLaneRetries = DEFAULTS.maxLaneRetries,
99
+ } = options;
100
+ if (kit !== undefined && config !== undefined) {
101
+ throw new ConfigInvalidError('Pass either `kit` or `config`, not both');
102
+ }
103
+ if (kit !== undefined && typeof kit?.coordinateBackground !== 'function') {
104
+ throw new ConfigInvalidError('kit must be a MigratorKit instance');
105
+ }
106
+ if (!queue || typeof queue.addBulk !== 'function') {
107
+ throw new ConfigInvalidError(
108
+ 'queue is required — the background queue the coordinators add their lanes to',
109
+ );
110
+ }
111
+ // The kit's own range for a caller's slice (background-spec's assertSliceMs).
112
+ if (
113
+ sliceMs !== undefined &&
114
+ (!Number.isSafeInteger(sliceMs) || sliceMs < 1 || sliceMs > MAX_SLICE_MS)
115
+ ) {
116
+ throw new ConfigInvalidError(`sliceMs must be an integer from 1 to ${MAX_SLICE_MS}`, {
117
+ sliceMs,
118
+ });
119
+ }
120
+ if (children !== 'auto' && children !== false) {
121
+ throw new ConfigInvalidError("children must be 'auto' or false", { children });
122
+ }
123
+ if (!Number.isSafeInteger(pollIntervalMs) || pollIntervalMs < 10) {
124
+ throw new ConfigInvalidError('pollIntervalMs must be an integer ≥ 10', { pollIntervalMs });
125
+ }
126
+ if (!Number.isSafeInteger(stallMs) || stallMs < 1000) {
127
+ throw new ConfigInvalidError('stallMs must be an integer of at least 1000', { stallMs });
128
+ }
129
+ if (!Number.isSafeInteger(maxLaneRetries) || maxLaneRetries < 0 || maxLaneRetries > 100) {
130
+ throw new ConfigInvalidError('maxLaneRetries must be an integer from 0 to 100', {
131
+ maxLaneRetries,
132
+ });
133
+ }
134
+ assertBackgroundJobOptions(jobOptions);
135
+ return { sliceMs, children, pollIntervalMs, stallMs, maxLaneRetries };
136
+ }
137
+
138
+ /**
139
+ * Build the function a BullMQ Worker on the background queue runs. Declared
140
+ * with exactly three parameters, so BullMQ hands it the cancellation signal.
141
+ */
142
+ function createBackgroundProcessor(options = {}) {
143
+ const settings = resolveBackgroundProcessorOptions(options);
144
+ const { kit: injectedKit, config, kitOptions, queue, jobOptions } = options;
145
+ const ownsKit = injectedKit === undefined;
146
+ const kit = injectedKit ?? new MigratorKit(config ?? {}, kitOptions);
147
+ const shutdownController = new AbortController();
148
+ const inFlight = new Set();
149
+ let warnedUnmovable = false;
150
+
151
+ /** A log row on the job — never allowed to fail it */
152
+ async function log(job, row) {
153
+ try {
154
+ await job.log?.(redactOutbound(row));
155
+ } catch {
156
+ // Redis is the job's problem, not the background migration's.
157
+ }
158
+ }
159
+
160
+ /** The job's progress, for a dashboard — never allowed to fail it either */
161
+ async function progress(job, value) {
162
+ try {
163
+ await job.updateProgress?.(value);
164
+ } catch {
165
+ // As above.
166
+ }
167
+ }
168
+
169
+ /**
170
+ * Continue this job later: its data updated first (when `data` is given),
171
+ * then moved to delayed — which BullMQ learns from the error's name. Outside
172
+ * a Worker (no token) there is nothing to move: the outcome is returned.
173
+ */
174
+ async function later(ctx, delayMs, result, data) {
175
+ const { job, token } = ctx;
176
+ if (typeof job.moveToDelayed !== 'function' || typeof token !== 'string') {
177
+ if (!warnedUnmovable) {
178
+ warnedUnmovable = true;
179
+ kit.logger.warn(
180
+ '⚠ Background jobs run outside a BullMQ Worker (no token to move them with): a ' +
181
+ 'coordinator takes one step and a lane one slice per job, and the rest waits for ' +
182
+ 'the next heal',
183
+ {},
184
+ );
185
+ }
186
+ return { ...result, retryAfterMs: delayMs };
187
+ }
188
+ if (data !== undefined) await job.updateData(data);
189
+ await job.moveToDelayed(Date.now() + Math.max(0, delayMs), token);
190
+ throw moved(DELAYED_ERROR_NAME);
191
+ }
192
+
193
+ /** Heal from MongoDB: a coordinator for every background migration with work to do */
194
+ async function heal(reason) {
195
+ try {
196
+ return await enqueueBackground(queue, kit, {
197
+ stallMs: settings.stallMs,
198
+ ...(jobOptions !== undefined ? { jobOptions } : {}),
199
+ });
200
+ } catch (error) {
201
+ kit.logger.warn(`⚠ Background heal (${reason}) failed: ${errorText(error)}`, {
202
+ error: errorText(error),
203
+ });
204
+ return { jobs: [] };
205
+ }
206
+ }
207
+
208
+ /** Whether this coordinator can spawn its lanes as children and wait for them */
209
+ function childrenFor(ctx) {
210
+ return (
211
+ settings.children !== false &&
212
+ typeof parentQueueOf(ctx.job) === 'string' &&
213
+ typeof ctx.job.moveToWaitingChildren === 'function' &&
214
+ typeof ctx.token === 'string'
215
+ );
216
+ }
217
+
218
+ /**
219
+ * Where a coordinator job lives, for its lanes' `parent` — the job's own
220
+ * queue, which is the one a lane must wake, whatever `queue` this
221
+ * processor was handed to add the lanes to.
222
+ */
223
+ function parentQueueOf(job) {
224
+ return typeof job.queueQualifiedName === 'string'
225
+ ? job.queueQualifiedName
226
+ : queue.qualifiedName;
227
+ }
228
+
229
+ async function runCoordinator(ctx, data) {
230
+ const { job } = ctx;
231
+ const name = data.migration;
232
+ const base = { kind: JOB_NAMES.BACKGROUND, migration: name };
233
+ // The round is the kit's to hand out (under the coordinator lock): a new
234
+ // chain asks without one, and keeps the one it gets in its data across
235
+ // its moves.
236
+ let round = data.round;
237
+ let spawn = data.spawn ?? 0;
238
+ const children = childrenFor(ctx);
239
+ const keep = () => ({ ...job.data, ...(round !== undefined ? { round } : {}), spawn });
240
+ const withRound = (result) => (round !== undefined ? { ...result, round } : result);
241
+
242
+ for (let step = 0; ; step++) {
243
+ if (shutdownController.signal.aborted) return later(ctx, 0, withRound(base), keep());
244
+ const answer = await kit.coordinateBackground(name, {
245
+ signal: ctx.abort,
246
+ driver: { kind: 'bullmq', ref: String(job.id), ...(round !== undefined ? { round } : {}) },
247
+ });
248
+ if (answer.round !== undefined) round = answer.round;
249
+ await progress(job, { status: answer.next, ...(round !== undefined ? { round } : {}) });
250
+ if (answer.next === 'done') {
251
+ await log(job, `✔ ${name}: ${answer.status}`);
252
+ // Its dependents may just have been unblocked.
253
+ if (answer.status === 'completed') await heal('completed');
254
+ return withRound({ ...base, status: answer.status });
255
+ }
256
+ if (answer.next === 'superseded') {
257
+ await log(job, `↷ ${name}: superseded by a newer coordinator`);
258
+ kit.logger.info(`↷ Background coordinator of ${name} superseded by a newer one`, {
259
+ background: name,
260
+ job: String(job.id),
261
+ });
262
+ return withRound({ ...base, status: 'superseded' });
263
+ }
264
+ if (answer.next !== 'process' || answer.lanes === 0) {
265
+ return later(
266
+ ctx,
267
+ answer.retryAfterMs ?? settings.pollIntervalMs,
268
+ withRound({ ...base, status: answer.next }),
269
+ keep(),
270
+ );
271
+ }
272
+ if (!children) {
273
+ // No parents in this BullMQ (or turned off): lanes deduplicated per
274
+ // slot, and the coordinator looks again after a while.
275
+ await addLanes(ctx, { name, answer, round, spawn });
276
+ return later(
277
+ ctx,
278
+ settings.pollIntervalMs,
279
+ withRound({ ...base, status: 'process' }),
280
+ keep(),
281
+ );
282
+ }
283
+ if (step >= MAX_INLINE_STEPS) return later(ctx, 0, withRound(base), keep());
284
+ spawn += 1;
285
+ // Before the lanes: their ids carry the spawn, and a new one must never
286
+ // repeat one a finished lane already has.
287
+ await job.updateData(keep());
288
+ await addLanes(ctx, {
289
+ name,
290
+ answer,
291
+ round,
292
+ spawn,
293
+ parent: { id: String(job.id), queue: parentQueueOf(job) },
294
+ });
295
+ if (await job.moveToWaitingChildren(ctx.token)) throw moved(WAITING_CHILDREN_ERROR_NAME);
296
+ // Every lane finished before this job could wait for them: look again now.
297
+ }
298
+ }
299
+
300
+ async function addLanes(ctx, { name, answer, round, spawn, parent }) {
301
+ const specs = [];
302
+ for (let lane = 0; lane < answer.lanes; lane++) {
303
+ specs.push(
304
+ buildLaneJob({
305
+ migration: name,
306
+ registration: answer.registration,
307
+ generation: answer.generation,
308
+ round,
309
+ spawn,
310
+ lane,
311
+ ...(parent !== undefined ? { parent } : {}),
312
+ jobOptions,
313
+ }),
314
+ );
315
+ }
316
+ await queue.addBulk(specs);
317
+ await log(
318
+ ctx.job,
319
+ `⇉ ${name}: ${specs.length} lane(s) for generation ${answer.generation} (round ${round})`,
320
+ );
321
+ }
322
+
323
+ async function runLane(ctx, data) {
324
+ const { job } = ctx;
325
+ const name = data.migration;
326
+ const base = { kind: JOB_NAMES.BACKGROUND_LANE, migration: name };
327
+ if (shutdownController.signal.aborted) return later(ctx, 0, { ...base, outcome: 'stopped' });
328
+ let slice;
329
+ try {
330
+ slice = await kit.runBackgroundSlice(name, {
331
+ signal: ctx.abort,
332
+ ...(settings.sliceMs !== undefined ? { sliceMs: settings.sliceMs } : {}),
333
+ });
334
+ } catch (error) {
335
+ if (shutdownController.signal.aborted) {
336
+ return later(ctx, 0, { ...base, outcome: 'stopped' });
337
+ }
338
+ // The failure is already counted on its partition, in MongoDB — which
339
+ // fails the partition after `maxSliceFailures`; this lane only backs off.
340
+ const retry = data.retry + 1;
341
+ const message = errorText(error);
342
+ if (retry > settings.maxLaneRetries) {
343
+ kit.logger.warn(`⚠ Background lane of ${name} gave up: ${message}`, {
344
+ background: name,
345
+ error: message,
346
+ });
347
+ await log(job, `✖ gave up after ${data.retry} retries: ${message}`);
348
+ return {
349
+ ...base,
350
+ outcome: 'gave-up',
351
+ ...(error instanceof MigronautError ? { code: error.code } : {}),
352
+ };
353
+ }
354
+ await log(job, `⚠ slice failed (${message}) — retry ${retry}`);
355
+ kit.logger.warn(
356
+ `⚠ Background lane of ${name}: a slice failed (${message}) — retry ${retry}`,
357
+ {
358
+ background: name,
359
+ job: String(job.id),
360
+ retry,
361
+ error: message,
362
+ },
363
+ );
364
+ return later(
365
+ ctx,
366
+ Math.min(MAX_LANE_BACKOFF_MS, 1000 * 2 ** (retry - 1)),
367
+ { ...base, outcome: 'retry' },
368
+ { ...job.data, retry },
369
+ );
370
+ }
371
+ const reset = data.retry > 0 ? { ...job.data, retry: 0 } : undefined;
372
+ await progress(job, { outcome: slice.outcome, counters: slice.counters ?? {} });
373
+ switch (slice.outcome) {
374
+ case 'yielded':
375
+ case 'stopped':
376
+ case 'lost':
377
+ // Work is left — continue as the same job, behind whatever waits.
378
+ return later(ctx, 0, { ...base, outcome: slice.outcome }, reset);
379
+ case 'busy':
380
+ return later(
381
+ ctx,
382
+ slice.retryAfterMs ?? settings.pollIntervalMs,
383
+ { ...base, outcome: 'busy' },
384
+ reset,
385
+ );
386
+ default:
387
+ return { ...base, outcome: slice.outcome, counters: slice.counters };
388
+ }
389
+ }
390
+
391
+ async function runVerify() {
392
+ const result = await kit.verifyBackground();
393
+ const healed = await heal('verify');
394
+ return {
395
+ kind: JOB_NAMES.BACKGROUND_VERIFY,
396
+ checked: result.checked,
397
+ skipped: result.skipped,
398
+ drift: result.drift,
399
+ enqueued: healed.jobs.length,
400
+ };
401
+ }
402
+
403
+ async function handle(job, token, signal) {
404
+ const signals = [shutdownController.signal];
405
+ if (signal) signals.push(signal);
406
+ const ctx = { job, token, abort: AbortSignal.any(signals) };
407
+ const data = parseBackgroundJobData(job);
408
+ // A job fetched while this process shuts down goes back for another worker.
409
+ if (shutdownController.signal.aborted) {
410
+ return later(ctx, 0, { kind: data.kind, outcome: 'stopped' });
411
+ }
412
+ await kit.connect();
413
+ if (data.kind === JOB_NAMES.BACKGROUND) return runCoordinator(ctx, data);
414
+ if (data.kind === JOB_NAMES.BACKGROUND_LANE) return runLane(ctx, data);
415
+ return runVerify();
416
+ }
417
+
418
+ // Three declared parameters, on purpose — see the factory's doc comment.
419
+ async function processor(job, token, signal) {
420
+ const run = handle(job, token, signal);
421
+ inFlight.add(run);
422
+ try {
423
+ return await run;
424
+ } catch (error) {
425
+ if (error?.name !== DELAYED_ERROR_NAME && error?.name !== WAITING_CHILDREN_ERROR_NAME) {
426
+ // A coordinator has a few attempts; a failure no retry can fix
427
+ // (an invalid payload) is told apart by name, as BullMQ checks it.
428
+ if (!isRetryableError(error) && (job?.opts?.attempts ?? 1) > 1) {
429
+ error.name = UNRECOVERABLE_ERROR_NAME;
430
+ }
431
+ prepareErrorForQueue(error);
432
+ }
433
+ throw error;
434
+ } finally {
435
+ inFlight.delete(run);
436
+ }
437
+ }
438
+
439
+ /**
440
+ * Stop: a lane stops at its next batch boundary, checkpoints, releases its
441
+ * lease and goes back to the queue (moved to delayed, for the next worker);
442
+ * a coordinator that is deciding bows out and comes back. Irreversible.
443
+ */
444
+ processor.shutdown = (reason = 'Background worker shutting down') => {
445
+ if (!shutdownController.signal.aborted) {
446
+ shutdownController.abort(new RunAbortedError(reason, { reason }));
447
+ }
448
+ };
449
+
450
+ /** Shut down, let the jobs in flight settle, disconnect a kit the processor created */
451
+ processor.close = async () => {
452
+ processor.shutdown();
453
+ await Promise.allSettled([...inFlight]);
454
+ if (ownsKit) await kit.disconnect();
455
+ };
456
+
457
+ /** Heal from MongoDB now — what a worker does when it starts */
458
+ processor.heal = () => heal('boot');
459
+
460
+ Object.defineProperty(processor, 'kit', { value: kit, enumerable: true });
461
+ return processor;
462
+ }
463
+
464
+ module.exports = {
465
+ BACKGROUND_PROCESSOR_DEFAULTS: DEFAULTS,
466
+ MAX_SLICE_MS,
467
+ createBackgroundProcessor,
468
+ resolveBackgroundProcessorOptions,
469
+ };