@alexify/migronaut 2.2.0 → 2.3.0

package/src/bullmq/jobs.js

@@ -25,14 +25,28 @@ const MIN_JOB_DATA_VERSION = 1;
 /**
  * One job name per kind of work — `up`/`down` carry one migration each, `sync`
  * plans and enqueues what is pending, `converge` brings the declared
- * collections to their declared state.
+ * collections to their declared state. The background names live on a queue
+ * of their own (`<queueName>-background`): `background` is a background
+ * migration's coordinator, `background-lane` one of its lanes (a child of the
+ * coordinator), `background-verify` the drift watch.
  */
-const JOB_NAMES = Object.freeze({ UP: 'up', DOWN: 'down', SYNC: 'sync', CONVERGE: 'converge' });
+const JOB_NAMES = Object.freeze({
+  UP: 'up',
+  DOWN: 'down',
+  SYNC: 'sync',
+  CONVERGE: 'converge',
+  BACKGROUND: 'background',
+  BACKGROUND_LANE: 'background-lane',
+  BACKGROUND_VERIFY: 'background-verify',
+});
 
 const DEFAULT_QUEUE_NAME = 'migronaut';
 /** No `:` — BullMQ rejects it in custom ids */
 const DEFAULT_SCHEDULER_ID = 'migronaut-sync';
 const DEFAULT_CONVERGE_SCHEDULER_ID = 'migronaut-converge';
+const DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID = 'migronaut-background-verify';
+/** The background queue is named after the migration queue it serves */
+const BACKGROUND_QUEUE_SUFFIX = '-background';
 
 /**
  * Forced onto every migration job, over anything the caller configured. A
@@ -55,7 +69,21 @@ const FORBIDDEN_JOB_OPTIONS = Object.freeze([
   'parent',
 ]);
 
+/**
+ * Refused in a background queue's `jobOptions` too: what a coordinator or a
+ * lane does when its children fail is the adapter's design, not a knob.
+ */
+const BACKGROUND_FORBIDDEN_JOB_OPTIONS = Object.freeze([
+  ...FORBIDDEN_JOB_OPTIONS,
+  'failParentOnFailure',
+  'continueParentOnFailure',
+  'ignoreDependencyOnFailure',
+  'removeDependencyOnFailure',
+]);
+
 const MAX_MIGRATION_NAME_LENGTH = 255;
+/** At most this many lanes — a background migration's `maxParallel` cap */
+const MAX_LANES = 64;
 /** A file checksum as migronaut computes it: a SHA-256 hex digest */
 const CHECKSUM_PATTERN = /^[0-9a-f]{64}$/;
 
@@ -77,13 +105,40 @@ const JOB_FIELDS = Object.freeze({
   ]),
   sync: new Set(['v', 'kind', 'to']),
   converge: new Set(['v', 'kind', 'groupId', 'ordered', 'requestedBy', 'reason']),
+  background: new Set([
+    'v',
+    'kind',
+    'migration',
+    'round',
+    'spawn',
+    'takeover',
+    'requestedBy',
+    'reason',
+  ]),
+  'background-lane': new Set([
+    'v',
+    'kind',
+    'migration',
+    'registration',
+    'generation',
+    'round',
+    'spawn',
+    'lane',
+    'retry',
+  ]),
+  'background-verify': new Set(['v', 'kind']),
 });
 /** The limit every migronaut id is minted under — a producer's own check and this one agree */
 const MAX_GROUP_ID_LENGTH = MAX_ID_LENGTH;
 
-const isPlainObject = (value) =>
+/**
+ * An object that is not an array — class instances included: a Queue the
+ * caller hands over is one (not a "plain object" in versioning/'s sense).
+ */
+const isObjectLike = (value) =>
   value !== null && typeof value === 'object' && !Array.isArray(value);
 const isPositiveInteger = (value) => Number.isSafeInteger(value) && value > 0;
+const isCount = (value) => Number.isSafeInteger(value) && value >= 0;
 
 /**
  * A migration name as an id fragment: letters, digits, `.`, `_` and `-` as
@@ -141,7 +196,7 @@ const DEFAULT_ALLOW = Object.freeze({ down: true, force: false, unordered: false
 /** Validate an `allow` option and fill in the defaults */
 function resolveAllow(allow) {
   if (allow === undefined) return DEFAULT_ALLOW;
-  if (!isPlainObject(allow)) {
+  if (!isObjectLike(allow)) {
     throw new ConfigInvalidError('allow must be an object', { allow: typeof allow });
   }
   for (const key of Object.keys(allow)) {
@@ -199,7 +254,7 @@ function invalid(job, issue) {
  * above all the migration name, which becomes a filesystem path.
  */
 function parseJobData(job) {
-  if (!isPlainObject(job)) throw invalid(job, 'job is not an object');
+  if (!isObjectLike(job)) throw invalid(job, 'job is not an object');
   const { name, data } = job;
   if (
     name !== JOB_NAMES.UP &&
@@ -209,7 +264,7 @@ function parseJobData(job) {
   ) {
     throw invalid(job, 'unknown job name');
   }
-  if (!isPlainObject(data)) throw invalid(job, 'data is not an object');
+  if (!isObjectLike(data)) throw invalid(job, 'data is not an object');
   if (!Number.isSafeInteger(data.v) || data.v < MIN_JOB_DATA_VERSION) {
     throw invalid(job, 'unsupported job data version');
   }
@@ -309,6 +364,87 @@ function parseJobData(job) {
   };
 }
 
+/** The version and field checks every job shares — then the fields of `kind` */
+function assertEnvelope(job, kind) {
+  const { data } = job;
+  if (!isObjectLike(data)) throw invalid(job, 'data is not an object');
+  if (!Number.isSafeInteger(data.v) || data.v < MIN_JOB_DATA_VERSION) {
+    throw invalid(job, 'unsupported job data version');
+  }
+  if (data.v > JOB_DATA_VERSION) {
+    throw invalid(
+      job,
+      `job data version ${data.v} is newer than this worker supports (${JOB_DATA_VERSION}) — ` +
+        'roll the workers out before the producers',
+    );
+  }
+  if (data.kind !== kind) throw invalid(job, 'kind does not match the job name');
+  for (const key of Object.keys(data)) {
+    if (!JOB_FIELDS[kind].has(key)) throw invalid(job, `unknown field "${key}"`);
+  }
+}
+
+/**
+ * Validate a job read back from a background queue and return a normalized
+ * copy — as untrusted as any other: the migration name becomes a path.
+ */
+function parseBackgroundJobData(job) {
+  if (!isObjectLike(job)) throw invalid(job, 'job is not an object');
+  const { name } = job;
+  if (
+    name !== JOB_NAMES.BACKGROUND &&
+    name !== JOB_NAMES.BACKGROUND_LANE &&
+    name !== JOB_NAMES.BACKGROUND_VERIFY
+  ) {
+    throw invalid(job, 'unknown background job name');
+  }
+  assertEnvelope(job, name);
+  const { data } = job;
+  if (name === JOB_NAMES.BACKGROUND_VERIFY) return { kind: name };
+  if (!isBareFilename(data.migration) || data.migration.length > MAX_MIGRATION_NAME_LENGTH) {
+    throw invalid(job, 'migration is not a bare filename');
+  }
+  for (const key of ['round', 'spawn', 'generation', 'retry']) {
+    if (data[key] !== undefined && !isCount(data[key])) {
+      throw invalid(job, `${key} is not a non-negative integer`);
+    }
+  }
+  if (name === JOB_NAMES.BACKGROUND) {
+    for (const key of ['requestedBy', 'reason']) {
+      const issue = actorIssue(key, data[key]);
+      if (issue) throw invalid(job, issue);
+    }
+    if (data.takeover !== undefined && data.takeover !== true) {
+      throw invalid(job, 'takeover is only valid as `true`');
+    }
+    return {
+      kind: name,
+      migration: data.migration,
+      ...(data.round !== undefined ? { round: data.round } : {}),
+      ...(data.spawn !== undefined ? { spawn: data.spawn } : {}),
+      ...(data.takeover ? { takeover: true } : {}),
+      ...pickActor(data),
+    };
+  }
+  if (!isGroupId(data.registration)) throw invalid(job, 'registration is not a short string');
+  for (const key of ['generation', 'round', 'spawn']) {
+    if (data[key] === undefined) throw invalid(job, `${key} is missing`);
+  }
+  if (!Number.isSafeInteger(data.lane) || data.lane < 0 || data.lane >= MAX_LANES) {
+    throw invalid(job, `lane is not an integer from 0 to ${MAX_LANES - 1}`);
+  }
+  return {
+    kind: name,
+    migration: data.migration,
+    registration: data.registration,
+    generation: data.generation,
+    round: data.round,
+    spawn: data.spawn,
+    lane: data.lane,
+    retry: data.retry ?? 0,
+  };
+}
+
 /**
  * Build one `up`/`down` job spec, ready for `queue.addBulk`. `ordered` is
  * always written: a job must say how it is to run, not leave it to whatever
@@ -427,8 +563,112 @@ function buildConvergeJobTemplate({ jobOptions } = {}) {
   };
 }
 
+/** The background queue that serves a migration queue */
+function backgroundQueueName(queueName) {
+  return `${queueName}${BACKGROUND_QUEUE_SUFFIX}`;
+}
+
+/**
+ * Options of a background queue's own jobs: the caller's passthrough, then a
+ * bounded retention where they set none — a lane per partition per spawn adds
+ * up — then what the contract owns.
+ */
+function backgroundJobOptions(jobOptions = {}, owned = {}) {
+  return {
+    removeOnComplete: TICK_RETENTION.removeOnComplete,
+    removeOnFail: TICK_RETENTION.removeOnFail,
+    ...jobOptions,
+    ...owned,
+  };
+}
+
+/**
+ * The coordinator job of a background migration. Deduplicated on its name, so
+ * every pod's heal and every sync tick collapse into the one coordinator
+ * chain that is alive (waiting, delayed or waiting for its lanes); a takeover
+ * of a stalled one gets an id of its own per round. A few attempts with a long
+ * backoff are the outer safety net — what it decides is all in MongoDB.
+ */
+function buildBackgroundJob({ migration, takeoverOf, jobOptions, requestedBy, reason }) {
+  const fragment = idFragment(migration);
+  return {
+    name: JOB_NAMES.BACKGROUND,
+    data: {
+      v: JOB_DATA_VERSION,
+      kind: JOB_NAMES.BACKGROUND,
+      migration,
+      ...(takeoverOf !== undefined ? { takeover: true } : {}),
+      ...pickActor({ requestedBy, reason }),
+    },
+    opts: backgroundJobOptions(jobOptions, {
+      attempts: 3,
+      backoff: { type: 'fixed', delay: 30_000 },
+      deduplication: {
+        id: takeoverOf === undefined ? `bg-${fragment}` : `bg-${fragment}-t${takeoverOf}`,
+      },
+    }),
+  };
+}
+
+/**
+ * One lane of a background migration. Under a coordinator (`parent`) its id
+ * must be new for every spawn — BullMQ will not move an existing job to
+ * another parent, and a job it already finished would never wake this one;
+ * with no parent (`children: false`) the id is deduplicated per lane slot.
+ */
+function buildLaneJob({
+  migration,
+  registration,
+  generation,
+  round,
+  spawn,
+  lane,
+  parent,
+  jobOptions,
+}) {
+  const base = `bgl-${idFragment(migration)}-${idFragment(registration)}-g${generation}`;
+  return {
+    name: JOB_NAMES.BACKGROUND_LANE,
+    data: {
+      v: JOB_DATA_VERSION,
+      kind: JOB_NAMES.BACKGROUND_LANE,
+      migration,
+      registration,
+      generation,
+      round,
+      spawn,
+      lane,
+    },
+    opts: backgroundJobOptions(
+      jobOptions,
+      parent === undefined
+        ? { attempts: 1, deduplication: { id: `${base}-l${lane}` } }
+        : {
+            attempts: 1,
+            jobId: `${base}-r${round}-s${spawn}-l${lane}`,
+            parent,
+            // A lane that fails still wakes its coordinator — which decides
+            // from MongoDB, never from how its lanes ended.
+            ignoreDependencyOnFailure: true,
+          },
+    ),
+  };
+}
+
+/** The job a drift-watch schedule produces — a tick like the `sync` one */
+function buildBackgroundVerifyJobTemplate({ jobOptions } = {}) {
+  return {
+    name: JOB_NAMES.BACKGROUND_VERIFY,
+    data: { v: JOB_DATA_VERSION, kind: JOB_NAMES.BACKGROUND_VERIFY },
+    opts: tickJobOptions(jobOptions),
+  };
+}
+
 module.exports = {
+  BACKGROUND_FORBIDDEN_JOB_OPTIONS,
+  BACKGROUND_QUEUE_SUFFIX,
   DEFAULT_ALLOW,
+  DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID,
   DEFAULT_CONVERGE_SCHEDULER_ID,
   DEFAULT_QUEUE_NAME,
   DEFAULT_SCHEDULER_ID,
@@ -439,15 +679,22 @@ module.exports = {
   MIN_JOB_DATA_VERSION,
   MIGRATION_JOB_OPTIONS,
   TICK_RETENTION,
+  MAX_LANES,
   assertAllowed,
+  backgroundQueueName,
+  buildBackgroundJob,
+  buildBackgroundVerifyJobTemplate,
   buildConvergeJob,
   buildConvergeJobTemplate,
   buildMigrationJob,
+  buildLaneJob,
   buildSyncJobTemplate,
   convergeDedupId,
   dedupId,
-  isPlainObject,
+  idFragment,
+  isObjectLike,
   migrationJobOptions,
+  parseBackgroundJobData,
   parseJobData,
   permissionsNeeded,
   resolveAllow,

package/src/bullmq/processor.js

@@ -11,14 +11,14 @@ const {
 const { pickActor } = require('../utils/actor.js');
 const { errorText } = require('../utils/error.js');
 const { redactDeep, redactOutbound } = require('../utils/redact.js');
+const { JOB_NAMES, assertAllowed, isObjectLike, parseJobData, resolveAllow } = require('./jobs.js');
 const {
-  JOB_NAMES,
-  assertAllowed,
-  isPlainObject,
-  parseJobData,
-  resolveAllow,
-} = require('./jobs.js');
-const { assertJobOptions, enqueueConverge, enqueueUp } = require('./producer.js');
+  assertBackgroundJobOptions,
+  assertJobOptions,
+  enqueueBackground,
+  enqueueConverge,
+  enqueueUp,
+} = require('./producer.js');
 
 /**
  * Failures a later attempt can get past without anything being fixed: the lock
@@ -136,10 +136,10 @@ function prepareErrorForQueue(error) {
  * opens any connection of its own.
  */
 function resolveProcessorOptions(options) {
-  if (!isPlainObject(options)) {
+  if (!isObjectLike(options)) {
     throw new ConfigInvalidError('createMigrationProcessor options must be an object');
   }
-  const { kit, config, lockWait = {}, jobOptions, ordered = true, allow } = options;
+  const { kit, config, lockWait = {}, jobOptions, ordered = true, allow, background } = options;
   if (kit !== undefined && config !== undefined) {
     throw new ConfigInvalidError('Pass either `kit` or `config`, not both');
   }
@@ -149,7 +149,7 @@ function resolveProcessorOptions(options) {
   if (typeof ordered !== 'boolean') {
     throw new ConfigInvalidError('ordered must be a boolean', { ordered });
   }
-  if (!isPlainObject(lockWait)) {
+  if (!isObjectLike(lockWait)) {
     throw new ConfigInvalidError('lockWait must be an object', { lockWait: typeof lockWait });
   }
   // Unlike runMigrations, waiting is the default: nothing is blocked on this
@@ -157,9 +157,26 @@ function resolveProcessorOptions(options) {
   const waitOptions = { onLockHeld: 'wait', ...lockWait };
   assertLockWaitOptions(waitOptions);
   assertJobOptions(jobOptions);
+  if (background !== undefined) assertBackgroundLink(background);
   return { waitOptions, defaultOrdered: ordered, allow: resolveAllow(allow) };
 }
 
+/**
+ * The background queue a migration processor hands what it registers to:
+ * `{ queue, jobOptions?, stallMs? }`.
+ */
+function assertBackgroundLink(background) {
+  if (!isObjectLike(background) || typeof background.queue?.addBulk !== 'function') {
+    throw new ConfigInvalidError('background must be { queue } — the background queue');
+  }
+  for (const key of Object.keys(background)) {
+    if (key !== 'queue' && key !== 'jobOptions' && key !== 'stallMs') {
+      throw new ConfigInvalidError(`background.${key} is not an option here`, { key });
+    }
+  }
+  assertBackgroundJobOptions(background.jobOptions);
+}
+
 /**
  * Build the function a BullMQ Worker runs for each migration job.
  *
@@ -173,7 +190,7 @@ function resolveProcessorOptions(options) {
  */
 function createMigrationProcessor(options = {}) {
   const { waitOptions, defaultOrdered, allow } = resolveProcessorOptions(options);
-  const { kit: injectedKit, config, kitOptions, queue, jobOptions } = options;
+  const { kit: injectedKit, config, kitOptions, queue, jobOptions, background } = options;
 
   const ownsKit = injectedKit === undefined;
   const kit = injectedKit ?? new MigratorKit(config ?? {}, kitOptions);
@@ -246,6 +263,9 @@ function createMigrationProcessor(options = {}) {
     'migration:skipped': (event) => {
       if (current) log(current, `⏭ Skipped ${event.migration} (${event.reason ?? 'skipped'})`);
     },
+    'background:registered': (event) => {
+      if (current) current.registered.push(event.migration);
+    },
     'converge:start': () => {
       if (!current) return;
       current.started = true;
@@ -288,6 +308,53 @@ function createMigrationProcessor(options = {}) {
   };
   for (const [event, listener] of Object.entries(listeners)) kit.on(event, listener);
 
+  /** The background queue options every enqueue from here shares */
+  const backgroundOptions = () => ({
+    ...(background.jobOptions !== undefined ? { jobOptions: background.jobOptions } : {}),
+    ...(background.stallMs !== undefined ? { stallMs: background.stallMs } : {}),
+  });
+
+  /**
+   * Hand what this job registered to the background queue. Never fails the
+   * job — it is applied; a coordinator that could not be added now is added
+   * by the next heal (a sync tick, a verify tick, a worker's start).
+   */
+  async function startBackground(ctx) {
+    if (background === undefined || ctx.registered.length === 0) return undefined;
+    const started = [];
+    for (const migration of ctx.registered) {
+      try {
+        const { jobs } = await enqueueBackground(background.queue, kit, {
+          migration,
+          ...backgroundOptions(),
+        });
+        for (const job of jobs) started.push({ migration, jobId: job.id });
+      } catch (error) {
+        kit.logger.warn(
+          `⚠ Could not enqueue background migration ${migration}: ${errorText(error)} — ` +
+            'the next heal will',
+          { ...jobIds(ctx), migration, error: errorText(error) },
+        );
+      }
+    }
+    for (const entry of started) log(ctx, `⧗ Background coordinator ${entry.jobId} enqueued`);
+    return started;
+  }
+
+  /** Every background migration with work to do gets its coordinator — a sync tick's heal */
+  async function healBackground() {
+    if (background === undefined) return undefined;
+    try {
+      const { jobs } = await enqueueBackground(background.queue, kit, backgroundOptions());
+      return jobs.length;
+    } catch (error) {
+      kit.logger.warn(`⚠ Background heal failed: ${errorText(error)}`, {
+        error: errorText(error),
+      });
+      return 0;
+    }
+  }
+
   function resultOf(ctx, rows, waitedMs) {
     const { data } = ctx;
     const row = rows[0];
@@ -319,7 +386,8 @@ function createMigrationProcessor(options = {}) {
 
     try {
       const { result, waitedMs } = await waitForLock(ctx, attempt, signal);
-      return resultOf(ctx, result, waitedMs);
+      const started = await startBackground(ctx);
+      return { ...resultOf(ctx, result, waitedMs), ...(started ? { background: started } : {}) };
     } catch (error) {
       // A duplicate rollback job: the first one already reverted it. Same
       // outcome as a duplicate `up` job, which the kit reports as skipped.
@@ -415,6 +483,9 @@ function createMigrationProcessor(options = {}) {
     };
   }
 
+  /** What the last sync tick found the line waiting for — `{ migration, waitsFor }` */
+  let lastWaiting;
+
   async function runSyncJob(ctx) {
     if (!queue) {
       throw new ConfigInvalidError(
@@ -422,6 +493,9 @@ function createMigrationProcessor(options = {}) {
       );
     }
     const { to } = ctx.data;
+    // Background migrations first: whatever this tick enqueues may wait for one.
+    const healed = await healBackground();
+    const backgroundField = healed !== undefined ? { background: { enqueued: healed } } : {};
     // The cheap probe first: a scheduler ticks far more often than there is
     // anything to do, and planning proper re-reads the whole directory.
     const pending = await kit.list('pending');
@@ -433,6 +507,7 @@ function createMigrationProcessor(options = {}) {
         enqueued: 0,
         upToDate: true,
         migrations: [],
+        ...backgroundField,
       };
       // With `convergeAfterUp`, a tick that finds no migration still checks
       // the declared collections — a deploy that only changed a definition
@@ -464,17 +539,34 @@ function createMigrationProcessor(options = {}) {
         upToDate: false,
         migrations: [],
         held,
+        ...backgroundField,
+      };
+    }
+    // Still waiting where the last tick found it waiting, for what is still
+    // not done: said again without planning — a background migration takes
+    // hours, and planning re-reads the whole directory on every tick.
+    if (lastWaiting?.migration === pending[0].file && (await stillWaiting(lastWaiting))) {
+      return {
+        kind: 'sync',
+        groupId: null,
+        batch: null,
+        enqueued: 0,
+        upToDate: false,
+        migrations: [],
+        waiting: lastWaiting,
+        ...backgroundField,
       };
     }
     const group = await enqueueUp(queue, kit, {
       ...(to !== undefined ? { to } : {}),
       ...(jobOptions ? { jobOptions } : {}),
     });
+    lastWaiting = group.waiting;
     const migrations = [];
     for (const job of group.jobs) migrations.push(job.migration);
     return {
       kind: 'sync',
-      groupId: group.upToDate ? null : group.groupId,
+      groupId: group.jobs.length === 0 ? null : group.groupId,
       batch: group.batch,
       enqueued: group.jobs.length,
       upToDate: group.upToDate,
@@ -482,9 +574,22 @@ function createMigrationProcessor(options = {}) {
       ...(group.converge
         ? { converge: { jobId: group.converge.id, deduplicated: group.converge.deduplicated } }
         : {}),
+      // Waiting is not held: `held` stays the circuit breaker on a failure.
+      ...(group.waiting ? { waiting: group.waiting } : {}),
+      ...backgroundField,
     };
   }
 
+  /** Whether a background migration `waiting` waits for is still not done */
+  async function stillWaiting(waiting) {
+    if (typeof kit.backgroundStatus !== 'function') return false;
+    for (const name of waiting.waitsFor) {
+      const status = await kit.backgroundStatus(name);
+      if (status?.status !== 'completed' || status.direction === 'revert') return true;
+    }
+    return false;
+  }
+
   /**
    * Put a job that a shutdown stopped before it started its work back at the
    * head of the queue, and return the error that tells BullMQ so — or
@@ -524,7 +629,14 @@ function createMigrationProcessor(options = {}) {
   }
 
   async function handle(job, token, signal) {
-    const ctx = { job, data: undefined, runId: undefined, started: false, writes: new Set() };
+    const ctx = {
+      job,
+      data: undefined,
+      runId: undefined,
+      started: false,
+      writes: new Set(),
+      registered: [],
+    };
     const startedAt = Date.now();
     const signals = [shutdownController.signal];
     if (signal) signals.push(signal);
@@ -624,9 +736,11 @@ function createMigrationProcessor(options = {}) {
 
 module.exports = {
   RETRYABLE_CODES,
+  UNRECOVERABLE_ERROR_NAME,
   WAITING_ERROR_NAME,
   createMigrationProcessor,
   isTransientForJob,
   isRetryableError,
+  prepareErrorForQueue,
   resolveProcessorOptions,
 };