@alexify/migronaut 2.2.0 → 2.4.0

package/src/bullmq/service.js

@@ -4,18 +4,32 @@ const { errorText } = require('../utils/error.js');
 const { isBareFilename } = require('../utils/migration-name.js');
 const { redactDeep, redactOutbound } = require('../utils/redact.js');
 const {
+  DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID,
   DEFAULT_CONVERGE_SCHEDULER_ID,
   DEFAULT_QUEUE_NAME,
   DEFAULT_SCHEDULER_ID,
   JOB_NAMES,
+  backgroundQueueName,
+  buildBackgroundVerifyJobTemplate,
   buildConvergeJobTemplate,
   permissionsNeeded,
   resolveAllow,
   buildSyncJobTemplate,
-  isPlainObject,
+  isObjectLike,
 } = require('./jobs.js');
+const {
+  createBackgroundProcessor,
+  resolveBackgroundProcessorOptions,
+} = require('./background-processor.js');
 const { createMigrationProcessor, resolveProcessorOptions } = require('./processor.js');
-const { assertJobOptions, enqueueConverge, enqueueDown, enqueueUp } = require('./producer.js');
+const {
+  DEFAULT_STALL_MS,
+  assertJobOptions,
+  enqueueBackground,
+  enqueueConverge,
+  enqueueDown,
+  enqueueUp,
+} = require('./producer.js');
 
 /**
  * Twice BullMQ's default job lock: its renewal (every half) then survives a
@@ -27,6 +41,98 @@ const DEFAULT_LOCK_DURATION_MS = 60_000;
 const DEFAULT_MAX_STALLED_COUNT = 1;
 /** The shortest interval `schedule({ every })` accepts */
 const MIN_SCHEDULE_EVERY_MS = 1000;
+/** Background jobs run side by side — a lane and a coordinator need not take turns */
+const DEFAULT_BACKGROUND_CONCURRENCY = 2;
+/** How often the drift watch runs on the background queue by default */
+const DEFAULT_BACKGROUND_VERIFY_MS = 600_000;
+/** Every key the `background` option accepts */
+const BACKGROUND_KEYS = new Set([
+  'queueName',
+  'queue',
+  'jobOptions',
+  'workerOptions',
+  'sliceMs',
+  'children',
+  'pollIntervalMs',
+  'stallMs',
+  'verifyIntervalMs',
+  'watch',
+  'maxLaneRetries',
+]);
+
+/**
+ * The `background` option, checked and filled in — or undefined when the
+ * queue has no background side. `true` takes every default.
+ */
+function resolveBackground(background, { queueName, QueueSource }) {
+  if (background === undefined || background === false) return undefined;
+  const options = background === true ? {} : background;
+  if (!isObjectLike(options)) {
+    throw new ConfigInvalidError('background must be true or an object');
+  }
+  for (const key of Object.keys(options)) {
+    if (!BACKGROUND_KEYS.has(key)) {
+      throw new ConfigInvalidError(`background.${key} is not a known option`, { key });
+    }
+  }
+  const queueIsInstance =
+    options.queue !== undefined &&
+    isObjectLike(options.queue) &&
+    typeof options.queue.addBulk === 'function';
+  if (options.queue !== undefined && !queueIsInstance) {
+    throw new ConfigInvalidError('background.queue must be a Queue instance');
+  }
+  if (!queueIsInstance && !isClass(QueueSource)) {
+    throw new ConfigInvalidError(
+      'background needs bullmq.Queue as a class to build its queue — or a background.queue',
+    );
+  }
+  const name =
+    options.queueName ??
+    (queueIsInstance ? options.queue.name : undefined) ??
+    backgroundQueueName(queueName);
+  assertName(name, 'background.queueName');
+  if (name === queueName) {
+    throw new ConfigInvalidError('background.queueName must differ from the migration queue', {
+      queueName: name,
+    });
+  }
+  if (options.workerOptions !== undefined && !isObjectLike(options.workerOptions)) {
+    throw new ConfigInvalidError('background.workerOptions must be an object');
+  }
+  assertBackgroundConcurrency(options.workerOptions?.concurrency);
+  const verifyIntervalMs = options.verifyIntervalMs ?? DEFAULT_BACKGROUND_VERIFY_MS;
+  if (
+    verifyIntervalMs !== false &&
+    (!Number.isSafeInteger(verifyIntervalMs) || verifyIntervalMs < MIN_SCHEDULE_EVERY_MS)
+  ) {
+    throw new ConfigInvalidError(
+      `background.verifyIntervalMs must be false or an integer ≥ ${MIN_SCHEDULE_EVERY_MS}`,
+      { verifyIntervalMs },
+    );
+  }
+  const { watch } = options;
+  if (watch !== undefined && typeof watch !== 'boolean' && !isObjectLike(watch)) {
+    throw new ConfigInvalidError('background.watch must be a boolean or the watcher options');
+  }
+  // Said explicitly, the interval is re-registered at every start; left to its
+  // default, a schedule set with schedule({ job: 'background-verify' }) stays.
+  return {
+    ...options,
+    name,
+    queueIsInstance,
+    verifyIntervalMs,
+    verifyIntervalGiven: options.verifyIntervalMs !== undefined,
+  };
+}
+
+function assertBackgroundConcurrency(concurrency) {
+  if (concurrency !== undefined && (!Number.isSafeInteger(concurrency) || concurrency < 1)) {
+    throw new ConfigInvalidError('background worker concurrency must be a positive integer', {
+      concurrency,
+    });
+  }
+}
 
 const isClass = (value) => typeof value === 'function';
 
@@ -90,9 +196,16 @@ class MigrationQueue {
   #globalConcurrency;
   #allow;
   #closing;
+  #background;
+  #backgroundQueue;
+  #ownsBackgroundQueue = false;
+  #backgroundProcessor;
+  #backgroundWorker;
+  #backgroundStarting;
+  #backgroundWatcher;
 
   constructor(options) {
-    if (!isPlainObject(options)) {
+    if (!isObjectLike(options)) {
       throw new ConfigInvalidError('createMigrationQueue options must be an object');
     }
     const {
@@ -108,9 +221,11 @@ class MigrationQueue {
       globalConcurrency = true,
       lockWait,
       allow,
+      background,
+      userlandLogRows,
     } = options;
 
-    if (!isPlainObject(bullmq)) {
+    if (!isObjectLike(bullmq)) {
       throw new ConfigInvalidError(
         'bullmq is required — pass { Queue, Worker, QueueEvents } from your own bullmq install',
       );
@@ -125,14 +240,14 @@ class MigrationQueue {
         { telemetry: typeof telemetry },
       );
     }
-    const queueIsInstance = isPlainObject(Queue) && typeof Queue.addBulk === 'function';
+    const queueIsInstance = isObjectLike(Queue) && typeof Queue.addBulk === 'function';
     if (!isClass(Queue) && !queueIsInstance) {
       throw new ConfigInvalidError('bullmq.Queue must be the Queue class or a Queue instance');
     }
     if (Worker !== undefined && !isClass(Worker)) {
       throw new ConfigInvalidError('bullmq.Worker must be the Worker class');
     }
-    const eventsIsInstance = isPlainObject(QueueEvents) && typeof QueueEvents.on === 'function';
+    const eventsIsInstance = isObjectLike(QueueEvents) && typeof QueueEvents.on === 'function';
     if (QueueEvents !== undefined && !isClass(QueueEvents) && !eventsIsInstance) {
       throw new ConfigInvalidError(
         'bullmq.QueueEvents must be the QueueEvents class or a QueueEvents instance',
@@ -155,13 +270,18 @@ class MigrationQueue {
     const resolvedPrefix = prefix ?? (queueIsInstance ? Queue.opts?.prefix : undefined);
     if (resolvedPrefix !== undefined) assertName(resolvedPrefix, 'prefix');
     if (queueIsInstance) {
-      MigrationQueue.#assertSameQueue('Queue', Queue, resolvedName, resolvedPrefix);
+      MigrationQueue.#assertSameQueue('bullmq.Queue', Queue, resolvedName, resolvedPrefix);
     }
     if (eventsIsInstance) {
-      MigrationQueue.#assertSameQueue('QueueEvents', QueueEvents, resolvedName, resolvedPrefix);
+      MigrationQueue.#assertSameQueue(
+        'bullmq.QueueEvents',
+        QueueEvents,
+        resolvedName,
+        resolvedPrefix,
+      );
     }
     assertJobOptions(jobOptions);
-    if (!isPlainObject(workerOptions)) {
+    if (!isObjectLike(workerOptions)) {
       throw new ConfigInvalidError('workerOptions must be an object');
     }
     MigrationQueue.#assertConcurrency(workerOptions.concurrency);
@@ -188,8 +308,30 @@ class MigrationQueue {
       ...(config !== undefined ? { config } : {}),
       ...(lockWait !== undefined ? { lockWait } : {}),
       ...(allow !== undefined ? { allow } : {}),
+      ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
     });
     this.#allow = resolveAllow(allow);
+    const backgroundSettings = resolveBackground(background, {
+      queueName: resolvedName,
+      QueueSource: Queue,
+    });
+    if (backgroundSettings?.queueIsInstance) {
+      // The same check as the migration queue's: a Worker built here on
+      // another name or prefix would listen to an empty queue.
+      MigrationQueue.#assertSameQueue(
+        'background.queue',
+        backgroundSettings.queue,
+        backgroundSettings.name,
+        resolvedPrefix,
+      );
+    }
+    if (backgroundSettings !== undefined) {
+      // Validated against a stand-in queue: the real one does not exist yet.
+      resolveBackgroundProcessorOptions({
+        ...MigrationQueue.#backgroundProcessorOptions(backgroundSettings),
+        queue: { addBulk() {} },
+      });
+    }
 
     this.#ownsKit = kit === undefined;
     this.#kit = kit ?? new MigratorKit(config ?? {}, kitOptions);
@@ -207,6 +349,29 @@ class MigrationQueue {
         error: errorText(error),
       }),
     );
+    if (backgroundSettings !== undefined) {
+      this.#background = backgroundSettings;
+      this.#ownsBackgroundQueue = !backgroundSettings.queueIsInstance;
+      this.#backgroundQueue = backgroundSettings.queueIsInstance
+        ? backgroundSettings.queue
+        : new Queue(backgroundSettings.name, {
+            connection,
+            ...(resolvedPrefix !== undefined ? { prefix: resolvedPrefix } : {}),
+            ...(telemetry !== undefined ? { telemetry } : {}),
+          });
+      this.#listen(this.#backgroundQueue, 'error', (error) =>
+        this.#kit.logger.error(`✖ Background queue error: ${errorText(error)}`, {
+          queue: backgroundSettings.name,
+          error: errorText(error),
+        }),
+      );
+      this.#backgroundProcessor = createBackgroundProcessor({
+        kit: this.#kit,
+        queue: this.#backgroundQueue,
+        ...MigrationQueue.#backgroundProcessorOptions(backgroundSettings),
+        ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
+      });
+    }
     this.#processor = createMigrationProcessor({
       kit: this.#kit,
       // `sync` jobs (and the converge jobs they add) enqueue into the queue
@@ -215,9 +380,64 @@ class MigrationQueue {
       ...(lockWait !== undefined ? { lockWait } : {}),
       ...(jobOptions !== undefined ? { jobOptions } : {}),
       ...(allow !== undefined ? { allow } : {}),
+      ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
+      // What an `up` registers starts on the background queue at once.
+      ...(this.#backgroundQueue !== undefined
+        ? {
+            background: {
+              queue: this.#backgroundQueue,
+              ...MigrationQueue.#backgroundEnqueueOptions(backgroundSettings),
+            },
+          }
+        : {}),
     });
   }
 
+  /** Whether the drift watch's schedule exists already — false when that cannot be told */
+  static async #hasScheduler(queue) {
+    if (typeof queue.getJobScheduler === 'function') {
+      return Boolean(await queue.getJobScheduler(DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID));
+    }
+    if (typeof queue.getJobSchedulers === 'function') {
+      for (const scheduler of await queue.getJobSchedulers()) {
+        if ((scheduler.id ?? scheduler.key) === DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID) return true;
+      }
+    }
+    return false;
+  }
+
+  /** The background processor's options out of the resolved `background` option */
+  static #backgroundProcessorOptions(settings) {
+    const picked = {};
+    for (const key of [
+      'jobOptions',
+      'sliceMs',
+      'children',
+      'pollIntervalMs',
+      'stallMs',
+      'maxLaneRetries',
+    ]) {
+      if (settings[key] !== undefined) picked[key] = settings[key];
+    }
+    return picked;
+  }
+
+  /** What every coordinator enqueue from this object carries */
+  static #backgroundEnqueueOptions(settings) {
+    return {
+      ...(settings.jobOptions !== undefined ? { jobOptions: settings.jobOptions } : {}),
+      stallMs: settings.stallMs ?? DEFAULT_STALL_MS,
+    };
+  }
+
+  #assertBackground(method) {
+    if (this.#background === undefined) {
+      throw new ConfigInvalidError(
+        `${method} needs the background queue — pass background: true to createMigrationQueue`,
+      );
+    }
+  }
+
   /**
    * Refuse, at the enqueue call, a request this object's own policy would
    * refuse on the worker — a job that can only fail is better not added. The
@@ -238,19 +458,15 @@ class MigrationQueue {
   /** An injected instance must be on the queue this object is configured for */
   static #assertSameQueue(label, instance, name, prefix) {
     if (typeof instance.name === 'string' && instance.name !== name) {
-      throw new ConfigInvalidError(
-        `bullmq.${label} is on queue "${instance.name}", not "${name}"`,
-        {
-          queueName: name,
-        },
-      );
+      throw new ConfigInvalidError(`${label} is on queue "${instance.name}", not "${name}"`, {
+        queueName: name,
+      });
     }
     const instancePrefix = instance.opts?.prefix;
     if (instancePrefix !== undefined && prefix !== undefined && instancePrefix !== prefix) {
-      throw new ConfigInvalidError(
-        `bullmq.${label} uses prefix "${instancePrefix}", not "${prefix}"`,
-        { prefix },
-      );
+      throw new ConfigInvalidError(`${label} uses prefix "${instancePrefix}", not "${prefix}"`, {
+        prefix,
+      });
     }
   }
 
@@ -301,6 +517,26 @@ class MigrationQueue {
     return this.#processor;
   }
 
+  /** The background queue (`background` option), if any */
+  get backgroundQueue() {
+    return this.#backgroundQueue;
+  }
+
+  /** The background worker started by {@link startBackgroundWorker}, if any */
+  get backgroundWorker() {
+    return this.#backgroundWorker;
+  }
+
+  /** The background queue's processor, for a Worker you construct yourself */
+  get backgroundProcessor() {
+    return this.#backgroundProcessor;
+  }
+
+  /** The live drift watcher `startBackgroundWorker()` started, if any */
+  get backgroundWatcher() {
+    return this.#backgroundWatcher;
+  }
+
   #ensureQueueEvents() {
     if (this.#queueEvents) return this.#queueEvents;
     // A connection opened after close() would have nobody to close it.
@@ -381,6 +617,48 @@ class MigrationQueue {
     );
   }
 
+  /**
+   * Enqueue the coordinator of one background migration — or of every one with
+   * work to do. Idempotent: a coordinator already alive absorbs the add.
+   * @experimental
+   */
+  async enqueueBackground(name, options = {}) {
+    this.#assertOpen();
+    this.#assertBackground('enqueueBackground()');
+    if (!isObjectLike(options)) {
+      throw new ConfigInvalidError('enqueueBackground options must be an object');
+    }
+    return enqueueBackground(this.#backgroundQueue, this.#kit, {
+      ...MigrationQueue.#backgroundEnqueueOptions(this.#background),
+      ...options,
+      ...(name !== undefined ? { migration: name } : {}),
+    });
+  }
+
+  /** A background migration's status (or every one's) — read from MongoDB */
+  async backgroundStatus(name) {
+    this.#assertOpen();
+    return this.#kit.backgroundStatus(name);
+  }
+
+  /**
+   * The drift watch, now — and, on a queue with a background side, a
+   * coordinator for whatever it reopened.
+   * @experimental
+   */
+  async verifyBackground(options = {}) {
+    this.#assertOpen();
+    const result = await this.#kit.verifyBackground(options);
+    if (this.#background !== undefined && result.drift.length > 0) {
+      await enqueueBackground(
+        this.#backgroundQueue,
+        this.#kit,
+        MigrationQueue.#backgroundEnqueueOptions(this.#background),
+      );
+    }
+    return result;
+  }
+
   /** Full migration status — read straight from MongoDB, not from the queue */
   async status() {
     this.#assertOpen();
@@ -414,7 +692,7 @@ class MigrationQueue {
    */
   async startWorker(overrides = {}) {
     this.#assertOpen();
-    if (!isPlainObject(overrides)) {
+    if (!isObjectLike(overrides)) {
       throw new ConfigInvalidError('startWorker options must be an object');
     }
     MigrationQueue.#assertConcurrency(overrides.concurrency);
@@ -481,6 +759,117 @@ class MigrationQueue {
     return worker;
   }
 
+  /**
+   * Start the background worker: coordinators and lanes of background
+   * migrations, side by side (concurrency 2 by default). Connects to MongoDB
+   * first, registers the drift watch's schedule (`verifyIntervalMs`), and
+   * heals — a coordinator for every background migration with work to do.
+   * Calling it again returns the same worker.
+   * @experimental
+   */
+  async startBackgroundWorker(overrides = {}) {
+    this.#assertOpen();
+    this.#assertBackground('startBackgroundWorker()');
+    if (!isObjectLike(overrides)) {
+      throw new ConfigInvalidError('startBackgroundWorker options must be an object');
+    }
+    assertBackgroundConcurrency(overrides.concurrency);
+    if (!this.#WorkerClass) {
+      throw new ConfigInvalidError(
+        'startBackgroundWorker() needs the Worker class — pass bullmq: { Queue, Worker }',
+      );
+    }
+    this.#backgroundStarting ??= this.#startBackgroundWorker(overrides).catch((error) => {
+      this.#backgroundStarting = undefined;
+      throw error;
+    });
+    return this.#backgroundStarting;
+  }
+
+  async #startBackgroundWorker(overrides) {
+    await this.#kit.connect();
+    const queue = this.#backgroundQueue;
+    const { verifyIntervalMs, verifyIntervalGiven, workerOptions = {} } = this.#background;
+    if (
+      verifyIntervalMs !== false &&
+      typeof queue.upsertJobScheduler === 'function' &&
+      (verifyIntervalGiven || !(await MigrationQueue.#hasScheduler(queue)))
+    ) {
+      await queue.upsertJobScheduler(
+        DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID,
+        { every: verifyIntervalMs },
+        buildBackgroundVerifyJobTemplate({ jobOptions: this.#background.jobOptions }),
+      );
+    }
+    this.#assertOpen();
+    const Worker = this.#WorkerClass;
+    // An injected background queue says where its keys live; its worker listens there.
+    const prefix = this.#background.queue?.opts?.prefix ?? this.#prefix;
+    const worker = new Worker(this.#background.name, this.#backgroundProcessor, {
+      connection: this.#connection,
+      ...(prefix !== undefined ? { prefix } : {}),
+      lockDuration: DEFAULT_LOCK_DURATION_MS,
+      maxStalledCount: DEFAULT_MAX_STALLED_COUNT,
+      ...(this.#telemetry !== undefined ? { telemetry: this.#telemetry } : {}),
+      concurrency: DEFAULT_BACKGROUND_CONCURRENCY,
+      ...workerOptions,
+      ...overrides,
+    });
+    this.#backgroundWorker = worker;
+    const fields = { queue: this.#background.name };
+    this.#listen(worker, 'error', (error) =>
+      this.#kit.logger.error(`✖ Background worker error: ${errorText(error)}`, {
+        ...fields,
+        error: errorText(error),
+      }),
+    );
+    this.#listen(worker, 'failed', (job, error) =>
+      this.#kit.logger.warn(
+        `✖ Background job failed${job?.id !== undefined ? ` (${job.id})` : ''}: ${errorText(error)}`,
+        { ...fields, ...failedJobFields(job, error), error: errorText(error) },
+      ),
+    );
+    await worker.waitUntilReady?.();
+    // Closing meanwhile: close() takes it from here — nothing more to start.
+    if (this.#closing) return worker;
+    await this.#backgroundProcessor.heal();
+    if (this.#closing) return worker;
+    await this.#startWatcher();
+    return worker;
+  }
+
+  /**
+   * The live drift watcher, in this process — when `background.watch` says
+   * so, or, unsaid, when `backgroundDrift` is `'stream'` or `'both'`. A
+   * watcher that cannot start (a standalone server) leaves the polling watch
+   * to it, with a warning: the worker itself is fine.
+   */
+  async #startWatcher() {
+    let wanted = this.#background.watch;
+    wanted ??= (await this.#kit.driftMode()) !== 'poll';
+    if (wanted === false) return;
+    const logger = this.#kit.logger;
+    try {
+      this.#backgroundWatcher = await this.#kit.watchBackground({
+        ...(typeof wanted === 'object' ? wanted : {}),
+        onError: (error, collection) =>
+          logger.warn(
+            `⚠ Drift watcher${collection ? ` (${collection})` : ''}: ${errorText(error)}`,
+            {
+              queue: this.#background.name,
+              ...(collection ? { collection } : {}),
+              error: errorText(error),
+            },
+          ),
+      });
+    } catch (error) {
+      logger.warn(`⚠ The live drift watcher did not start: ${errorText(error)}`, {
+        queue: this.#background.name,
+        error: errorText(error),
+      });
+    }
+  }
+
   /** Stop workers from picking up new jobs. The job in flight finishes */
   async pause() {
     this.#assertOpen();
@@ -532,17 +921,32 @@ class MigrationQueue {
    */
   async schedule(options = {}) {
     this.#assertOpen();
-    if (!isPlainObject(options)) {
+    if (!isObjectLike(options)) {
       throw new ConfigInvalidError('schedule options must be an object');
     }
     const { job = JOB_NAMES.SYNC, every, pattern, tz, to } = options;
-    if (job !== JOB_NAMES.SYNC && job !== JOB_NAMES.CONVERGE) {
-      throw new ConfigInvalidError("schedule job must be 'sync' or 'converge'", { job });
+    if (
+      job !== JOB_NAMES.SYNC &&
+      job !== JOB_NAMES.CONVERGE &&
+      job !== JOB_NAMES.BACKGROUND_VERIFY
+    ) {
+      throw new ConfigInvalidError(
+        "schedule job must be 'sync', 'converge' or 'background-verify'",
+        { job },
+      );
     }
     const converge = job === JOB_NAMES.CONVERGE;
-    const { id = converge ? DEFAULT_CONVERGE_SCHEDULER_ID : DEFAULT_SCHEDULER_ID } = options;
+    const verify = job === JOB_NAMES.BACKGROUND_VERIFY;
+    if (verify) this.#assertBackground("schedule({ job: 'background-verify' })");
+    const {
+      id = verify
+        ? DEFAULT_BACKGROUND_VERIFY_SCHEDULER_ID
+        : converge
+          ? DEFAULT_CONVERGE_SCHEDULER_ID
+          : DEFAULT_SCHEDULER_ID,
+    } = options;
     assertName(id, 'id');
-    if (converge && to !== undefined) {
+    if (job !== JOB_NAMES.SYNC && to !== undefined) {
       throw new ConfigInvalidError('to only applies to a sync schedule', { to });
     }
     if ((every === undefined) === (pattern === undefined)) {
@@ -566,17 +970,24 @@ class MigrationQueue {
     if (to !== undefined && !isBareFilename(to)) {
       throw new ConfigInvalidError('to must be a migration filename', { to });
     }
-    if (typeof this.#queue.upsertJobScheduler !== 'function') {
+    const queue = verify ? this.#backgroundQueue : this.#queue;
+    if (typeof queue.upsertJobScheduler !== 'function') {
       throw new ConfigInvalidError(
         'schedule() needs job schedulers (queue.upsertJobScheduler) — BullMQ 5.16 or newer',
       );
     }
-    await this.#queue.upsertJobScheduler(
+    let template;
+    if (verify) {
+      template = buildBackgroundVerifyJobTemplate({ jobOptions: this.#background.jobOptions });
+    } else if (converge) {
+      template = buildConvergeJobTemplate({ jobOptions: this.#jobOptions });
+    } else {
+      template = buildSyncJobTemplate({ to, jobOptions: this.#jobOptions });
+    }
+    await queue.upsertJobScheduler(
       id,
       { ...(every !== undefined ? { every } : { pattern }), ...(tz !== undefined ? { tz } : {}) },
-      converge
-        ? buildConvergeJobTemplate({ jobOptions: this.#jobOptions })
-        : buildSyncJobTemplate({ to, jobOptions: this.#jobOptions }),
+      template,
     );
   }
 
@@ -593,7 +1004,11 @@ class MigrationQueue {
         'unschedule() needs job schedulers (queue.removeJobScheduler) — BullMQ 5.16 or newer',
       );
     }
-    return Boolean(await this.#queue.removeJobScheduler(id));
+    const removed = Boolean(await this.#queue.removeJobScheduler(id));
+    // A schedule of the background queue (the drift watch) goes the same way.
+    const background = this.#backgroundQueue;
+    if (typeof background?.removeJobScheduler !== 'function') return removed;
+    return Boolean(await background.removeJobScheduler(id)) || removed;
   }
 
   /**
@@ -616,31 +1031,55 @@ class MigrationQueue {
         failures.push(error);
       }
     };
-    // The worker stops fetching first: a job the shutdown below puts back in
-    // the queue must go to the next worker, not straight back to this one.
+    // The workers stop fetching first, together: a job the shutdown below
+    // puts back in the queue must go to the next worker, not straight back
+    // to this one. Both processors are told to stop: a lane checkpoints at
+    // its next batch and goes back to the queue. Nothing waits for a start
+    // in progress before that — one can hang on Redis for good.
     const worker = this.#worker;
-    const workerClosed = worker ? attempt(() => worker.close(force)) : undefined;
+    const backgroundWorker = this.#backgroundWorker;
+    const workersClosed = [
+      worker ? attempt(() => worker.close(force)) : undefined,
+      backgroundWorker ? attempt(() => backgroundWorker.close(force)) : undefined,
+    ];
     this.#processor.shutdown('Migration queue closing');
-    await workerClosed;
-    // A worker still starting is closed too, not orphaned — and a start that
-    // failed is that call's failure, not this one's.
+    this.#backgroundProcessor?.shutdown('Migration queue closing');
+    // The watcher: its streams close, its last positions are saved, its
+    // locks go to the next pod's watcher.
+    const watcher = this.#backgroundWatcher;
+    if (watcher) await attempt(() => watcher.stop());
+    await Promise.all(workersClosed);
+    // A worker still starting is closed too, not orphaned — the start sees
+    // the close at its next step — and a start that failed is that call's
+    // failure, not this one's.
     await this.#workerStarting?.catch(() => undefined);
     if (this.#worker && this.#worker !== worker) {
       await attempt(() => this.#worker.close(force));
     }
+    await this.#backgroundStarting?.catch(() => undefined);
+    if (this.#backgroundWorker && this.#backgroundWorker !== backgroundWorker) {
+      await attempt(() => this.#backgroundWorker.close(force));
+    }
+    if (this.#backgroundWatcher && this.#backgroundWatcher !== watcher) {
+      await attempt(() => this.#backgroundWatcher.stop());
+    }
+    const processors = [this.#processor];
+    if (this.#backgroundProcessor) processors.push(this.#backgroundProcessor);
+    if (!force) {
+      for (const processor of processors) await attempt(() => processor.close());
+    }
     if (this.#queueEvents && this.#ownsQueueEvents) await attempt(() => this.#queueEvents.close());
     if (this.#ownsQueue) await attempt(() => this.#queue.close());
+    if (this.#ownsBackgroundQueue) await attempt(() => this.#backgroundQueue.close());
     if (force) {
-      // The migration in flight is not waited for — but it keeps its
-      // connection until it ends: a kit this object created is disconnected
-      // only once the processor has settled.
-      this.#processor
-        .close()
+      // The work in flight is not waited for — but it keeps its connection
+      // until it ends: a kit this object created is disconnected only once
+      // the processors have settled.
+      Promise.allSettled(processors.map((processor) => processor.close()))
         .then(() => (this.#ownsKit ? this.#kit.disconnect() : undefined))
         .catch(() => undefined);
-    } else {
-      await attempt(() => this.#processor.close());
-      if (this.#ownsKit) await attempt(() => this.#kit.disconnect());
+    } else if (this.#ownsKit) {
+      await attempt(() => this.#kit.disconnect());
     }
     if (failures.length > 0) throw failures[0];
   }