@lenne.tech/nest-server 12.1.0 → 12.2.0

package/src/core/common/services/core-cron-jobs.service.ts

@@ -8,6 +8,7 @@ import { CRON_LOCK_COLLECTION, cronMongoLocks, defaultSqlLockStore } from '../pe
 import { Falsy } from '../types/falsy.type.js';
 import { getCronJobsInfrastructure } from './core-cron-jobs.registry.js';
 
+import type { LockStore } from '../persistence/lock-store.interface.js';
 import type { CoreRedisService } from './core-redis.service.js';
 import type { Redis } from 'ioredis';
 import type { Connection } from 'mongoose';
@@ -44,6 +45,28 @@ const CRON_LOCK_TTL_SECONDS = 3600;
  */
 const CRON_INIT_LOCK_TTL_SECONDS = 300;
 
+/**
+ * Lifetime of the lease a `runParallel: false` job holds while it RUNS, renewed every third of it.
+ *
+ * Short, so a replica that dies mid-run frees the job within a minute; the heartbeat keeps it held for as long as the
+ * run takes.
+ */
+const CRON_RUNNING_LOCK_TTL_SECONDS = 60;
+
+/** Extends a Redis lease only for its owner */
+const REDIS_RENEW_SCRIPT =
+  "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end";
+
+/** Deletes a Redis lease only for its owner */
+const REDIS_RELEASE_SCRIPT =
+  "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end";
+
+/**
+ * Jobs one `clean()` call removes when the backlog of earlier versions is trimmed: about 6.5 ms of Redis time each, so
+ * Redis keeps answering other clients between the calls
+ */
+const CRON_JOB_TRIM_BATCH = 1000;
+
 /**
  * Name of the shared BullMQ queue carrying all distributed cron jobs
  */
@@ -78,11 +101,13 @@ export interface CoreCronJobsOptions {
 /**
  * Cron jobs service to extend
  *
- * Jobs are deduplicated across replicas unless `distributed: false` is set:
+ * Jobs are deduplicated across replicas when they are `distributed` — the default with `redis` configured; without
+ * it a job runs on every replica unless it sets `distributed: true`:
  * - Redis configured + `bullmq` installed: the job becomes a BullMQ job scheduler,
  *   so exactly one worker in the cluster picks up each tick.
  * - Otherwise: the local cron timer stays, but each tick first acquires a lease
- *   (Redis `SET NX` when available, else a `cron-locks` document in MongoDB).
+ *   (Redis `SET NX` when available, else the SQL lease table of the default source, else a `cron-locks` document in
+ *   MongoDB).
  */
 export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicationShutdown {
   /**
@@ -98,9 +123,39 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
    */
   runningJobs: Record<string, Date[]> = {};
 
+  /**
+   * Options of every job a BullMQ scheduler creates (DEV-3896). One attempt: a failed tick is not repeated, as with the
+   * local timer — the next tick runs on schedule, and a retry could repeat side effects of a run that failed halfway.
+   * Completed and failed jobs are trimmed, one job per tick would otherwise stay in Redis for good. A subclass may set
+   * its own.
+   */
+  protected bullJobOptions: Record<string, unknown> = {
+    attempts: 1,
+    removeOnComplete: { count: 100 },
+    removeOnFail: { count: 500 },
+  };
+
   /** Shared BullMQ queue, created only in BullMQ mode */
   protected bullQueue?: any;
 
+  /** Redis connection of the shared queue; its `ready` event triggers the scheduler check after a reconnect */
+  protected bullQueueConnection?: any;
+
+  /** Names of the jobs registered as BullMQ job schedulers, i.e. the schedules that must exist in Redis */
+  protected bullScheduledJobs = new Set<string>();
+
+  /** Running scheduler check, so a reconnect and the timer never check twice at once */
+  protected bullSchedulerCheck?: Promise<void>;
+
+  /**
+   * How often to check that every BullMQ scheduler still exists in Redis, in ms. `0` switches the time-based check
+   * off; the check after a reconnect of the queue connection stays. See restoreBullSchedulers().
+   */
+  protected bullSchedulerCheckIntervalMs = 60_000;
+
+  /** Timer of the periodic scheduler check */
+  protected bullSchedulerCheckTimer?: ReturnType<typeof setInterval>;
+
   /** Shared BullMQ worker, created only in BullMQ mode */
   protected bullWorker?: any;
 
@@ -120,6 +175,14 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
    */
   protected readonly leaseOwner = `cron:${process.pid}:${Math.random().toString(36).slice(2)}`;
 
+  /**
+   * Listener for the `ready` event of the queue connection. An arrow function, so shutdown can remove exactly this
+   * listener. `void` is safe: restoreBullSchedulers() catches every failure per job and never rejects.
+   */
+  protected readonly onBullQueueReady = (): void => {
+    void this.restoreBullSchedulers();
+  };
+
   // ===================================================================================================================
   // Initializations
   // ===================================================================================================================
@@ -156,6 +219,9 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
    * Close the BullMQ worker and queue
    */
   async onApplicationShutdown(): Promise<void> {
+    this.stopBullSchedulerCheck();
+    // A check already running finishes its current job before the queue closes; it never rejects.
+    await this.bullSchedulerCheck;
     await this.drainRunningJobs();
 
     // Settled, not sequential-and-throwing: a rejection from the worker's close would abort the
@@ -181,7 +247,12 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     }
 
     const bullMqActive = await this.initBullMq();
-    const bullScheduled = new Set<string>();
+    if (bullMqActive) {
+      await this.trimBullJobBacklog();
+    }
+    this.bullScheduledJobs.clear();
+    /** Jobs that run on every replica because no redis is configured and they did not choose either way */
+    const localByDefault: string[] = [];
 
     // Init cron jobs
     for (const [name, CronExpressionOrConfig] of Object.entries(this.cronJobs)) {
@@ -239,7 +310,7 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
       if (bullMqActive && distributed) {
         if (typeof config.cronTime === 'string' && !config.utcOffset) {
           await this.registerBullJob(name, config);
-          bullScheduled.add(name);
+          this.bullScheduledJobs.add(name);
           continue;
         }
         cronLogger.debug(
@@ -251,10 +322,15 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
 
       // Init local cron job
       this.registerLocalJob(name, config, distributed);
+      if (!distributed && conf.distributed === undefined) {
+        localByDefault.push(name);
+      }
     }
 
-    await this.removeStaleBullSchedulers(bullScheduled);
+    this.logLocalJobsHint(localByDefault);
+    await this.removeStaleBullSchedulers(this.bullScheduledJobs);
     this.startBullWorker();
+    this.startBullSchedulerCheck();
   }
 
   /**
@@ -352,6 +428,9 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
         .createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 })
         .then(() => undefined)
         .catch((e) => {
+          // Tried again on the next tick: cached as done, the index would never exist and `cron-locks` would grow
+          // without bound (DEV-3896). The lease itself does not need it — expiry is decided by expiresAt.
+          this.leaseIndexReady = undefined;
           cronLogger.error(`Could not create TTL index on cron lock collection: ${errorMessage(e)}`, errorStack(e));
         });
     }
@@ -436,11 +515,17 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     // same outage a real rejection that the caller can report.
     const queueOptions = { commandTimeout: 10_000, maxRetriesPerRequest: 3 };
 
+    const queueConnection = service.createClient('cron-queue', queueOptions);
     this.bullQueue = new bullmq.Queue(CRON_QUEUE_NAME, {
-      connection: service.createClient('cron-queue', queueOptions),
+      connection: queueConnection,
       prefix,
     });
 
+    // A Redis or Valkey restart without persistence loses every scheduler, and the reconnect is the first moment a
+    // replica can notice it — see restoreBullSchedulers().
+    this.bullQueueConnection = queueConnection;
+    queueConnection.on?.('ready', this.onBullQueueReady);
+
     const workerConnection = service.createClient('cron-worker', blockingOptions);
     // autorun: false — initBullMq() runs BEFORE the loop that fills `jobConfigs`, so a worker
     // that starts consuming here would hand `runTick` a job it has no config for. That path
@@ -489,6 +574,44 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     }
   }
 
+  /**
+   * Trim the finished cron jobs earlier versions kept, in bounded batches, before the schedulers get `bullJobOptions`.
+   *
+   * Up to 12.1.0 every completed and failed tick stayed in Redis. BullMQ trims by `count` in ONE Lua call without a
+   * limit, so the first tick to finish under the new options would delete the whole backlog at once — measured 2 s for
+   * 300k jobs and 4.4 s for 700k, during which Redis answers nobody and every command of every replica times out
+   * (DEV-3896). `clean()` removes the oldest first and honours its limit. Once the backlog is gone this is one count
+   * per type at bootstrap. A failure is reported and does not stop the jobs.
+   */
+  protected async trimBullJobBacklog(): Promise<void> {
+    const keep = {
+      completed: (this.bullJobOptions.removeOnComplete as { count?: number } | undefined)?.count,
+      failed: (this.bullJobOptions.removeOnFail as { count?: number } | undefined)?.count,
+    };
+    for (const type of ['completed', 'failed'] as const) {
+      const limit = keep[type];
+      if (typeof limit !== 'number' || limit < 0) {
+        continue;
+      }
+      try {
+        let excess = (await this.bullQueue.getJobCountByTypes(type)) - limit;
+        if (excess <= 0) {
+          continue;
+        }
+        cronLogger.log(`Trimming ${excess} ${type} cron jobs an earlier version kept in Redis`);
+        while (excess > 0) {
+          const removed: string[] = await this.bullQueue.clean(0, Math.min(CRON_JOB_TRIM_BATCH, excess), type);
+          if (!removed?.length) {
+            break;
+          }
+          excess -= removed.length;
+        }
+      } catch (error) {
+        cronLogger.warn(`Trimming the ${type} cron jobs failed: ${errorMessage(error)}`);
+      }
+    }
+  }
+
   /**
    * Drop schedulers in the shared queue that this version no longer registers.
    *
@@ -513,6 +636,7 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
       for (const scheduler of await this.bullQueue.getJobSchedulers()) {
         if (scheduler?.key && known.has(scheduler.key) && !active.has(scheduler.key)) {
           await this.bullQueue.removeJobScheduler(scheduler.key);
+          await this.markBullSchedulerRemoved(scheduler.key);
           cronLogger.log(`CronJob scheduler ${scheduler.key} removed — no longer configured`);
         }
       }
@@ -522,6 +646,121 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     }
   }
 
+  /**
+   * Register every BullMQ scheduler again that Redis no longer has.
+   *
+   * `upsertJobScheduler` writes a schedule into Redis once, at bootstrap. A Redis or Valkey restart without
+   * persistence, a FLUSHDB or an eviction under `maxmemory` removes it, and before 12.1.1 nothing registered it
+   * again: no replica ran the job until the next deploy, without a log line, because the worker kept polling an empty
+   * queue. This runs after every reconnect of the queue connection and every `bullSchedulerCheckIntervalMs`.
+   *
+   * Only MISSING schedulers are registered. An upsert of an existing one is not free: BullMQ removes the job the
+   * scheduler planned last, also one that is already due and waits for a free worker slot, and plans the next one
+   * from now, so the due tick would be lost. Replicas that find a scheduler missing at the same moment may both
+   * register it; the second upsert replaces the first one's job with an identical one.
+   *
+   * Which schedulers exist comes from the LISTING, not from `getJobScheduler()`: that method does not exist before
+   * bullmq 5.23, and up to 5.40.2 it answers a deleted scheduler with an object instead of `undefined` — both inside
+   * the peer range. A scheduler a newer version removed on purpose (see markBullSchedulerRemoved()) stays removed.
+   *
+   * Only the schedule comes back: `runOnInit` and the registry entry belong to bootstrap. A tick that fell due while
+   * the scheduler was gone is not run.
+   */
+  protected restoreBullSchedulers(): Promise<void> {
+    if (!this.bullSchedulerCheck) {
+      this.bullSchedulerCheck = (async () => {
+        if (!this.bullQueue || !this.bullScheduledJobs.size) {
+          return;
+        }
+        const present = await this.listBullSchedulerKeys();
+        if (!present) {
+          return;
+        }
+        for (const name of this.bullScheduledJobs) {
+          const config = this.jobConfigs[name];
+          if (!this.bullQueue || !config || present.has(name)) {
+            continue;
+          }
+          try {
+            if (await this.isBullSchedulerRemoved(name)) {
+              continue;
+            }
+            await this.upsertBullScheduler(name, config);
+            cronLogger.warn(`CronJob scheduler ${name} was missing in Redis and has been registered again`);
+          } catch (error) {
+            cronLogger.warn(`Failed to check the BullMQ scheduler of CronJob ${name}: ${errorMessage(error)}`);
+          }
+        }
+      })().finally(() => {
+        this.bullSchedulerCheck = undefined;
+      });
+    }
+    return this.bullSchedulerCheck;
+  }
+
+  /** Keys of the schedulers the shared queue holds, or `undefined` when the listing failed (logged) */
+  protected async listBullSchedulerKeys(): Promise<Set<string> | undefined> {
+    try {
+      const schedulers: { key?: string }[] = await this.bullQueue.getJobSchedulers();
+      return new Set(schedulers.map((scheduler) => scheduler?.key).filter((key): key is string => !!key));
+    } catch (error) {
+      cronLogger.warn(`Failed to list the BullMQ job schedulers: ${errorMessage(error)}`);
+      return undefined;
+    }
+  }
+
+  /**
+   * Redis key marking a scheduler that removeStaleBullSchedulers() removed on purpose
+   *
+   * In a rolling deploy the new version's bootstrap removes the scheduler of a job it no longer runs through BullMQ,
+   * while replicas of the old version still have that job in `bullScheduledJobs`. Their check finds the scheduler
+   * missing and would register it again; for a job moved to a local timer that is a second run every tick until the
+   * next deploy. The marker tells them the scheduler is gone on purpose. registerBullJob() deletes it, and a real data
+   * loss takes it along with the schedulers, so restoring still works then.
+   */
+  protected bullSchedulerRemovedKey(name: string): string | undefined {
+    return this.getRedisService()?.key('cron-scheduler-removed', name);
+  }
+
+  /** Whether a newer configuration removed this job's scheduler on purpose; see bullSchedulerRemovedKey() */
+  protected async isBullSchedulerRemoved(name: string): Promise<boolean> {
+    const key = this.bullSchedulerRemovedKey(name);
+    return !!key && !!(await this.getRedisClient()?.get(key));
+  }
+
+  /** Record that this job's scheduler was removed on purpose; see bullSchedulerRemovedKey() */
+  protected async markBullSchedulerRemoved(name: string): Promise<void> {
+    const key = this.bullSchedulerRemovedKey(name);
+    if (key) {
+      await this.getRedisClient()?.set(key, new Date().toISOString());
+    }
+  }
+
+  /**
+   * Arm the periodic scheduler check. Only the reconnect notices a Redis restart; a FLUSHDB or an eviction closes no
+   * connection, so only a timer finds those. Unref'd, so the timer never keeps a process alive on its own.
+   */
+  protected startBullSchedulerCheck(): void {
+    if (!this.bullQueue || !this.bullScheduledJobs.size || !(this.bullSchedulerCheckIntervalMs > 0)) {
+      return;
+    }
+    this.bullSchedulerCheckTimer = setInterval(this.onBullQueueReady, this.bullSchedulerCheckIntervalMs);
+    this.bullSchedulerCheckTimer.unref?.();
+  }
+
+  /**
+   * Stop the scheduler check before the queue closes, so neither the timer nor a reconnect touches a closed queue
+   */
+  protected stopBullSchedulerCheck(): void {
+    if (this.bullSchedulerCheckTimer) {
+      clearInterval(this.bullSchedulerCheckTimer);
+      this.bullSchedulerCheckTimer = undefined;
+    }
+    this.bullQueueConnection?.off?.('ready', this.onBullQueueReady);
+    this.bullQueueConnection = undefined;
+    this.bullScheduledJobs.clear();
+  }
+
   /**
    * Start consuming once every job config is registered.
    *
@@ -552,11 +791,12 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
    * Register a job as BullMQ job scheduler
    */
   protected async registerBullJob(name: string, config: CronJobConfigWithTimeZone): Promise<void> {
-    await this.bullQueue.upsertJobScheduler(
-      name,
-      { pattern: config.cronTime as string, tz: config.timeZone ?? undefined },
-      { name },
-    );
+    // This version runs the job through BullMQ, so a removal marker an earlier configuration left no longer applies.
+    const removedKey = this.bullSchedulerRemovedKey(name);
+    if (removedKey) {
+      await this.getRedisClient()?.del(removedKey);
+    }
+    await this.upsertBullScheduler(name, config);
 
     // A BullMQ scheduler never fires immediately, so runOnInit is emulated locally under a lease.
     // Not awaited, so bootstrap is not blocked by the first run — same as the local timer path.
@@ -594,14 +834,31 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     }
   }
 
+  /**
+   * Write the job's schedule into Redis — at bootstrap and when restoreBullSchedulers() finds it missing
+   */
+  protected upsertBullScheduler(name: string, config: CronJobConfigWithTimeZone): Promise<unknown> {
+    return this.bullQueue.upsertJobScheduler(
+      name,
+      { pattern: config.cronTime as string, tz: config.timeZone ?? undefined },
+      { name, opts: this.bullJobOptions },
+    );
+  }
+
   /**
    * Register a job as local cron timer
    */
   protected registerLocalJob(name: string, config: CronJobConfigWithTimeZone, distributed: boolean): void {
     // Filled after construction; runOnInit fires DURING construction, where the fire time is simply "now"
     const ref: { job?: CronJob } = {};
-    /** Fire time of the last SCHEDULED tick, to tell a manual trigger apart from a repeat */
-    let lastLeasedFireTime: number | undefined;
+    /**
+     * The schedule instant the next scheduled fire belongs to, read when the job is armed and after every scheduled
+     * fire. cron's lastDate() is NOT that instant: cron 4 sets it to the moment its timer fired, so two replicas whose
+     * timers fire on either side of a second boundary computed different lease keys and both ran the tick (DEV-3896).
+     */
+    let scheduledFor: number | undefined;
+    /** lastDate() as of the last scheduled fire, to tell the next scheduled fire from a manual one */
+    let lastFired: number | undefined;
 
     const job = new CronJob(
       config.cronTime,
@@ -612,21 +869,23 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
           // the fleet-wide init lease, because "startup" is not an instant the replicas
           // agree on; a scheduled tick takes a lease keyed on the schedule instant, which
           // every replica computes identically.
-          const scheduled = ref.job?.lastDate();
-          if (!scheduled) {
+          if (!ref.job) {
             this.runInitTick(name);
             return;
           }
-          // `cron` only advances lastDate() on a SCHEDULED fire, so a manual fireOnTick() (the
-          // Hub's "Run now") reports the previous tick's instant — whose lease this replica
-          // already holds. Reusing it would lose the race against itself and silently do
-          // nothing, while the operator sees a success response. A manual run gets its own key.
-          if (scheduled.getTime() === lastLeasedFireTime) {
+          // cron advances lastDate() only when its timer fires, never before the instant; a manual fireOnTick() (the
+          // Hub's "Run now") leaves it as it was. Keyed on an instant, a manual run would take a lease another replica
+          // may already hold (the operator's run silently skipped) or the next tick's (that tick skipped everywhere),
+          // so it gets its own key.
+          const fired = ref.job.lastDate()?.getTime();
+          if (fired === undefined || fired === lastFired || scheduledFor === undefined || fired < scheduledFor) {
             await this.runDistributedTick(name, 'manual');
             return;
           }
-          lastLeasedFireTime = scheduled.getTime();
-          await this.runDistributedTick(name, scheduled);
+          lastFired = fired;
+          const instant = this.currentScheduledInstant(ref.job, scheduledFor, fired);
+          scheduledFor = this.nextScheduledInstant(ref.job);
+          await this.runDistributedTick(name, new Date(instant));
           return;
         }
         await this.runTick(name);
@@ -640,6 +899,7 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
       config.unrefTimeout,
     );
     ref.job = job;
+    scheduledFor = this.nextScheduledInstant(job);
     this.schedulerRegistry.addCronJob(name, job);
     if (this.config.log && this.schedulerRegistry.getCronJob(name)) {
       cronLogger.log(`CronJob ${name} initialized with "${config.cronTime}"`);
@@ -697,21 +957,175 @@ export abstract class CoreCronJobs implements OnApplicationBootstrap, OnApplicat
     const date = new Date();
     this.runningJobs[name].push(date);
 
-    // Execute the job and wait until job process is done
+    let lease: false | undefined | { release: () => Promise<void> };
     try {
+      // runParallel above is per process. A job that must not run twice at once holds a lease for the length of the
+      // run, so a second replica skips its next tick instead of starting it in parallel (DEV-3896).
+      if (this.runsExclusively(config)) {
+        lease = await this.acquireRunningLease(name);
+        if (lease === false) {
+          this.logSkippedTick(`${name}:running`);
+          return;
+        }
+      }
+
+      // Execute the job and wait until job process is done
       await this[name]();
     } catch (e) {
-      // Remove job from running list
-      this.runningJobs[name] = this.runningJobs[name].filter((item) => item !== date);
       if (config.throwException) {
         throw e;
       } else {
         cronLogger.error(`CronJob ${name} failed: ${errorMessage(e)}`, errorStack(e));
       }
+    } finally {
+      // Remove job from running list
+      this.runningJobs[name] = this.runningJobs[name].filter((item) => item !== date);
+      if (lease) {
+        await lease.release();
+      }
+    }
+  }
+
+  /**
+   * Whether a job may run only once at a time across all replicas: a distributed job with `runParallel: false` (or
+   * `1`). A larger `runParallel` stays a per-process limit.
+   */
+  protected runsExclusively(config: CronJobConfigWithTimeZone): boolean {
+    return config.distributed !== false && (!config.runParallel || config.runParallel === 1);
+  }
+
+  /**
+   * Take the lease a running job holds, kept alive by a heartbeat until `release()`.
+   *
+   * Returns `false` when another replica runs the job. Without a lease backend, or when the backend fails, the job runs
+   * anyway (fail open, like the tick lease): a missing lock must never stop the job.
+   */
+  protected async acquireRunningLease(name: string): Promise<false | { release: () => Promise<void> }> {
+    const key = `${name}:running`;
+    const unlocked = { release: async () => undefined };
+    let store: LockStore | undefined;
+    try {
+      store = await this.leaseStore();
+      if (!store) {
+        this.warnDedupInactive();
+        return unlocked;
+      }
+      if (!(await store.tryAcquire(key, this.leaseOwner, CRON_RUNNING_LOCK_TTL_SECONDS))) {
+        return false;
+      }
+    } catch (e) {
+      cronLogger.error(`Cron running lease for ${key} failed, running anyway: ${errorMessage(e)}`, errorStack(e));
+      return unlocked;
     }
 
-    // Remove job from running list
-    this.runningJobs[name] = this.runningJobs[name].filter((item) => item !== date);
+    const held = store;
+    const heartbeat = setInterval(
+      () => {
+        held
+          .renew(key, this.leaseOwner, CRON_RUNNING_LOCK_TTL_SECONDS)
+          .then((renewed) => {
+            if (!renewed) {
+              cronLogger.warn(`Cron running lease for ${key} was lost; another replica may start the job`);
+            }
+          })
+          .catch((e) => cronLogger.warn(`Renewing the cron running lease for ${key} failed: ${errorMessage(e)}`));
+      },
+      (CRON_RUNNING_LOCK_TTL_SECONDS * 1000) / 3,
+    );
+    heartbeat.unref?.();
+
+    return {
+      release: async () => {
+        clearInterval(heartbeat);
+        await held
+          .release(key, this.leaseOwner)
+          .catch((e) => cronLogger.warn(`Releasing the cron running lease for ${key} failed: ${errorMessage(e)}`));
+      },
+    };
+  }
+
+  /**
+   * The store for leases with an owner: Redis when available, else the SQL lease table of the default source, else
+   * `cron-locks` in MongoDB; `undefined` when there is none
+   */
+  protected async leaseStore(): Promise<LockStore | undefined> {
+    const redis = this.getRedisClient();
+    if (redis) {
+      return this.redisLeaseStore(redis);
+    }
+    const sqlLocks = await defaultSqlLockStore('cron');
+    if (sqlLocks) {
+      return sqlLocks;
+    }
+    const connection = this.getConnection();
+    if (!connection) {
+      return undefined;
+    }
+    await this.ensureLeaseIndex(connection);
+    return cronMongoLocks(connection.db.collection(CRON_LOCK_COLLECTION));
+  }
+
+  /** Leases in Redis under `<keyPrefix>:cron-lock:<key>`, renewed and released only by their owner */
+  protected redisLeaseStore(redis: Redis): LockStore {
+    const redisKey = (key: string) => this.getRedisService().key('cron-lock', key);
+    return {
+      release: async (key, owner) => (await redis.eval(REDIS_RELEASE_SCRIPT, 1, redisKey(key), owner)) === 1,
+      renew: async (key, owner, ttlSeconds) =>
+        (await redis.eval(REDIS_RENEW_SCRIPT, 1, redisKey(key), owner, ttlSeconds * 1000)) === 1,
+      tryAcquire: async (key, owner, ttlSeconds) =>
+        (await redis.set(redisKey(key), owner, 'EX', ttlSeconds, 'NX')) === 'OK',
+    };
+  }
+
+  /**
+   * The instant a scheduled fire belongs to: the remembered instant, moved forward to the last one at or before the
+   * fire. It stays as remembered on every regular fire; it lags only when the job did not fire for a while (stopped
+   * in the Hub and started again), and keyed on that old instant the first tick after a restart would run next to
+   * the replica that runs the current one. Not moved for a `utcOffset` job or a one-shot date, whose schedule this
+   * cannot step through.
+   */
+  protected currentScheduledInstant(job: CronJob, remembered: number, fired: number): number {
+    const cronTime = job.cronTime;
+    if (cronTime.utcOffset !== undefined || cronTime.realDate) {
+      return remembered;
+    }
+    let instant = remembered;
+    try {
+      // Bounded: a job stopped for a long time on a per-second schedule must not keep the tick busy
+      for (let step = 0; step < 10_000; step++) {
+        const next = cronTime.getNextDateFrom(new Date(instant), cronTime.timeZone).toMillis();
+        if (next > fired) {
+          break;
+        }
+        instant = next;
+      }
+    } catch {
+      // cron could not step the schedule: keep what is known
+    }
+    return instant;
+  }
+
+  /** The next instant the job's schedule fires at, in ms; `undefined` when cron cannot say (a past one-shot date) */
+  protected nextScheduledInstant(job: CronJob): number | undefined {
+    try {
+      return job.nextDate().toMillis();
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Say once at startup which jobs run on every replica because no redis is configured and they did not choose with
+   * `distributed`. Info, not a warning: most projects run one replica, where this is right (DEV-3896).
+   */
+  protected logLocalJobsHint(names: string[]): void {
+    if (!names.length || !this.config.log) {
+      return;
+    }
+    cronLogger.log(
+      `CronJobs ${names.join(', ')} run on every replica: no redis is configured. With more than one replica, ` +
+        'configure redis, or set distributed: true to take a lease in MongoDB or the SQL default source.',
+    );
   }
 
   /**

package/src/core/common/services/core-redis-pubsub.ts

@@ -75,14 +75,18 @@ export class CoreRedisPubSub extends PubSubEngine implements OnModuleDestroy {
     const channel = this.channel(triggerName);
     let channelHandlers = this.handlers.get(channel);
 
-    // First handler for this channel: subscribe on Redis
+    // First handler for this channel: subscribe on Redis. Registered with the Redis service, so the subscription is
+    // established again after every reconnect.
     if (!channelHandlers) {
       channelHandlers = new Map();
       this.handlers.set(channel, channelHandlers);
       try {
-        await this.getSubscriber().subscribe(channel);
+        this.getSubscriber();
+        await this.redisService.subscribeChannel(channel);
       } catch (error) {
+        // The caller gets the error, as before, so the channel is not kept for a later reconnect either.
         this.handlers.delete(channel);
+        await this.redisService.releaseChannel(channel).catch(() => undefined);
         throw error;
       }
     }
@@ -109,7 +113,7 @@ export class CoreRedisPubSub extends PubSubEngine implements OnModuleDestroy {
     // Last handler for this channel: unsubscribe on Redis
     if (channelHandlers.size === 0) {
       this.handlers.delete(channel);
-      await this.getSubscriber().unsubscribe(channel);
+      await this.redisService.releaseChannel(channel);
     }
   }
 
@@ -163,9 +167,7 @@ export class CoreRedisPubSub extends PubSubEngine implements OnModuleDestroy {
         subscriber.off('message', this.messageListener);
         this.messageListener = undefined;
       }
-      if (channels.length) {
-        await subscriber.unsubscribe(...channels);
-      }
+      await Promise.all(channels.map((channel) => this.redisService.releaseChannel(channel)));
     } catch (error) {
       // Redis already gone: the connection is being torn down anyway.
       this.logger.debug(`PubSub teardown skipped: ${error instanceof Error ? error.message : 'Unknown error'}`);