@orion-js/dogs 4.5.1 → 4.6.2

package/dist/index.js

@@ -71,6 +71,11 @@ var JobRecordSchema = schemaWithName("JobRecord", {
   nextRunAt: { type: "date" },
   lastRunAt: { type: "date", optional: true },
   lockedUntil: { type: "date", optional: true },
+  /**
+   * Identifies the execution that currently owns the lock. Mutations from an execution must match
+   * this value so a stale execution cannot modify a newer claim.
+   */
+  lockId: { type: "string", optional: true },
   tries: { type: "number", optional: true },
   params: { type: "blackbox", optional: true },
   /**
@@ -93,6 +98,35 @@ var JobRecordSchema = schemaWithName("JobRecord", {
 var DEFAULT_MAX_TRIES_REACHED_RETENTION_MS = 1e3 * 60 * 60 * 24 * 7;
 
 // src/repos/JobsRepo.ts
+var JOB_ACQUISITION_HINTS = {
+  byJobName: {
+    jobName: 1,
+    priority: -1,
+    nextRunAt: 1
+  },
+  globalPriority: {
+    priority: -1,
+    nextRunAt: 1
+  }
+};
+var INITIAL_JOB_ACQUISITION_HINT = "byJobName";
+var CANDIDATE_JOB_ACQUISITION_HINT = "globalPriority";
+var JOB_ACQUISITION_SORT = {
+  priority: -1,
+  nextRunAt: 1
+};
+function getJobAcquisitionSelector(jobNames, now) {
+  return {
+    jobName: { $in: jobNames },
+    nextRunAt: { $lte: now },
+    $and: [
+      { $or: [{ lockedUntil: { $exists: false } }, { lockedUntil: { $lte: now } }] },
+      // Exclude event jobs that have reached max tries. Recurrent jobs keep running even
+      // if old records still have this status from previous versions.
+      { $or: [{ type: { $ne: "event" } }, { status: { $ne: "maxTriesReached" } }] }
+    ]
+  };
+}
 var _jobs_dec, _JobsRepo_decorators, _init;
 _JobsRepo_decorators = [Repository()], _jobs_dec = [MongoCollection({
   idGeneration: "uuid",
@@ -100,11 +134,10 @@ _JobsRepo_decorators = [Repository()], _jobs_dec = [MongoCollection({
   schema: JobRecordSchema,
   indexes: [
     {
-      keys: {
-        jobName: 1,
-        priority: -1,
-        nextRunAt: 1
-      }
+      keys: JOB_ACQUISITION_HINTS.byJobName
+    },
+    {
+      keys: JOB_ACQUISITION_HINTS.globalPriority
     },
     {
       keys: {
@@ -137,29 +170,20 @@ var JobsRepo = class {
   constructor() {
     __publicField(this, "jobs", __runInitializers(_init, 8, this)), __runInitializers(_init, 11, this);
   }
-  async getJobAndLock(jobNames, lockTime) {
-    const lockedUntil = new Date(Date.now() + lockTime);
+  async getJobAndLock(jobNames, lockTime, hint = INITIAL_JOB_ACQUISITION_HINT) {
+    const executionId = generateId();
+    const now = /* @__PURE__ */ new Date();
+    const lockedUntil = new Date(now.getTime() + lockTime);
     const job = await this.jobs.findOneAndUpdate(
+      getJobAcquisitionSelector(jobNames, now),
       {
-        jobName: { $in: jobNames },
-        nextRunAt: { $lte: /* @__PURE__ */ new Date() },
-        $and: [
-          { $or: [{ lockedUntil: { $exists: false } }, { lockedUntil: { $lte: /* @__PURE__ */ new Date() } }] },
-          // Exclude event jobs that have reached max tries. Recurrent jobs keep running even
-          // if old records still have this status from previous versions.
-          { $or: [{ type: { $ne: "event" } }, { status: { $ne: "maxTriesReached" } }] }
-        ]
-      },
-      {
-        $set: { lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
+        $set: { lockId: executionId, lockedUntil, lastRunAt: now },
         $inc: { tries: 1 }
       },
       {
         mongoOptions: {
-          sort: {
-            priority: -1,
-            nextRunAt: 1
-          },
+          hint: JOB_ACQUISITION_HINTS[hint],
+          sort: JOB_ACQUISITION_SORT,
           returnDocument: "before"
         }
       }
@@ -172,7 +196,8 @@ var JobsRepo = class {
     }
     return {
       jobId: job._id,
-      executionId: generateId(),
+      executionId,
+      lockId: executionId,
       name: job.jobName,
       params: job.params,
       type: job.type,
@@ -183,8 +208,24 @@ var JobsRepo = class {
       wasStale
     };
   }
-  async setJobRecordPriority(jobId, priority) {
-    await this.jobs.updateOne(jobId, { $set: { priority } });
+  async getJobAcquisitionExecutionTime(jobNames, hint) {
+    var _a;
+    const explain = await this.jobs.find(getJobAcquisitionSelector(jobNames, /* @__PURE__ */ new Date()), {
+      hint: JOB_ACQUISITION_HINTS[hint],
+      maxTimeMS: 1e3,
+      readPreference: "primary"
+    }).sort(JOB_ACQUISITION_SORT).limit(1).explain("executionStats");
+    const executionTimeMillis = (_a = explain.executionStats) == null ? void 0 : _a.executionTimeMillis;
+    if (typeof executionTimeMillis !== "number" || !Number.isFinite(executionTimeMillis)) {
+      throw new Error(`Explain for acquisition hint "${hint}" returned no execution time`);
+    }
+    return executionTimeMillis;
+  }
+  async setJobRecordPriority(jobId, priority, lockId) {
+    const result = await this.jobs.updateOne(lockId ? { _id: jobId, lockId } : jobId, {
+      $set: { priority }
+    });
+    return result.matchedCount > 0;
   }
   async scheduleNextRun(options) {
     const updator = {
@@ -193,22 +234,31 @@ var JobsRepo = class {
         priority: options.priority,
         ...options.resetTries ? { tries: 0 } : {}
       },
-      $unset: { lockedUntil: "" }
+      $unset: { lockId: "", lockedUntil: "" }
     };
-    await this.jobs.updateOne(options.jobId, updator);
+    const result = await this.jobs.updateOne(
+      options.lockId ? { _id: options.jobId, lockId: options.lockId } : options.jobId,
+      updator
+    );
+    return result.matchedCount > 0;
   }
-  async deleteEventJob(jobId) {
-    await this.jobs.deleteOne({ _id: jobId, type: "event" });
+  async deleteEventJob(jobId, lockId) {
+    const result = await this.jobs.deleteOne({
+      _id: jobId,
+      type: "event",
+      ...lockId ? { lockId } : {}
+    });
+    return result.deletedCount > 0;
   }
   /**
    * Marks an event job as having reached its maximum tries limit.
    * The job won't be picked up for execution and MongoDB will delete it after
    * the configured retention period.
    */
-  async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS) {
+  async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS, lockId) {
     const maxTriesReachedAt = /* @__PURE__ */ new Date();
-    await this.jobs.updateOne(
-      { _id: jobId, type: "event" },
+    const result = await this.jobs.updateOne(
+      { _id: jobId, type: "event", ...lockId ? { lockId } : {} },
       {
         $set: {
           status: "maxTriesReached",
@@ -216,11 +266,13 @@ var JobsRepo = class {
           ...retentionMs === null ? {} : { expiresAt: new Date(maxTriesReachedAt.getTime() + retentionMs) }
         },
         $unset: {
+          lockId: "",
           lockedUntil: "",
           ...retentionMs === null ? { expiresAt: "" } : {}
         }
       }
     );
+    return result.matchedCount > 0;
   }
   /**
    * Applies the current retention configuration to existing maxTriesReached
@@ -249,23 +301,22 @@ var JobsRepo = class {
       }
     ]);
   }
-  async extendLockTime(jobId, extraTime) {
-    await this.updateLockTime(jobId, extraTime);
+  async extendLockTime(jobId, extraTime, lockId) {
+    return this.updateLockTime(jobId, extraTime, lockId);
   }
   /**
    * Updates the lock time for a job to the specified duration from now.
    * Can be used to both extend or shorten the lock time.
    */
-  async updateLockTime(jobId, lockDuration) {
+  async updateLockTime(jobId, lockDuration, lockId) {
     const lockedUntil = new Date(Date.now() + lockDuration);
-    await this.jobs.updateOne(
-      {
-        _id: jobId
-      },
+    const result = await this.jobs.updateOne(
+      { _id: jobId, ...lockId ? { lockId } : {} },
       {
         $set: { lockedUntil }
       }
     );
+    return result.matchedCount > 0;
   }
   async unlockAllJobs() {
     const result = await this.jobs.updateMany(
@@ -273,7 +324,7 @@ var JobsRepo = class {
         lockedUntil: { $exists: true }
       },
       {
-        $unset: { lockedUntil: "" }
+        $unset: { lockId: "", lockedUntil: "" }
       }
     );
     return result.modifiedCount;
@@ -579,11 +630,15 @@ var Executor = class {
   getEffectiveLockTime(job, jobToRun) {
     return job.lockTime ?? jobToRun.lockTime;
   }
-  getContext(job, jobToRun, onStale) {
+  getContext(job, jobToRun, onStale, isExecutionActive) {
     var _a;
     const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
     let staleTimeout = setTimeout(() => onStale(), effectiveLockTime);
     (_a = staleTimeout.unref) == null ? void 0 : _a.call(staleTimeout);
+    const contextLogger = logger3.addMetadata({
+      jobName: jobToRun.name,
+      jobId: jobToRun.jobId
+    });
     return {
       definition: job,
       record: jobToRun,
@@ -591,15 +646,26 @@ var Executor = class {
       clearStaleTimeout: () => clearTimeout(staleTimeout),
       extendLockTime: async (extraTime) => {
         var _a2;
+        if (!isExecutionActive()) {
+          contextLogger.warn(`Will not extend lock for stale execution "${jobToRun.executionId}"`);
+          return;
+        }
         clearTimeout(staleTimeout);
+        let didExtend = false;
+        try {
+          didExtend = await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime, jobToRun.lockId);
+        } catch (error) {
+          onStale();
+          throw error;
+        }
+        if (!didExtend) {
+          onStale();
+          return;
+        }
         staleTimeout = setTimeout(() => onStale(), extraTime);
         (_a2 = staleTimeout.unref) == null ? void 0 : _a2.call(staleTimeout);
-        await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime);
       },
-      logger: logger3.addMetadata({
-        jobName: jobToRun.name,
-        jobId: jobToRun.jobId
-      })
+      logger: contextLogger
     };
   }
   getJobDefinition(jobToRun, jobs) {
@@ -631,8 +697,16 @@ var Executor = class {
     jobLogger.warn(
       `Job "${jobToRun.name}" has exceeded max tries (${jobToRun.tries}). Marking as maxTriesReached.`
     );
-    await this.jobsRepo.markJobAsMaxTriesReached(jobToRun.jobId, retentionMs);
-    if (!onMaxTriesReached) return;
+    const didMark = await this.jobsRepo.markJobAsMaxTriesReached(
+      jobToRun.jobId,
+      retentionMs,
+      jobToRun.lockId
+    );
+    if (!didMark) {
+      jobLogger.warn(`Could not mark job "${jobToRun.name}" because its lock is no longer owned`);
+      return false;
+    }
+    if (!onMaxTriesReached) return true;
     try {
       await onMaxTriesReached(jobToRun);
     } catch (callbackError) {
@@ -640,30 +714,33 @@ var Executor = class {
         error: callbackError
       });
     }
+    return true;
   }
   async onError(error, job, jobToRun, context) {
     const scheduleRecurrent = async () => {
       if (job.type === "recurrent") {
-        await this.jobsRepo.scheduleNextRun({
+        return this.jobsRepo.scheduleNextRun({
           jobId: jobToRun.jobId,
           nextRunAt: getNextRunDate(job),
           resetTries: true,
-          priority: job.priority
+          priority: job.priority,
+          lockId: jobToRun.lockId
         });
       }
+      return true;
     };
     const handleRetry = async (nextRunAt) => {
-      await this.jobsRepo.scheduleNextRun({
+      return this.jobsRepo.scheduleNextRun({
         jobId: jobToRun.jobId,
         nextRunAt,
         resetTries: false,
-        priority: job.type === "recurrent" ? job.priority : jobToRun.priority
+        priority: job.type === "recurrent" ? job.priority : jobToRun.priority,
+        lockId: jobToRun.lockId
       });
     };
     if (!job.onError) {
       context.logger.error(`Error executing job "${jobToRun.name}"`, { error });
-      await scheduleRecurrent();
-      return;
+      return scheduleRecurrent();
     }
     context.logger.info(`Error executing job "${jobToRun.name}"`, { error });
     const result = await job.onError(
@@ -672,12 +749,12 @@ var Executor = class {
       context
     );
     if (result.action === "dismiss") {
-      await scheduleRecurrent();
-      return;
+      return scheduleRecurrent();
     }
     if (result.action === "retry") {
-      await handleRetry(getNextRunDate(result));
+      return handleRetry(getNextRunDate(result));
     }
+    return true;
   }
   async saveExecution(options) {
     const { startedAt, status, errorMessage, result, job, jobToRun } = options;
@@ -707,19 +784,21 @@ var Executor = class {
   async afterExecutionSuccess(job, jobToRun, context) {
     if (job.type === "recurrent") {
       context.logger.debug(`Scheduling next run for recurrent job "${jobToRun.name}"`);
-      await this.jobsRepo.scheduleNextRun({
+      return this.jobsRepo.scheduleNextRun({
         jobId: jobToRun.jobId,
         nextRunAt: getNextRunDate(job),
         resetTries: true,
-        priority: job.priority
+        priority: job.priority,
+        lockId: jobToRun.lockId
       });
     }
     if (job.type === "event") {
       context.logger.debug(`Removing event job after success "${jobToRun.name}"`);
-      await this.jobsRepo.deleteEventJob(jobToRun.jobId);
+      return this.jobsRepo.deleteEventJob(jobToRun.jobId, jobToRun.lockId);
     }
+    return false;
   }
-  async executeJob(config, jobToRun, respawnWorker) {
+  async executeJob(config, jobToRun, onExecutionStale) {
     const job = this.getJobDefinition(jobToRun, config.jobs);
     if (!job) return;
     if (job.type === "event") {
@@ -735,33 +814,64 @@ var Executor = class {
     }
     const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
     if (effectiveLockTime !== jobToRun.lockTime) {
-      await this.jobsRepo.updateLockTime(jobToRun.jobId, effectiveLockTime);
+      const didUpdateLock = await this.jobsRepo.updateLockTime(
+        jobToRun.jobId,
+        effectiveLockTime,
+        jobToRun.lockId
+      );
+      if (!didUpdateLock) {
+        logger3.warn(`Will not execute job "${jobToRun.name}" because its lock is no longer owned`);
+        return;
+      }
     }
     const tracer = trace.getTracer("orionjs.dogs", "1.0");
     await tracer.startActiveSpan(`job.${jobToRun.name}.${jobToRun.executionId}`, async (span) => {
       try {
         const startedAt = /* @__PURE__ */ new Date();
-        const onStale = async () => {
-          if (job.onStale) {
-            context.logger.info(`Job "${jobToRun.name}" is stale`);
-            job.onStale(jobToRun.params, context);
-          } else {
-            context.logger.error(`Job "${jobToRun.name}" is stale`);
-          }
-          await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0);
-          void respawnWorker();
-          void this.saveExecution({
-            startedAt,
-            status: "stale",
-            result: null,
-            errorMessage: null,
-            job,
-            jobToRun
-          }).catch((error) => {
-            context.logger.error("Error saving stale execution history", { error });
+        let executionStatus = "running";
+        let staleHandlingPromise;
+        let context;
+        const markAsStale = () => {
+          if (executionStatus === "stale") return staleHandlingPromise;
+          executionStatus = "stale";
+          context.clearStaleTimeout();
+          staleHandlingPromise = (async () => {
+            context.logger.warn(`Job "${jobToRun.name}" is stale`);
+            if (job.onStale) {
+              void Promise.resolve(job.onStale(jobToRun.params, context)).catch((error) => {
+                context.logger.error(`Error in onStale callback for job "${jobToRun.name}"`, {
+                  error
+                });
+              });
+            }
+            try {
+              await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0, jobToRun.lockId);
+            } catch (error) {
+              context.logger.error(`Error lowering priority for stale job "${jobToRun.name}"`, {
+                error
+              });
+            } finally {
+              onExecutionStale();
+            }
+            await this.saveExecution({
+              startedAt,
+              status: "stale",
+              result: null,
+              errorMessage: null,
+              job,
+              jobToRun
+            });
+          })().catch((error) => {
+            context.logger.error("Error handling stale execution", { error });
           });
+          return staleHandlingPromise;
         };
-        const context = this.getContext(job, jobToRun, onStale);
+        context = this.getContext(
+          job,
+          jobToRun,
+          () => void markAsStale(),
+          () => executionStatus === "running"
+        );
         const extraContext = {
           controllerType: "job",
           jobName: jobToRun.name,
@@ -777,6 +887,18 @@ var Executor = class {
             });
             const result = await job.resolve(jobToRun.params, context);
             context.clearStaleTimeout();
+            if (executionStatus === "stale") {
+              await staleHandlingPromise;
+              context.logger.warn(
+                `Ignoring late success from stale execution "${jobToRun.executionId}"`
+              );
+              return;
+            }
+            const stillOwnsJob = await this.afterExecutionSuccess(job, jobToRun, context);
+            if (!stillOwnsJob) {
+              await markAsStale();
+              return;
+            }
             void this.saveExecution({
               startedAt,
               status: "success",
@@ -787,9 +909,21 @@ var Executor = class {
             }).catch((error) => {
               context.logger.error("Error saving successful execution history", { error });
             });
-            await this.afterExecutionSuccess(job, jobToRun, context);
           } catch (error) {
             context.clearStaleTimeout();
+            if (executionStatus === "stale") {
+              await staleHandlingPromise;
+              context.logger.warn(
+                `Ignoring late error from stale execution "${jobToRun.executionId}"`,
+                { error }
+              );
+              return;
+            }
+            const stillOwnsJob = await this.onError(error, job, jobToRun, context);
+            if (!stillOwnsJob) {
+              await markAsStale();
+              return;
+            }
             void this.saveExecution({
               startedAt,
               status: "error",
@@ -800,7 +934,6 @@ var Executor = class {
             }).catch((saveError) => {
               context.logger.error("Error saving failed execution history", { error: saveError });
             });
-            await this.onError(error, job, jobToRun, context);
           }
         });
       } catch (error) {
@@ -826,6 +959,7 @@ var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
 _WorkerService_decorators = [Service3()], _jobsRepo_dec3 = [Inject3(() => JobsRepo)], _executor_dec = [Inject3(() => Executor)];
 var WorkerService = class {
   constructor() {
+    __publicField(this, "jobAcquisitionHintProbeIntervalMs", 30 * 60 * 1e3);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -860,87 +994,242 @@ var WorkerService = class {
     }
     workersInstance.runningJobsByName.set(jobName, currentExecutions - 1);
   }
-  async withJobAcquisitionLock(workersInstance, callback) {
-    const previousLock = workersInstance.jobAcquisitionLock;
-    let releaseLock;
-    workersInstance.jobAcquisitionLock = new Promise((resolve) => {
-      releaseLock = resolve;
-    });
-    await previousLock;
-    try {
-      return await callback();
-    } finally {
-      releaseLock();
+  releaseExecutionCapacity(workersInstance, execution, nextStatus) {
+    if (execution.status !== "running") return;
+    execution.status = nextStatus;
+    workersInstance.activeExecutions.delete(execution.job.executionId);
+    this.releaseJobExecution(workersInstance, execution.job.name);
+    execution.releaseCapacity();
+    if (nextStatus === "stale") {
+      workersInstance.staleExecutions.set(execution.job.executionId, execution);
     }
   }
-  async getJobAndReserveExecution(config, workersInstance, jobNames) {
-    return this.withJobAcquisitionLock(workersInstance, async () => {
-      const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
-      if (availableJobNames.length === 0) return;
-      const jobToRun = await this.jobsRepo.getJobAndLock(availableJobNames, config.defaultLockTime);
-      if (!jobToRun) return;
-      this.reserveJobExecution(workersInstance, jobToRun.name);
-      return jobToRun;
+  startExecution(config, workersInstance, jobToRun) {
+    let releaseCapacity;
+    const capacityPromise = new Promise((resolve) => {
+      releaseCapacity = resolve;
+    });
+    const execution = {
+      job: jobToRun,
+      promise: Promise.resolve(),
+      capacityPromise,
+      releaseCapacity,
+      status: "running"
+    };
+    this.reserveJobExecution(workersInstance, jobToRun.name);
+    workersInstance.activeExecutions.set(jobToRun.executionId, execution);
+    logger4.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
+    const executionPromise = this.executor.executeJob(config, jobToRun, () => {
+      this.releaseExecutionCapacity(workersInstance, execution, "stale");
+    });
+    execution.promise = executionPromise;
+    void executionPromise.catch((error) => {
+      logger4.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
+    }).finally(() => {
+      if (execution.status === "running") {
+        this.releaseExecutionCapacity(workersInstance, execution, "finished");
+      } else if (execution.status === "stale") {
+        workersInstance.staleExecutions.delete(jobToRun.executionId);
+      }
     });
   }
-  async runWorkerLoop(config, workersInstance, workerInstance, jobNames, executeConfig) {
-    const jobToRun = await this.getJobAndReserveExecution(config, workersInstance, jobNames);
-    if (!jobToRun) {
-      logger4.debug("No job to run");
-      return false;
-    }
-    logger4.debug(`Got job [w${workerInstance.workerIndex}] to run:`, jobToRun);
-    try {
-      await this.executor.executeJob(executeConfig, jobToRun, workerInstance.respawn);
-    } finally {
-      this.releaseJobExecution(workersInstance, jobToRun.name);
-    }
-    return true;
+  async waitForPollOrStop(config, workersInstance) {
+    await Promise.race([sleep(config.pollInterval), workersInstance.stopSignal]);
   }
-  async startWorker(config, workersInstance, workerInstance) {
-    const names = this.getJobNames(config.jobs);
-    logger4.debug(
-      `Running worker loop [w${workerInstance.workerIndex}] for jobs "${names.join(", ")}"...`
+  async waitForCapacityOrStop(workersInstance) {
+    const capacityPromises = Array.from(
+      workersInstance.activeExecutions.values(),
+      (execution) => execution.capacityPromise
     );
+    if (capacityPromises.length === 0) return;
+    await Promise.race([...capacityPromises, workersInstance.stopSignal]);
+  }
+  async runSchedulerLoop(config, workersInstance) {
+    const jobNames = this.getJobNames(config.jobs);
     const executeConfig = {
       jobs: config.jobs,
       maxTries: config.maxTries,
       maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
       onMaxTriesReached: config.onMaxTriesReached
     };
-    while (true) {
-      if (!workerInstance.running) {
-        logger4.info(`Got signal to stop. Stopping worker [w${workerInstance.workerIndex}]...`);
-        return;
-      }
+    logger4.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
+    while (workersInstance.running) {
+      let didMiss = false;
+      let allJobNamesAtCapacity = false;
       try {
-        const didRun = await this.runWorkerLoop(
-          config,
-          workersInstance,
-          workerInstance,
-          names,
-          executeConfig
-        );
-        if (!didRun) await sleep(config.pollInterval);
-        if (didRun) await sleep(config.cooldownPeriod);
+        while (workersInstance.activeExecutions.size < config.workersCount) {
+          const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
+          if (availableJobNames.length === 0) {
+            allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
+            break;
+          }
+          const jobToRun = await this.jobsRepo.getJobAndLock(
+            availableJobNames,
+            config.defaultLockTime,
+            workersInstance.jobAcquisitionHint
+          );
+          if (!jobToRun) {
+            logger4.debug("No job to run");
+            didMiss = true;
+            break;
+          }
+          this.startExecution(executeConfig, workersInstance, jobToRun);
+          if (!workersInstance.running) break;
+        }
       } catch (error) {
-        logger4.error("Error in job runner.", { error });
-        await sleep(config.pollInterval);
+        logger4.error("Error in job scheduler.", { error });
+        didMiss = true;
+      }
+      if (!workersInstance.running) return;
+      const concurrencyIsFull = workersInstance.activeExecutions.size >= config.workersCount && workersInstance.activeExecutions.size > 0;
+      if (concurrencyIsFull || allJobNamesAtCapacity) {
+        await this.waitForCapacityOrStop(workersInstance);
+      } else if (didMiss || workersInstance.activeExecutions.size === 0) {
+        await this.waitForPollOrStop(config, workersInstance);
+      }
+    }
+  }
+  getOtherJobAcquisitionHint(hint) {
+    return hint === INITIAL_JOB_ACQUISITION_HINT ? CANDIDATE_JOB_ACQUISITION_HINT : INITIAL_JOB_ACQUISITION_HINT;
+  }
+  getMedian(values) {
+    const sortedValues = [...values].sort((left, right) => left - right);
+    return sortedValues[Math.floor(sortedValues.length / 2)];
+  }
+  resetPendingJobAcquisitionHint(workersInstance) {
+    workersInstance.pendingJobAcquisitionHint = void 0;
+    workersInstance.pendingJobAcquisitionHintWins = 0;
+  }
+  async runJobAcquisitionHintProbe(config, workersInstance) {
+    const jobNames = this.getJobNames(config.jobs);
+    if (jobNames.length === 0 || !workersInstance.running) return;
+    const byHint = {
+      [INITIAL_JOB_ACQUISITION_HINT]: [],
+      [CANDIDATE_JOB_ACQUISITION_HINT]: []
+    };
+    const startsWithInitialHint = workersInstance.jobAcquisitionHintProbeCycle % 2 === 0;
+    const firstHint = startsWithInitialHint ? INITIAL_JOB_ACQUISITION_HINT : CANDIDATE_JOB_ACQUISITION_HINT;
+    const secondHint = this.getOtherJobAcquisitionHint(firstHint);
+    workersInstance.jobAcquisitionHintProbeCycle++;
+    try {
+      for (let sample = 0; sample < 3; sample++) {
+        for (const hint of [firstHint, secondHint]) {
+          if (!workersInstance.running) return;
+          const executionTimeMillis = await this.jobsRepo.getJobAcquisitionExecutionTime(
+            jobNames,
+            hint
+          );
+          if (!workersInstance.running) return;
+          byHint[hint].push(executionTimeMillis);
+        }
       }
+    } catch (error) {
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      if (workersInstance.running) {
+        logger4.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+      }
+      return;
+    }
+    const currentHint = workersInstance.jobAcquisitionHint;
+    const otherHint = this.getOtherJobAcquisitionHint(currentHint);
+    const currentMedian = this.getMedian(byHint[currentHint]);
+    const otherMedian = this.getMedian(byHint[otherHint]);
+    const winner = otherMedian < currentMedian ? otherHint : currentHint;
+    if (!workersInstance.hasCompletedInitialJobAcquisitionHintProbe) {
+      workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
+      workersInstance.jobAcquisitionHint = winner;
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      logger4.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      return;
+    }
+    if (winner === currentHint) {
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      logger4.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      return;
+    }
+    if (workersInstance.pendingJobAcquisitionHint === winner) {
+      workersInstance.pendingJobAcquisitionHintWins++;
+    } else {
+      workersInstance.pendingJobAcquisitionHint = winner;
+      workersInstance.pendingJobAcquisitionHintWins = 1;
+    }
+    if (workersInstance.pendingJobAcquisitionHintWins < 2) {
+      logger4.debug(
+        `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
+      );
+      return;
+    }
+    workersInstance.jobAcquisitionHint = winner;
+    this.resetPendingJobAcquisitionHint(workersInstance);
+    logger4.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+  }
+  async waitForNextJobAcquisitionHintProbe(workersInstance) {
+    await new Promise((resolve) => {
+      let resolved = false;
+      const finishWait = () => {
+        if (resolved) return;
+        resolved = true;
+        clearTimeout(workersInstance.jobAcquisitionHintProbeTimer);
+        workersInstance.jobAcquisitionHintProbeTimer = void 0;
+        workersInstance.cancelJobAcquisitionHintProbeWait = void 0;
+        resolve();
+      };
+      workersInstance.jobAcquisitionHintProbeTimer = setTimeout(
+        finishWait,
+        this.jobAcquisitionHintProbeIntervalMs
+      );
+      workersInstance.cancelJobAcquisitionHintProbeWait = finishWait;
+    });
+  }
+  async runJobAcquisitionHintProbeLoop(config, workersInstance) {
+    if (this.getJobNames(config.jobs).length === 0) return;
+    while (workersInstance.running) {
+      await this.runJobAcquisitionHintProbe(config, workersInstance);
+      if (!workersInstance.running) return;
+      await this.waitForNextJobAcquisitionHintProbe(workersInstance);
     }
   }
   createWorkersInstanceDefinition(config) {
+    let signalStop;
+    const stopSignal = new Promise((resolve) => {
+      signalStop = resolve;
+    });
     const workersInstance = {
       running: true,
       workersCount: config.workersCount,
-      workers: [],
+      activeExecutions: /* @__PURE__ */ new Map(),
+      staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
-      jobAcquisitionLock: Promise.resolve(),
+      jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
+      jobAcquisitionHintProbeCycle: 0,
+      hasCompletedInitialJobAcquisitionHintProbe: false,
+      pendingJobAcquisitionHintWins: 0,
+      jobAcquisitionHintProbePromise: Promise.resolve(),
+      schedulerPromise: Promise.resolve(),
+      stopSignal,
+      signalStop,
+      get runningExecutions() {
+        return this.activeExecutions.size;
+      },
       stop: async () => {
-        logger4.info("Stopping workers...");
-        workersInstance.running = false;
-        const stopingPromises = workersInstance.workers.map((worker) => worker.stop());
-        await Promise.all(stopingPromises);
+        if (workersInstance.stopPromise) return workersInstance.stopPromise;
+        workersInstance.stopPromise = (async () => {
+          var _a;
+          logger4.info("Stopping job scheduler...");
+          workersInstance.running = false;
+          workersInstance.signalStop();
+          (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
+          await Promise.all([
+            workersInstance.schedulerPromise,
+            workersInstance.jobAcquisitionHintProbePromise
+          ]);
+          const activeCapacity = Array.from(
+            workersInstance.activeExecutions.values(),
+            (execution) => execution.capacityPromise
+          );
+          await Promise.all(activeCapacity);
+        })();
+        return workersInstance.stopPromise;
       }
     };
     return workersInstance;
@@ -954,44 +1243,17 @@ var WorkerService = class {
       })
     );
   }
-  async startANewWorker(config, workersInstance, workerIndex = workersInstance.workers.length) {
-    if (!workersInstance.running) {
-      return;
-    }
-    const workerInstance = {
-      running: true,
-      workerIndex,
-      stop: async () => {
-        logger4.info(`Stopping worker [w${workerIndex}]...`);
-        workerInstance.running = false;
-        await workerInstance.promise;
-      },
-      respawn: async () => {
-        logger4.info(`Respawning worker [w${workerIndex}]...`);
-        workerInstance.running = false;
-        await this.startANewWorker(config, workersInstance, workerIndex);
-      }
-    };
-    const workerPromise = this.startWorker(config, workersInstance, workerInstance);
-    workerInstance.promise = workerPromise;
-    workersInstance.workers[workerIndex] = workerInstance;
-  }
-  async runWorkers(config, workersInstance) {
+  async runScheduler(config, workersInstance) {
     logger4.debug("Will ensure maxTriesReached retention and TTL index");
-    const ensureRetentionPromise = this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger4.error("Error ensuring maxTriesReached retention", { error }));
+    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger4.error("Error ensuring maxTriesReached retention", { error }));
     logger4.debug("Will ensure records for recurrent jobs");
     await this.ensureRecords(config);
-    const workersCount = config.workersCount;
-    const workerWord = workersCount === 1 ? "worker" : "workers";
-    logger4.info(`Starting ${workersCount} ${workerWord}`);
-    for (let workerIndex = 0; workerIndex < workersCount; workerIndex++) {
-      this.startANewWorker(config, workersInstance, workerIndex);
-    }
-    await ensureRetentionPromise;
+    logger4.info(`Starting job scheduler with concurrency ${config.workersCount}`);
+    await this.runSchedulerLoop(config, workersInstance);
   }
   /**
-   * Starts the job workers with the provided configuration.
-   * @param userConfig - Configuration for the workers. Required field: jobs
+   * Starts one job scheduler with the provided concurrency configuration.
+   * @param userConfig - Configuration for the scheduler. Required field: jobs
    */
   startWorkers(userConfig) {
     const maxTriesReachedRetentionMs = userConfig.maxTriesReachedRetentionMs === void 0 ? DEFAULT_MAX_TRIES_REACHED_RETENTION_MS : userConfig.maxTriesReachedRetentionMs;
@@ -999,7 +1261,6 @@ var WorkerService = class {
       throw new Error("maxTriesReachedRetentionMs must be null, zero, or a positive number");
     }
     const config = {
-      cooldownPeriod: 100,
       pollInterval: 3e3,
       workersCount: 4,
       defaultLockTime: 30 * 1e3,
@@ -1008,8 +1269,18 @@ var WorkerService = class {
     };
     setNameToJobs(config.jobs);
     const workersInstance = this.createWorkersInstanceDefinition(config);
-    logger4.debug("Starting workers", config);
-    this.runWorkers(config, workersInstance);
+    logger4.debug("Starting job scheduler", config);
+    workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
+      config,
+      workersInstance
+    );
+    workersInstance.schedulerPromise = this.runScheduler(config, workersInstance).catch((error) => {
+      var _a;
+      workersInstance.running = false;
+      workersInstance.signalStop();
+      (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
+      logger4.error("Job scheduler stopped after an unexpected error.", { error });
+    });
     return workersInstance;
   }
 };