@orion-js/dogs 4.6.1 → 4.6.2

package/dist/index.js

@@ -98,6 +98,35 @@ var JobRecordSchema = schemaWithName("JobRecord", {
 var DEFAULT_MAX_TRIES_REACHED_RETENTION_MS = 1e3 * 60 * 60 * 24 * 7;
 
 // src/repos/JobsRepo.ts
+var JOB_ACQUISITION_HINTS = {
+  byJobName: {
+    jobName: 1,
+    priority: -1,
+    nextRunAt: 1
+  },
+  globalPriority: {
+    priority: -1,
+    nextRunAt: 1
+  }
+};
+var INITIAL_JOB_ACQUISITION_HINT = "byJobName";
+var CANDIDATE_JOB_ACQUISITION_HINT = "globalPriority";
+var JOB_ACQUISITION_SORT = {
+  priority: -1,
+  nextRunAt: 1
+};
+function getJobAcquisitionSelector(jobNames, now) {
+  return {
+    jobName: { $in: jobNames },
+    nextRunAt: { $lte: now },
+    $and: [
+      { $or: [{ lockedUntil: { $exists: false } }, { lockedUntil: { $lte: now } }] },
+      // Exclude event jobs that have reached max tries. Recurrent jobs keep running even
+      // if old records still have this status from previous versions.
+      { $or: [{ type: { $ne: "event" } }, { status: { $ne: "maxTriesReached" } }] }
+    ]
+  };
+}
 var _jobs_dec, _JobsRepo_decorators, _init;
 _JobsRepo_decorators = [Repository()], _jobs_dec = [MongoCollection({
   idGeneration: "uuid",
@@ -105,11 +134,10 @@ _JobsRepo_decorators = [Repository()], _jobs_dec = [MongoCollection({
   schema: JobRecordSchema,
   indexes: [
     {
-      keys: {
-        jobName: 1,
-        priority: -1,
-        nextRunAt: 1
-      }
+      keys: JOB_ACQUISITION_HINTS.byJobName
+    },
+    {
+      keys: JOB_ACQUISITION_HINTS.globalPriority
     },
     {
       keys: {
@@ -142,30 +170,20 @@ var JobsRepo = class {
   constructor() {
     __publicField(this, "jobs", __runInitializers(_init, 8, this)), __runInitializers(_init, 11, this);
   }
-  async getJobAndLock(jobNames, lockTime) {
+  async getJobAndLock(jobNames, lockTime, hint = INITIAL_JOB_ACQUISITION_HINT) {
     const executionId = generateId();
-    const lockedUntil = new Date(Date.now() + lockTime);
+    const now = /* @__PURE__ */ new Date();
+    const lockedUntil = new Date(now.getTime() + lockTime);
     const job = await this.jobs.findOneAndUpdate(
+      getJobAcquisitionSelector(jobNames, now),
       {
-        jobName: { $in: jobNames },
-        nextRunAt: { $lte: /* @__PURE__ */ new Date() },
-        $and: [
-          { $or: [{ lockedUntil: { $exists: false } }, { lockedUntil: { $lte: /* @__PURE__ */ new Date() } }] },
-          // Exclude event jobs that have reached max tries. Recurrent jobs keep running even
-          // if old records still have this status from previous versions.
-          { $or: [{ type: { $ne: "event" } }, { status: { $ne: "maxTriesReached" } }] }
-        ]
-      },
-      {
-        $set: { lockId: executionId, lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
+        $set: { lockId: executionId, lockedUntil, lastRunAt: now },
         $inc: { tries: 1 }
       },
       {
         mongoOptions: {
-          sort: {
-            priority: -1,
-            nextRunAt: 1
-          },
+          hint: JOB_ACQUISITION_HINTS[hint],
+          sort: JOB_ACQUISITION_SORT,
           returnDocument: "before"
         }
       }
@@ -190,6 +208,19 @@ var JobsRepo = class {
       wasStale
     };
   }
+  async getJobAcquisitionExecutionTime(jobNames, hint) {
+    var _a;
+    const explain = await this.jobs.find(getJobAcquisitionSelector(jobNames, /* @__PURE__ */ new Date()), {
+      hint: JOB_ACQUISITION_HINTS[hint],
+      maxTimeMS: 1e3,
+      readPreference: "primary"
+    }).sort(JOB_ACQUISITION_SORT).limit(1).explain("executionStats");
+    const executionTimeMillis = (_a = explain.executionStats) == null ? void 0 : _a.executionTimeMillis;
+    if (typeof executionTimeMillis !== "number" || !Number.isFinite(executionTimeMillis)) {
+      throw new Error(`Explain for acquisition hint "${hint}" returned no execution time`);
+    }
+    return executionTimeMillis;
+  }
   async setJobRecordPriority(jobId, priority, lockId) {
     const result = await this.jobs.updateOne(lockId ? { _id: jobId, lockId } : jobId, {
       $set: { priority }
@@ -928,6 +959,7 @@ var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
 _WorkerService_decorators = [Service3()], _jobsRepo_dec3 = [Inject3(() => JobsRepo)], _executor_dec = [Inject3(() => Executor)];
 var WorkerService = class {
   constructor() {
+    __publicField(this, "jobAcquisitionHintProbeIntervalMs", 30 * 60 * 1e3);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -1033,7 +1065,8 @@ var WorkerService = class {
           }
           const jobToRun = await this.jobsRepo.getJobAndLock(
             availableJobNames,
-            config.defaultLockTime
+            config.defaultLockTime,
+            workersInstance.jobAcquisitionHint
           );
           if (!jobToRun) {
             logger4.debug("No job to run");
@@ -1056,6 +1089,106 @@ var WorkerService = class {
       }
     }
   }
+  getOtherJobAcquisitionHint(hint) {
+    return hint === INITIAL_JOB_ACQUISITION_HINT ? CANDIDATE_JOB_ACQUISITION_HINT : INITIAL_JOB_ACQUISITION_HINT;
+  }
+  getMedian(values) {
+    const sortedValues = [...values].sort((left, right) => left - right);
+    return sortedValues[Math.floor(sortedValues.length / 2)];
+  }
+  resetPendingJobAcquisitionHint(workersInstance) {
+    workersInstance.pendingJobAcquisitionHint = void 0;
+    workersInstance.pendingJobAcquisitionHintWins = 0;
+  }
+  async runJobAcquisitionHintProbe(config, workersInstance) {
+    const jobNames = this.getJobNames(config.jobs);
+    if (jobNames.length === 0 || !workersInstance.running) return;
+    const byHint = {
+      [INITIAL_JOB_ACQUISITION_HINT]: [],
+      [CANDIDATE_JOB_ACQUISITION_HINT]: []
+    };
+    const startsWithInitialHint = workersInstance.jobAcquisitionHintProbeCycle % 2 === 0;
+    const firstHint = startsWithInitialHint ? INITIAL_JOB_ACQUISITION_HINT : CANDIDATE_JOB_ACQUISITION_HINT;
+    const secondHint = this.getOtherJobAcquisitionHint(firstHint);
+    workersInstance.jobAcquisitionHintProbeCycle++;
+    try {
+      for (let sample = 0; sample < 3; sample++) {
+        for (const hint of [firstHint, secondHint]) {
+          if (!workersInstance.running) return;
+          const executionTimeMillis = await this.jobsRepo.getJobAcquisitionExecutionTime(
+            jobNames,
+            hint
+          );
+          if (!workersInstance.running) return;
+          byHint[hint].push(executionTimeMillis);
+        }
+      }
+    } catch (error) {
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      if (workersInstance.running) {
+        logger4.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+      }
+      return;
+    }
+    const currentHint = workersInstance.jobAcquisitionHint;
+    const otherHint = this.getOtherJobAcquisitionHint(currentHint);
+    const currentMedian = this.getMedian(byHint[currentHint]);
+    const otherMedian = this.getMedian(byHint[otherHint]);
+    const winner = otherMedian < currentMedian ? otherHint : currentHint;
+    if (!workersInstance.hasCompletedInitialJobAcquisitionHintProbe) {
+      workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
+      workersInstance.jobAcquisitionHint = winner;
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      logger4.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      return;
+    }
+    if (winner === currentHint) {
+      this.resetPendingJobAcquisitionHint(workersInstance);
+      logger4.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      return;
+    }
+    if (workersInstance.pendingJobAcquisitionHint === winner) {
+      workersInstance.pendingJobAcquisitionHintWins++;
+    } else {
+      workersInstance.pendingJobAcquisitionHint = winner;
+      workersInstance.pendingJobAcquisitionHintWins = 1;
+    }
+    if (workersInstance.pendingJobAcquisitionHintWins < 2) {
+      logger4.debug(
+        `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
+      );
+      return;
+    }
+    workersInstance.jobAcquisitionHint = winner;
+    this.resetPendingJobAcquisitionHint(workersInstance);
+    logger4.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+  }
+  async waitForNextJobAcquisitionHintProbe(workersInstance) {
+    await new Promise((resolve) => {
+      let resolved = false;
+      const finishWait = () => {
+        if (resolved) return;
+        resolved = true;
+        clearTimeout(workersInstance.jobAcquisitionHintProbeTimer);
+        workersInstance.jobAcquisitionHintProbeTimer = void 0;
+        workersInstance.cancelJobAcquisitionHintProbeWait = void 0;
+        resolve();
+      };
+      workersInstance.jobAcquisitionHintProbeTimer = setTimeout(
+        finishWait,
+        this.jobAcquisitionHintProbeIntervalMs
+      );
+      workersInstance.cancelJobAcquisitionHintProbeWait = finishWait;
+    });
+  }
+  async runJobAcquisitionHintProbeLoop(config, workersInstance) {
+    if (this.getJobNames(config.jobs).length === 0) return;
+    while (workersInstance.running) {
+      await this.runJobAcquisitionHintProbe(config, workersInstance);
+      if (!workersInstance.running) return;
+      await this.waitForNextJobAcquisitionHintProbe(workersInstance);
+    }
+  }
   createWorkersInstanceDefinition(config) {
     let signalStop;
     const stopSignal = new Promise((resolve) => {
@@ -1067,6 +1200,11 @@ var WorkerService = class {
       activeExecutions: /* @__PURE__ */ new Map(),
       staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
+      jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
+      jobAcquisitionHintProbeCycle: 0,
+      hasCompletedInitialJobAcquisitionHintProbe: false,
+      pendingJobAcquisitionHintWins: 0,
+      jobAcquisitionHintProbePromise: Promise.resolve(),
       schedulerPromise: Promise.resolve(),
       stopSignal,
       signalStop,
@@ -1076,10 +1214,15 @@ var WorkerService = class {
       stop: async () => {
         if (workersInstance.stopPromise) return workersInstance.stopPromise;
         workersInstance.stopPromise = (async () => {
+          var _a;
           logger4.info("Stopping job scheduler...");
           workersInstance.running = false;
           workersInstance.signalStop();
-          await workersInstance.schedulerPromise;
+          (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
+          await Promise.all([
+            workersInstance.schedulerPromise,
+            workersInstance.jobAcquisitionHintProbePromise
+          ]);
           const activeCapacity = Array.from(
             workersInstance.activeExecutions.values(),
             (execution) => execution.capacityPromise
@@ -1127,8 +1270,15 @@ var WorkerService = class {
     setNameToJobs(config.jobs);
     const workersInstance = this.createWorkersInstanceDefinition(config);
     logger4.debug("Starting job scheduler", config);
+    workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
+      config,
+      workersInstance
+    );
     workersInstance.schedulerPromise = this.runScheduler(config, workersInstance).catch((error) => {
+      var _a;
       workersInstance.running = false;
+      workersInstance.signalStop();
+      (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
       logger4.error("Job scheduler stopped after an unexpected error.", { error });
     });
     return workersInstance;