@orion-js/dogs 4.6.7 → 4.6.9

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Orionjs Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.cjs

@@ -524,6 +524,17 @@ var JobsRepo = class {
     );
     return result.matchedCount > 0;
   }
+  async rescheduleAfterExecutionGuardAcquisitionFailure(jobId, lockId, nextRunAt) {
+    const result = await this.jobs.updateOne(
+      { _id: jobId, lockId, tries: { $gte: 1 } },
+      {
+        $set: { nextRunAt },
+        $inc: { tries: -1 },
+        $unset: { lockId: "", lockedUntil: "", claimWasStale: "" }
+      }
+    );
+    return result.matchedCount > 0;
+  }
   async deleteEventJob(jobId, lockId) {
     const result = await this.jobs.deleteOne({
       _id: jobId,
@@ -872,7 +883,7 @@ function scheduleJobsInternal(jobs, nPartitions) {
 
 // src/services/WorkerService.ts
 var import_helpers2 = require("@orion-js/helpers");
-var import_logger4 = require("@orion-js/logger");
+var import_logger5 = require("@orion-js/logger");
 var import_services4 = require("@orion-js/services");
 
 // src/services/Executor.ts
@@ -1124,6 +1135,42 @@ var Executor = class {
     }
     return false;
   }
+  async handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context) {
+    context.logger.error(`Error acquiring execution guard for job "${jobToRun.name}"`, { error });
+    const nextRunAt = new Date(Date.now() + (config.executionGuardRetryDelayMs ?? 1e3));
+    try {
+      const didReschedule = await this.jobsRepo.rescheduleAfterExecutionGuardAcquisitionFailure(
+        jobToRun.jobId,
+        jobToRun.lockId,
+        nextRunAt
+      );
+      if (!didReschedule) {
+        context.logger.warn(
+          `Could not requeue job "${jobToRun.name}" after execution guard acquisition failed because its lock is no longer owned`
+        );
+      }
+    } catch (rescheduleError) {
+      context.logger.error(
+        `Error requeueing job "${jobToRun.name}" after execution guard acquisition failed`,
+        { error: rescheduleError }
+      );
+    }
+  }
+  async releaseExecutionGuardLease(lease, jobToRun, context) {
+    try {
+      await lease.release();
+    } catch (error) {
+      context.logger.error(`Error releasing execution guard for job "${jobToRun.name}"`, { error });
+    }
+  }
+  async runGuardedResolver(job, jobToRun, context, lease) {
+    try {
+      return await job.resolve(jobToRun.params, context);
+    } finally {
+      context.clearStaleTimeout();
+      await this.releaseExecutionGuardLease(lease, jobToRun, context);
+    }
+  }
   async executeJob(config, jobToRun, onExecutionStale) {
     const job = this.getJobDefinition(jobToRun, config.jobs);
     if (!job) return;
@@ -1205,13 +1252,36 @@ var Executor = class {
           params: jobToRun.params
         };
         await (0, import_logger3.runWithOrionAsyncContext)(extraContext, async () => {
-          try {
+          let executionGuardLease;
+          if (config.executionGuard) {
             (0, import_logger3.updateOrionAsyncContext)({
               jobName: jobToRun.name,
               jobId: jobToRun.jobId,
               params: jobToRun.params
             });
-            const result = await job.resolve(jobToRun.params, context);
+            try {
+              executionGuardLease = await config.executionGuard.acquire(jobToRun);
+            } catch (error) {
+              context.clearStaleTimeout();
+              await this.handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context);
+              if (executionStatus === "stale") await staleHandlingPromise;
+              return;
+            }
+            if (executionStatus === "stale") {
+              await staleHandlingPromise;
+              await this.releaseExecutionGuardLease(executionGuardLease, jobToRun, context);
+              return;
+            }
+          }
+          try {
+            if (!config.executionGuard) {
+              (0, import_logger3.updateOrionAsyncContext)({
+                jobName: jobToRun.name,
+                jobId: jobToRun.jobId,
+                params: jobToRun.params
+              });
+            }
+            const result = executionGuardLease ? await this.runGuardedResolver(job, jobToRun, context, executionGuardLease) : await job.resolve(jobToRun.params, context);
             context.clearStaleTimeout();
             if (executionStatus === "stale") {
               await staleHandlingPromise;
@@ -1280,6 +1350,76 @@ __decorateElement(_init4, 5, "jobsHistoryRepo", _jobsHistoryRepo_dec, Executor);
 Executor = __decorateElement(_init4, 0, "Executor", _Executor_decorators, Executor);
 __runInitializers(_init4, 1, Executor);
 
+// src/services/jobFilters.ts
+var import_logger4 = require("@orion-js/logger");
+var DEFAULT_JOBS_FILTER_TIMEOUT_MS = 5e3;
+var warnedUnknownJobNames = /* @__PURE__ */ new Set();
+async function withTimeout(promise, timeoutMs, name) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`${name} timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function runCheck(options, name, run) {
+  if (options.pendingChecks.has(name)) {
+    throw new Error(`${name} is still running from a previous claim`);
+  }
+  const promise = Promise.resolve().then(run);
+  options.pendingChecks.set(name, promise);
+  void promise.then(
+    () => options.pendingChecks.delete(name),
+    () => options.pendingChecks.delete(name)
+  );
+  return await withTimeout(promise, options.timeoutMs, name);
+}
+function warnUnknownJobNames(jobs, filter) {
+  for (const jobName of [...filter.allow ?? [], ...filter.deny ?? []]) {
+    if (jobs[jobName] || warnedUnknownJobNames.has(jobName)) continue;
+    warnedUnknownJobNames.add(jobName);
+    import_logger4.logger.warn(`getJobsFilter returned unknown job name "${jobName}"; it will be ignored.`);
+  }
+}
+async function filterByJobsFilter(options, getJobsFilter) {
+  try {
+    const filter = await runCheck(options, "getJobsFilter", getJobsFilter) ?? {};
+    warnUnknownJobNames(options.jobs, filter);
+    return options.jobNames.filter((jobName) => {
+      var _a;
+      if (filter.allow && !filter.allow.includes(jobName)) return false;
+      if ((_a = filter.deny) == null ? void 0 : _a.includes(jobName)) return false;
+      return true;
+    });
+  } catch (error) {
+    import_logger4.logger.error("Error in getJobsFilter; no jobs will be taken on this claim.", { error });
+    return [];
+  }
+}
+async function canTakeJob(options, jobName) {
+  const canTakeJobs = options.jobs[jobName].canTakeJobs;
+  if (!canTakeJobs) return true;
+  try {
+    return await runCheck(options, `canTakeJobs of "${jobName}"`, canTakeJobs) === true;
+  } catch (error) {
+    import_logger4.logger.error(`Error in canTakeJobs of job "${jobName}"; it will not be taken.`, { error });
+    return false;
+  }
+}
+async function getTakeableJobNames(inputOptions) {
+  const options = {
+    timeoutMs: DEFAULT_JOBS_FILTER_TIMEOUT_MS,
+    pendingChecks: /* @__PURE__ */ new Map(),
+    ...inputOptions
+  };
+  const jobNames = inputOptions.getJobsFilter ? await filterByJobsFilter(options, inputOptions.getJobsFilter) : options.jobNames;
+  const checks = await Promise.all(jobNames.map((jobName) => canTakeJob(options, jobName)));
+  return jobNames.filter((_, index) => checks[index]);
+}
+
 // src/services/WorkerService.ts
 var FILTERED_JOB_NAMES_ACQUISITION_HINT = "byJobName";
 var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
@@ -1291,6 +1431,7 @@ var WorkerService = class {
     __publicField(this, "partitionReconcilerJitterMs", 30 * 1e3);
     __publicField(this, "partitionReconcilerBatchPauseMs", 250);
     __publicField(this, "partitionReconcilerBatchSize", 500);
+    __publicField(this, "jobsFilterTimeoutMs", DEFAULT_JOBS_FILTER_TIMEOUT_MS);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -1349,13 +1490,13 @@ var WorkerService = class {
     };
     this.reserveJobExecution(workersInstance, jobToRun.name);
     workersInstance.activeExecutions.set(jobToRun.executionId, execution);
-    import_logger4.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
+    import_logger5.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
     const executionPromise = this.executor.executeJob(config, jobToRun, () => {
       this.releaseExecutionCapacity(workersInstance, execution, "stale");
     });
     execution.promise = executionPromise;
     void executionPromise.catch((error) => {
-      import_logger4.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
+      import_logger5.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
     }).finally(() => {
       if (execution.status === "running") {
         this.releaseExecutionCapacity(workersInstance, execution, "finished");
@@ -1390,20 +1531,39 @@ var WorkerService = class {
       jobs: config.jobs,
       maxTries: config.maxTries,
       maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
-      onMaxTriesReached: config.onMaxTriesReached
+      onMaxTriesReached: config.onMaxTriesReached,
+      ...config.executionGuard ? {
+        executionGuard: config.executionGuard,
+        executionGuardRetryDelayMs: Math.max(config.pollInterval, 1e3)
+      } : {}
     };
-    import_logger4.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
+    import_logger5.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
     let consecutivePartitionMisses = 0;
     while (workersInstance.running) {
       let didMiss = false;
       let allJobNamesAtCapacity = false;
       try {
         while (workersInstance.activeExecutions.size < config.workersCount) {
-          const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
-          if (availableJobNames.length === 0) {
+          const jobNamesWithCapacity = this.getAvailableJobNames(config, workersInstance, jobNames);
+          if (jobNamesWithCapacity.length === 0) {
             allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
             break;
           }
+          const availableJobNames = await Promise.race([
+            getTakeableJobNames({
+              jobs: config.jobs,
+              getJobsFilter: config.getJobsFilter,
+              jobNames: jobNamesWithCapacity,
+              timeoutMs: this.jobsFilterTimeoutMs,
+              pendingChecks: workersInstance.pendingFilterChecks
+            }),
+            workersInstance.stopSignal.then(() => [])
+          ]);
+          if (!workersInstance.running) break;
+          if (availableJobNames.length === 0) {
+            didMiss = true;
+            break;
+          }
           const jobAcquisitionHint = availableJobNames.length < jobNames.length ? FILTERED_JOB_NAMES_ACQUISITION_HINT : workersInstance.jobAcquisitionHint;
           const partition = this.getNextPartition(config, workersInstance);
           const jobToRun = await this.jobsRepo.getJobAndLock(
@@ -1414,7 +1574,7 @@ var WorkerService = class {
           );
           if (!jobToRun) {
             consecutivePartitionMisses++;
-            import_logger4.logger.debug(`No job to run in partition ${partition}`);
+            import_logger5.logger.debug(`No job to run in partition ${partition}`);
             if (consecutivePartitionMisses >= config.nPartitions) {
               consecutivePartitionMisses = 0;
               didMiss = true;
@@ -1427,7 +1587,7 @@ var WorkerService = class {
           if (!workersInstance.running) break;
         }
       } catch (error) {
-        import_logger4.logger.error("Error in job scheduler.", { error });
+        import_logger5.logger.error("Error in job scheduler.", { error });
         consecutivePartitionMisses = 0;
         didMiss = true;
       }
@@ -1479,7 +1639,7 @@ var WorkerService = class {
     } catch (error) {
       this.resetPendingJobAcquisitionHint(workersInstance);
       if (workersInstance.running) {
-        import_logger4.logger.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+        import_logger5.logger.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
       }
       return;
     }
@@ -1492,12 +1652,12 @@ var WorkerService = class {
       workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
       workersInstance.jobAcquisitionHint = winner;
       this.resetPendingJobAcquisitionHint(workersInstance);
-      import_logger4.logger.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      import_logger5.logger.info(`Selected job acquisition hint "${winner}" after startup probe.`);
       return;
     }
     if (winner === currentHint) {
       this.resetPendingJobAcquisitionHint(workersInstance);
-      import_logger4.logger.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      import_logger5.logger.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
       return;
     }
     if (workersInstance.pendingJobAcquisitionHint === winner) {
@@ -1507,14 +1667,14 @@ var WorkerService = class {
       workersInstance.pendingJobAcquisitionHintWins = 1;
     }
     if (workersInstance.pendingJobAcquisitionHintWins < 2) {
-      import_logger4.logger.debug(
+      import_logger5.logger.debug(
         `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
       );
       return;
     }
     workersInstance.jobAcquisitionHint = winner;
     this.resetPendingJobAcquisitionHint(workersInstance);
-    import_logger4.logger.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+    import_logger5.logger.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
   }
   async waitForNextJobAcquisitionHintProbe(workersInstance) {
     await new Promise((resolve) => {
@@ -1551,7 +1711,7 @@ var WorkerService = class {
         batchSize: this.partitionReconcilerBatchSize
       });
     } catch (error) {
-      import_logger4.logger.warn("Job partition reconciliation failed; it will be retried.", { error });
+      import_logger5.logger.warn("Job partition reconciliation failed; it will be retried.", { error });
       return 0;
     }
   }
@@ -1591,6 +1751,7 @@ var WorkerService = class {
       activeExecutions: /* @__PURE__ */ new Map(),
       staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
+      pendingFilterChecks: /* @__PURE__ */ new Map(),
       jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
       jobAcquisitionHintProbeCycle: 0,
       hasCompletedInitialJobAcquisitionHintProbe: false,
@@ -1609,7 +1770,7 @@ var WorkerService = class {
         if (workersInstance.stopPromise) return workersInstance.stopPromise;
         workersInstance.stopPromise = (async () => {
           var _a, _b;
-          import_logger4.logger.info("Stopping job scheduler...");
+          import_logger5.logger.info("Stopping job scheduler...");
           workersInstance.running = false;
           workersInstance.signalStop();
           (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
@@ -1634,17 +1795,17 @@ var WorkerService = class {
     const jobs = this.getJobs(config.jobs);
     await Promise.all(
       jobs.filter((job) => job.type === "recurrent").map(async (job) => {
-        import_logger4.logger.debug(`Ensuring records for job "${job.name}"...`);
+        import_logger5.logger.debug(`Ensuring records for job "${job.name}"...`);
         await this.jobsRepo.ensureJobRecord(job, getRandomPartition(config.nPartitions));
       })
     );
   }
   async runScheduler(config, workersInstance) {
-    import_logger4.logger.debug("Will ensure maxTriesReached retention and TTL index");
-    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger4.logger.error("Error ensuring maxTriesReached retention", { error }));
-    import_logger4.logger.debug("Will ensure records for recurrent jobs");
+    import_logger5.logger.debug("Will ensure maxTriesReached retention and TTL index");
+    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger5.logger.error("Error ensuring maxTriesReached retention", { error }));
+    import_logger5.logger.debug("Will ensure records for recurrent jobs");
     await this.ensureRecords(config);
-    import_logger4.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
+    import_logger5.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
     await this.runSchedulerLoop(config, workersInstance);
   }
   /**
@@ -1670,7 +1831,7 @@ var WorkerService = class {
     setNameToJobs(config.jobs, config.nPartitions);
     (_b = (_a = this.jobsRepo).configureJobPartitions) == null ? void 0 : _b.call(_a, this.getJobNames(config.jobs), config.nPartitions);
     const workersInstance = this.createWorkersInstanceDefinition(config);
-    import_logger4.logger.debug("Starting job scheduler", config);
+    import_logger5.logger.debug("Starting job scheduler", config);
     workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
       config,
       workersInstance
@@ -1687,7 +1848,7 @@ var WorkerService = class {
       workersInstance.signalStop();
       (_a2 = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a2.call(workersInstance);
       (_b2 = workersInstance.cancelPartitionReconcilerWait) == null ? void 0 : _b2.call(workersInstance);
-      import_logger4.logger.error("Job scheduler stopped after an unexpected error.", { error });
+      import_logger5.logger.error("Job scheduler stopped after an unexpected error.", { error });
     });
     return workersInstance;
   }