@orion-js/dogs 4.6.7 → 4.6.9

package/dist/index.js

@@ -480,6 +480,17 @@ var JobsRepo = class {
     );
     return result.matchedCount > 0;
   }
+  async rescheduleAfterExecutionGuardAcquisitionFailure(jobId, lockId, nextRunAt) {
+    const result = await this.jobs.updateOne(
+      { _id: jobId, lockId, tries: { $gte: 1 } },
+      {
+        $set: { nextRunAt },
+        $inc: { tries: -1 },
+        $unset: { lockId: "", lockedUntil: "", claimWasStale: "" }
+      }
+    );
+    return result.matchedCount > 0;
+  }
   async deleteEventJob(jobId, lockId) {
     const result = await this.jobs.deleteOne({
       _id: jobId,
@@ -828,7 +839,7 @@ function scheduleJobsInternal(jobs, nPartitions) {
 
 // src/services/WorkerService.ts
 import { sleep } from "@orion-js/helpers";
-import { logger as logger4 } from "@orion-js/logger";
+import { logger as logger5 } from "@orion-js/logger";
 import { Inject as Inject3, Service as Service3 } from "@orion-js/services";
 
 // src/services/Executor.ts
@@ -1080,6 +1091,42 @@ var Executor = class {
     }
     return false;
   }
+  async handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context) {
+    context.logger.error(`Error acquiring execution guard for job "${jobToRun.name}"`, { error });
+    const nextRunAt = new Date(Date.now() + (config.executionGuardRetryDelayMs ?? 1e3));
+    try {
+      const didReschedule = await this.jobsRepo.rescheduleAfterExecutionGuardAcquisitionFailure(
+        jobToRun.jobId,
+        jobToRun.lockId,
+        nextRunAt
+      );
+      if (!didReschedule) {
+        context.logger.warn(
+          `Could not requeue job "${jobToRun.name}" after execution guard acquisition failed because its lock is no longer owned`
+        );
+      }
+    } catch (rescheduleError) {
+      context.logger.error(
+        `Error requeueing job "${jobToRun.name}" after execution guard acquisition failed`,
+        { error: rescheduleError }
+      );
+    }
+  }
+  async releaseExecutionGuardLease(lease, jobToRun, context) {
+    try {
+      await lease.release();
+    } catch (error) {
+      context.logger.error(`Error releasing execution guard for job "${jobToRun.name}"`, { error });
+    }
+  }
+  async runGuardedResolver(job, jobToRun, context, lease) {
+    try {
+      return await job.resolve(jobToRun.params, context);
+    } finally {
+      context.clearStaleTimeout();
+      await this.releaseExecutionGuardLease(lease, jobToRun, context);
+    }
+  }
   async executeJob(config, jobToRun, onExecutionStale) {
     const job = this.getJobDefinition(jobToRun, config.jobs);
     if (!job) return;
@@ -1161,13 +1208,36 @@ var Executor = class {
           params: jobToRun.params
         };
         await runWithOrionAsyncContext(extraContext, async () => {
-          try {
+          let executionGuardLease;
+          if (config.executionGuard) {
             updateOrionAsyncContext({
               jobName: jobToRun.name,
               jobId: jobToRun.jobId,
               params: jobToRun.params
             });
-            const result = await job.resolve(jobToRun.params, context);
+            try {
+              executionGuardLease = await config.executionGuard.acquire(jobToRun);
+            } catch (error) {
+              context.clearStaleTimeout();
+              await this.handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context);
+              if (executionStatus === "stale") await staleHandlingPromise;
+              return;
+            }
+            if (executionStatus === "stale") {
+              await staleHandlingPromise;
+              await this.releaseExecutionGuardLease(executionGuardLease, jobToRun, context);
+              return;
+            }
+          }
+          try {
+            if (!config.executionGuard) {
+              updateOrionAsyncContext({
+                jobName: jobToRun.name,
+                jobId: jobToRun.jobId,
+                params: jobToRun.params
+              });
+            }
+            const result = executionGuardLease ? await this.runGuardedResolver(job, jobToRun, context, executionGuardLease) : await job.resolve(jobToRun.params, context);
             context.clearStaleTimeout();
             if (executionStatus === "stale") {
               await staleHandlingPromise;
@@ -1236,6 +1306,76 @@ __decorateElement(_init4, 5, "jobsHistoryRepo", _jobsHistoryRepo_dec, Executor);
 Executor = __decorateElement(_init4, 0, "Executor", _Executor_decorators, Executor);
 __runInitializers(_init4, 1, Executor);
 
+// src/services/jobFilters.ts
+import { logger as logger4 } from "@orion-js/logger";
+var DEFAULT_JOBS_FILTER_TIMEOUT_MS = 5e3;
+var warnedUnknownJobNames = /* @__PURE__ */ new Set();
+async function withTimeout(promise, timeoutMs, name) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`${name} timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function runCheck(options, name, run) {
+  if (options.pendingChecks.has(name)) {
+    throw new Error(`${name} is still running from a previous claim`);
+  }
+  const promise = Promise.resolve().then(run);
+  options.pendingChecks.set(name, promise);
+  void promise.then(
+    () => options.pendingChecks.delete(name),
+    () => options.pendingChecks.delete(name)
+  );
+  return await withTimeout(promise, options.timeoutMs, name);
+}
+function warnUnknownJobNames(jobs, filter) {
+  for (const jobName of [...filter.allow ?? [], ...filter.deny ?? []]) {
+    if (jobs[jobName] || warnedUnknownJobNames.has(jobName)) continue;
+    warnedUnknownJobNames.add(jobName);
+    logger4.warn(`getJobsFilter returned unknown job name "${jobName}"; it will be ignored.`);
+  }
+}
+async function filterByJobsFilter(options, getJobsFilter) {
+  try {
+    const filter = await runCheck(options, "getJobsFilter", getJobsFilter) ?? {};
+    warnUnknownJobNames(options.jobs, filter);
+    return options.jobNames.filter((jobName) => {
+      var _a;
+      if (filter.allow && !filter.allow.includes(jobName)) return false;
+      if ((_a = filter.deny) == null ? void 0 : _a.includes(jobName)) return false;
+      return true;
+    });
+  } catch (error) {
+    logger4.error("Error in getJobsFilter; no jobs will be taken on this claim.", { error });
+    return [];
+  }
+}
+async function canTakeJob(options, jobName) {
+  const canTakeJobs = options.jobs[jobName].canTakeJobs;
+  if (!canTakeJobs) return true;
+  try {
+    return await runCheck(options, `canTakeJobs of "${jobName}"`, canTakeJobs) === true;
+  } catch (error) {
+    logger4.error(`Error in canTakeJobs of job "${jobName}"; it will not be taken.`, { error });
+    return false;
+  }
+}
+async function getTakeableJobNames(inputOptions) {
+  const options = {
+    timeoutMs: DEFAULT_JOBS_FILTER_TIMEOUT_MS,
+    pendingChecks: /* @__PURE__ */ new Map(),
+    ...inputOptions
+  };
+  const jobNames = inputOptions.getJobsFilter ? await filterByJobsFilter(options, inputOptions.getJobsFilter) : options.jobNames;
+  const checks = await Promise.all(jobNames.map((jobName) => canTakeJob(options, jobName)));
+  return jobNames.filter((_, index) => checks[index]);
+}
+
 // src/services/WorkerService.ts
 var FILTERED_JOB_NAMES_ACQUISITION_HINT = "byJobName";
 var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
@@ -1247,6 +1387,7 @@ var WorkerService = class {
     __publicField(this, "partitionReconcilerJitterMs", 30 * 1e3);
     __publicField(this, "partitionReconcilerBatchPauseMs", 250);
     __publicField(this, "partitionReconcilerBatchSize", 500);
+    __publicField(this, "jobsFilterTimeoutMs", DEFAULT_JOBS_FILTER_TIMEOUT_MS);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -1305,13 +1446,13 @@ var WorkerService = class {
     };
     this.reserveJobExecution(workersInstance, jobToRun.name);
     workersInstance.activeExecutions.set(jobToRun.executionId, execution);
-    logger4.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
+    logger5.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
     const executionPromise = this.executor.executeJob(config, jobToRun, () => {
       this.releaseExecutionCapacity(workersInstance, execution, "stale");
     });
     execution.promise = executionPromise;
     void executionPromise.catch((error) => {
-      logger4.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
+      logger5.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
     }).finally(() => {
       if (execution.status === "running") {
         this.releaseExecutionCapacity(workersInstance, execution, "finished");
@@ -1346,20 +1487,39 @@ var WorkerService = class {
       jobs: config.jobs,
       maxTries: config.maxTries,
       maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
-      onMaxTriesReached: config.onMaxTriesReached
+      onMaxTriesReached: config.onMaxTriesReached,
+      ...config.executionGuard ? {
+        executionGuard: config.executionGuard,
+        executionGuardRetryDelayMs: Math.max(config.pollInterval, 1e3)
+      } : {}
     };
-    logger4.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
+    logger5.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
     let consecutivePartitionMisses = 0;
     while (workersInstance.running) {
       let didMiss = false;
       let allJobNamesAtCapacity = false;
       try {
         while (workersInstance.activeExecutions.size < config.workersCount) {
-          const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
-          if (availableJobNames.length === 0) {
+          const jobNamesWithCapacity = this.getAvailableJobNames(config, workersInstance, jobNames);
+          if (jobNamesWithCapacity.length === 0) {
             allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
             break;
           }
+          const availableJobNames = await Promise.race([
+            getTakeableJobNames({
+              jobs: config.jobs,
+              getJobsFilter: config.getJobsFilter,
+              jobNames: jobNamesWithCapacity,
+              timeoutMs: this.jobsFilterTimeoutMs,
+              pendingChecks: workersInstance.pendingFilterChecks
+            }),
+            workersInstance.stopSignal.then(() => [])
+          ]);
+          if (!workersInstance.running) break;
+          if (availableJobNames.length === 0) {
+            didMiss = true;
+            break;
+          }
           const jobAcquisitionHint = availableJobNames.length < jobNames.length ? FILTERED_JOB_NAMES_ACQUISITION_HINT : workersInstance.jobAcquisitionHint;
           const partition = this.getNextPartition(config, workersInstance);
           const jobToRun = await this.jobsRepo.getJobAndLock(
@@ -1370,7 +1530,7 @@ var WorkerService = class {
           );
           if (!jobToRun) {
             consecutivePartitionMisses++;
-            logger4.debug(`No job to run in partition ${partition}`);
+            logger5.debug(`No job to run in partition ${partition}`);
             if (consecutivePartitionMisses >= config.nPartitions) {
               consecutivePartitionMisses = 0;
               didMiss = true;
@@ -1383,7 +1543,7 @@ var WorkerService = class {
           if (!workersInstance.running) break;
         }
       } catch (error) {
-        logger4.error("Error in job scheduler.", { error });
+        logger5.error("Error in job scheduler.", { error });
         consecutivePartitionMisses = 0;
         didMiss = true;
       }
@@ -1435,7 +1595,7 @@ var WorkerService = class {
     } catch (error) {
       this.resetPendingJobAcquisitionHint(workersInstance);
       if (workersInstance.running) {
-        logger4.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+        logger5.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
       }
       return;
     }
@@ -1448,12 +1608,12 @@ var WorkerService = class {
       workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
       workersInstance.jobAcquisitionHint = winner;
       this.resetPendingJobAcquisitionHint(workersInstance);
-      logger4.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      logger5.info(`Selected job acquisition hint "${winner}" after startup probe.`);
       return;
     }
     if (winner === currentHint) {
       this.resetPendingJobAcquisitionHint(workersInstance);
-      logger4.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      logger5.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
       return;
     }
     if (workersInstance.pendingJobAcquisitionHint === winner) {
@@ -1463,14 +1623,14 @@ var WorkerService = class {
       workersInstance.pendingJobAcquisitionHintWins = 1;
     }
     if (workersInstance.pendingJobAcquisitionHintWins < 2) {
-      logger4.debug(
+      logger5.debug(
         `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
       );
       return;
     }
     workersInstance.jobAcquisitionHint = winner;
     this.resetPendingJobAcquisitionHint(workersInstance);
-    logger4.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+    logger5.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
   }
   async waitForNextJobAcquisitionHintProbe(workersInstance) {
     await new Promise((resolve) => {
@@ -1507,7 +1667,7 @@ var WorkerService = class {
         batchSize: this.partitionReconcilerBatchSize
       });
     } catch (error) {
-      logger4.warn("Job partition reconciliation failed; it will be retried.", { error });
+      logger5.warn("Job partition reconciliation failed; it will be retried.", { error });
       return 0;
     }
   }
@@ -1547,6 +1707,7 @@ var WorkerService = class {
       activeExecutions: /* @__PURE__ */ new Map(),
       staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
+      pendingFilterChecks: /* @__PURE__ */ new Map(),
       jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
       jobAcquisitionHintProbeCycle: 0,
       hasCompletedInitialJobAcquisitionHintProbe: false,
@@ -1565,7 +1726,7 @@ var WorkerService = class {
         if (workersInstance.stopPromise) return workersInstance.stopPromise;
         workersInstance.stopPromise = (async () => {
           var _a, _b;
-          logger4.info("Stopping job scheduler...");
+          logger5.info("Stopping job scheduler...");
           workersInstance.running = false;
           workersInstance.signalStop();
           (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
@@ -1590,17 +1751,17 @@ var WorkerService = class {
     const jobs = this.getJobs(config.jobs);
     await Promise.all(
       jobs.filter((job) => job.type === "recurrent").map(async (job) => {
-        logger4.debug(`Ensuring records for job "${job.name}"...`);
+        logger5.debug(`Ensuring records for job "${job.name}"...`);
         await this.jobsRepo.ensureJobRecord(job, getRandomPartition(config.nPartitions));
       })
     );
   }
   async runScheduler(config, workersInstance) {
-    logger4.debug("Will ensure maxTriesReached retention and TTL index");
-    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger4.error("Error ensuring maxTriesReached retention", { error }));
-    logger4.debug("Will ensure records for recurrent jobs");
+    logger5.debug("Will ensure maxTriesReached retention and TTL index");
+    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger5.error("Error ensuring maxTriesReached retention", { error }));
+    logger5.debug("Will ensure records for recurrent jobs");
     await this.ensureRecords(config);
-    logger4.info(`Starting job scheduler with concurrency ${config.workersCount}`);
+    logger5.info(`Starting job scheduler with concurrency ${config.workersCount}`);
     await this.runSchedulerLoop(config, workersInstance);
   }
   /**
@@ -1626,7 +1787,7 @@ var WorkerService = class {
     setNameToJobs(config.jobs, config.nPartitions);
     (_b = (_a = this.jobsRepo).configureJobPartitions) == null ? void 0 : _b.call(_a, this.getJobNames(config.jobs), config.nPartitions);
     const workersInstance = this.createWorkersInstanceDefinition(config);
-    logger4.debug("Starting job scheduler", config);
+    logger5.debug("Starting job scheduler", config);
     workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
       config,
       workersInstance
@@ -1643,7 +1804,7 @@ var WorkerService = class {
       workersInstance.signalStop();
       (_a2 = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a2.call(workersInstance);
       (_b2 = workersInstance.cancelPartitionReconcilerWait) == null ? void 0 : _b2.call(workersInstance);
-      logger4.error("Job scheduler stopped after an unexpected error.", { error });
+      logger5.error("Job scheduler stopped after an unexpected error.", { error });
     });
     return workersInstance;
   }