@orion-js/dogs 4.6.6 → 4.6.8

package/dist/index.cjs

@@ -524,6 +524,17 @@ var JobsRepo = class {
     );
     return result.matchedCount > 0;
   }
+  async rescheduleAfterExecutionGuardAcquisitionFailure(jobId, lockId, nextRunAt) {
+    const result = await this.jobs.updateOne(
+      { _id: jobId, lockId, tries: { $gte: 1 } },
+      {
+        $set: { nextRunAt },
+        $inc: { tries: -1 },
+        $unset: { lockId: "", lockedUntil: "", claimWasStale: "" }
+      }
+    );
+    return result.matchedCount > 0;
+  }
   async deleteEventJob(jobId, lockId) {
     const result = await this.jobs.deleteOne({
       _id: jobId,
@@ -894,39 +905,102 @@ var Executor = class {
     return job.lockTime ?? jobToRun.lockTime;
   }
   getContext(job, jobToRun, onStale, isExecutionActive) {
-    var _a;
     const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
-    let staleTimeout = setTimeout(() => onStale(), effectiveLockTime);
-    (_a = staleTimeout.unref) == null ? void 0 : _a.call(staleTimeout);
     const contextLogger = import_logger3.logger.addMetadata({
       jobName: jobToRun.name,
       jobId: jobToRun.jobId
     });
+    let staleTimeout;
+    let watchdogCleared = false;
+    let staleReported = false;
+    let pendingExtraTime;
+    let renewalRunning = false;
+    let renewalPromise;
+    const clearCurrentStaleTimeout = () => {
+      if (staleTimeout === void 0) return;
+      clearTimeout(staleTimeout);
+      staleTimeout = void 0;
+    };
+    const reportStale = () => {
+      if (staleReported) return;
+      staleReported = true;
+      watchdogCleared = true;
+      pendingExtraTime = void 0;
+      clearCurrentStaleTimeout();
+      onStale();
+    };
+    const armStaleTimeout = (lockedUntil) => {
+      var _a;
+      if (watchdogCleared || staleReported || !isExecutionActive()) return;
+      clearCurrentStaleTimeout();
+      staleTimeout = setTimeout(
+        () => {
+          staleTimeout = void 0;
+          reportStale();
+        },
+        Math.max(0, lockedUntil - Date.now())
+      );
+      (_a = staleTimeout.unref) == null ? void 0 : _a.call(staleTimeout);
+    };
+    const runRenewals = async () => {
+      try {
+        while (pendingExtraTime !== void 0) {
+          const extraTime = pendingExtraTime;
+          pendingExtraTime = void 0;
+          if (watchdogCleared || staleReported || !isExecutionActive()) {
+            clearCurrentStaleTimeout();
+            return;
+          }
+          clearCurrentStaleTimeout();
+          const lockedUntil = Date.now() + extraTime;
+          let didExtend = false;
+          try {
+            didExtend = await this.jobsRepo.extendLockTime(
+              jobToRun.jobId,
+              extraTime,
+              jobToRun.lockId
+            );
+          } catch (error) {
+            reportStale();
+            throw error;
+          }
+          if (!didExtend) {
+            reportStale();
+            return;
+          }
+          if (watchdogCleared || staleReported || !isExecutionActive()) {
+            pendingExtraTime = void 0;
+            clearCurrentStaleTimeout();
+            return;
+          }
+          armStaleTimeout(lockedUntil);
+        }
+      } finally {
+        renewalRunning = false;
+        renewalPromise = void 0;
+      }
+    };
+    armStaleTimeout(Date.now() + effectiveLockTime);
     return {
       definition: job,
       record: jobToRun,
       tries: jobToRun.tries || 0,
-      clearStaleTimeout: () => clearTimeout(staleTimeout),
-      extendLockTime: async (extraTime) => {
-        var _a2;
-        if (!isExecutionActive()) {
+      clearStaleTimeout: () => {
+        watchdogCleared = true;
+        pendingExtraTime = void 0;
+        clearCurrentStaleTimeout();
+      },
+      extendLockTime: (extraTime) => {
+        if (watchdogCleared || staleReported || !isExecutionActive()) {
           contextLogger.warn(`Will not extend lock for stale execution "${jobToRun.executionId}"`);
-          return;
-        }
-        clearTimeout(staleTimeout);
-        let didExtend = false;
-        try {
-          didExtend = await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime, jobToRun.lockId);
-        } catch (error) {
-          onStale();
-          throw error;
+          return Promise.resolve();
         }
-        if (!didExtend) {
-          onStale();
-          return;
+        pendingExtraTime = Math.max(pendingExtraTime ?? extraTime, extraTime);
+        if (!renewalRunning) {
+          renewalRunning = true;
+          renewalPromise = runRenewals();
         }
-        staleTimeout = setTimeout(() => onStale(), extraTime);
-        (_a2 = staleTimeout.unref) == null ? void 0 : _a2.call(staleTimeout);
+        return renewalPromise ?? Promise.resolve();
       },
       logger: contextLogger
     };
@@ -1061,6 +1135,42 @@ var Executor = class {
     }
     return false;
   }
+  async handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context) {
+    context.logger.error(`Error acquiring execution guard for job "${jobToRun.name}"`, { error });
+    const nextRunAt = new Date(Date.now() + (config.executionGuardRetryDelayMs ?? 1e3));
+    try {
+      const didReschedule = await this.jobsRepo.rescheduleAfterExecutionGuardAcquisitionFailure(
+        jobToRun.jobId,
+        jobToRun.lockId,
+        nextRunAt
+      );
+      if (!didReschedule) {
+        context.logger.warn(
+          `Could not requeue job "${jobToRun.name}" after execution guard acquisition failed because its lock is no longer owned`
+        );
+      }
+    } catch (rescheduleError) {
+      context.logger.error(
+        `Error requeueing job "${jobToRun.name}" after execution guard acquisition failed`,
+        { error: rescheduleError }
+      );
+    }
+  }
+  async releaseExecutionGuardLease(lease, jobToRun, context) {
+    try {
+      await lease.release();
+    } catch (error) {
+      context.logger.error(`Error releasing execution guard for job "${jobToRun.name}"`, { error });
+    }
+  }
+  async runGuardedResolver(job, jobToRun, context, lease) {
+    try {
+      return await job.resolve(jobToRun.params, context);
+    } finally {
+      context.clearStaleTimeout();
+      await this.releaseExecutionGuardLease(lease, jobToRun, context);
+    }
+  }
   async executeJob(config, jobToRun, onExecutionStale) {
     const job = this.getJobDefinition(jobToRun, config.jobs);
     if (!job) return;
@@ -1142,13 +1252,36 @@ var Executor = class {
           params: jobToRun.params
         };
         await (0, import_logger3.runWithOrionAsyncContext)(extraContext, async () => {
-          try {
+          let executionGuardLease;
+          if (config.executionGuard) {
             (0, import_logger3.updateOrionAsyncContext)({
               jobName: jobToRun.name,
               jobId: jobToRun.jobId,
               params: jobToRun.params
             });
-            const result = await job.resolve(jobToRun.params, context);
+            try {
+              executionGuardLease = await config.executionGuard.acquire(jobToRun);
+            } catch (error) {
+              context.clearStaleTimeout();
+              await this.handleExecutionGuardAcquisitionFailure(error, config, jobToRun, context);
+              if (executionStatus === "stale") await staleHandlingPromise;
+              return;
+            }
+            if (executionStatus === "stale") {
+              await staleHandlingPromise;
+              await this.releaseExecutionGuardLease(executionGuardLease, jobToRun, context);
+              return;
+            }
+          }
+          try {
+            if (!config.executionGuard) {
+              (0, import_logger3.updateOrionAsyncContext)({
+                jobName: jobToRun.name,
+                jobId: jobToRun.jobId,
+                params: jobToRun.params
+              });
+            }
+            const result = executionGuardLease ? await this.runGuardedResolver(job, jobToRun, context, executionGuardLease) : await job.resolve(jobToRun.params, context);
             context.clearStaleTimeout();
             if (executionStatus === "stale") {
               await staleHandlingPromise;
@@ -1327,7 +1460,11 @@ var WorkerService = class {
       jobs: config.jobs,
       maxTries: config.maxTries,
       maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
-      onMaxTriesReached: config.onMaxTriesReached
+      onMaxTriesReached: config.onMaxTriesReached,
+      ...config.executionGuard ? {
+        executionGuard: config.executionGuard,
+        executionGuardRetryDelayMs: Math.max(config.pollInterval, 1e3)
+      } : {}
     };
     import_logger4.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
     let consecutivePartitionMisses = 0;