@orion-js/dogs 4.6.8 → 4.6.9

package/dist/index.cjs

@@ -883,7 +883,7 @@ function scheduleJobsInternal(jobs, nPartitions) {
 
 // src/services/WorkerService.ts
 var import_helpers2 = require("@orion-js/helpers");
-var import_logger4 = require("@orion-js/logger");
+var import_logger5 = require("@orion-js/logger");
 var import_services4 = require("@orion-js/services");
 
 // src/services/Executor.ts
@@ -1350,6 +1350,76 @@ __decorateElement(_init4, 5, "jobsHistoryRepo", _jobsHistoryRepo_dec, Executor);
 Executor = __decorateElement(_init4, 0, "Executor", _Executor_decorators, Executor);
 __runInitializers(_init4, 1, Executor);
 
+// src/services/jobFilters.ts
+var import_logger4 = require("@orion-js/logger");
+var DEFAULT_JOBS_FILTER_TIMEOUT_MS = 5e3;
+var warnedUnknownJobNames = /* @__PURE__ */ new Set();
+async function withTimeout(promise, timeoutMs, name) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`${name} timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function runCheck(options, name, run) {
+  if (options.pendingChecks.has(name)) {
+    throw new Error(`${name} is still running from a previous claim`);
+  }
+  const promise = Promise.resolve().then(run);
+  options.pendingChecks.set(name, promise);
+  void promise.then(
+    () => options.pendingChecks.delete(name),
+    () => options.pendingChecks.delete(name)
+  );
+  return await withTimeout(promise, options.timeoutMs, name);
+}
+function warnUnknownJobNames(jobs, filter) {
+  for (const jobName of [...filter.allow ?? [], ...filter.deny ?? []]) {
+    if (jobs[jobName] || warnedUnknownJobNames.has(jobName)) continue;
+    warnedUnknownJobNames.add(jobName);
+    import_logger4.logger.warn(`getJobsFilter returned unknown job name "${jobName}"; it will be ignored.`);
+  }
+}
+async function filterByJobsFilter(options, getJobsFilter) {
+  try {
+    const filter = await runCheck(options, "getJobsFilter", getJobsFilter) ?? {};
+    warnUnknownJobNames(options.jobs, filter);
+    return options.jobNames.filter((jobName) => {
+      var _a;
+      if (filter.allow && !filter.allow.includes(jobName)) return false;
+      if ((_a = filter.deny) == null ? void 0 : _a.includes(jobName)) return false;
+      return true;
+    });
+  } catch (error) {
+    import_logger4.logger.error("Error in getJobsFilter; no jobs will be taken on this claim.", { error });
+    return [];
+  }
+}
+async function canTakeJob(options, jobName) {
+  const canTakeJobs = options.jobs[jobName].canTakeJobs;
+  if (!canTakeJobs) return true;
+  try {
+    return await runCheck(options, `canTakeJobs of "${jobName}"`, canTakeJobs) === true;
+  } catch (error) {
+    import_logger4.logger.error(`Error in canTakeJobs of job "${jobName}"; it will not be taken.`, { error });
+    return false;
+  }
+}
+async function getTakeableJobNames(inputOptions) {
+  const options = {
+    timeoutMs: DEFAULT_JOBS_FILTER_TIMEOUT_MS,
+    pendingChecks: /* @__PURE__ */ new Map(),
+    ...inputOptions
+  };
+  const jobNames = inputOptions.getJobsFilter ? await filterByJobsFilter(options, inputOptions.getJobsFilter) : options.jobNames;
+  const checks = await Promise.all(jobNames.map((jobName) => canTakeJob(options, jobName)));
+  return jobNames.filter((_, index) => checks[index]);
+}
+
 // src/services/WorkerService.ts
 var FILTERED_JOB_NAMES_ACQUISITION_HINT = "byJobName";
 var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
@@ -1361,6 +1431,7 @@ var WorkerService = class {
     __publicField(this, "partitionReconcilerJitterMs", 30 * 1e3);
     __publicField(this, "partitionReconcilerBatchPauseMs", 250);
     __publicField(this, "partitionReconcilerBatchSize", 500);
+    __publicField(this, "jobsFilterTimeoutMs", DEFAULT_JOBS_FILTER_TIMEOUT_MS);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -1419,13 +1490,13 @@ var WorkerService = class {
     };
     this.reserveJobExecution(workersInstance, jobToRun.name);
     workersInstance.activeExecutions.set(jobToRun.executionId, execution);
-    import_logger4.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
+    import_logger5.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
     const executionPromise = this.executor.executeJob(config, jobToRun, () => {
       this.releaseExecutionCapacity(workersInstance, execution, "stale");
     });
     execution.promise = executionPromise;
     void executionPromise.catch((error) => {
-      import_logger4.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
+      import_logger5.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
     }).finally(() => {
       if (execution.status === "running") {
         this.releaseExecutionCapacity(workersInstance, execution, "finished");
@@ -1466,18 +1537,33 @@ var WorkerService = class {
         executionGuardRetryDelayMs: Math.max(config.pollInterval, 1e3)
       } : {}
     };
-    import_logger4.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
+    import_logger5.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
     let consecutivePartitionMisses = 0;
     while (workersInstance.running) {
       let didMiss = false;
       let allJobNamesAtCapacity = false;
       try {
         while (workersInstance.activeExecutions.size < config.workersCount) {
-          const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
-          if (availableJobNames.length === 0) {
+          const jobNamesWithCapacity = this.getAvailableJobNames(config, workersInstance, jobNames);
+          if (jobNamesWithCapacity.length === 0) {
             allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
             break;
           }
+          const availableJobNames = await Promise.race([
+            getTakeableJobNames({
+              jobs: config.jobs,
+              getJobsFilter: config.getJobsFilter,
+              jobNames: jobNamesWithCapacity,
+              timeoutMs: this.jobsFilterTimeoutMs,
+              pendingChecks: workersInstance.pendingFilterChecks
+            }),
+            workersInstance.stopSignal.then(() => [])
+          ]);
+          if (!workersInstance.running) break;
+          if (availableJobNames.length === 0) {
+            didMiss = true;
+            break;
+          }
           const jobAcquisitionHint = availableJobNames.length < jobNames.length ? FILTERED_JOB_NAMES_ACQUISITION_HINT : workersInstance.jobAcquisitionHint;
           const partition = this.getNextPartition(config, workersInstance);
           const jobToRun = await this.jobsRepo.getJobAndLock(
@@ -1488,7 +1574,7 @@ var WorkerService = class {
           );
           if (!jobToRun) {
             consecutivePartitionMisses++;
-            import_logger4.logger.debug(`No job to run in partition ${partition}`);
+            import_logger5.logger.debug(`No job to run in partition ${partition}`);
             if (consecutivePartitionMisses >= config.nPartitions) {
               consecutivePartitionMisses = 0;
               didMiss = true;
@@ -1501,7 +1587,7 @@ var WorkerService = class {
           if (!workersInstance.running) break;
         }
       } catch (error) {
-        import_logger4.logger.error("Error in job scheduler.", { error });
+        import_logger5.logger.error("Error in job scheduler.", { error });
         consecutivePartitionMisses = 0;
         didMiss = true;
       }
@@ -1553,7 +1639,7 @@ var WorkerService = class {
     } catch (error) {
       this.resetPendingJobAcquisitionHint(workersInstance);
       if (workersInstance.running) {
-        import_logger4.logger.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+        import_logger5.logger.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
       }
       return;
     }
@@ -1566,12 +1652,12 @@ var WorkerService = class {
       workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
       workersInstance.jobAcquisitionHint = winner;
       this.resetPendingJobAcquisitionHint(workersInstance);
-      import_logger4.logger.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      import_logger5.logger.info(`Selected job acquisition hint "${winner}" after startup probe.`);
       return;
     }
     if (winner === currentHint) {
       this.resetPendingJobAcquisitionHint(workersInstance);
-      import_logger4.logger.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      import_logger5.logger.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
       return;
     }
     if (workersInstance.pendingJobAcquisitionHint === winner) {
@@ -1581,14 +1667,14 @@ var WorkerService = class {
       workersInstance.pendingJobAcquisitionHintWins = 1;
     }
     if (workersInstance.pendingJobAcquisitionHintWins < 2) {
-      import_logger4.logger.debug(
+      import_logger5.logger.debug(
         `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
       );
       return;
     }
     workersInstance.jobAcquisitionHint = winner;
     this.resetPendingJobAcquisitionHint(workersInstance);
-    import_logger4.logger.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+    import_logger5.logger.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
   }
   async waitForNextJobAcquisitionHintProbe(workersInstance) {
     await new Promise((resolve) => {
@@ -1625,7 +1711,7 @@ var WorkerService = class {
         batchSize: this.partitionReconcilerBatchSize
       });
     } catch (error) {
-      import_logger4.logger.warn("Job partition reconciliation failed; it will be retried.", { error });
+      import_logger5.logger.warn("Job partition reconciliation failed; it will be retried.", { error });
       return 0;
     }
   }
@@ -1665,6 +1751,7 @@ var WorkerService = class {
       activeExecutions: /* @__PURE__ */ new Map(),
       staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
+      pendingFilterChecks: /* @__PURE__ */ new Map(),
       jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
       jobAcquisitionHintProbeCycle: 0,
       hasCompletedInitialJobAcquisitionHintProbe: false,
@@ -1683,7 +1770,7 @@ var WorkerService = class {
         if (workersInstance.stopPromise) return workersInstance.stopPromise;
         workersInstance.stopPromise = (async () => {
           var _a, _b;
-          import_logger4.logger.info("Stopping job scheduler...");
+          import_logger5.logger.info("Stopping job scheduler...");
           workersInstance.running = false;
           workersInstance.signalStop();
           (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
@@ -1708,17 +1795,17 @@ var WorkerService = class {
     const jobs = this.getJobs(config.jobs);
     await Promise.all(
       jobs.filter((job) => job.type === "recurrent").map(async (job) => {
-        import_logger4.logger.debug(`Ensuring records for job "${job.name}"...`);
+        import_logger5.logger.debug(`Ensuring records for job "${job.name}"...`);
         await this.jobsRepo.ensureJobRecord(job, getRandomPartition(config.nPartitions));
       })
     );
   }
   async runScheduler(config, workersInstance) {
-    import_logger4.logger.debug("Will ensure maxTriesReached retention and TTL index");
-    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger4.logger.error("Error ensuring maxTriesReached retention", { error }));
-    import_logger4.logger.debug("Will ensure records for recurrent jobs");
+    import_logger5.logger.debug("Will ensure maxTriesReached retention and TTL index");
+    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger5.logger.error("Error ensuring maxTriesReached retention", { error }));
+    import_logger5.logger.debug("Will ensure records for recurrent jobs");
     await this.ensureRecords(config);
-    import_logger4.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
+    import_logger5.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
     await this.runSchedulerLoop(config, workersInstance);
   }
   /**
@@ -1744,7 +1831,7 @@ var WorkerService = class {
     setNameToJobs(config.jobs, config.nPartitions);
     (_b = (_a = this.jobsRepo).configureJobPartitions) == null ? void 0 : _b.call(_a, this.getJobNames(config.jobs), config.nPartitions);
     const workersInstance = this.createWorkersInstanceDefinition(config);
-    import_logger4.logger.debug("Starting job scheduler", config);
+    import_logger5.logger.debug("Starting job scheduler", config);
     workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
       config,
       workersInstance
@@ -1761,7 +1848,7 @@ var WorkerService = class {
       workersInstance.signalStop();
       (_a2 = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a2.call(workersInstance);
       (_b2 = workersInstance.cancelPartitionReconcilerWait) == null ? void 0 : _b2.call(workersInstance);
-      import_logger4.logger.error("Job scheduler stopped after an unexpected error.", { error });
+      import_logger5.logger.error("Job scheduler stopped after an unexpected error.", { error });
     });
     return workersInstance;
   }